From 883d8213eb4def1ddd3fdc1ad6c7544dd661193e Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:02:17 -0700 Subject: [PATCH 1/8] fix(cua): stabilize Linux window previews and turn cleanup --- apps/desktop/src/cua/DesktopCuaDriver.test.ts | 3 + apps/desktop/src/cua/DesktopCuaDriver.ts | 41 +++- apps/server/src/cua/CuaDriver.test.ts | 6 + apps/server/src/cua/CuaDriver.ts | 22 +- apps/server/src/cua/CuaWindowPreview.test.ts | 226 +++++++++++++++++- apps/server/src/cua/CuaWindowPreview.ts | 125 +++++----- apps/web/src/components/ChatView.tsx | 23 +- .../preview/ThreadPreviewMiniPlayer.tsx | 40 +++- .../src/server/cuaToolPresentation.test.ts | 27 +++ .../src/server/cuaToolPresentation.ts | 14 +- packages/shared/package.json | 4 + .../shared/src/cuaDriverHostOptions.test.ts | 33 +++ packages/shared/src/cuaDriverHostOptions.ts | 26 ++ 13 files changed, 491 insertions(+), 99 deletions(-) create mode 100644 packages/shared/src/cuaDriverHostOptions.test.ts create mode 100644 packages/shared/src/cuaDriverHostOptions.ts diff --git a/apps/desktop/src/cua/DesktopCuaDriver.test.ts b/apps/desktop/src/cua/DesktopCuaDriver.test.ts index adb9f1ab77733..9b6258b0afde6 100644 --- a/apps/desktop/src/cua/DesktopCuaDriver.test.ts +++ b/apps/desktop/src/cua/DesktopCuaDriver.test.ts @@ -64,6 +64,9 @@ const fixture = Effect.fn(function* ( }), loadEmbedded: async () => ({ EmbeddedCuaDriverHost: class { + static withOptions() { + return new this(); + } async start() { starts++; Queue.offerUnsafe(starting, undefined); diff --git a/apps/desktop/src/cua/DesktopCuaDriver.ts b/apps/desktop/src/cua/DesktopCuaDriver.ts index 9283a7944580c..51126fb135ce2 100644 --- a/apps/desktop/src/cua/DesktopCuaDriver.ts +++ b/apps/desktop/src/cua/DesktopCuaDriver.ts @@ -1,5 +1,10 @@ import { CuaDriverMcpConfiguration, type DesktopCuaDriverRequest } from "@t3tools/contracts"; -import type { EmbeddedCuaDriverHost, EmbeddedDriverConnection } from "@trycua/cua-driver/embedded"; +import { cuaDriverHostOptions } from "@t3tools/shared/cuaDriverHostOptions"; +import type { + EmbeddedCuaDriverHost, + EmbeddedDriverConnection, + EmbeddedDriverHostOptions, +} from "@trycua/cua-driver/embedded"; import * as Cause from "effect/Cause"; import * as Exit from "effect/Exit"; import * as Context from "effect/Context"; @@ -16,10 +21,11 @@ import * as DesktopTelemetryPublisher from "../telemetry/DesktopTelemetryPublish export interface DesktopCuaDriverDependencies { readonly loadEmbedded: () => Promise<{ - readonly EmbeddedCuaDriverHost: new ( - path: string, - hostBundleId: string, - ) => Pick; + readonly EmbeddedCuaDriverHost: { + readonly withOptions: ( + options: EmbeddedDriverHostOptions, + ) => Pick; + }; }>; readonly loadElectron: () => Promise< Pick< @@ -33,7 +39,21 @@ const decodeMcpConfiguration = Schema.decodeEffect(CuaDriverMcpConfiguration); // The SDK loads its native libraries from the archive's .unpacked sibling // through our @ubjs/node patch, so a packaged app imports it in place. const defaultDependencies: DesktopCuaDriverDependencies = { - loadEmbedded: () => import("@trycua/cua-driver/embedded"), + loadEmbedded: async () => { + const module = await import("@trycua/cua-driver/embedded"); + return { + EmbeddedCuaDriverHost: { + withOptions: (options) => { + const host = module.EmbeddedCuaDriverHost.withOptions(options); + // The generated factory's interface omits the native object's disposal method. + if (!(host instanceof module.EmbeddedCuaDriverHost)) { + throw new Error("Cua Driver returned an unexpected host object."); + } + return host; + }, + }, + }; + }, loadElectron: () => import("@trycua/cua-driver/electron"), }; @@ -217,7 +237,14 @@ export const make = Effect.fn("desktop.cuaDriver.make")(function* ( const module = yield* Effect.tryPromise(dependencies.loadEmbedded); const owned = yield* Effect.try(() => { const owned: OwnedHost = { - host: new module.EmbeddedCuaDriverHost(binaryPath.value, environment.appUserModelId), + host: module.EmbeddedCuaDriverHost.withOptions( + cuaDriverHostOptions( + binaryPath.value, + environment.appUserModelId, + environment.platform, + process.env, + ), + ), abort: new AbortController(), }; hosts.add(owned); diff --git a/apps/server/src/cua/CuaDriver.test.ts b/apps/server/src/cua/CuaDriver.test.ts index ba423c3569472..ea3e1252c4aba 100644 --- a/apps/server/src/cua/CuaDriver.test.ts +++ b/apps/server/src/cua/CuaDriver.test.ts @@ -56,6 +56,9 @@ const standaloneFixture = Effect.fn(function* (failFirstDestroy = false) { Effect.succeed("/driver"), Effect.succeed({ EmbeddedCuaDriverHost: class { + static withOptions() { + return new this(); + } constructor() { creates++; } @@ -127,6 +130,9 @@ describe("Cua Driver failure boundaries", () => { Effect.succeed("/driver"), Effect.succeed({ EmbeddedCuaDriverHost: class { + static withOptions() { + return new this(); + } constructor() { throw cause; } diff --git a/apps/server/src/cua/CuaDriver.ts b/apps/server/src/cua/CuaDriver.ts index 7d8a3117ae44e..b1501f17d3787 100644 --- a/apps/server/src/cua/CuaDriver.ts +++ b/apps/server/src/cua/CuaDriver.ts @@ -6,7 +6,12 @@ import type { DesktopCuaDriverReport, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; -import type { EmbeddedCuaDriverHost, EmbeddedDriverConnection } from "@trycua/cua-driver/embedded"; +import { cuaDriverHostOptions } from "@t3tools/shared/cuaDriverHostOptions"; +import type { + EmbeddedCuaDriverHost, + EmbeddedDriverConnection, + EmbeddedDriverHostOptions, +} from "@trycua/cua-driver/embedded"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as Deferred from "effect/Deferred"; @@ -314,10 +319,11 @@ export const make = Effect.fn("CuaDriver.make")(function* ( }); type StandaloneHostModule = { - readonly EmbeddedCuaDriverHost: new ( - path: string, - hostBundleId: string, - ) => Pick; + readonly EmbeddedCuaDriverHost: { + readonly withOptions: ( + options: EmbeddedDriverHostOptions, + ) => Pick; + }; }; // A Node single-executable cannot `import()` the external SDK from disk, and the SDK exposes @@ -351,6 +357,7 @@ export const makeStandaloneHostFactory = Effect.fn("CuaDriver.standaloneHostFact resolveBinary: Effect.Effect, loadEmbedded: Effect.Effect, ) { + const platform = yield* HostProcess.Platform; const mutex = yield* Semaphore.make(1); let occupied = false; // Return a lazy acquisition effect so one factory owns all subsequent host attempts. @@ -362,7 +369,10 @@ export const makeStandaloneHostFactory = Effect.fn("CuaDriver.standaloneHostFact const binaryPath = yield* resolveBinary; const { EmbeddedCuaDriverHost } = yield* loadEmbedded; const host = yield* Effect.try({ - try: () => new EmbeddedCuaDriverHost(binaryPath, "com.t3tools.t3code.server"), + try: () => + EmbeddedCuaDriverHost.withOptions( + cuaDriverHostOptions(binaryPath, "com.t3tools.t3code.server", platform, process.env), + ), catch: (cause) => new CuaDriverHostCreateError({ binaryPath, cause }), }); occupied = true; diff --git a/apps/server/src/cua/CuaWindowPreview.test.ts b/apps/server/src/cua/CuaWindowPreview.test.ts index 1c1db464e3ef4..149fe513d630e 100644 --- a/apps/server/src/cua/CuaWindowPreview.test.ts +++ b/apps/server/src/cua/CuaWindowPreview.test.ts @@ -9,11 +9,15 @@ import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { describe, expect } from "vite-plus/test"; +import { + clearCuaToolContext, + cuaToolPresentation, +} from "@t3tools/provider-core/server/cuaToolPresentation"; import type { McpProviderSessionConfig } from "@t3tools/provider-core/server/mcpSession"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; -import { cuaToolPresentation } from "@t3tools/provider-core/server/cuaToolPresentation"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as CuaWindowPreview from "./CuaWindowPreview.ts"; @@ -27,22 +31,31 @@ const turnEvent = (type: "turn.started" | "turn.completed") => payload: { status: type === "turn.started" ? "running" : "completed" }, }) as unknown as OrchestrationV2DomainEvent; -const makeHarness = Effect.fn(function* () { +const makeHarness = Effect.fn(function* ( + windowFailure?: "missing" | "throw" | "transient", + onWindowCapture?: () => void | Promise, +) { const events = yield* PubSub.unbounded(); const captures: string[] = []; + const windows: Array<{ pid: number; windowId: bigint }> = []; let clients = 0; let destroyed = 0; const fakeClient = { - getWindowState: (input: { pid: number; windowId: bigint }) => - Promise.resolve({ + getWindowState: async (input: { pid: number; windowId: bigint }) => { + windows.push(input); + await onWindowCapture?.(); + if (windowFailure === "throw" || (windowFailure === "transient" && windows.length < 3)) + return Promise.reject(new Error("Window capture interrupted")); + return Promise.resolve({ pid: input.pid, windowId: input.windowId, appName: "Calendar", windowTitle: "December 2026", screenshotWidth: 640, screenshotHeight: 400, - images: [{ mimeType: "image/png", dataBase64: png }], - }), + images: windowFailure === "missing" ? [] : [{ mimeType: "image/png", dataBase64: png }], + }); + }, getDesktopState: () => { captures.push("desktop"); return Promise.resolve({ @@ -80,6 +93,7 @@ const makeHarness = Effect.fn(function* () { service, emit: (event: OrchestrationV2DomainEvent) => PubSub.publish(events, event), captures, + windows, counts: () => ({ clients, destroyed }), }; }); @@ -97,10 +111,61 @@ const setCuaSession = Effect.flatMap(McpProviderSessions.McpProviderSessions, (s ); describe("CuaWindowPreview", () => { - it.live("captures only while a client subscribes during a Cua turn", () => + it.effect("never overlaps captures and backs off after a slow frame", () => Effect.gen(function* () { + clearCuaToolContext(threadId); yield* setCuaSession; + yield* Effect.addFinalizer(() => Effect.sync(() => clearCuaToolContext(threadId))); + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 42, window_id: 7 }, + status: "completed", + }); + const gate = Promise.withResolvers(); + const firstStarted = yield* Deferred.make(); + const secondStarted = yield* Deferred.make(); + const firstPublished = yield* Deferred.make(); + let calls = 0; + const harness = yield* makeHarness(undefined, () => { + calls += 1; + Deferred.doneUnsafe(calls === 1 ? firstStarted : secondStarted, Effect.void); + return calls === 1 ? gate.promise : undefined; + }); + yield* harness.service.stream(threadId).pipe( + Stream.runForEach((state) => + state.frame ? Deferred.succeed(firstPublished, undefined) : Effect.void, + ), + Effect.forkScoped, + ); + yield* Effect.yieldNow; + yield* harness.emit(turnEvent("turn.started")); + yield* Deferred.await(firstStarted); + yield* TestClock.adjust(40); + expect(calls).toBe(1); + gate.resolve(); + yield* Deferred.await(firstPublished); + yield* TestClock.adjust(39); + expect(calls).toBe(1); + yield* TestClock.adjust(1); + yield* Deferred.await(secondStarted); + expect(calls).toBe(2); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("stops capture at turn end and waits for a fresh window in the next turn", () => + Effect.gen(function* () { + yield* setCuaSession; + clearCuaToolContext(threadId); + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 42, window_id: 7 }, + status: "completed", + }); const harness = yield* makeHarness(); + const waiting = yield* Deferred.make(); + const resumed = yield* Deferred.make(); const live = yield* Deferred.make(); const idle = yield* Deferred.make(); const states: string[] = []; @@ -108,7 +173,13 @@ describe("CuaWindowPreview", () => { Stream.tap((state) => Effect.gen(function* () { states.push(state.status); - if (state.status === "live") yield* Deferred.succeed(live, undefined); + if (state.status === "live" && state.frame) { + yield* Deferred.succeed(live, undefined); + if (harness.windows.at(-1)?.windowId === 8n) + yield* Deferred.succeed(resumed, undefined); + } + if (state.status === "live" && !state.frame) + yield* Deferred.succeed(waiting, undefined); if (state.status === "idle" && states.length > 1) yield* Deferred.succeed(idle, undefined); }), @@ -123,11 +194,25 @@ describe("CuaWindowPreview", () => { yield* harness.emit(turnEvent("turn.started")); yield* Deferred.await(live); expect(harness.counts().clients).toBe(1); - expect(harness.captures.length).toBeGreaterThan(0); + expect(harness.windows.length).toBeGreaterThan(0); yield* harness.emit(turnEvent("turn.completed")); yield* Deferred.await(idle); expect(harness.counts().destroyed).toBe(1); + const capturesAtIdle = harness.windows.length; + yield* harness.emit(turnEvent("turn.started")); + yield* Deferred.await(waiting); + expect(harness.windows.length).toBe(capturesAtIdle); + expect(harness.captures).toEqual([]); + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 43, window_id: 8 }, + status: "completed", + }); + yield* Deferred.await(resumed); + expect(harness.windows.at(-1)).toMatchObject({ pid: 43, windowId: 8n }); + expect(states).not.toContain("unavailable"); yield* Fiber.interrupt(subscriber); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -172,4 +257,127 @@ describe("CuaWindowPreview", () => { expect(harness.counts().clients).toBe(0); }).pipe(Effect.scoped, Effect.provide(layer)), ); + + it.live("follows window selection and switching after capture has started", () => + Effect.gen(function* () { + clearCuaToolContext(threadId); + yield* setCuaSession; + yield* Effect.addFinalizer(() => Effect.sync(() => clearCuaToolContext(threadId))); + const harness = yield* makeHarness(); + const waiting = yield* Deferred.make(); + const firstWindow = yield* Deferred.make(); + const secondWindow = yield* Deferred.make(); + yield* harness.service.stream(threadId).pipe( + Stream.runForEach((state) => + Effect.gen(function* () { + if (state.status !== "live") return; + const window = harness.windows.at(-1); + if (!window) yield* Deferred.succeed(waiting, undefined); + else if (window.windowId === 7n) yield* Deferred.succeed(firstWindow, undefined); + else if (window.windowId === 8n) yield* Deferred.succeed(secondWindow, undefined); + }), + ), + Effect.forkScoped, + ); + yield* Effect.yieldNow; + yield* harness.emit(turnEvent("turn.started")); + yield* Deferred.await(waiting); + expect(harness.captures).toEqual([]); + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 42, window_id: 7 }, + status: "completed", + }); + yield* Deferred.await(firstWindow); + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 43, window_id: 8 }, + status: "completed", + }); + yield* Deferred.await(secondWindow); + expect(harness.windows.at(-1)).toMatchObject({ pid: 43, windowId: 8n }); + expect(harness.counts().clients).toBe(1); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("discards a completed capture after the agent switches windows", () => + Effect.gen(function* () { + yield* setCuaSession; + yield* Effect.addFinalizer(() => Effect.sync(() => clearCuaToolContext(threadId))); + const selectWindow = (windowId: number) => + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 42, window_id: windowId }, + status: "completed", + }); + selectWindow(7); + const harness = yield* makeHarness(undefined, () => { + selectWindow(8); + }); + const first = yield* harness.service.stream(threadId).pipe( + Stream.filter((state) => state.status !== "idle"), + Stream.runHead, + Effect.forkScoped, + ); + yield* Effect.yieldNow; + yield* harness.emit(turnEvent("turn.started")); + const state = yield* Fiber.join(first); + expect(state._tag === "Some" ? state.value.status : null).toBe("live"); + expect(harness.windows.map((window) => window.windowId)).toEqual([7n, 8n]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("retries transient capture failures without publishing unavailable", () => + Effect.gen(function* () { + yield* setCuaSession; + yield* Effect.addFinalizer(() => Effect.sync(() => clearCuaToolContext(threadId))); + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 42, window_id: 7 }, + status: "completed", + }); + const harness = yield* makeHarness("transient"); + const first = yield* harness.service.stream(threadId).pipe( + Stream.filter((state) => state.status !== "idle"), + Stream.runHead, + Effect.forkScoped, + ); + yield* Effect.yieldNow; + yield* harness.emit(turnEvent("turn.started")); + const state = yield* Fiber.join(first); + expect(state._tag === "Some" ? state.value.status : null).toBe("live"); + expect(harness.windows).toHaveLength(3); + expect(harness.captures).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live.each(["missing", "throw"] as const)( + "does not substitute desktop pixels when window capture is %s", + (failure) => + Effect.gen(function* () { + yield* setCuaSession; + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/click", + args: { pid: 42, window_id: "7", x: 1, y: 1 }, + status: "completed", + }); + const harness = yield* makeHarness(failure); + const unavailable = yield* harness.service.stream(threadId).pipe( + Stream.filter((state) => state.status !== "idle"), + Stream.runHead, + Effect.forkScoped, + ); + yield* Effect.yieldNow; + yield* harness.emit(turnEvent("turn.started")); + const state = yield* Fiber.join(unavailable); + expect(state._tag === "Some" ? state.value.status : null).toBe("unavailable"); + expect(state._tag === "Some" ? state.value.frame : null).toBeUndefined(); + expect(harness.captures).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); }); diff --git a/apps/server/src/cua/CuaWindowPreview.ts b/apps/server/src/cua/CuaWindowPreview.ts index 987b3ac23eede..af3b042365b73 100644 --- a/apps/server/src/cua/CuaWindowPreview.ts +++ b/apps/server/src/cua/CuaWindowPreview.ts @@ -7,6 +7,7 @@ import type { ThreadId, } from "@t3tools/contracts"; import type * as CuaSdk from "@trycua/cua-driver"; +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; @@ -14,7 +15,6 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PubSub from "effect/PubSub"; import * as Schedule from "effect/Schedule"; @@ -26,6 +26,7 @@ import * as Stream from "effect/Stream"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import { + clearCuaWindowTarget, readCuaWindowTarget, type CuaWindowTarget, } from "@t3tools/provider-core/server/cuaToolPresentation"; @@ -56,18 +57,12 @@ export class CuaWindowPreview extends Context.Service< } >()("t3/cua/CuaWindowPreview") {} -export type SdkModule = Pick< - typeof CuaSdk, - "CuaDriver" | "GetWindowStateInput" | "GetDesktopStateInput" ->; +export type SdkModule = Pick; // `connect` is typed as the interface, but every implementation is the // generated class with the native handle to release. -type SdkClient = Pick< - CuaSdk.CuaDriver, - "getWindowState" | "getDesktopState" | "shutdown" | "uniffiDestroy" ->; +type SdkClient = Pick; -const REFRESH_INTERVAL = Duration.seconds(1.5); +const REFRESH_INTERVAL = Duration.millis(250); /** Long edge of the preview frame; small enough for a floating card over the app WS. */ const MAX_DIMENSION = 800; const IDLE_STATE: CuaWindowPreviewState = { status: "idle" }; @@ -94,58 +89,38 @@ const loadSdk = Effect.fn("CuaWindowPreview.loadSdk")(function* () { return yield* new CuaWindowPreviewSdkError({ cause: "@trycua/cua-driver is not installed." }); }); -/** - * Captures the window the agent last targeted, or the whole desktop before it - * has named one. A window that disappeared falls back to the desktop rather - * than failing, so the card keeps showing something while the agent moves on. - */ +/** Captures only the selected window; desktop pixels cannot stand in for it. */ const captureFrame = Effect.fn("CuaWindowPreview.capture")(function* ( sdk: SdkModule, client: SdkClient, - target: CuaWindowTarget | undefined, + target: CuaWindowTarget, ) { const capturedAt = DateTime.formatIso(yield* DateTime.now); - if (target) { - const window = yield* Effect.tryPromise({ - try: (signal) => - client.getWindowState( - sdk.GetWindowStateInput.new({ - pid: target.pid, - windowId: target.windowId, - includeAccessibilityTree: false, - includeScreenshot: true, - maxDimension: MAX_DIMENSION, - }), - { signal }, - ), - catch: (cause) => new CuaWindowPreviewCaptureError({ cause }), - }).pipe(Effect.option); - const image = Option.isSome(window) ? window.value.images[0] : undefined; - if (Option.isSome(window) && image) { - return { - ...(window.value.appName ? { appName: window.value.appName } : {}), - ...(window.value.windowTitle ? { windowTitle: window.value.windowTitle } : {}), - width: window.value.screenshotWidth ?? 0, - height: window.value.screenshotHeight ?? 0, - mimeType: image.mimeType, - dataBase64: image.dataBase64, - capturedAt, - } satisfies CuaWindowPreviewFrame; - } - } - const desktop = yield* Effect.tryPromise({ - try: (signal) => client.getDesktopState(sdk.GetDesktopStateInput.new({}), { signal }), + const window = yield* Effect.tryPromise({ + try: (signal) => + client.getWindowState( + sdk.GetWindowStateInput.new({ + pid: target.pid, + windowId: target.windowId, + includeAccessibilityTree: false, + includeScreenshot: true, + maxDimension: MAX_DIMENSION, + }), + { signal }, + ), catch: (cause) => new CuaWindowPreviewCaptureError({ cause }), }); - const image = desktop.images[0]; - if (desktop.isError || !image) { + const image = window.images[0]; + if (!image) { return yield* new CuaWindowPreviewCaptureError({ - cause: desktop.errorCode ?? desktop.text ?? "no image", + cause: "Window capture is unavailable.", }); } return { - width: 0, - height: 0, + ...(window.appName ? { appName: window.appName } : {}), + ...(window.windowTitle ? { windowTitle: window.windowTitle } : {}), + width: window.screenshotWidth ?? 0, + height: window.screenshotHeight ?? 0, mimeType: image.mimeType, dataBase64: image.dataBase64, capturedAt, @@ -226,17 +201,42 @@ export const make = Effect.fn("CuaWindowPreview.make")(function* ( Effect.andThen(Effect.sync(() => client.uniffiDestroy())), ), ); - yield* captureFrame(module, client, readCuaWindowTarget(threadId)).pipe( - Effect.flatMap((frame) => publish(preview, { status: "live", frame })), - Effect.catchTags({ - CuaWindowPreviewCaptureError: (error) => - publish(preview, { - status: "unavailable", - detail: typeof error.cause === "string" ? error.cause : error.message, - }), - }), - Effect.repeat(Schedule.spaced(refreshInterval)), - ); + const capture = Effect.suspend(() => { + const target = readCuaWindowTarget(threadId); + if (!target) return publish(preview, { status: "live" }); + const isCurrent = () => { + const current = readCuaWindowTarget(threadId); + return current?.pid === target?.pid && current?.windowId === target?.windowId; + }; + return captureFrame(module, client, target).pipe( + // Compositor transitions can briefly fail a frame. Keep the current + // frame during bounded retries, but report persistent capture loss. + Effect.retry({ times: 2, schedule: Schedule.spaced("150 millis"), while: isCurrent }), + Effect.flatMap((frame) => + isCurrent() ? publish(preview, { status: "live", frame }) : Effect.void, + ), + Effect.catchTags({ + CuaWindowPreviewCaptureError: (error) => + isCurrent() + ? publish(preview, { + status: "unavailable", + detail: typeof error.cause === "string" ? error.cause : error.message, + }) + : Effect.void, + }), + ); + }); + return yield* Effect.gen(function* () { + const startedAt = yield* Clock.currentTimeMillis; + yield* capture; + const elapsed = Math.max(0, (yield* Clock.currentTimeMillis) - startedAt); + // Cap at 4 FPS, with at least as much idle time as capture time. + // Slow captures lower the rate instead of filling a queue or busy-looping. + const minimumInterval = readCuaWindowTarget(threadId) + ? Duration.toMillis(refreshInterval) + : 1500; + yield* Effect.sleep(Math.max(minimumInterval - elapsed, elapsed)); + }).pipe(Effect.forever); }, (effect, _threadId, preview) => effect.pipe( @@ -287,6 +287,7 @@ export const make = Effect.fn("CuaWindowPreview.make")(function* ( preview.live = (yield* mcpSessions.read(event.threadId))?.cuaDriver?.socketPath !== undefined; yield* reconcile(event.threadId); } else if (status !== "pending") { + clearCuaWindowTarget(event.threadId); const preview = threads.get(event.threadId); if (!preview) return; preview.live = false; diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 496a0228d6919..42d876f81e341 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -1950,17 +1950,30 @@ export default function ChatView(props: ChatViewProps) { // Each new call reopens a card the user closed; a browser or device already floating stays. const computerUse = selectLatestComputerUse(serverProjection?.turnItems ?? []); const lastShownComputerUseItemId = useRef(null); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Floats the computer card when the shown thread's projection reports a new computer use call. + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Floats the computer card for a new computer use call in the shown thread's projection and dismisses it when the run settles. useEffect(() => { - if (!activeThreadRef || !computerUse?.inProgress || shouldUsePlanSidebarSheet) return; + if (!activeThreadRef) return; + const current = activePreviewMiniPlayer; + if (latestRunSettled) { + if (current?.source.kind === "computer") { + usePreviewMiniPlayerStore.getState().close(activeThreadRef); + } + return; + } + if (!computerUse?.inProgress || shouldUsePlanSidebarSheet) return; if (lastShownComputerUseItemId.current === computerUse.itemId) return; lastShownComputerUseItemId.current = computerUse.itemId; if (!getClientSettings().browserAutoShowFloatingPreview) return; - const current = - usePreviewMiniPlayerStore.getState().byThreadKey[scopedThreadKey(activeThreadRef)]; if (current && current.source.kind !== "computer") return; usePreviewMiniPlayerStore.getState().open(activeThreadRef, COMPUTER_MINI_PLAYER_SOURCE); - }, [activeThreadRef, computerUse, shouldUsePlanSidebarSheet]); + }, [ + activePreviewMiniPlayer, + activeThreadRef, + computerUse, + lastShownComputerUseItemId, + latestRunSettled, + shouldUsePlanSidebarSheet, + ]); // The thread's own change request, placed against the project it belongs to. Without a // project there is nothing to resolve it against, so the caller falls back to the browser. const activeProjectRepository = sourceControlClients.repositorySelector( diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index f6e35d1f202a1..bcc438c48f09a 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -245,9 +245,9 @@ export function ThreadPreviewMiniPlayer({ /** * The window an agent drives on the host computer. While a computer use turn - * runs, the server captures that window every second and a half and streams - * it here; the capture runs only while this card is mounted. Between turns the - * card keeps the last frame instead of going blank. + * runs, the server captures that window at an adaptive rate and streams + * it here; the capture runs only while this card is mounted. Chat dismisses + * the card when the turn settles. */ function ComputerMiniPlayer({ threadRef, @@ -255,11 +255,35 @@ function ComputerMiniPlayer({ preview, }: Props & { readonly preview: ComputerUsePreview | null }) { const live = useCuaWindowPreview(threadRef); - const [lastFrame, setLastFrame] = useState(null); + const [decoded, setDecoded] = useState<{ + frame: CuaWindowPreviewFrame; + src: string; + } | null>(null); const liveFrame = live.status === "live" ? (live.frame ?? null) : null; - if (liveFrame !== null && liveFrame !== lastFrame) setLastFrame(liveFrame); - const frame = liveFrame ?? lastFrame; - const src = frame ? `data:${frame.mimeType};base64,${frame.dataBase64}` : null; + if (live.status === "live" && !liveFrame && decoded !== null) setDecoded(null); + useEffect(() => { + if (!liveFrame) return; + let cancelled = false; + const image = new Image(); + const src = `data:${liveFrame.mimeType};base64,${liveFrame.dataBase64}`; + image.src = src; + // Keep the visible frame until the replacement is ready to paint. + void image.decode().then( + () => { + if (!cancelled) setDecoded({ frame: liveFrame, src }); + }, + () => { + // A malformed frame must not erase the last successfully decoded one. + }, + ); + return () => { + cancelled = true; + }; + }, [liveFrame]); + const visible = + live.status === "unavailable" || (live.status === "live" && !liveFrame) ? null : decoded; + const frame = visible?.frame ?? null; + const src = visible?.src ?? null; const sourceSize = frame && frame.width > 0 && frame.height > 0 ? { width: frame.width, height: frame.height } @@ -302,7 +326,7 @@ function ComputerMiniPlayer({ {caption} diff --git a/packages/provider-core/src/server/cuaToolPresentation.test.ts b/packages/provider-core/src/server/cuaToolPresentation.test.ts index f91110bb01ca7..2fad60dc2fdd3 100644 --- a/packages/provider-core/src/server/cuaToolPresentation.test.ts +++ b/packages/provider-core/src/server/cuaToolPresentation.test.ts @@ -252,6 +252,33 @@ const CUA_DRIVER_TOOLS = [ ]; describe("readCuaWindowTarget", () => { + it("keeps the capture target when restoring focus or a later tool fails", () => { + clearCuaToolContext(threadId); + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 42, window_id: 7 }, + status: "completed", + }); + for (const status of ["inProgress", "completed"] as const) { + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/bring_to_front", + args: { pid: 99, window_id: 8 }, + status, + }); + NodeAssert.deepEqual(readCuaWindowTarget(threadId), { pid: 42, windowId: 7n }); + } + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 99, window_id: 8 }, + status: "failed", + }); + NodeAssert.deepEqual(readCuaWindowTarget(threadId), { pid: 42, windowId: 7n }); + clearCuaToolContext(threadId); + }); + it("remembers the last pid and window id a tool call addressed", () => { clearCuaToolContext(threadId); NodeAssert.equal(readCuaWindowTarget(threadId), undefined); diff --git a/packages/provider-core/src/server/cuaToolPresentation.ts b/packages/provider-core/src/server/cuaToolPresentation.ts index 22530ef0f47d1..7b5f1c25fab27 100644 --- a/packages/provider-core/src/server/cuaToolPresentation.ts +++ b/packages/provider-core/src/server/cuaToolPresentation.ts @@ -83,6 +83,12 @@ export function clearCuaToolContext(threadId: ThreadId): void { directories.delete(threadId); } +/** End the turn's capture target without forgetting learned app identities. */ +export function clearCuaWindowTarget(threadId: ThreadId): void { + const directory = directories.get(threadId); + if (directory) directory.lastWindow = undefined; +} + /** Last window the agent targeted in this thread, if any tool call named one. */ export function readCuaWindowTarget(threadId: ThreadId): CuaWindowTarget | undefined { return directories.get(threadId)?.lastWindow; @@ -210,6 +216,7 @@ function humanizeBundleId(bundleId: string): string | undefined { function resolveApp( threadId: ThreadId, args: Record | undefined, + trackWindow: boolean, ): CuaApp | undefined { const directory = directories.get(threadId); const pid = asPid(args?.pid); @@ -217,7 +224,7 @@ function resolveApp( const name = asAppName(args?.app_name) ?? asAppName(args?.app); const known = pid !== undefined ? directory?.byPid.get(pid) : undefined; const windowId = asWindowId(args?.window_id); - if (pid !== undefined && windowId !== undefined) { + if (trackWindow && pid !== undefined && windowId !== undefined) { directoryFor(threadId).lastWindow = { pid, windowId }; } const app: CuaApp = { @@ -465,7 +472,10 @@ export function cuaToolPresentation(input: { if (!tool) return undefined; const args = asRecord(input.args); const targetArgs = tool === "run_steps" ? runStepsTarget(args) : args; - const app = TOOLS_WITHOUT_APP.has(tool) ? undefined : resolveApp(input.threadId, targetArgs); + // Restoring the user's foreground window must not redirect the live preview. + const app = TOOLS_WITHOUT_APP.has(tool) + ? undefined + : resolveApp(input.threadId, targetArgs, tool !== "bring_to_front" && input.status !== "failed"); const appName = app?.name ?? (app?.bundleId ? humanizeBundleId(app.bundleId) : undefined); const reference = app ? nativeApp(app) : undefined; const icon = reference ? ({ _tag: "native-app", app: reference } as const) : undefined; diff --git a/packages/shared/package.json b/packages/shared/package.json index 1b86c7965d347..a851f62eb3d82 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -515,6 +515,10 @@ "types": "./src/cloneUrl.ts", "import": "./src/cloneUrl.ts" }, + "./cuaDriverHostOptions": { + "types": "./src/cuaDriverHostOptions.ts", + "import": "./src/cuaDriverHostOptions.ts" + }, "./cuaDriverRelease": { "types": "./src/cuaDriverRelease.ts", "import": "./src/cuaDriverRelease.ts" diff --git a/packages/shared/src/cuaDriverHostOptions.test.ts b/packages/shared/src/cuaDriverHostOptions.test.ts new file mode 100644 index 0000000000000..6107e666970c2 --- /dev/null +++ b/packages/shared/src/cuaDriverHostOptions.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { cuaDriverHostOptions } from "./cuaDriverHostOptions.ts"; + +describe("Cua host backend selection", () => { + it("uses Wayland even when the compositor also exposes XWayland", () => { + const options = cuaDriverHostOptions("/driver", "t3", "linux", { + WAYLAND_DISPLAY: "wayland-0", + DISPLAY: ":0", + }); + expect(options.environment).toEqual([{ name: "CUA_DRIVER_RS_ENABLE_WAYLAND", value: "1" }]); + }); + + it.each(["0", "false", ""])("preserves an explicit opt-out (%j)", (value) => { + const environment = { WAYLAND_DISPLAY: "wayland-0", CUA_DRIVER_RS_ENABLE_WAYLAND: value }; + expect(cuaDriverHostOptions("/driver", "t3", "linux", environment).environment).toEqual([ + { name: "CUA_DRIVER_RS_ENABLE_WAYLAND", value }, + ]); + }); + + it("does not enable Wayland for X11, headless, or non-Linux hosts", () => { + expect(cuaDriverHostOptions("/driver", "t3", "linux", { DISPLAY: ":0" }).environment).toEqual( + [], + ); + expect(cuaDriverHostOptions("/driver", "t3", "linux", {}).environment).toEqual([]); + for (const platform of ["darwin", "win32"] as const) { + expect( + cuaDriverHostOptions("/driver", "t3", platform, { WAYLAND_DISPLAY: "wayland-0" }) + .environment, + ).toEqual([]); + } + }); +}); diff --git a/packages/shared/src/cuaDriverHostOptions.ts b/packages/shared/src/cuaDriverHostOptions.ts new file mode 100644 index 0000000000000..ca1dd8a4621b7 --- /dev/null +++ b/packages/shared/src/cuaDriverHostOptions.ts @@ -0,0 +1,26 @@ +/** Select the native Wayland backend for graphical Linux hosts, preserving explicit opt-outs. */ +export function cuaDriverHostOptions( + binaryPath: string, + hostBundleId: string, + platform: NodeJS.Platform, + environment: NodeJS.ProcessEnv, +) { + const wayland = platform === "linux" && Boolean(environment.WAYLAND_DISPLAY?.trim()); + return { + binaryPath, + hostBundleId, + approveCapabilityManifest: false, + approveSessionPolicy: false, + dangerouslyBypassApprovals: false, + inheritStderr: true, + noOverlay: false, + environment: wayland + ? [ + { + name: "CUA_DRIVER_RS_ENABLE_WAYLAND", + value: environment.CUA_DRIVER_RS_ENABLE_WAYLAND ?? "1", + }, + ] + : [], + }; +} From 9308edeff3aa84153c09f9c201330b0e7cb12934 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:03:37 -0700 Subject: [PATCH 2/8] fix(cua): preserve native Linux preview transport patch --- patches/cua-driver-linux/0.34.0.patch | 997 ++++++++++++++++++++++++++ patches/cua-driver-linux/README.md | 33 + 2 files changed, 1030 insertions(+) create mode 100644 patches/cua-driver-linux/0.34.0.patch create mode 100644 patches/cua-driver-linux/README.md diff --git a/patches/cua-driver-linux/0.34.0.patch b/patches/cua-driver-linux/0.34.0.patch new file mode 100644 index 0000000000000..4ed81c76d7e1f --- /dev/null +++ b/patches/cua-driver-linux/0.34.0.patch @@ -0,0 +1,997 @@ +diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs +index 1883e39..d599610 100644 +--- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs ++++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs +@@ -860,6 +860,7 @@ pub(crate) fn allowed_environment_name(name: &str) -> bool { + | "PROGRAMDATA" + | "DISPLAY" + | "WAYLAND_DISPLAY" ++ | "CUA_DRIVER_RS_ENABLE_WAYLAND" + | "XDG_RUNTIME_DIR" + | "XDG_SESSION_TYPE" + | "DBUS_SESSION_BUS_ADDRESS" +@@ -1306,6 +1307,41 @@ mod tests { + .any(|variable| variable.value == "forged-lock")); + } + ++ #[test] ++ fn wayland_backend_opt_in_reaches_embedded_driver() { ++ let name = "CUA_DRIVER_RS_ENABLE_WAYLAND"; ++ assert!(allowed_environment_name(name)); ++ let inherited = [(name.to_owned(), "1".to_owned())]; ++ let values = merge_safe_environment(inherited.clone(), &[]); ++ assert!(values ++ .iter() ++ .any(|variable| variable.name == name && variable.value == "1")); ++ ++ // Hosts can opt in without changing their own process environment. ++ let enabled = EmbeddedEnvironmentVariable { ++ name: name.into(), ++ value: "1".into(), ++ }; ++ let values = merge_safe_environment(std::iter::empty(), &[enabled]); ++ assert!(values ++ .iter() ++ .any(|variable| variable.name == name && variable.value == "1")); ++ ++ // An explicit opt-out still wins over the inherited opt-in. ++ let disabled = EmbeddedEnvironmentVariable { ++ name: name.into(), ++ value: "0".into(), ++ }; ++ let values = merge_safe_environment(inherited, &[disabled]); ++ assert!(values ++ .iter() ++ .any(|variable| variable.name == name && variable.value == "0")); ++ assert!(!allowed_environment_name("LD_PRELOAD")); ++ assert!(!allowed_environment_name( ++ "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS" ++ )); ++ } ++ + #[test] + fn interactive_linux_session_environment_is_inherited() { + let values = merge_safe_environment( +diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs +index 1d35cd5..06035e6 100644 +--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs ++++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs +@@ -483,12 +483,51 @@ fn start_shell_helper_idle_watch() { + }); + } + ++// GNOME's helper cursor is a stage actor, not a child of the target window. ++// Until the helper can bind it to a surface, only desktop actions may draw it. ++#[cfg(target_os = "linux")] ++static SHELL_HELPER_DESKTOP_ACTIONS: Mutex>> = Mutex::new(None); ++ ++#[cfg(target_os = "linux")] ++fn shell_helper_action_visible(actions: &mut HashMap, msg: &OverlayMsg) -> bool { ++ match msg { ++ OverlayMsg::Cmd(command) => { ++ match &command.cmd { ++ OverlayCommand::BeginAction { target, .. } => { ++ actions.insert( ++ command.key.clone(), ++ target.is_some_and(|target| target.as_str() == "desktop"), ++ ); ++ } ++ OverlayCommand::EndAction(_) => { ++ actions.remove(&command.key); ++ } ++ _ => {} ++ } ++ actions.get(&command.key).copied().unwrap_or(false) ++ } ++ OverlayMsg::Remove(key) | OverlayMsg::Revive(key) => { ++ actions.remove(key); ++ false ++ } ++ OverlayMsg::Wake => false, ++ } ++} ++ + #[cfg(target_os = "linux")] + fn dispatch_wayland_overlay_message(msg: &OverlayMsg) -> WaylandOverlayBackend { + let backend = wayland_overlay_backend(); + match backend { + WaylandOverlayBackend::SemanticShellHelper | WaylandOverlayBackend::LegacyShellHelper => { + let semantic = backend == WaylandOverlayBackend::SemanticShellHelper; ++ let visible = { ++ let mut actions = SHELL_HELPER_DESKTOP_ACTIONS.lock().unwrap(); ++ shell_helper_action_visible(actions.get_or_insert_with(HashMap::new), msg) ++ }; ++ if !visible { ++ crate::wayland::shell_helper::hide_cursor(); ++ return backend; ++ } + if shell_helper_surface_event(msg, Instant::now()) { + if let OverlayMsg::Cmd(command) = msg { + dispatch_shell_helper_command(&command.key, &command.cmd, semantic); +@@ -533,6 +572,9 @@ fn dispatch_shell_helper_command(key: &str, cmd: &OverlayCommand, semantic: bool + } + OverlayCommand::EndAction(action) if semantic => { + crate::wayland::shell_helper::set_cursor_state(action.as_str(), "", "", false); ++ // The foreground transaction has restored the user's app. Leaving ++ // the stage-wide cursor visible would attribute input to that app. ++ crate::wayland::shell_helper::hide_cursor(); + } + OverlayCommand::SetSessionLabel(label) if semantic => { + crate::wayland::shell_helper::set_session_label( +@@ -5323,6 +5365,37 @@ fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver OverlayMsg { + OverlayMsg::Cmd(KeyedOverlayCommand { + key: key.to_owned(), +diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +index ca0dec4..464c322 100644 +--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs ++++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +@@ -1337,7 +1337,12 @@ impl Tool for GetWindowStateTool { + window_overlays(pid, xid) + }; + let screenshot = if should_capture { +- let captured = if overlays.covers_window { ++ let preview = if !want_tree && max_dim > 0 && crate::wayland::is_wayland() { ++ crate::wayland::shell_helper::screenshot_window_preview(xid, pid, max_dim) ++ } else { None }; ++ let captured = if let Some((png, width, height)) = preview { ++ Ok((png, Some((width, height)))) ++ } else if overlays.covers_window { + overlays + .window_rect + .ok_or_else(|| anyhow::anyhow!("window geometry unavailable")) +@@ -1345,12 +1350,16 @@ impl Tool for GetWindowStateTool { + crate::capture::screenshot_root_region_png(x, y, w, h) + }) + .or_else(|_| crate::wayland::screenshot_dispatch_with_pid(xid, pid)) ++ .map(|png| (png, None)) + } else { +- crate::wayland::screenshot_dispatch_with_pid(xid, pid) ++ crate::wayland::screenshot_dispatch_with_pid(xid, pid).map(|png| (png, None)) + }; + match captured { +- Ok(raw) => { +- let (orig_w, orig_h) = crate::capture::png_dimensions_pub(&raw)?; ++ Ok((raw, original_size)) => { ++ let (orig_w, orig_h) = match original_size { ++ Some(size) => size, ++ None => crate::capture::png_dimensions_pub(&raw)?, ++ }; + let png = crate::capture::resize_png_if_needed(&raw, max_dim)?; + let (w, h) = crate::capture::png_dimensions_pub(&png)?; + // An explicit per-call max_image_dimension (0 = native) +diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs +index a2691a0..4ab3a74 100644 +--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs ++++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs +@@ -17,8 +17,7 @@ + //! handshake → connection.seat events → seat.bind(capabilities) → + //! device.frame + emulate events. + //! 4. Public API sends commands over a crossbeam-channel and blocks +-//! until the worker reports the request was flushed to the EIS +-//! server. ++//! until EIS acknowledges the preceding input through ei_connection.sync. + //! + //! Persistence: ashpd `PersistMode::ExplicitlyRevoked` keeps the user's consent + //! until they revoke it in desktop settings. The restore token is stored at +@@ -607,6 +606,16 @@ fn run_calloop(context: reis::ei::Context, rx: Receiver) -> anyhow::Result< + pending = still; + } + ++ state.pending_sync.retain(|_, (reply, sent)| { ++ if sent.elapsed() < std::time::Duration::from_secs(5) { ++ return true; ++ } ++ let _ = reply.send(Err(anyhow::anyhow!( ++ "EIS did not acknowledge input within 5s; delivery is unknown" ++ ))); ++ false ++ }); ++ + // The handle_command path may have queued frame() requests; flush + // them once per loop iteration so the EIS server sees them. + if let Some(ctx) = state.context.as_ref() { +@@ -620,6 +629,8 @@ fn run_calloop(context: reis::ei::Context, rx: Receiver) -> anyhow::Result< + #[derive(Default)] + struct EisState { + context: Option, ++ connection: Option, ++ pending_sync: HashMap>, std::time::Instant)>, + seats: HashMap, + devices: HashMap, + sequence: u32, +@@ -737,17 +748,25 @@ impl EisState { + handshake.finish(); + } + } +- Event::Connection(_conn, ev) => match ev { +- reis::ei::connection::Event::Seat { seat } => { +- self.seats.insert(seat, SeatData::default()); ++ Event::Callback(callback, reis::ei::callback::Event::Done { .. }) => { ++ if let Some((reply, _)) = self.pending_sync.remove(&callback) { ++ let _ = reply.send(Ok(())); + } +- // The EIS server pings to check liveness; failing to answer +- // makes it drop the connection. Mirror reis's examples. +- reis::ei::connection::Event::Ping { ping } => { +- ping.done(0); ++ } ++ Event::Connection(conn, ev) => { ++ self.connection = Some(conn); ++ match ev { ++ reis::ei::connection::Event::Seat { seat } => { ++ self.seats.insert(seat, SeatData::default()); ++ } ++ // The EIS server pings to check liveness; failing to answer ++ // makes it drop the connection. Mirror reis's examples. ++ reis::ei::connection::Event::Ping { ping } => { ++ ping.done(0); ++ } ++ _ => {} + } +- _ => {} +- }, ++ } + Event::Seat(seat, ev) => { + let data = self.seats.entry(seat.clone()).or_default(); + match ev { +@@ -945,33 +964,36 @@ impl EisState { + + fn handle_command(&mut self, cmd: Cmd) { + let result = self.run_command(&cmd); +- // Send the reply on whichever channel this command carries. +- match cmd { +- Cmd::WaitReady { reply, .. } => { +- let _ = reply.send(result); +- } +- Cmd::Click { reply, .. } => { +- let _ = reply.send(result); +- } +- Cmd::MoveAbsolute { reply, .. } => { +- let _ = reply.send(result); +- } +- Cmd::Scroll { reply, .. } => { +- let _ = reply.send(result); +- } +- Cmd::TypeText { reply, .. } => { +- let _ = reply.send(result); +- } +- Cmd::PressKey { reply, .. } => { +- let _ = reply.send(result); +- } +- Cmd::KeySequence { reply, .. } => { +- let _ = reply.send(result); +- } +- Cmd::Drag { reply, .. } => { +- let _ = reply.send(result); ++ let needs_sync = !matches!(cmd, Cmd::WaitReady { .. }); ++ let reply = match cmd { ++ Cmd::WaitReady { reply, .. } ++ | Cmd::Click { reply, .. } ++ | Cmd::MoveAbsolute { reply, .. } ++ | Cmd::Scroll { reply, .. } ++ | Cmd::TypeText { reply, .. } ++ | Cmd::PressKey { reply, .. } ++ | Cmd::KeySequence { reply, .. } ++ | Cmd::Drag { reply, .. } => reply, ++ Cmd::Shutdown => return, ++ }; ++ if result.is_ok() && needs_sync { ++ if let (Some(connection), Some(context)) = (&self.connection, &self.context) { ++ // A socket flush is not delivery. Hold the caller's focus guard ++ // until EIS acknowledges all preceding input frames. ++ let callback = connection.sync(1); ++ if let Err(error) = context.flush() { ++ let _ = reply.send(Err(error.into())); ++ return; ++ } ++ self.pending_sync ++ .insert(callback, (reply, std::time::Instant::now())); ++ } else { ++ let _ = reply.send(Err(anyhow::anyhow!( ++ "EIS connection unavailable for input acknowledgement" ++ ))); + } +- Cmd::Shutdown => {} ++ } else { ++ let _ = reply.send(result); + } + } + +diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs +index 0909579..86d050e 100644 +--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs ++++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs +@@ -1044,17 +1044,13 @@ fn wayland_window_crop(xid: u64) -> anyhow::Result { + window.visible, + ); + } +- if let Some(window) = shell_helper::list_windows(None) +- .and_then(|windows| windows.into_iter().find(|window| window.xid == xid)) +- { +- return attested_wayland_crop( ++ if shell_helper::available() { ++ // GetRects proves geometry, not occlusion. Cropping the Shell stage can ++ // return a covering app (and uses physical pixels with logical bounds). ++ return Err(surface_identity_unproven( + xid, +- window.x, +- window.y, +- window.width, +- window.height, +- window.is_on_screen, +- ); ++ "GNOME stage capture cannot attest an individual window; use desktop capture", ++ )); + } + Err(surface_identity_unproven( + xid, +@@ -1106,6 +1102,14 @@ fn screenshot_dispatch_for_pid(xid: u64, pid: Option) -> anyhow::Result) -> Vec { + } + }; + } ++ // A verified GNOME helper already supplies exact IDs and ownership. ++ // Do not probe unrelated compositor protocols for every preview frame. ++ if let Some(ws) = shell_helper::list_windows(filter_pid) { ++ return listed_windows(ws); ++ } + if let Some(ws) = kwin_helper::list_window_infos() { + return listed_windows(apply_pid_filter(ws, filter_pid)); + } +diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs +index 111b4ed..8483142 100644 +--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs ++++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs +@@ -18,12 +18,11 @@ + //! + //! Everything here is **best-effort**: if the extension isn't installed/enabled + //! the calls return `None` / no-op and callers keep the prior behaviour (no +-//! screen coords, no Wayland cursor). Uses a short-lived `gdbus` subprocess so +-//! there's no zbus blocking-feature or async-context coupling — the calls are +-//! infrequent (once per `get_window_state`, a few per click). ++//! screen coords, no Wayland cursor). Reuses one session-bus connection on a ++//! dedicated runtime; preview frames do not spawn subprocesses or poll for exit. + + use std::os::unix::fs::{MetadataExt, PermissionsExt}; +-use std::process::Command; ++mod transport; + use std::time::Duration; + + use crate::x11::WindowInfo; +@@ -36,6 +35,7 @@ const DBUS_PATH: &str = "/org/freedesktop/DBus"; + const DBUS_IFACE: &str = "org.freedesktop.DBus"; + const BROWSER_HELPER_API_VERSION: u32 = 4; + const SEMANTIC_CURSOR_API_VERSION: u32 = 8; ++const WINDOW_CAPTURE_API_VERSION: u32 = 9; + + #[derive(Debug, Clone)] + struct ShellWindow { +@@ -85,30 +85,7 @@ fn gdbus_call_to( + args: &[String], + timeout: Duration, + ) -> Option { +- let mut cmd = Command::new("gdbus"); +- cmd.arg("call") +- .arg("--session") +- .arg("--dest") +- .arg(destination) +- .arg("--object-path") +- .arg(object_path) +- .arg("--method") +- .arg(method); +- for a in args { +- cmd.arg(a); +- } +- // gdbus is local IPC; cap it so a wedged shell can't stall the caller. +- let child = cmd +- .stdin(std::process::Stdio::null()) +- .stdout(std::process::Stdio::piped()) +- .stderr(std::process::Stdio::null()) +- .spawn() +- .ok()?; +- let out = wait_timeout(child, timeout)?; +- if !out.status.success() { +- return None; +- } +- Some(String::from_utf8_lossy(&out.stdout).into_owned()) ++ transport::call(destination, object_path, method, args, timeout) + } + + /// Resolve the helper's immutable unique bus name and prove that it is hosted +@@ -210,6 +187,42 @@ fn is_trusted_gnome_shell(pid: u32) -> bool { + .is_some_and(|meta| meta.uid() == 0 && meta.permissions().mode() & 0o022 == 0) + } + ++/// Capture the exact compositor actor without activating it. Older helpers ++/// cannot prove window pixels, so callers must not fall back to stage crops. ++pub fn screenshot_window(id: u64, pid: Option) -> Option> { ++ let id = u32::try_from(id).ok()?; ++ let owner = shell_owner_with_min_version(Some(WINDOW_CAPTURE_API_VERSION))?; ++ let raw = gdbus_call_to( ++ &owner, ++ PATH, ++ &format!("{IFACE}.CaptureWindow"), ++ &[id.to_string(), pid.unwrap_or(0).to_string()], ++ Duration::from_secs(5), ++ )?; ++ decode_capture(&raw) ++} ++ ++/// Preview-sized pixels plus the original action-frame dimensions. ++/// API 9 helpers fall back to the existing full-size capture path. ++pub fn screenshot_window_preview(id: u64, pid: u32, max_dimension: u32) -> Option<(Vec, u32, u32)> { ++ let id = u32::try_from(id).ok()?; ++ let owner = shell_owner_with_min_version(Some(10))?; ++ let raw = gdbus_call_to( ++ &owner, PATH, &format!("{IFACE}.CaptureWindowPreview"), ++ &[id.to_string(), pid.to_string(), max_dimension.to_string()], ++ Duration::from_secs(5), ++ )?; ++ let png = decode_capture(&raw)?; ++ let tail = &raw[raw.rfind('\'')? + 1..]; ++ let dims: Vec = tail.split(',').filter_map(|part| ++ part.trim().trim_end_matches(')').trim().trim_start_matches("uint32 ").parse().ok() ++ ).collect(); ++ match dims.as_slice() { ++ [width, height] if *width > 0 && *height > 0 => Some((png, *width, *height)), ++ _ => None, ++ } ++} ++ + /// Capture the GNOME stage through the compositor helper. + /// + /// Mutter does not expose wlroots screencopy protocols, and its one-shot +@@ -247,53 +260,6 @@ fn decode_capture(raw: &str) -> Option> { + B64.decode(&raw[start..end]).ok() + } + +-/// `Child::wait` with a deadline (no extra crates). Kills + reaps on timeout. +-fn wait_timeout(mut child: std::process::Child, dur: Duration) -> Option { +- use std::io::Read; +- +- // Drain stdout while the child is running. Capture() returns a base64 PNG +- // that readily exceeds a pipe's ~64 KiB capacity; waiting for exit before +- // reading deadlocks the child on a full pipe and turns a healthy Shell +- // response into a false timeout. +- let stdout = child.stdout.take()?; +- let reader = std::thread::spawn(move || { +- let mut stdout = stdout; +- let mut bytes = Vec::new(); +- stdout.read_to_end(&mut bytes).ok()?; +- Some(bytes) +- }); +- let deadline = std::time::Instant::now() + dur; +- let status = loop { +- match child.try_wait() { +- Ok(Some(status)) => break status, +- Ok(None) => { +- if std::time::Instant::now() >= deadline { +- let _ = child.kill(); +- let status = child.wait().ok()?; +- let _ = reader.join(); +- if !status.success() { +- return None; +- } +- return None; +- } +- std::thread::sleep(Duration::from_millis(15)); +- } +- Err(_) => { +- let _ = child.kill(); +- let _ = child.wait(); +- let _ = reader.join(); +- return None; +- } +- } +- }; +- let stdout = reader.join().ok().flatten()?; +- Some(std::process::Output { +- status, +- stdout, +- stderr: Vec::new(), +- }) +-} +- + /// Screen origin of the compositor frame backing `pid`. + /// + /// `screenshot_window_dispatch` crops the Shell stage to this same frame +@@ -643,9 +609,9 @@ mod tests { + } + + #[test] +- fn bundled_helper_v8_uses_host_owned_modifier_badge_chips() { ++ fn bundled_helper_uses_host_owned_modifier_badge_chips() { + assert!(EXTENSION_SOURCE.contains("GetVersion()")); +- assert!(EXTENSION_SOURCE.contains("return 8;")); ++ assert!(EXTENSION_SOURCE.contains("return 10;")); + assert!(EXTENSION_SOURCE.contains("SetCursorState")); + assert!(EXTENSION_SOURCE.contains("SetCursorColor")); + assert!(EXTENSION_SOURCE.contains("SetSessionLabel")); +@@ -670,7 +636,7 @@ mod tests { + assert!(!EXTENSION_SOURCE.contains("function drawModifiers")); + let metadata: serde_json::Value = + serde_json::from_str(EXTENSION_METADATA).expect("valid bundled helper metadata"); +- assert_eq!(metadata["version"], 8); ++ assert_eq!(metadata["version"], 10); + + for action in [ + "idle", "observe", "click", "drag", "scroll", "text", "key", "navigate", "app", +diff --git a/libs/cua-driver/wayland-helper/winrects@cua/extension.js b/libs/cua-driver/wayland-helper/winrects@cua/extension.js +index 743acc2..cbc2888 100644 +--- a/libs/cua-driver/wayland-helper/winrects@cua/extension.js ++++ b/libs/cua-driver/wayland-helper/winrects@cua/extension.js +@@ -3,16 +3,20 @@ import GLib from 'gi://GLib'; + import Shell from 'gi://Shell'; + import St from 'gi://St'; + import Clutter from 'gi://Clutter'; ++import Cogl from 'gi://Cogl'; + import Cairo from 'cairo'; + import * as Main from 'resource:///org/gnome/shell/ui/main.js'; + import { Extension } from 'resource:///org/gnome/shell/extensions/extension.js'; + + Gio._promisify(Shell.Screenshot.prototype, 'screenshot'); ++Gio._promisify(Shell.Screenshot, 'composite_to_stream'); + + const IFACE = ` + + + ++ ++ + + + +@@ -680,6 +684,7 @@ function drawBadgeChip(cr, glyph, filled, fillColor) { + + export default class WinRectsExtension extends Extension { + enable() { ++ this._captureClones = new Map(); + this._impl = Gio.DBusExportedObject.wrapJSObject(IFACE, this); + this._impl.export(Gio.DBus.session, '/org/cua/WinRects'); + this._nameId = Gio.bus_own_name( +@@ -822,6 +827,11 @@ export default class WinRectsExtension extends Extension { + }); + } + disable() { ++ for (const entry of this._captureClones.values()) { ++ GLib.source_remove(entry.timeout); ++ entry.clone.destroy(); ++ } ++ this._captureClones.clear(); + if (this._frameId) { + GLib.source_remove(this._frameId); + this._frameId = 0; +@@ -851,7 +861,7 @@ export default class WinRectsExtension extends Extension { + } + } + GetVersion() { +- return 8; ++ return 10; + } + GetRects() { + const actors = global.get_window_actors(); +@@ -891,6 +901,96 @@ export default class WinRectsExtension extends Extension { + } + return JSON.stringify(out); + } ++ CaptureWindowAsync([id, pid], invocation) { ++ return this._captureWindow(id, pid, 0, invocation); ++ } ++ CaptureWindowPreviewAsync([id, pid, maxDimension], invocation) { ++ if (maxDimension < 1 || maxDimension > 4096) { ++ invocation.return_dbus_error('org.cua.WinRects.InvalidSize', 'Invalid preview size'); ++ return; ++ } ++ return this._captureWindow(id, pid, maxDimension, invocation); ++ } ++ async _captureWindow(id, pid, maxDimension, invocation) { ++ let stream; ++ try { ++ if (Main.sessionMode.isLocked || Main.sessionMode.isGreeter) ++ throw new Error('Desktop is unavailable'); ++ const actor = global.get_window_actors().find( ++ candidate => candidate.meta_window?.get_stable_sequence() === id); ++ const window = actor?.meta_window; ++ if (!window || (pid !== 0 && window.get_pid() !== pid) || window.minimized) ++ throw new Error('Target window is unavailable'); ++ // A mapped clone keeps Mutter from suspending a fully covered source. ++ // It paints no pixels and expires when capture requests stop. ++ let entry = this._captureClones.get(id); ++ if (entry) GLib.source_remove(entry.timeout); ++ else { ++ const clone = new Clutter.Clone({source: actor, opacity: 0, width: 1, height: 1, reactive: false}); ++ global.stage.add_child(clone); ++ entry = {clone, timeout: 0}; ++ this._captureClones.set(id, entry); ++ } ++ entry.timeout = GLib.timeout_add(GLib.PRIORITY_DEFAULT, 4000, () => { ++ entry.clone.destroy(); ++ this._captureClones.delete(id); ++ return GLib.SOURCE_REMOVE; ++ }); ++ // Snapshot this actor, never the stage. Covered windows retain their own ++ // content; neither activation nor a desktop crop is needed. ++ const frame = window.get_frame_rect(); ++ const buffer = window.get_buffer_rect(); ++ const scale = actor.get_resource_scale(); ++ const content = actor.paint_to_content(null); ++ const texture = content?.get_texture(); ++ if (!texture || !Number.isFinite(scale) || scale <= 0) ++ throw new Error('Window content is unavailable'); ++ const x = Math.round((frame.x - buffer.x) * scale); ++ const y = Math.round((frame.y - buffer.y) * scale); ++ const width = Math.round(frame.width * scale); ++ const height = Math.round(frame.height * scale); ++ if (x < 0 || y < 0 || width <= 0 || height <= 0 || ++ x + width > texture.get_width() || y + height > texture.get_height()) ++ throw new Error('Window content does not match its frame'); ++ stream = Gio.MemoryOutputStream.new_resizable(); ++ let captureTexture = texture; ++ let captureX = x, captureY = y, captureWidth = width, captureHeight = height; ++ if (maxDimension > 0 && Math.max(width, height) > maxDimension) { ++ const ratio = maxDimension / Math.max(width, height); ++ captureWidth = Math.max(1, Math.round(width * ratio)); ++ captureHeight = Math.max(1, Math.round(height * ratio)); ++ const context = global.stage.get_context().get_backend().get_cogl_context(); ++ captureTexture = Cogl.Texture2D.new_with_size(context, captureWidth, captureHeight); ++ const framebuffer = Cogl.Offscreen.new_with_texture(captureTexture); ++ framebuffer.allocate(); ++ framebuffer.set_viewport(0, 0, captureWidth, captureHeight); ++ framebuffer.orthographic(0, 0, captureWidth, captureHeight, -1, 1); ++ const pipeline = Cogl.Pipeline.new(context); ++ pipeline.set_layer_texture(0, texture); ++ framebuffer.draw_textured_rectangle(pipeline, 0, 0, captureWidth, captureHeight, ++ x / texture.get_width(), y / texture.get_height(), ++ (x + width) / texture.get_width(), (y + height) / texture.get_height()); ++ framebuffer.finish(); ++ captureX = 0; ++ captureY = 0; ++ } ++ await Shell.Screenshot.composite_to_stream( ++ captureTexture, captureX, captureY, captureWidth, captureHeight, 1, null, 0, 0, 1, stream); ++ stream.close(null); ++ if (Main.sessionMode.isLocked || Main.sessionMode.isGreeter) ++ throw new Error('Desktop became unavailable'); ++ if (stream.get_data_size() > 32 * 1024 * 1024) ++ throw new Error('Window capture is too large'); ++ const encoded = GLib.base64_encode(stream.steal_as_bytes().get_data()); ++ invocation.return_value(maxDimension > 0 ++ ? new GLib.Variant('(suu)', [encoded, width, height]) ++ : new GLib.Variant('(s)', [encoded])); ++ } catch (error) { ++ invocation.return_dbus_error('org.cua.WinRects.CaptureFailed', String(error)); ++ } finally { ++ if (stream && !stream.is_closed()) stream.close(null); ++ } ++ } + async CaptureAsync(_params, invocation) { + try { + const shooter = new Shell.Screenshot(); +diff --git a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json +index 3faf86b..a2843ca 100644 +--- a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json ++++ b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json +@@ -3,5 +3,5 @@ + "name": "WinRects", + "description": "Expose window geometry, capture, cursor, and verified activation for cua-driver on Wayland.", + "shell-version": ["45", "46", "47", "48", "49", "50"], +- "version": 8 ++ "version": 10 + } +diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs +new file mode 100644 +index 0000000..a054366 +--- /dev/null ++++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs +@@ -0,0 +1,262 @@ ++//! Persistent session-bus transport for synchronous compositor helpers. ++//! A separate runtime also serves callers already inside Tokio without nested block_on. ++use std::sync::{mpsc, OnceLock}; ++use std::time::Duration; ++use tokio::sync::Mutex; ++use zbus::{Connection, Message}; ++ ++static CONNECTION: Mutex> = Mutex::const_new(None); ++ ++fn runtime() -> &'static tokio::runtime::Runtime { ++ static RUNTIME: OnceLock = OnceLock::new(); ++ RUNTIME.get_or_init(|| { ++ tokio::runtime::Builder::new_multi_thread() ++ .worker_threads(1) ++ .enable_all() ++ .build() ++ .expect("compositor D-Bus runtime") ++ }) ++} ++ ++pub(super) fn call( ++ destination: &str, ++ path: &str, ++ method: &str, ++ args: &[String], ++ timeout: Duration, ++) -> Option { ++ let destination = destination.to_owned(); ++ let path = path.to_owned(); ++ let method = method.to_owned(); ++ let args = args.to_vec(); ++ let (tx, rx) = mpsc::sync_channel(1); ++ runtime().spawn(async move { ++ let result = tokio::time::timeout(timeout, async { ++ let connection = { ++ let mut shared = CONNECTION.lock().await; ++ if shared.is_none() { ++ *shared = Some(Connection::session().await.ok()?); ++ } ++ shared.as_ref()?.clone() ++ }; ++ invoke(&connection, &destination, &path, &method, &args).await ++ }) ++ .await ++ .ok() ++ .flatten(); ++ // A failed or timed-out call may mean the session bus disconnected. ++ // Reconnect on the next call; never replay a possibly applied mutation. ++ if result.is_none() { ++ CONNECTION.lock().await.take(); ++ } ++ let _ = tx.send(result); ++ }); ++ rx.recv_timeout(timeout + Duration::from_millis(100)) ++ .ok() ++ .flatten() ++} ++ ++// Preserve the small textual reply contract consumed by shell_helper's existing ++// parsers. Bodies on the wire are typed; strings are never shell or GVariant code. ++async fn invoke( ++ connection: &Connection, ++ destination: &str, ++ path: &str, ++ method: &str, ++ args: &[String], ++) -> Option { ++ let (interface, member) = method.rsplit_once('.')?; ++ macro_rules! send { ++ ($body:expr) => { ++ connection ++ .call_method(Some(destination), path, Some(interface), member, &$body) ++ .await ++ .ok()? ++ }; ++ } ++ let reply: Message = match (member, args) { ++ ( ++ "GetNameOwner" ++ | "GetConnectionUnixProcessID" ++ | "GetConnectionUnixUser" ++ | "SetCursorColor" ++ | "SetSessionLabel", ++ [value], ++ ) => send!((value.as_str(),)), ++ ("GetRects" | "GetVersion" | "Capture" | "HideCursor", []) => send!(()), ++ ("Activate", [id]) => send!((id.parse::().ok()?,)), ++ ("MoveCursor" | "ClickPulse", [x, y]) => { ++ send!((x.parse::().ok()?, y.parse::().ok()?)) ++ } ++ ("CaptureWindow", [id, pid]) => send!((id.parse::().ok()?, pid.parse::().ok()?)), ++ ("CaptureWindowPreview", [id, pid, max]) => send!(( ++ id.parse::().ok()?, ++ pid.parse::().ok()?, ++ max.parse::().ok()? ++ )), ++ ("SetCursorState", [action, delivery, target, active]) => send!(( ++ action.as_str(), ++ delivery.as_str(), ++ target.as_str(), ++ active.parse::().ok()? ++ )), ++ _ => return None, ++ }; ++ let body = reply.body(); ++ match member { ++ "GetRects" => body.deserialize::().ok(), ++ "GetNameOwner" | "Capture" | "CaptureWindow" => { ++ Some(format!("('{}',)", body.deserialize::().ok()?)) ++ } ++ "GetVersion" | "GetConnectionUnixProcessID" | "GetConnectionUnixUser" => { ++ Some(format!("(uint32 {},)", body.deserialize::().ok()?)) ++ } ++ "Activate" => Some(format!("({},)", body.deserialize::().ok()?)), ++ "CaptureWindowPreview" => { ++ let (png, width, height) = body.deserialize::<(String, u32, u32)>().ok()?; ++ Some(format!("('{png}', uint32 {width}, uint32 {height})")) ++ } ++ _ => { ++ body.deserialize::<()>().ok()?; ++ Some("()".into()) ++ } ++ } ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use std::io::BufRead; ++ use std::process::{Child, Command, Stdio}; ++ ++ struct PrivateBus(Child); ++ impl Drop for PrivateBus { ++ fn drop(&mut self) { ++ let _ = self.0.kill(); ++ let _ = self.0.wait(); ++ } ++ } ++ struct Helper; ++ #[zbus::interface(name = "org.cua.WinRects")] ++ impl Helper { ++ fn get_version(&self) -> u32 { ++ 10 ++ } ++ fn get_rects(&self) -> String { ++ r#"[{"title":"Joe's window","pid":42}]"#.into() ++ } ++ fn capture_window_preview( ++ &self, ++ id: u32, ++ pid: u32, ++ max: u32, ++ ) -> zbus::fdo::Result<(String, u32, u32)> { ++ if id != 7 || pid != 42 || max != 800 { ++ return Err(zbus::fdo::Error::InvalidArgs("wrong target".into())); ++ } ++ Ok(("cG5n".into(), 1600, 900)) ++ } ++ fn activate(&self, id: u32) -> bool { ++ id == 7 ++ } ++ fn set_cursor_state( ++ &self, ++ action: &str, ++ delivery: &str, ++ target: &str, ++ active: bool, ++ ) -> zbus::fdo::Result<()> { ++ if (action, delivery, target, active) != ("observe", "", "window", false) { ++ return Err(zbus::fdo::Error::InvalidArgs("wrong state".into())); ++ } ++ Ok(()) ++ } ++ } ++ ++ #[tokio::test] ++ async fn typed_roundtrips_on_one_private_connection() { ++ let mut bus = PrivateBus( ++ Command::new("dbus-daemon") ++ .args(["--session", "--nofork", "--print-address=1"]) ++ .stdout(Stdio::piped()) ++ .spawn() ++ .expect("private test bus"), ++ ); ++ let mut address = String::new(); ++ std::io::BufReader::new(bus.0.stdout.take().unwrap()) ++ .read_line(&mut address) ++ .unwrap(); ++ let service = zbus::connection::Builder::address(address.trim()) ++ .unwrap() ++ .name("org.cua.WinRects") ++ .unwrap() ++ .serve_at("/org/cua/WinRects", Helper) ++ .unwrap() ++ .build() ++ .await ++ .unwrap(); ++ let client = zbus::connection::Builder::address(address.trim()) ++ .unwrap() ++ .build() ++ .await ++ .unwrap(); ++ let call = |method: &'static str, args: Vec| { ++ let client = client.clone(); ++ async move { ++ invoke( ++ &client, ++ "org.cua.WinRects", ++ "/org/cua/WinRects", ++ &format!("org.cua.WinRects.{method}"), ++ &args, ++ ) ++ .await ++ } ++ }; ++ for _ in 0..3 { ++ assert_eq!( ++ call("GetVersion", vec![]).await.as_deref(), ++ Some("(uint32 10,)") ++ ); ++ assert_eq!( ++ call("GetRects", vec![]).await.as_deref(), ++ Some(r#"[{"title":"Joe's window","pid":42}]"#) ++ ); ++ let raw = call( ++ "CaptureWindowPreview", ++ vec!["7".into(), "42".into(), "800".into()], ++ ) ++ .await ++ .unwrap(); ++ assert_eq!(super::super::decode_capture(&raw), Some(b"png".to_vec())); ++ assert_eq!(raw, "('cG5n', uint32 1600, uint32 900)"); ++ } ++ assert_eq!( ++ call("Activate", vec!["7".into()]).await.as_deref(), ++ Some("(true,)") ++ ); ++ assert_eq!( ++ call("Activate", vec!["8".into()]).await.as_deref(), ++ Some("(false,)") ++ ); ++ assert_eq!( ++ call( ++ "SetCursorState", ++ vec!["observe".into(), "".into(), "window".into(), "false".into()] ++ ) ++ .await ++ .as_deref(), ++ Some("()") ++ ); ++ assert_eq!( ++ call( ++ "CaptureWindowPreview", ++ vec!["7".into(), "43".into(), "800".into()] ++ ) ++ .await, ++ None ++ ); ++ assert_eq!(call("Activate", vec!["-1".into()]).await, None); ++ drop(service); ++ } ++} diff --git a/patches/cua-driver-linux/README.md b/patches/cua-driver-linux/README.md new file mode 100644 index 0000000000000..f14c212b3addc --- /dev/null +++ b/patches/cua-driver-linux/README.md @@ -0,0 +1,33 @@ +# Linux Cua Driver test build + +`0.34.0.patch` contains the native changes used by the Linux PiP test installer +`0.0.46-preview.20261007.2755`. It applies to trycua/cua commit +`b0968e1b12834e485dda68789541a3cc57664a9f` (`cua-driver-rs-v0.34.0`). +This is a source patch, not a pnpm dependency patch. Standard T3 builds still use +upstream release binaries; this file does not silently replace those binaries. + +The patch includes the Wayland host opt-in, exact-window GNOME capture and +preview downscaling (helper API 10), and persistent D-Bus transport. It retains +owner attestation on each call and reconnects after failed calls without +replaying mutations. The transport has a separate runtime because synchronous +helper callers can already be inside Tokio. + +To reproduce the native binaries in a separate checkout: + +```sh +git clone https://github.com/trycua/cua.git cua-linux-preview +cd cua-linux-preview +git checkout b0968e1b12834e485dda68789541a3cc57664a9f +git apply /absolute/path/to/t3code/patches/cua-driver-linux/0.34.0.patch +cd libs/cua-driver/rust +cargo test -p platform-linux wayland::shell_helper --lib +cargo build --locked -p cua-driver -p cua-driver-sdk +``` + +The test installer uses `target/debug/cua-driver` and +`target/debug/libcua_driver_sdk.so`, plus the patched files in +`libs/cua-driver/wayland-helper/winrects@cua`. Both the desktop driver bundle and +the SDK npm package's native library must use the matching build. Do not replace +files inside a running T3 installation. This debug build does not enable the +optional `portal-input` feature; GNOME fallback mouse/keyboard injection remains +unavailable, although supported accessibility actions and window capture work. From 58470283e177f4bdb0dd8ffb6cc9510533459ee4 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:23:38 -0700 Subject: [PATCH 3/8] perf(cua): reduce work for display-only and static previews --- apps/server/src/cua/CuaWindowPreview.test.ts | 122 ++++++++++++++++- apps/server/src/cua/CuaWindowPreview.ts | 134 +++++++++++++++---- patches/cua-driver-linux/0.34.0.patch | 83 +++++++++++- patches/cua-driver-linux/README.md | 9 +- 4 files changed, 314 insertions(+), 34 deletions(-) diff --git a/apps/server/src/cua/CuaWindowPreview.test.ts b/apps/server/src/cua/CuaWindowPreview.test.ts index 149fe513d630e..961cd99012085 100644 --- a/apps/server/src/cua/CuaWindowPreview.test.ts +++ b/apps/server/src/cua/CuaWindowPreview.test.ts @@ -34,13 +34,44 @@ const turnEvent = (type: "turn.started" | "turn.completed") => const makeHarness = Effect.fn(function* ( windowFailure?: "missing" | "throw" | "transient", onWindowCapture?: () => void | Promise, + options: { displayOnly?: boolean; image?: () => string } = {}, ) { const events = yield* PubSub.unbounded(); const captures: string[] = []; const windows: Array<{ pid: number; windowId: bigint }> = []; let clients = 0; let destroyed = 0; + const rawCalls: unknown[] = []; const fakeClient = { + listToolsJson: async () => + JSON.stringify({ + tools: [ + { + name: "get_window_state", + input_schema: { + properties: options.displayOnly ? { display_only: { type: "boolean" } } : {}, + }, + }, + ], + }), + callTool: async (name: string, args: string) => { + const input = JSON.parse(args) as { pid: number; window_id: number }; + rawCalls.push({ name, ...input }); + const window = await fakeClient.getWindowState({ + pid: input.pid, + windowId: BigInt(input.window_id), + }); + return { + images: window.images, + isError: false, + structuredJson: JSON.stringify({ + app_name: window.appName, + window_title: window.windowTitle, + screenshot_width: window.screenshotWidth, + screenshot_height: window.screenshotHeight, + }), + }; + }, getWindowState: async (input: { pid: number; windowId: bigint }) => { windows.push(input); await onWindowCapture?.(); @@ -53,7 +84,10 @@ const makeHarness = Effect.fn(function* ( windowTitle: "December 2026", screenshotWidth: 640, screenshotHeight: 400, - images: windowFailure === "missing" ? [] : [{ mimeType: "image/png", dataBase64: png }], + images: + windowFailure === "missing" + ? [] + : [{ mimeType: "image/png", dataBase64: options.image?.() ?? png }], }); }, getDesktopState: () => { @@ -94,6 +128,7 @@ const makeHarness = Effect.fn(function* ( emit: (event: OrchestrationV2DomainEvent) => PubSub.publish(events, event), captures, windows, + rawCalls, counts: () => ({ clients, destroyed }), }; }); @@ -111,6 +146,78 @@ const setCuaSession = Effect.flatMap(McpProviderSessions.McpProviderSessions, (s ); describe("CuaWindowPreview", () => { + it.effect.each([false, true])( + "deduplicates static frames, backs off, and resumes on changes (display-only: %s)", + (displayOnly) => + Effect.gen(function* () { + clearCuaToolContext(threadId); + McpProviderSession.setMcpProviderSession({ + threadId, + endpoint: "http://127.0.0.1/mcp", + authorizationHeader: "Bearer x", + capabilities: new Set(), + cuaDriver: { command: "/driver", args: [], environment: [], socketPath: "/tmp/cua.sock" }, + } as unknown as McpProviderSession.McpProviderSessionConfig); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + McpProviderSession.clearMcpProviderSession(threadId); + clearCuaToolContext(threadId); + }), + ); + const select = (windowId: number) => + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 42, window_id: windowId }, + status: "completed", + }); + select(7); + let pixels = png; + const harness = yield* makeHarness(undefined, undefined, { + displayOnly, + image: () => pixels, + }); + const firstPublished = yield* Deferred.make(); + const frames: string[] = []; + yield* harness.service.stream(threadId).pipe( + Stream.runForEach((state) => { + if (!state.frame) return Effect.void; + frames.push(state.frame.dataBase64); + return Deferred.succeed(firstPublished, undefined); + }), + Effect.forkScoped, + ); + yield* Effect.yieldNow; + yield* harness.emit(turnEvent("turn.started")); + yield* Deferred.await(firstPublished); + for (let i = 0; i < 3; i++) yield* TestClock.adjust(20); + expect(harness.windows).toHaveLength(4); + expect(frames).toEqual([png]); + yield* TestClock.adjust(79); + expect(harness.windows).toHaveLength(4); + pixels = "changed"; + yield* TestClock.adjust(1); + expect(harness.windows).toHaveLength(5); + expect(frames).toEqual([png, "changed"]); + yield* TestClock.adjust(20); + expect(harness.windows).toHaveLength(6); + // Identical pixels in a different window still publish the new target. + select(8); + yield* TestClock.adjust(20); + expect(frames).toEqual([png, "changed", "changed"]); + expect(harness.windows.at(-1)?.windowId).toBe(8n); + expect(harness.rawCalls).toHaveLength(displayOnly ? 7 : 0); + if (displayOnly) + expect(harness.rawCalls[0]).toMatchObject({ + name: "get_window_state", + display_only: true, + include_accessibility_tree: false, + include_screenshot: true, + max_dimension: 800, + }); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + it.effect("never overlaps captures and backs off after a slow frame", () => Effect.gen(function* () { clearCuaToolContext(threadId); @@ -355,9 +462,14 @@ describe("CuaWindowPreview", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); - it.live.each(["missing", "throw"] as const)( - "does not substitute desktop pixels when window capture is %s", - (failure) => + it.live.each([ + ["missing", false], + ["throw", false], + ["missing", true], + ["throw", true], + ] as const)( + "does not substitute desktop pixels when window capture is %s (display-only: %s)", + ([failure, displayOnly]) => Effect.gen(function* () { yield* setCuaSession; cuaToolPresentation({ @@ -366,7 +478,7 @@ describe("CuaWindowPreview", () => { args: { pid: 42, window_id: "7", x: 1, y: 1 }, status: "completed", }); - const harness = yield* makeHarness(failure); + const harness = yield* makeHarness(failure, undefined, { displayOnly }); const unavailable = yield* harness.service.stream(threadId).pipe( Stream.filter((state) => state.status !== "idle"), Stream.runHead, diff --git a/apps/server/src/cua/CuaWindowPreview.ts b/apps/server/src/cua/CuaWindowPreview.ts index af3b042365b73..2f7dd2d6d6f73 100644 --- a/apps/server/src/cua/CuaWindowPreview.ts +++ b/apps/server/src/cua/CuaWindowPreview.ts @@ -60,7 +60,10 @@ export class CuaWindowPreview extends Context.Service< export type SdkModule = Pick; // `connect` is typed as the interface, but every implementation is the // generated class with the native handle to release. -type SdkClient = Pick; +type SdkClient = Pick< + CuaSdk.CuaDriver, + "getWindowState" | "callTool" | "listToolsJson" | "shutdown" | "uniffiDestroy" +>; const REFRESH_INTERVAL = Duration.millis(250); /** Long edge of the preview frame; small enough for a floating card over the app WS. */ @@ -89,27 +92,75 @@ const loadSdk = Effect.fn("CuaWindowPreview.loadSdk")(function* () { return yield* new CuaWindowPreviewSdkError({ cause: "@trycua/cua-driver is not installed." }); }); +// Runtime schemas can advertise native extensions without changing the SDK ABI. +const previewTools = Schema.fromJsonString( + Schema.Struct({ + tools: Schema.Array( + Schema.Struct({ + name: Schema.String, + input_schema: Schema.Struct({ properties: Schema.Record(Schema.String, Schema.Unknown) }), + }), + ), + }), +); +const previewMetadata = Schema.fromJsonString( + Schema.Struct({ + app_name: Schema.optionalKey(Schema.String), + window_title: Schema.optionalKey(Schema.String), + screenshot_width: Schema.Number, + screenshot_height: Schema.Number, + }), +); + +const encodePreviewArgs = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); + /** Captures only the selected window; desktop pixels cannot stand in for it. */ const captureFrame = Effect.fn("CuaWindowPreview.capture")(function* ( sdk: SdkModule, client: SdkClient, target: CuaWindowTarget, + displayOnly: boolean, ) { const capturedAt = DateTime.formatIso(yield* DateTime.now); - const window = yield* Effect.tryPromise({ - try: (signal) => - client.getWindowState( - sdk.GetWindowStateInput.new({ - pid: target.pid, - windowId: target.windowId, - includeAccessibilityTree: false, - includeScreenshot: true, - maxDimension: MAX_DIMENSION, - }), - { signal }, - ), - catch: (cause) => new CuaWindowPreviewCaptureError({ cause }), - }); + const window = yield* Effect.gen(function* () { + if (displayOnly && target.windowId <= BigInt(Number.MAX_SAFE_INTEGER)) { + const args = yield* encodePreviewArgs({ + pid: target.pid, + window_id: Number(target.windowId), + include_accessibility_tree: false, + include_screenshot: true, + max_dimension: MAX_DIMENSION, + display_only: true, + }); + const result = yield* Effect.tryPromise({ + try: (signal) => client.callTool("get_window_state", args, { signal }), + catch: (cause) => new CuaWindowPreviewCaptureError({ cause }), + }); + if (result.isError) return yield* new CuaWindowPreviewCaptureError({ cause: result.text }); + const metadata = yield* Schema.decodeUnknownEffect(previewMetadata)(result.structuredJson); + return { + appName: metadata.app_name, + windowTitle: metadata.window_title, + screenshotWidth: metadata.screenshot_width, + screenshotHeight: metadata.screenshot_height, + images: result.images, + }; + } + return yield* Effect.tryPromise({ + try: (signal) => + client.getWindowState( + sdk.GetWindowStateInput.new({ + pid: target.pid, + windowId: target.windowId, + includeAccessibilityTree: false, + includeScreenshot: true, + maxDimension: MAX_DIMENSION, + }), + { signal }, + ), + catch: (cause) => new CuaWindowPreviewCaptureError({ cause }), + }); + }).pipe(Effect.mapError((cause) => new CuaWindowPreviewCaptureError({ cause }))); const image = window.images[0]; if (!image) { return yield* new CuaWindowPreviewCaptureError({ @@ -201,28 +252,62 @@ export const make = Effect.fn("CuaWindowPreview.make")(function* ( Effect.andThen(Effect.sync(() => client.uniffiDestroy())), ), ); + const displayOnly = yield* Effect.tryPromise({ + try: (signal) => client.listToolsJson({ signal }), + catch: (cause) => new CuaWindowPreviewSdkError({ cause }), + }).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(previewTools)), + Effect.map((inventory) => + inventory.tools.some( + (tool) => + tool.name === "get_window_state" && "display_only" in tool.input_schema.properties, + ), + ), + Effect.orElseSucceed(() => false), + ); + let previousTarget: CuaWindowTarget | undefined; + let unchangedFrames = 0; const capture = Effect.suspend(() => { const target = readCuaWindowTarget(threadId); + if (target?.pid !== previousTarget?.pid || target?.windowId !== previousTarget?.windowId) { + unchangedFrames = 0; + previousTarget = undefined; + } if (!target) return publish(preview, { status: "live" }); const isCurrent = () => { const current = readCuaWindowTarget(threadId); return current?.pid === target?.pid && current?.windowId === target?.windowId; }; - return captureFrame(module, client, target).pipe( + return captureFrame(module, client, target, displayOnly).pipe( // Compositor transitions can briefly fail a frame. Keep the current // frame during bounded retries, but report persistent capture loss. Effect.retry({ times: 2, schedule: Schedule.spaced("150 millis"), while: isCurrent }), - Effect.flatMap((frame) => - isCurrent() ? publish(preview, { status: "live", frame }) : Effect.void, - ), + Effect.flatMap((frame) => { + if (!isCurrent()) return Effect.void; + const previous = preview.state.status === "live" ? preview.state.frame : undefined; + const unchanged = + previousTarget !== undefined && + previous && + previous.dataBase64 === frame.dataBase64 && + previous.mimeType === frame.mimeType && + previous.width === frame.width && + previous.height === frame.height && + previous.appName === frame.appName && + previous.windowTitle === frame.windowTitle; + previousTarget = target; + unchangedFrames = unchanged ? Math.min(unchangedFrames + 1, 3) : 0; + return unchanged ? Effect.void : publish(preview, { status: "live", frame }); + }), Effect.catchTags({ - CuaWindowPreviewCaptureError: (error) => - isCurrent() + CuaWindowPreviewCaptureError: (error) => { + unchangedFrames = 0; + return isCurrent() ? publish(preview, { status: "unavailable", detail: typeof error.cause === "string" ? error.cause : error.message, }) - : Effect.void, + : Effect.void; + }, }), ); }); @@ -230,10 +315,11 @@ export const make = Effect.fn("CuaWindowPreview.make")(function* ( const startedAt = yield* Clock.currentTimeMillis; yield* capture; const elapsed = Math.max(0, (yield* Clock.currentTimeMillis) - startedAt); - // Cap at 4 FPS, with at least as much idle time as capture time. + // Poll static windows at 1 FPS; changed frames immediately restore 4 FPS. + // Leave at least as much idle time as capture time. // Slow captures lower the rate instead of filling a queue or busy-looping. const minimumInterval = readCuaWindowTarget(threadId) - ? Duration.toMillis(refreshInterval) + ? Duration.toMillis(refreshInterval) * (unchangedFrames >= 3 ? 4 : 1) : 1500; yield* Effect.sleep(Math.max(minimumInterval - elapsed, elapsed)); }).pipe(Effect.forever); diff --git a/patches/cua-driver-linux/0.34.0.patch b/patches/cua-driver-linux/0.34.0.patch index 4ed81c76d7e1f..9680d22b131eb 100644 --- a/patches/cua-driver-linux/0.34.0.patch +++ b/patches/cua-driver-linux/0.34.0.patch @@ -157,10 +157,50 @@ index 1d35cd5..06035e6 100644 OverlayMsg::Cmd(KeyedOverlayCommand { key: key.to_owned(), diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs -index ca0dec4..464c322 100644 +index ca0dec4..b0d4dc7 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs -@@ -1337,7 +1337,12 @@ impl Tool for GetWindowStateTool { +@@ -1090,6 +1090,7 @@ impl Tool for GetWindowStateTool { + "pid":{"type":"integer","description":"Process ID that owns the window."}, + "window_id":{"type":"integer","description":"Native window identifier from list_windows, or the `popup.window_id` a click / right_click result named (an open context menu / popover; its menu items then get element indices). Omitted: the pid's open popup menu when one is mapped, else its focused / active / largest window."}, + "capture_mode": cua_driver_core::capture_mode::capture_mode_schema(), ++ "display_only":{"type":"boolean","description":"Default false. With include_accessibility_tree:false, returns pixels for display without registering an action capture or replacing the agent snapshot. The image cannot ground input actions."}, + "include_accessibility_tree":{"type":"boolean", + "description":"Default true — walk the AT-SPI tree and return `elements` + `tree_markdown` alongside the screenshot. Set false to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Mirrors include_screenshot. Setting BOTH include_accessibility_tree:false AND include_screenshot:false is an error (nothing to return)."}, + "include_screenshot":{"type":"boolean", +@@ -1108,6 +1109,15 @@ impl Tool for GetWindowStateTool { + } + + async fn invoke(&self, args: Value) -> ToolResult { ++ let display_only = args.get("display_only").and_then(Value::as_bool) == Some(true); ++ if display_only ++ && args ++ .get("include_accessibility_tree") ++ .and_then(Value::as_bool) ++ != Some(false) ++ { ++ return ToolResult::error("display_only requires include_accessibility_tree:false"); ++ } + use cua_driver_core::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { + Ok(v) => v, +@@ -1288,10 +1298,11 @@ impl Tool for GetWindowStateTool { + screenshot_out_file to force a capture.", + ); + } +- let observation_only = args +- .get("_observation_only") +- .and_then(|value| value.as_bool()) +- == Some(true); ++ let observation_only = display_only ++ || args ++ .get("_observation_only") ++ .and_then(|value| value.as_bool()) ++ == Some(true); + let state = self.state.clone(); + let state_for_capture = state.clone(); + let query_for_walk = query.clone(); +@@ -1337,7 +1348,12 @@ impl Tool for GetWindowStateTool { window_overlays(pid, xid) }; let screenshot = if should_capture { @@ -174,7 +214,7 @@ index ca0dec4..464c322 100644 overlays .window_rect .ok_or_else(|| anyhow::anyhow!("window geometry unavailable")) -@@ -1345,12 +1350,16 @@ impl Tool for GetWindowStateTool { +@@ -1345,12 +1361,16 @@ impl Tool for GetWindowStateTool { crate::capture::screenshot_root_region_png(x, y, w, h) }) .or_else(|_| crate::wayland::screenshot_dispatch_with_pid(xid, pid)) @@ -194,6 +234,43 @@ index ca0dec4..464c322 100644 let png = crate::capture::resize_png_if_needed(&raw, max_dim)?; let (w, h) = crate::capture::png_dimensions_pub(&png)?; // An explicit per-call max_image_dimension (0 = native) +@@ -1734,6 +1754,10 @@ impl Tool for GetWindowStateTool { + window_bounds is where it sits on the screen. Pass scope:\"desktop\" only \ + for get_desktop_state pixels." + ); ++ if display_only { ++ structured["display_only"] = json!(true); ++ structured["frame_note"] = json!("Display-only preview; not registered for input actions. Obtain a normal get_window_state screenshot before acting."); ++ } + + // The capture-only path (include_accessibility_tree:false) leaves + // `content` empty when the screenshot was also unavailable — most +@@ -14542,3 +14566,25 @@ mod cursor_hook_emission_tests { + assert!(seen[1].pressed); + } + } ++ ++#[cfg(test)] ++#[tokio::test] ++async fn display_only_refuses_accessibility_snapshot_requests() { ++ let tool = GetWindowStateTool { ++ state: ToolState::new(), ++ }; ++ for tree in [None, Some(true)] { ++ let mut args = json!({"pid": 42, "window_id": 7, "display_only": true}); ++ if let Some(tree) = tree { ++ args["include_accessibility_tree"] = json!(tree); ++ } ++ let result = tool.invoke(args).await; ++ assert_eq!( ++ serde_json::to_value(result).unwrap(), ++ serde_json::to_value(ToolResult::error( ++ "display_only requires include_accessibility_tree:false" ++ )) ++ .unwrap() ++ ); ++ } ++} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs index a2691a0..4ab3a74 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs diff --git a/patches/cua-driver-linux/README.md b/patches/cua-driver-linux/README.md index f14c212b3addc..74041d998083c 100644 --- a/patches/cua-driver-linux/README.md +++ b/patches/cua-driver-linux/README.md @@ -1,13 +1,17 @@ # Linux Cua Driver test build `0.34.0.patch` contains the native changes used by the Linux PiP test installer -`0.0.46-preview.20261007.2755`. It applies to trycua/cua commit +`0.0.46-preview.20261007.2756`. It applies to trycua/cua commit `b0968e1b12834e485dda68789541a3cc57664a9f` (`cua-driver-rs-v0.34.0`). This is a source patch, not a pnpm dependency patch. Standard T3 builds still use upstream release binaries; this file does not silently replace those binaries. The patch includes the Wayland host opt-in, exact-window GNOME capture and -preview downscaling (helper API 10), and persistent D-Bus transport. It retains +preview downscaling (helper API 10), persistent D-Bus transport, and an opt-in +`display_only` screenshot path that does not register action captures or replace +agent snapshots. T3 discovers the Linux extension through the daemon tool schema; +older drivers and other platforms retain the typed screenshot path. Static-frame +deduplication and polling backoff apply on every platform. It retains owner attestation on each call and reconnects after failed calls without replaying mutations. The transport has a separate runtime because synchronous helper callers can already be inside Tokio. @@ -21,6 +25,7 @@ git checkout b0968e1b12834e485dda68789541a3cc57664a9f git apply /absolute/path/to/t3code/patches/cua-driver-linux/0.34.0.patch cd libs/cua-driver/rust cargo test -p platform-linux wayland::shell_helper --lib +cargo test -p platform-linux display_only_refuses --lib cargo build --locked -p cua-driver -p cua-driver-sdk ``` From 3b3d1821a3c8c3ab2b1aaf38988417dfc83f8843 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:22:33 -0700 Subject: [PATCH 4/8] fix(cua): ignore GNOME timestamps when deduplicating previews --- patches/cua-driver-linux/0.34.0.patch | 57 ++++++++++++++++++++++++--- patches/cua-driver-linux/README.md | 6 ++- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/patches/cua-driver-linux/0.34.0.patch b/patches/cua-driver-linux/0.34.0.patch index 9680d22b131eb..d2d8e40680bf8 100644 --- a/patches/cua-driver-linux/0.34.0.patch +++ b/patches/cua-driver-linux/0.34.0.patch @@ -465,7 +465,7 @@ index 0909579..86d050e 100644 return listed_windows(apply_pid_filter(ws, filter_pid)); } diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs -index 111b4ed..8483142 100644 +index 111b4ed..a36c240 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs @@ -18,12 +18,11 @@ @@ -524,7 +524,7 @@ index 111b4ed..8483142 100644 } /// Resolve the helper's immutable unique bus name and prove that it is hosted -@@ -210,6 +187,42 @@ fn is_trusted_gnome_shell(pid: u32) -> bool { +@@ -210,6 +187,62 @@ fn is_trusted_gnome_shell(pid: u32) -> bool { .is_some_and(|meta| meta.uid() == 0 && meta.permissions().mode() & 0o022 == 0) } @@ -553,7 +553,7 @@ index 111b4ed..8483142 100644 + &[id.to_string(), pid.to_string(), max_dimension.to_string()], + Duration::from_secs(5), + )?; -+ let png = decode_capture(&raw)?; ++ let png = strip_preview_timestamp(decode_capture(&raw)?)?; + let tail = &raw[raw.rfind('\'')? + 1..]; + let dims: Vec = tail.split(',').filter_map(|part| + part.trim().trim_end_matches(')').trim().trim_start_matches("uint32 ").parse().ok() @@ -563,11 +563,31 @@ index 111b4ed..8483142 100644 + _ => None, + } +} ++ ++/// GNOME embeds a wall-clock timestamp even when pixels are identical. Remove ++/// only that ancillary chunk so preview deduplication needs no pixel decoding. ++fn strip_preview_timestamp(mut png: Vec) -> Option> { ++ if !png.starts_with(b"\x89PNG\r\n\x1a\n") { return None; } ++ let mut offset = 8usize; ++ loop { ++ let header = png.get(offset..offset.checked_add(8)?)?; ++ let size = u32::from_be_bytes(header[..4].try_into().ok()?) as usize; ++ let end = offset.checked_add(12)?.checked_add(size)?; ++ let chunk = png.get(offset..end)?; ++ if &chunk[4..8] == b"tEXt" && chunk[8..8 + size].starts_with(b"Creation Time\0") { ++ png.drain(offset..end); ++ } else if &chunk[4..8] == b"IEND" { ++ return (size == 0 && end == png.len()).then_some(png); ++ } else { ++ offset = end; ++ } ++ } ++} + /// Capture the GNOME stage through the compositor helper. /// /// Mutter does not expose wlroots screencopy protocols, and its one-shot -@@ -247,53 +260,6 @@ fn decode_capture(raw: &str) -> Option> { +@@ -247,53 +280,6 @@ fn decode_capture(raw: &str) -> Option> { B64.decode(&raw[start..end]).ok() } @@ -621,7 +641,32 @@ index 111b4ed..8483142 100644 /// Screen origin of the compositor frame backing `pid`. /// /// `screenshot_window_dispatch` crops the Shell stage to this same frame -@@ -643,9 +609,9 @@ mod tests { +@@ -585,6 +571,24 @@ pub fn hide_cursor() { + mod tests { + use super::*; + ++ #[test] ++ fn preview_timestamp_does_not_change_pixels_or_frame_identity() { ++ use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; ++ let first = B64.decode("iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAE3RFWHRDcmVhdGlvbiBUaW1lAGZpcnN0muU4QwAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAAAUSURBVHicY5SLqvjPwMDAwMQABQAczgHzr55lzgAAAABJRU5ErkJggg==").unwrap(); ++ let second = B64.decode("iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFHRFWHRDcmVhdGlvbiBUaW1lAHNlY29uZKzNUcIAAAAZdEVYdFNvZnR3YXJlAGdub21lLXNjcmVlbnNob3TvA78+AAAAFElEQVR4nGOUi6r4z8DAwMDEAAUAHM4B86+eZc4AAAAASUVORK5CYII=").unwrap(); ++ let stable = strip_preview_timestamp(first.clone()).unwrap(); ++ assert_eq!(stable, strip_preview_timestamp(second).unwrap()); ++ assert_eq!(image::load_from_memory(&stable).unwrap().to_rgba8(), ++ image::load_from_memory(&first).unwrap().to_rgba8()); ++ assert!(stable.windows(8).any(|bytes| bytes == b"Software")); ++ assert_eq!(Some(stable.clone()), strip_preview_timestamp(stable)); ++ assert!(strip_preview_timestamp(first[..first.len() - 1].to_vec()).is_none()); ++ let mut malformed = first; ++ malformed[8..12].copy_from_slice(&u32::MAX.to_be_bytes()); ++ assert!(strip_preview_timestamp(malformed).is_none()); ++ } ++ ++ + const EXTENSION_SOURCE: &str = + include_str!("../../../../../wayland-helper/winrects@cua/extension.js"); + const EXTENSION_METADATA: &str = +@@ -643,9 +647,9 @@ mod tests { } #[test] @@ -633,7 +678,7 @@ index 111b4ed..8483142 100644 assert!(EXTENSION_SOURCE.contains("SetCursorState")); assert!(EXTENSION_SOURCE.contains("SetCursorColor")); assert!(EXTENSION_SOURCE.contains("SetSessionLabel")); -@@ -670,7 +636,7 @@ mod tests { +@@ -670,7 +674,7 @@ mod tests { assert!(!EXTENSION_SOURCE.contains("function drawModifiers")); let metadata: serde_json::Value = serde_json::from_str(EXTENSION_METADATA).expect("valid bundled helper metadata"); diff --git a/patches/cua-driver-linux/README.md b/patches/cua-driver-linux/README.md index 74041d998083c..45d3b61e109c2 100644 --- a/patches/cua-driver-linux/README.md +++ b/patches/cua-driver-linux/README.md @@ -1,7 +1,7 @@ # Linux Cua Driver test build `0.34.0.patch` contains the native changes used by the Linux PiP test installer -`0.0.46-preview.20261007.2756`. It applies to trycua/cua commit +`0.0.46-preview.20261007.2757`. It applies to trycua/cua commit `b0968e1b12834e485dda68789541a3cc57664a9f` (`cua-driver-rs-v0.34.0`). This is a source patch, not a pnpm dependency patch. Standard T3 builds still use upstream release binaries; this file does not silently replace those binaries. @@ -11,7 +11,9 @@ preview downscaling (helper API 10), persistent D-Bus transport, and an opt-in `display_only` screenshot path that does not register action captures or replace agent snapshots. T3 discovers the Linux extension through the daemon tool schema; older drivers and other platforms retain the typed screenshot path. Static-frame -deduplication and polling backoff apply on every platform. It retains +deduplication and polling backoff apply on every platform. GNOME preview PNGs +omit the changing `Creation Time` text chunk so identical pixels produce +identical bytes without decoding and re-encoding the image. It retains owner attestation on each call and reconnects after failed calls without replaying mutations. The transport has a separate runtime because synchronous helper callers can already be inside Tokio. From 126ea5f481b3521e78d93c9624471806d8ab999b Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:13:26 -0700 Subject: [PATCH 5/8] fix(cua): ship patched Linux driver in standard builds --- .github/workflows/release-desktop.yml | 11 ++- .../src/cua/CuaDriverInstallation.test.ts | 42 ++++++++++- apps/server/src/cua/CuaDriverInstallation.ts | 40 +++++++++- apps/server/src/cua/CuaWindowPreview.test.ts | 15 +--- patches/cua-driver-linux/README.md | 75 ++++++++++--------- scripts/build-cli-archive.ts | 18 +++++ scripts/build-cua-linux.sh | 67 +++++++++++++++++ scripts/build-desktop-artifact.test.ts | 33 +++++++- scripts/build-desktop-artifact.ts | 21 ++++++ scripts/lib/cua-linux-bundle.test.ts | 49 ++++++++++++ scripts/lib/cua-linux-bundle.ts | 35 +++++++++ 11 files changed, 349 insertions(+), 57 deletions(-) create mode 100755 scripts/build-cua-linux.sh create mode 100644 scripts/lib/cua-linux-bundle.test.ts create mode 100644 scripts/lib/cua-linux-bundle.ts diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 5a16a9b4c170f..9d37f8e10c201 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -203,8 +203,15 @@ jobs: native/hyprland-snap-shot/target/${{ inputs.rust_target }}/release/t3-hyprland-snap-shot key: linux-capture-helpers-${{ inputs.rust_target }}-${{ hashFiles('native/kde-snap-shot/Cargo.lock', 'native/kde-snap-shot/Cargo.toml', 'native/kde-snap-shot/src/**', 'native/hyprland-snap-shot/Cargo.lock', 'native/hyprland-snap-shot/Cargo.toml', 'native/hyprland-snap-shot/src/**', 'native/hyprland-snap-shot/protocols/**') }} + - name: Cache patched Linux Cua build + if: inputs.platform == 'linux' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: node_modules/.cache/t3code/cua-linux + key: cua-linux-${{ inputs.arch }}-${{ hashFiles('scripts/build-cua-linux.sh', 'patches/cua-driver-linux/0.34.0.patch') }} + - name: Setup Rust - if: steps.resource_monitor_cache.outputs.cache-hit != 'true' || (inputs.platform == 'linux' && steps.capture_helper_cache.outputs.cache-hit != 'true') + if: inputs.platform == 'linux' || steps.resource_monitor_cache.outputs.cache-hit != 'true' uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable with: toolchain: stable @@ -447,7 +454,7 @@ jobs: shell: bash run: | sudo apt-get update - sudo apt-get install -y libsecret-1-dev pkg-config + sudo apt-get install -y libsecret-1-dev pkg-config libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev libxkbcommon-dev if ! command -v magick >/dev/null 2>&1 && ! command -v convert >/dev/null 2>&1; then sudo apt-get install -y imagemagick fi diff --git a/apps/server/src/cua/CuaDriverInstallation.test.ts b/apps/server/src/cua/CuaDriverInstallation.test.ts index 3060fd78b57fa..927b481a95d61 100644 --- a/apps/server/src/cua/CuaDriverInstallation.test.ts +++ b/apps/server/src/cua/CuaDriverInstallation.test.ts @@ -55,7 +55,11 @@ const FILES = ["cua-driver.exe", "cua-driver-uia.exe", "cua-cursor-theme.exe"]; const archive = makeZip(FILES.map((name) => ({ name, data: name }))); const makeHarness = Effect.fn("test.makeCuaDriverInstallation")(function* ( - options: { readonly sha256?: string; readonly archive?: Buffer } = {}, + options: { + readonly sha256?: string; + readonly archive?: Buffer; + readonly bundled?: "complete" | "missing"; + } = {}, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -63,8 +67,22 @@ const makeHarness = Effect.fn("test.makeCuaDriverInstallation")(function* ( const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cua-install-test-" }); const body = options.archive ?? archive; const requests: Array = []; + const bundledDir = path.join(baseDir, "bundled-cua"); + if (options.bundled === "complete") { + for (const file of [ + "cua-driver", + "libcua_driver_sdk.so", + "cua-cursor-theme", + "wayland-helper/winrects@cua/extension.js", + ]) { + const destination = path.join(bundledDir, file); + yield* fs.makeDirectory(path.dirname(destination), { recursive: true }); + yield* fs.writeFileString(destination, "patched"); + } + } const installation = yield* CuaDriverInstallation.makeCuaDriverInstallation({ baseDir, + ...(options.bundled ? { bundledDir } : {}), release: { version: "9.9.9", archiveName: "cua-driver-fixture.zip", @@ -91,6 +109,7 @@ const makeHarness = Effect.fn("test.makeCuaDriverInstallation")(function* ( ); return { installation, + bundledDir, fs, path, installRoot: path.join(baseDir, "tools", "cua-driver"), @@ -99,6 +118,27 @@ const makeHarness = Effect.fn("test.makeCuaDriverInstallation")(function* ( }); it.layer(NodeServices.layer)("CuaDriverInstallation", (it) => { + it.effect( + "uses the shipped patched driver without fetching or accepting an older cached release", + () => + Effect.gen(function* () { + const h = yield* makeHarness({ bundled: "complete" }); + yield* h.fs.makeDirectory(h.path.join(h.installRoot, "9.9.9"), { recursive: true }); + yield* h.fs.writeFileString(h.path.join(h.installRoot, "9.9.9/cua-driver.exe"), "upstream"); + expect(yield* h.installation.executable).toBe(h.path.join(h.bundledDir, "cua-driver")); + expect(h.requests).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect("refuses an incomplete shipped bundle instead of downloading upstream", () => + Effect.gen(function* () { + const h = yield* makeHarness({ bundled: "missing" }); + const error = yield* Effect.flip(h.installation.executable); + expect(error.detail).toContain("bundled Cua Driver is incomplete"); + expect(h.requests).toEqual([]); + }).pipe(Effect.scoped), + ); + it.effect("installs the verified release once and replaces older versions", () => Effect.gen(function* () { const { installation, fs, path, installRoot, requests } = yield* makeHarness(); diff --git a/apps/server/src/cua/CuaDriverInstallation.ts b/apps/server/src/cua/CuaDriverInstallation.ts index ab43f0484a47f..85e7b2eab4efa 100644 --- a/apps/server/src/cua/CuaDriverInstallation.ts +++ b/apps/server/src/cua/CuaDriverInstallation.ts @@ -35,8 +35,8 @@ const wrapFailure = (detail: string) => (cause: unknown) => /** * The pinned Cua Driver for a server that runs without the desktop app. The - * desktop bundles the same release in its resources; a standalone `t3` server - * downloads it into T3's home the first time a session needs it. + * desktop and Linux CLI bundle their driver. Other standalone servers download + * the pinned upstream release into T3's home when a session first needs it. */ export class CuaDriverInstallation extends Context.Service< CuaDriverInstallation, @@ -47,7 +47,11 @@ export class CuaDriverInstallation extends Context.Service< >()("t3/cua/CuaDriverInstallation") {} export const makeCuaDriverInstallation = Effect.fn("CuaDriverInstallation.make")( - function* (options: { readonly baseDir: string; readonly release?: CuaDriverRelease | null }) { + function* (options: { + readonly baseDir: string; + readonly release?: CuaDriverRelease | null; + readonly bundledDir?: string; + }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const http = yield* HttpClient.HttpClient; @@ -214,6 +218,27 @@ export const makeCuaDriverInstallation = Effect.fn("CuaDriverInstallation.make") const executable = gate.withPermit( Effect.gen(function* () { + if (options.bundledDir !== undefined) { + for (const file of [ + "cua-driver", + "libcua_driver_sdk.so", + "cua-cursor-theme", + "wayland-helper/winrects@cua/extension.js", + ]) { + const filePath = path.join(options.bundledDir, file); + if ( + !(yield* fs.stat(filePath).pipe( + Effect.map((info) => info.type === "File"), + Effect.orElseSucceed(() => false), + )) + ) { + return yield* new CuaDriverInstallError({ + detail: `The bundled Cua Driver is incomplete: ${filePath}. Reinstall T3 Code.`, + }); + } + } + return path.join(options.bundledDir, "cua-driver"); + } if (release === null) { return yield* new CuaDriverInstallError({ detail: `Cua publishes no driver for ${platform}-${arch}.`, @@ -234,6 +259,13 @@ export const layer = Layer.effect( CuaDriverInstallation, Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; - return yield* makeCuaDriverInstallation({ baseDir: config.baseDir }); + const path = yield* Path.Path; + const bundled = (yield* HostProcess.Platform) === "linux" && (yield* HostProcess.IsExecutable); + return yield* makeCuaDriverInstallation({ + baseDir: config.baseDir, + ...(bundled + ? { bundledDir: path.join(path.dirname(yield* HostProcess.ExecutablePath), "cua-driver") } + : {}), + }); }), ); diff --git a/apps/server/src/cua/CuaWindowPreview.test.ts b/apps/server/src/cua/CuaWindowPreview.test.ts index 961cd99012085..22c77f6da4af3 100644 --- a/apps/server/src/cua/CuaWindowPreview.test.ts +++ b/apps/server/src/cua/CuaWindowPreview.test.ts @@ -151,19 +151,8 @@ describe("CuaWindowPreview", () => { (displayOnly) => Effect.gen(function* () { clearCuaToolContext(threadId); - McpProviderSession.setMcpProviderSession({ - threadId, - endpoint: "http://127.0.0.1/mcp", - authorizationHeader: "Bearer x", - capabilities: new Set(), - cuaDriver: { command: "/driver", args: [], environment: [], socketPath: "/tmp/cua.sock" }, - } as unknown as McpProviderSession.McpProviderSessionConfig); - yield* Effect.addFinalizer(() => - Effect.sync(() => { - McpProviderSession.clearMcpProviderSession(threadId); - clearCuaToolContext(threadId); - }), - ); + yield* setCuaSession; + yield* Effect.addFinalizer(() => Effect.sync(() => clearCuaToolContext(threadId))); const select = (windowId: number) => cuaToolPresentation({ threadId, diff --git a/patches/cua-driver-linux/README.md b/patches/cua-driver-linux/README.md index 45d3b61e109c2..0fd6348049df7 100644 --- a/patches/cua-driver-linux/README.md +++ b/patches/cua-driver-linux/README.md @@ -1,40 +1,43 @@ -# Linux Cua Driver test build - -`0.34.0.patch` contains the native changes used by the Linux PiP test installer -`0.0.46-preview.20261007.2757`. It applies to trycua/cua commit -`b0968e1b12834e485dda68789541a3cc57664a9f` (`cua-driver-rs-v0.34.0`). -This is a source patch, not a pnpm dependency patch. Standard T3 builds still use -upstream release binaries; this file does not silently replace those binaries. - -The patch includes the Wayland host opt-in, exact-window GNOME capture and -preview downscaling (helper API 10), persistent D-Bus transport, and an opt-in -`display_only` screenshot path that does not register action captures or replace -agent snapshots. T3 discovers the Linux extension through the daemon tool schema; -older drivers and other platforms retain the typed screenshot path. Static-frame -deduplication and polling backoff apply on every platform. GNOME preview PNGs -omit the changing `Creation Time` text chunk so identical pixels produce -identical bytes without decoding and re-encoding the image. It retains -owner attestation on each call and reconnects after failed calls without -replaying mutations. The transport has a separate runtime because synchronous -helper callers can already be inside Tokio. - -To reproduce the native binaries in a separate checkout: +# Patched Linux Cua Driver + +Linux desktop packages and Linux CLI archives build Cua Driver from trycua/cua +commit `b0968e1b12834e485dda68789541a3cc57664a9f` (`cua-driver-rs-v0.34.0`), applying +`0.34.0.patch`. This is a source patch, not a pnpm dependency patch. + +The normal packaging commands invoke `scripts/build-cua-linux.sh`. It builds the +driver, cursor helper, and SDK library together in release mode, with upstream's +`cua-driver/portal-input` feature. Packaging also ships the patched GNOME helper +(API 10) and replaces the SDK npm package's native library. Replacing only the +daemon would leave the embedded host running upstream code. + +Linux CLI archives carry their own driver beside the `t3` executable. They use +that bundle before any previously downloaded version and fail if it is +incomplete, rather than silently downloading an unpatched driver. macOS and +Windows keep their upstream native binaries. Source-run development servers +retain the upstream downloader unless `T3CODE_CUA_DRIVER_PATH` selects a custom +driver; the source patch is applied during Linux release packaging. + +Build on Linux matching the target architecture (`x64` or `arm64`), with Rustup, +Git, a C/C++ toolchain, pkg-config, and the development packages `libx11-dev`, +`libxi-dev`, `libxtst-dev`, `libxext-dev`, `libwayland-dev`, and `libxkbcommon-dev`. +CI installs these dependencies. The upstream checkout pins its Rust toolchain. + +To build just the native bundle from the repository root: ```sh -git clone https://github.com/trycua/cua.git cua-linux-preview -cd cua-linux-preview -git checkout b0968e1b12834e485dda68789541a3cc57664a9f -git apply /absolute/path/to/t3code/patches/cua-driver-linux/0.34.0.patch -cd libs/cua-driver/rust -cargo test -p platform-linux wayland::shell_helper --lib -cargo test -p platform-linux display_only_refuses --lib -cargo build --locked -p cua-driver -p cua-driver-sdk +bash scripts/build-cua-linux.sh "$PWD" x64 /tmp/t3-cua-linux ``` -The test installer uses `target/debug/cua-driver` and -`target/debug/libcua_driver_sdk.so`, plus the patched files in -`libs/cua-driver/wayland-helper/winrects@cua`. Both the desktop driver bundle and -the SDK npm package's native library must use the matching build. Do not replace -files inside a running T3 installation. This debug build does not enable the -optional `portal-input` feature; GNOME fallback mouse/keyboard injection remains -unavailable, although supported accessibility actions and window capture work. +Builds are cached by architecture, source commit, patch, and build script; cached +bundle files are checksum-verified. Desktop and CLI packaging share that cache. +A native build or SDK staging failure stops packaging. + +The patch preserves Wayland host opt-in, compositor-attested window capture, +preview downscaling, persistent D-Bus transport, and display-only screenshots +that do not replace agent snapshots or register action captures. It removes +GNOME's changing PNG `Creation Time` chunk without decoding pixels, allowing +static-frame deduplication. D-Bus calls retain owner attestation and reconnect +after failure without replaying mutations. + +Build 2757's earlier local validation used debug binaries without `portal-input`. +Normal release packaging uses the optimized build described above. diff --git a/scripts/build-cli-archive.ts b/scripts/build-cli-archive.ts index 3c74b6714ddeb..4f318c938eb5f 100644 --- a/scripts/build-cli-archive.ts +++ b/scripts/build-cli-archive.ts @@ -21,12 +21,14 @@ import rootPackageJson from "../package.json" with { type: "json" }; import serverPackageJson from "../apps/server/package.json" with { type: "json" }; import { + stageCuaDriverExecutable, createStagePatchedDependencies, createStageWorkspaceConfig, resolveFffNativeDependencies, STAGE_INSTALL_ARGS, } from "./build-desktop-artifact.ts"; import { assertLinuxArm64PageCompatibility } from "./lib/linux-arm64-sea.ts"; +import { stageCuaLinuxSdk } from "./lib/cua-linux-bundle.ts"; import { selectCliRuntimeExternalDependencies } from "./lib/cli-external-packages.ts"; import { signWithJsign } from "./lib/windows-jsign.ts"; import { windowsSystemTar } from "./lib/windows-tar.ts"; @@ -530,6 +532,22 @@ const buildCliArchive = Effect.fn("buildCliArchive")(function* (input: { version: input.version, }); + if (input.platform === "linux") { + yield* stageCuaDriverExecutable({ + platform: input.platform, + arch: input.arch, + repoRoot, + stageRoot, + stageResourcesDir: contentDir, + verbose: true, + }); + yield* stageCuaLinuxSdk({ + arch: input.arch, + bundleDir: path.join(contentDir, "cua-driver"), + nodeModulesDir: path.join(contentDir, "node_modules"), + }); + } + const executablePath = path.join(contentDir, executableName); if (input.platform === "linux") { yield* bundleLinuxLibatomic({ repoRoot, contentDir, executablePath }); diff --git a/scripts/build-cua-linux.sh b/scripts/build-cua-linux.sh new file mode 100755 index 0000000000000..5c7b0f7e6905e --- /dev/null +++ b/scripts/build-cua-linux.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Build the Linux driver and SDK from the same patched source. Build hosts need +# Rust/rustup, git, and the X11/Wayland development libraries used by upstream. +set -euo pipefail + +repo_root=$(cd "$1" && pwd) +arch=$2 +output_dir=$3 +version=${4:-0.34.0} +if [[ "$version" != "0.34.0" ]]; then + echo "Update the Linux source pin and patch before shipping Cua Driver $version." >&2 + exit 1 +fi +source_commit=b0968e1b12834e485dda68789541a3cc57664a9f +patch_file="$repo_root/patches/cua-driver-linux/0.34.0.patch" +case "$(uname -s):$(uname -m):$arch" in + Linux:x86_64:x64|Linux:aarch64:arm64) ;; + *) echo 'Patched Cua Driver must be built on a Linux host of the target architecture.' >&2; exit 1 ;; +esac +patch_hash=$(sha256sum "$patch_file" | cut -d ' ' -f 1) +cache_root="$repo_root/node_modules/.cache/t3code/cua-linux" +build_hash=$(cat "$patch_file" "$repo_root/scripts/build-cua-linux.sh" | sha256sum | cut -d ' ' -f 1) +cache_dir="$cache_root/$arch-$source_commit-$build_hash" +mkdir -p "$cache_root" +# Desktop and CLI packaging can share the build without racing a checkout or copy. +exec 9>"$cache_root/build.lock" +flock 9 + +if [[ ! -f "$cache_dir/bundle/SHA256SUMS" ]] || ! (cd "$cache_dir/bundle" && sha256sum --check --status SHA256SUMS); then + mkdir -p "$cache_dir" + source_dir="$cache_dir/source" + if [[ ! -f "$source_dir/.t3-patched" ]]; then + work_dir=$(mktemp -d "$cache_dir/source-XXXXXX") + trap 'rm -rf "$work_dir"' EXIT + git -C "$work_dir" init --quiet + git -C "$work_dir" fetch --quiet --depth=1 https://github.com/trycua/cua.git "$source_commit" + git -C "$work_dir" checkout --quiet --detach FETCH_HEAD + git -C "$work_dir" apply --check "$patch_file" + git -C "$work_dir" apply "$patch_file" + touch "$work_dir/.t3-patched" + mv "$work_dir" "$source_dir" + trap - EXIT + fi + ( + cd "$source_dir/libs/cua-driver/rust" + CARGO_TARGET_DIR="$cache_root/target-$arch" cargo build --locked --release \ + -p cua-driver -p cua-driver-sdk -p cursor-theme-cli --features cua-driver/portal-input + ) + bundle_dir=$(mktemp -d "$cache_dir/bundle-XXXXXX") + trap 'rm -rf "$bundle_dir"' EXIT + chmod 755 "$bundle_dir" + for binary in cua-driver cua-cursor-theme libcua_driver_sdk.so; do + cp "$cache_root/target-$arch/release/$binary" "$bundle_dir/$binary" + done + mkdir -p "$bundle_dir/wayland-helper" + cp -a "$source_dir/libs/cua-driver/wayland-helper/winrects@cua" "$bundle_dir/wayland-helper/" + cp "$source_dir/LICENSE.md" "$bundle_dir/LICENSE" + cp "$source_dir/libs/cua-driver/rust/THIRD_PARTY_NOTICES.md" "$bundle_dir/THIRD_PARTY_NOTICES.md" + (cd "$bundle_dir" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) + # A failed build never publishes a partially updated bundle. + rm -rf "$cache_dir/bundle" + mv "$bundle_dir" "$cache_dir/bundle" + trap - EXIT +fi +mkdir -p "$output_dir" +cp -a "$cache_dir/bundle/." "$output_dir/" +echo "Staged patched Cua Driver 0.34.0 ($arch, patch $patch_hash)." diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index 8f6e218d1eb15..d1f8e81098d2f 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -259,7 +259,7 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { ); const error = yield* Effect.flip( stageCuaDriverExecutable({ - platform: "linux", + platform: "mac", arch: "x64", repoRoot: root, stageRoot: root, @@ -276,6 +276,37 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { ), ); + it.effect("does not fall back to upstream when the patched Linux build fails", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cua-build-failure-" }); + const commands: string[] = []; + const error = yield* Effect.flip( + stageCuaDriverExecutable({ + platform: "linux", + arch: "arm64", + repoRoot: root, + stageRoot: root, + stageResourcesDir: `${root}/resources`, + verbose: false, + }).pipe( + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => { + commands.push((command as unknown as { command: string }).command); + return Effect.succeed(mockProcess(42)); + }), + ), + ), + ); + assert.instanceOf(error, BuildCommandFailedError); + assert.deepStrictEqual(commands, ["bash"]); + assert.isFalse(yield* fs.exists(`${root}/resources/cua-driver`)); + }), + ), + ); + it("bundles the universal Cua Driver on every macOS arch", () => { for (const arch of ["arm64", "x64", "universal"] as const) { assert.include(resolveCuaDriverAsset("mac", arch).archiveName, "darwin-universal-binary"); diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 6e06df50a2935..489794ab74c15 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -30,6 +30,7 @@ import { releasePackageFiles, updateReleasePackageVersions, } from "./update-release-package-versions.ts"; +import { stageCuaLinuxSdk } from "./lib/cua-linux-bundle.ts"; import { applyWebBrandAssets } from "./apply-web-brand-assets.ts"; import { BRAND_ASSET_PATHS, @@ -3059,6 +3060,19 @@ export const stageCuaDriverExecutable = Effect.fn("stageCuaDriverExecutable")(fu }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; + if (input.platform === "linux") { + yield* runCommand( + ChildProcess.make("bash", [ + path.join(input.repoRoot, "scripts/build-cua-linux.sh"), + input.repoRoot, + input.arch, + path.join(input.stageResourcesDir, "cua-driver"), + CUA_DRIVER_VERSION, + ]), + { label: "build patched Linux Cua Driver", verbose: input.verbose }, + ); + return; + } const asset = resolveCuaDriverAsset(input.platform, input.arch); const cacheDir = path.join(input.repoRoot, "node_modules/.cache/t3code/cua-driver"); const archivePath = path.join(cacheDir, asset.archiveName); @@ -4092,6 +4106,13 @@ const buildDesktopArtifactUnlocked = Effect.fn("buildDesktopArtifactUnlocked")(f }), { label: "vp install --prod", verbose: options.verbose }, ); + if (options.platform === "linux") { + yield* stageCuaLinuxSdk({ + arch: options.arch, + bundleDir: path.join(stageProdResourcesDir, "cua-driver"), + nodeModulesDir: path.join(stageAppDir, "node_modules"), + }); + } yield* stageClerkPasskeyNativeBinaries(stageAppDir, options.platform, options.arch); yield* stageKeyringNativeBinaries(stageAppDir, options.platform, options.arch); diff --git a/scripts/lib/cua-linux-bundle.test.ts b/scripts/lib/cua-linux-bundle.test.ts new file mode 100644 index 0000000000000..1857688eb1a4b --- /dev/null +++ b/scripts/lib/cua-linux-bundle.test.ts @@ -0,0 +1,49 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import { stageCuaLinuxSdk } from "./cua-linux-bundle.ts"; + +it.layer(NodeServices.layer)("stageCuaLinuxSdk", (it) => { + it.effect.each(["hoisted", "pnpm"] as const)( + "replaces the %s SDK without changing a shared store file", + (layout) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "cua-sdk-stage-" }); + const nodeModulesDir = path.join(root, "node_modules"); + const deps = + layout === "pnpm" ? path.join(nodeModulesDir, ".pnpm/sdk/node_modules") : nodeModulesDir; + const sdk = path.join(deps, "@trycua/cua-driver"); + const native = path.join(deps, "@trycua/cua-driver-linux-x64-gnu"); + yield* fs.makeDirectory(sdk, { recursive: true }); + yield* fs.makeDirectory(native, { recursive: true }); + yield* fs.writeFileString(path.join(sdk, "package.json"), '{"name":"@trycua/cua-driver"}'); + yield* fs.writeFileString( + path.join(native, "package.json"), + '{"name":"@trycua/cua-driver-linux-x64-gnu"}', + ); + if (layout === "pnpm") { + yield* fs.makeDirectory(path.join(nodeModulesDir, "@trycua"), { recursive: true }); + yield* fs.symlink(sdk, path.join(nodeModulesDir, "@trycua/cua-driver")); + } + const shared = path.join(root, "store-library"); + yield* fs.writeFileString(shared, "upstream"); + const library = path.join(native, "libcua_driver_sdk.so"); + yield* fs.link(shared, library); + const bundleDir = path.join(root, "bundle"); + yield* fs.makeDirectory(bundleDir); + yield* fs.writeFileString(path.join(bundleDir, "libcua_driver_sdk.so"), "patched"); + yield* stageCuaLinuxSdk({ arch: "x64", bundleDir, nodeModulesDir }); + expect(yield* fs.readFileString(library)).toBe("patched"); + expect(yield* fs.readFileString(shared)).toBe("upstream"); + yield* fs.remove(library); + const error = yield* Effect.flip( + stageCuaLinuxSdk({ arch: "x64", bundleDir, nodeModulesDir }), + ); + expect(error._tag).toBe("CuaLinuxSdkMissingError"); + }).pipe(Effect.scoped), + ); +}); diff --git a/scripts/lib/cua-linux-bundle.ts b/scripts/lib/cua-linux-bundle.ts new file mode 100644 index 0000000000000..80419235999da --- /dev/null +++ b/scripts/lib/cua-linux-bundle.ts @@ -0,0 +1,35 @@ +// @effect-diagnostics nodeBuiltinImport:off - Resolve the native dependency through its installed SDK package. +import * as NodeModule from "node:module"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +export class CuaLinuxSdkMissingError extends Schema.TaggedError()( + "CuaLinuxSdkMissingError", + { filePath: Schema.String }, +) {} + +/** Replace the SDK's native library as well as the daemon; they are one build. */ +export const stageCuaLinuxSdk = Effect.fn("stageCuaLinuxSdk")(function* (input: { + readonly arch: string; + readonly bundleDir: string; + readonly nodeModulesDir: string; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sdkDir = yield* fs.realPath(path.join(input.nodeModulesDir, "@trycua/cua-driver")); + const nativePackage = `@trycua/cua-driver-linux-${input.arch}-gnu/package.json`; + const nativeManifest = yield* Effect.try({ + try: () => NodeModule.createRequire(path.join(sdkDir, "package.json")).resolve(nativePackage), + catch: () => new CuaLinuxSdkMissingError({ filePath: nativePackage }), + }); + const destination = path.join(path.dirname(nativeManifest), "libcua_driver_sdk.so"); + if (!(yield* fs.exists(destination))) { + return yield* new CuaLinuxSdkMissingError({ filePath: destination }); + } + // pnpm may hardlink this file into its store. Unlink before replacing it so + // packaging cannot modify another checkout's upstream SDK. + yield* fs.remove(destination); + yield* fs.copyFile(path.join(input.bundleDir, "libcua_driver_sdk.so"), destination); +}); From 01f4e4b0f230d17240aabe30fc4a48fad2406fa9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 9 Oct 2026 10:55:52 -0700 Subject: [PATCH 6/8] build(cua): ship upstream Cua Driver on Linux Cua Driver 0.34.1 and later include our Linux fixes: the Wayland opt-in reaches embedded hosts, GNOME windows are captured from their own actor (WinRects API 10), the GNOME helper reuses one session-bus connection, and get_window_state accepts display_only. Linux packaging now bundles the published release like macOS and Windows, so the source patch, the native build script, SDK library replacement, the Linux CLI bundle lookup, and the extra CI toolchain are gone. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release-desktop.yml | 11 +- .../src/cua/CuaDriverInstallation.test.ts | 42 +- apps/server/src/cua/CuaDriverInstallation.ts | 40 +- patches/cua-driver-linux/0.34.0.patch | 1119 ----------------- patches/cua-driver-linux/README.md | 43 - scripts/build-cli-archive.ts | 18 - scripts/build-cua-linux.sh | 67 - scripts/build-desktop-artifact.test.ts | 33 +- scripts/build-desktop-artifact.ts | 21 - scripts/lib/cua-linux-bundle.test.ts | 49 - scripts/lib/cua-linux-bundle.ts | 35 - 11 files changed, 8 insertions(+), 1470 deletions(-) delete mode 100644 patches/cua-driver-linux/0.34.0.patch delete mode 100644 patches/cua-driver-linux/README.md delete mode 100755 scripts/build-cua-linux.sh delete mode 100644 scripts/lib/cua-linux-bundle.test.ts delete mode 100644 scripts/lib/cua-linux-bundle.ts diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 9d37f8e10c201..5a16a9b4c170f 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -203,15 +203,8 @@ jobs: native/hyprland-snap-shot/target/${{ inputs.rust_target }}/release/t3-hyprland-snap-shot key: linux-capture-helpers-${{ inputs.rust_target }}-${{ hashFiles('native/kde-snap-shot/Cargo.lock', 'native/kde-snap-shot/Cargo.toml', 'native/kde-snap-shot/src/**', 'native/hyprland-snap-shot/Cargo.lock', 'native/hyprland-snap-shot/Cargo.toml', 'native/hyprland-snap-shot/src/**', 'native/hyprland-snap-shot/protocols/**') }} - - name: Cache patched Linux Cua build - if: inputs.platform == 'linux' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: node_modules/.cache/t3code/cua-linux - key: cua-linux-${{ inputs.arch }}-${{ hashFiles('scripts/build-cua-linux.sh', 'patches/cua-driver-linux/0.34.0.patch') }} - - name: Setup Rust - if: inputs.platform == 'linux' || steps.resource_monitor_cache.outputs.cache-hit != 'true' + if: steps.resource_monitor_cache.outputs.cache-hit != 'true' || (inputs.platform == 'linux' && steps.capture_helper_cache.outputs.cache-hit != 'true') uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable with: toolchain: stable @@ -454,7 +447,7 @@ jobs: shell: bash run: | sudo apt-get update - sudo apt-get install -y libsecret-1-dev pkg-config libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev libxkbcommon-dev + sudo apt-get install -y libsecret-1-dev pkg-config if ! command -v magick >/dev/null 2>&1 && ! command -v convert >/dev/null 2>&1; then sudo apt-get install -y imagemagick fi diff --git a/apps/server/src/cua/CuaDriverInstallation.test.ts b/apps/server/src/cua/CuaDriverInstallation.test.ts index 927b481a95d61..3060fd78b57fa 100644 --- a/apps/server/src/cua/CuaDriverInstallation.test.ts +++ b/apps/server/src/cua/CuaDriverInstallation.test.ts @@ -55,11 +55,7 @@ const FILES = ["cua-driver.exe", "cua-driver-uia.exe", "cua-cursor-theme.exe"]; const archive = makeZip(FILES.map((name) => ({ name, data: name }))); const makeHarness = Effect.fn("test.makeCuaDriverInstallation")(function* ( - options: { - readonly sha256?: string; - readonly archive?: Buffer; - readonly bundled?: "complete" | "missing"; - } = {}, + options: { readonly sha256?: string; readonly archive?: Buffer } = {}, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -67,22 +63,8 @@ const makeHarness = Effect.fn("test.makeCuaDriverInstallation")(function* ( const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cua-install-test-" }); const body = options.archive ?? archive; const requests: Array = []; - const bundledDir = path.join(baseDir, "bundled-cua"); - if (options.bundled === "complete") { - for (const file of [ - "cua-driver", - "libcua_driver_sdk.so", - "cua-cursor-theme", - "wayland-helper/winrects@cua/extension.js", - ]) { - const destination = path.join(bundledDir, file); - yield* fs.makeDirectory(path.dirname(destination), { recursive: true }); - yield* fs.writeFileString(destination, "patched"); - } - } const installation = yield* CuaDriverInstallation.makeCuaDriverInstallation({ baseDir, - ...(options.bundled ? { bundledDir } : {}), release: { version: "9.9.9", archiveName: "cua-driver-fixture.zip", @@ -109,7 +91,6 @@ const makeHarness = Effect.fn("test.makeCuaDriverInstallation")(function* ( ); return { installation, - bundledDir, fs, path, installRoot: path.join(baseDir, "tools", "cua-driver"), @@ -118,27 +99,6 @@ const makeHarness = Effect.fn("test.makeCuaDriverInstallation")(function* ( }); it.layer(NodeServices.layer)("CuaDriverInstallation", (it) => { - it.effect( - "uses the shipped patched driver without fetching or accepting an older cached release", - () => - Effect.gen(function* () { - const h = yield* makeHarness({ bundled: "complete" }); - yield* h.fs.makeDirectory(h.path.join(h.installRoot, "9.9.9"), { recursive: true }); - yield* h.fs.writeFileString(h.path.join(h.installRoot, "9.9.9/cua-driver.exe"), "upstream"); - expect(yield* h.installation.executable).toBe(h.path.join(h.bundledDir, "cua-driver")); - expect(h.requests).toEqual([]); - }).pipe(Effect.scoped), - ); - - it.effect("refuses an incomplete shipped bundle instead of downloading upstream", () => - Effect.gen(function* () { - const h = yield* makeHarness({ bundled: "missing" }); - const error = yield* Effect.flip(h.installation.executable); - expect(error.detail).toContain("bundled Cua Driver is incomplete"); - expect(h.requests).toEqual([]); - }).pipe(Effect.scoped), - ); - it.effect("installs the verified release once and replaces older versions", () => Effect.gen(function* () { const { installation, fs, path, installRoot, requests } = yield* makeHarness(); diff --git a/apps/server/src/cua/CuaDriverInstallation.ts b/apps/server/src/cua/CuaDriverInstallation.ts index 85e7b2eab4efa..ab43f0484a47f 100644 --- a/apps/server/src/cua/CuaDriverInstallation.ts +++ b/apps/server/src/cua/CuaDriverInstallation.ts @@ -35,8 +35,8 @@ const wrapFailure = (detail: string) => (cause: unknown) => /** * The pinned Cua Driver for a server that runs without the desktop app. The - * desktop and Linux CLI bundle their driver. Other standalone servers download - * the pinned upstream release into T3's home when a session first needs it. + * desktop bundles the same release in its resources; a standalone `t3` server + * downloads it into T3's home the first time a session needs it. */ export class CuaDriverInstallation extends Context.Service< CuaDriverInstallation, @@ -47,11 +47,7 @@ export class CuaDriverInstallation extends Context.Service< >()("t3/cua/CuaDriverInstallation") {} export const makeCuaDriverInstallation = Effect.fn("CuaDriverInstallation.make")( - function* (options: { - readonly baseDir: string; - readonly release?: CuaDriverRelease | null; - readonly bundledDir?: string; - }) { + function* (options: { readonly baseDir: string; readonly release?: CuaDriverRelease | null }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const http = yield* HttpClient.HttpClient; @@ -218,27 +214,6 @@ export const makeCuaDriverInstallation = Effect.fn("CuaDriverInstallation.make") const executable = gate.withPermit( Effect.gen(function* () { - if (options.bundledDir !== undefined) { - for (const file of [ - "cua-driver", - "libcua_driver_sdk.so", - "cua-cursor-theme", - "wayland-helper/winrects@cua/extension.js", - ]) { - const filePath = path.join(options.bundledDir, file); - if ( - !(yield* fs.stat(filePath).pipe( - Effect.map((info) => info.type === "File"), - Effect.orElseSucceed(() => false), - )) - ) { - return yield* new CuaDriverInstallError({ - detail: `The bundled Cua Driver is incomplete: ${filePath}. Reinstall T3 Code.`, - }); - } - } - return path.join(options.bundledDir, "cua-driver"); - } if (release === null) { return yield* new CuaDriverInstallError({ detail: `Cua publishes no driver for ${platform}-${arch}.`, @@ -259,13 +234,6 @@ export const layer = Layer.effect( CuaDriverInstallation, Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; - const path = yield* Path.Path; - const bundled = (yield* HostProcess.Platform) === "linux" && (yield* HostProcess.IsExecutable); - return yield* makeCuaDriverInstallation({ - baseDir: config.baseDir, - ...(bundled - ? { bundledDir: path.join(path.dirname(yield* HostProcess.ExecutablePath), "cua-driver") } - : {}), - }); + return yield* makeCuaDriverInstallation({ baseDir: config.baseDir }); }), ); diff --git a/patches/cua-driver-linux/0.34.0.patch b/patches/cua-driver-linux/0.34.0.patch deleted file mode 100644 index d2d8e40680bf8..0000000000000 --- a/patches/cua-driver-linux/0.34.0.patch +++ /dev/null @@ -1,1119 +0,0 @@ -diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs -index 1883e39..d599610 100644 ---- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs -+++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/embedded.rs -@@ -860,6 +860,7 @@ pub(crate) fn allowed_environment_name(name: &str) -> bool { - | "PROGRAMDATA" - | "DISPLAY" - | "WAYLAND_DISPLAY" -+ | "CUA_DRIVER_RS_ENABLE_WAYLAND" - | "XDG_RUNTIME_DIR" - | "XDG_SESSION_TYPE" - | "DBUS_SESSION_BUS_ADDRESS" -@@ -1306,6 +1307,41 @@ mod tests { - .any(|variable| variable.value == "forged-lock")); - } - -+ #[test] -+ fn wayland_backend_opt_in_reaches_embedded_driver() { -+ let name = "CUA_DRIVER_RS_ENABLE_WAYLAND"; -+ assert!(allowed_environment_name(name)); -+ let inherited = [(name.to_owned(), "1".to_owned())]; -+ let values = merge_safe_environment(inherited.clone(), &[]); -+ assert!(values -+ .iter() -+ .any(|variable| variable.name == name && variable.value == "1")); -+ -+ // Hosts can opt in without changing their own process environment. -+ let enabled = EmbeddedEnvironmentVariable { -+ name: name.into(), -+ value: "1".into(), -+ }; -+ let values = merge_safe_environment(std::iter::empty(), &[enabled]); -+ assert!(values -+ .iter() -+ .any(|variable| variable.name == name && variable.value == "1")); -+ -+ // An explicit opt-out still wins over the inherited opt-in. -+ let disabled = EmbeddedEnvironmentVariable { -+ name: name.into(), -+ value: "0".into(), -+ }; -+ let values = merge_safe_environment(inherited, &[disabled]); -+ assert!(values -+ .iter() -+ .any(|variable| variable.name == name && variable.value == "0")); -+ assert!(!allowed_environment_name("LD_PRELOAD")); -+ assert!(!allowed_environment_name( -+ "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS" -+ )); -+ } -+ - #[test] - fn interactive_linux_session_environment_is_inherited() { - let values = merge_safe_environment( -diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs -index 1d35cd5..06035e6 100644 ---- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs -+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs -@@ -483,12 +483,51 @@ fn start_shell_helper_idle_watch() { - }); - } - -+// GNOME's helper cursor is a stage actor, not a child of the target window. -+// Until the helper can bind it to a surface, only desktop actions may draw it. -+#[cfg(target_os = "linux")] -+static SHELL_HELPER_DESKTOP_ACTIONS: Mutex>> = Mutex::new(None); -+ -+#[cfg(target_os = "linux")] -+fn shell_helper_action_visible(actions: &mut HashMap, msg: &OverlayMsg) -> bool { -+ match msg { -+ OverlayMsg::Cmd(command) => { -+ match &command.cmd { -+ OverlayCommand::BeginAction { target, .. } => { -+ actions.insert( -+ command.key.clone(), -+ target.is_some_and(|target| target.as_str() == "desktop"), -+ ); -+ } -+ OverlayCommand::EndAction(_) => { -+ actions.remove(&command.key); -+ } -+ _ => {} -+ } -+ actions.get(&command.key).copied().unwrap_or(false) -+ } -+ OverlayMsg::Remove(key) | OverlayMsg::Revive(key) => { -+ actions.remove(key); -+ false -+ } -+ OverlayMsg::Wake => false, -+ } -+} -+ - #[cfg(target_os = "linux")] - fn dispatch_wayland_overlay_message(msg: &OverlayMsg) -> WaylandOverlayBackend { - let backend = wayland_overlay_backend(); - match backend { - WaylandOverlayBackend::SemanticShellHelper | WaylandOverlayBackend::LegacyShellHelper => { - let semantic = backend == WaylandOverlayBackend::SemanticShellHelper; -+ let visible = { -+ let mut actions = SHELL_HELPER_DESKTOP_ACTIONS.lock().unwrap(); -+ shell_helper_action_visible(actions.get_or_insert_with(HashMap::new), msg) -+ }; -+ if !visible { -+ crate::wayland::shell_helper::hide_cursor(); -+ return backend; -+ } - if shell_helper_surface_event(msg, Instant::now()) { - if let OverlayMsg::Cmd(command) = msg { - dispatch_shell_helper_command(&command.key, &command.cmd, semantic); -@@ -533,6 +572,9 @@ fn dispatch_shell_helper_command(key: &str, cmd: &OverlayCommand, semantic: bool - } - OverlayCommand::EndAction(action) if semantic => { - crate::wayland::shell_helper::set_cursor_state(action.as_str(), "", "", false); -+ // The foreground transaction has restored the user's app. Leaving -+ // the stage-wide cursor visible would attribute input to that app. -+ crate::wayland::shell_helper::hide_cursor(); - } - OverlayCommand::SetSessionLabel(label) if semantic => { - crate::wayland::shell_helper::set_session_label( -@@ -5323,6 +5365,37 @@ fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver OverlayMsg { - OverlayMsg::Cmd(KeyedOverlayCommand { - key: key.to_owned(), -diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs -index ca0dec4..b0d4dc7 100644 ---- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs -+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs -@@ -1090,6 +1090,7 @@ impl Tool for GetWindowStateTool { - "pid":{"type":"integer","description":"Process ID that owns the window."}, - "window_id":{"type":"integer","description":"Native window identifier from list_windows, or the `popup.window_id` a click / right_click result named (an open context menu / popover; its menu items then get element indices). Omitted: the pid's open popup menu when one is mapped, else its focused / active / largest window."}, - "capture_mode": cua_driver_core::capture_mode::capture_mode_schema(), -+ "display_only":{"type":"boolean","description":"Default false. With include_accessibility_tree:false, returns pixels for display without registering an action capture or replacing the agent snapshot. The image cannot ground input actions."}, - "include_accessibility_tree":{"type":"boolean", - "description":"Default true — walk the AT-SPI tree and return `elements` + `tree_markdown` alongside the screenshot. Set false to SKIP the AT-SPI walk entirely and return just the screenshot plus window metadata (window_bounds, app_name, window_title) — the capture-only path for a live window preview / picture-in-picture. Mirrors include_screenshot. Setting BOTH include_accessibility_tree:false AND include_screenshot:false is an error (nothing to return)."}, - "include_screenshot":{"type":"boolean", -@@ -1108,6 +1109,15 @@ impl Tool for GetWindowStateTool { - } - - async fn invoke(&self, args: Value) -> ToolResult { -+ let display_only = args.get("display_only").and_then(Value::as_bool) == Some(true); -+ if display_only -+ && args -+ .get("include_accessibility_tree") -+ .and_then(Value::as_bool) -+ != Some(false) -+ { -+ return ToolResult::error("display_only requires include_accessibility_tree:false"); -+ } - use cua_driver_core::tool_args::ArgsExt; - let pid = match args.require_u32("pid") { - Ok(v) => v, -@@ -1288,10 +1298,11 @@ impl Tool for GetWindowStateTool { - screenshot_out_file to force a capture.", - ); - } -- let observation_only = args -- .get("_observation_only") -- .and_then(|value| value.as_bool()) -- == Some(true); -+ let observation_only = display_only -+ || args -+ .get("_observation_only") -+ .and_then(|value| value.as_bool()) -+ == Some(true); - let state = self.state.clone(); - let state_for_capture = state.clone(); - let query_for_walk = query.clone(); -@@ -1337,7 +1348,12 @@ impl Tool for GetWindowStateTool { - window_overlays(pid, xid) - }; - let screenshot = if should_capture { -- let captured = if overlays.covers_window { -+ let preview = if !want_tree && max_dim > 0 && crate::wayland::is_wayland() { -+ crate::wayland::shell_helper::screenshot_window_preview(xid, pid, max_dim) -+ } else { None }; -+ let captured = if let Some((png, width, height)) = preview { -+ Ok((png, Some((width, height)))) -+ } else if overlays.covers_window { - overlays - .window_rect - .ok_or_else(|| anyhow::anyhow!("window geometry unavailable")) -@@ -1345,12 +1361,16 @@ impl Tool for GetWindowStateTool { - crate::capture::screenshot_root_region_png(x, y, w, h) - }) - .or_else(|_| crate::wayland::screenshot_dispatch_with_pid(xid, pid)) -+ .map(|png| (png, None)) - } else { -- crate::wayland::screenshot_dispatch_with_pid(xid, pid) -+ crate::wayland::screenshot_dispatch_with_pid(xid, pid).map(|png| (png, None)) - }; - match captured { -- Ok(raw) => { -- let (orig_w, orig_h) = crate::capture::png_dimensions_pub(&raw)?; -+ Ok((raw, original_size)) => { -+ let (orig_w, orig_h) = match original_size { -+ Some(size) => size, -+ None => crate::capture::png_dimensions_pub(&raw)?, -+ }; - let png = crate::capture::resize_png_if_needed(&raw, max_dim)?; - let (w, h) = crate::capture::png_dimensions_pub(&png)?; - // An explicit per-call max_image_dimension (0 = native) -@@ -1734,6 +1754,10 @@ impl Tool for GetWindowStateTool { - window_bounds is where it sits on the screen. Pass scope:\"desktop\" only \ - for get_desktop_state pixels." - ); -+ if display_only { -+ structured["display_only"] = json!(true); -+ structured["frame_note"] = json!("Display-only preview; not registered for input actions. Obtain a normal get_window_state screenshot before acting."); -+ } - - // The capture-only path (include_accessibility_tree:false) leaves - // `content` empty when the screenshot was also unavailable — most -@@ -14542,3 +14566,25 @@ mod cursor_hook_emission_tests { - assert!(seen[1].pressed); - } - } -+ -+#[cfg(test)] -+#[tokio::test] -+async fn display_only_refuses_accessibility_snapshot_requests() { -+ let tool = GetWindowStateTool { -+ state: ToolState::new(), -+ }; -+ for tree in [None, Some(true)] { -+ let mut args = json!({"pid": 42, "window_id": 7, "display_only": true}); -+ if let Some(tree) = tree { -+ args["include_accessibility_tree"] = json!(tree); -+ } -+ let result = tool.invoke(args).await; -+ assert_eq!( -+ serde_json::to_value(result).unwrap(), -+ serde_json::to_value(ToolResult::error( -+ "display_only requires include_accessibility_tree:false" -+ )) -+ .unwrap() -+ ); -+ } -+} -diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs -index a2691a0..4ab3a74 100644 ---- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs -+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs -@@ -17,8 +17,7 @@ - //! handshake → connection.seat events → seat.bind(capabilities) → - //! device.frame + emulate events. - //! 4. Public API sends commands over a crossbeam-channel and blocks --//! until the worker reports the request was flushed to the EIS --//! server. -+//! until EIS acknowledges the preceding input through ei_connection.sync. - //! - //! Persistence: ashpd `PersistMode::ExplicitlyRevoked` keeps the user's consent - //! until they revoke it in desktop settings. The restore token is stored at -@@ -607,6 +606,16 @@ fn run_calloop(context: reis::ei::Context, rx: Receiver) -> anyhow::Result< - pending = still; - } - -+ state.pending_sync.retain(|_, (reply, sent)| { -+ if sent.elapsed() < std::time::Duration::from_secs(5) { -+ return true; -+ } -+ let _ = reply.send(Err(anyhow::anyhow!( -+ "EIS did not acknowledge input within 5s; delivery is unknown" -+ ))); -+ false -+ }); -+ - // The handle_command path may have queued frame() requests; flush - // them once per loop iteration so the EIS server sees them. - if let Some(ctx) = state.context.as_ref() { -@@ -620,6 +629,8 @@ fn run_calloop(context: reis::ei::Context, rx: Receiver) -> anyhow::Result< - #[derive(Default)] - struct EisState { - context: Option, -+ connection: Option, -+ pending_sync: HashMap>, std::time::Instant)>, - seats: HashMap, - devices: HashMap, - sequence: u32, -@@ -737,17 +748,25 @@ impl EisState { - handshake.finish(); - } - } -- Event::Connection(_conn, ev) => match ev { -- reis::ei::connection::Event::Seat { seat } => { -- self.seats.insert(seat, SeatData::default()); -+ Event::Callback(callback, reis::ei::callback::Event::Done { .. }) => { -+ if let Some((reply, _)) = self.pending_sync.remove(&callback) { -+ let _ = reply.send(Ok(())); - } -- // The EIS server pings to check liveness; failing to answer -- // makes it drop the connection. Mirror reis's examples. -- reis::ei::connection::Event::Ping { ping } => { -- ping.done(0); -+ } -+ Event::Connection(conn, ev) => { -+ self.connection = Some(conn); -+ match ev { -+ reis::ei::connection::Event::Seat { seat } => { -+ self.seats.insert(seat, SeatData::default()); -+ } -+ // The EIS server pings to check liveness; failing to answer -+ // makes it drop the connection. Mirror reis's examples. -+ reis::ei::connection::Event::Ping { ping } => { -+ ping.done(0); -+ } -+ _ => {} - } -- _ => {} -- }, -+ } - Event::Seat(seat, ev) => { - let data = self.seats.entry(seat.clone()).or_default(); - match ev { -@@ -945,33 +964,36 @@ impl EisState { - - fn handle_command(&mut self, cmd: Cmd) { - let result = self.run_command(&cmd); -- // Send the reply on whichever channel this command carries. -- match cmd { -- Cmd::WaitReady { reply, .. } => { -- let _ = reply.send(result); -- } -- Cmd::Click { reply, .. } => { -- let _ = reply.send(result); -- } -- Cmd::MoveAbsolute { reply, .. } => { -- let _ = reply.send(result); -- } -- Cmd::Scroll { reply, .. } => { -- let _ = reply.send(result); -- } -- Cmd::TypeText { reply, .. } => { -- let _ = reply.send(result); -- } -- Cmd::PressKey { reply, .. } => { -- let _ = reply.send(result); -- } -- Cmd::KeySequence { reply, .. } => { -- let _ = reply.send(result); -- } -- Cmd::Drag { reply, .. } => { -- let _ = reply.send(result); -+ let needs_sync = !matches!(cmd, Cmd::WaitReady { .. }); -+ let reply = match cmd { -+ Cmd::WaitReady { reply, .. } -+ | Cmd::Click { reply, .. } -+ | Cmd::MoveAbsolute { reply, .. } -+ | Cmd::Scroll { reply, .. } -+ | Cmd::TypeText { reply, .. } -+ | Cmd::PressKey { reply, .. } -+ | Cmd::KeySequence { reply, .. } -+ | Cmd::Drag { reply, .. } => reply, -+ Cmd::Shutdown => return, -+ }; -+ if result.is_ok() && needs_sync { -+ if let (Some(connection), Some(context)) = (&self.connection, &self.context) { -+ // A socket flush is not delivery. Hold the caller's focus guard -+ // until EIS acknowledges all preceding input frames. -+ let callback = connection.sync(1); -+ if let Err(error) = context.flush() { -+ let _ = reply.send(Err(error.into())); -+ return; -+ } -+ self.pending_sync -+ .insert(callback, (reply, std::time::Instant::now())); -+ } else { -+ let _ = reply.send(Err(anyhow::anyhow!( -+ "EIS connection unavailable for input acknowledgement" -+ ))); - } -- Cmd::Shutdown => {} -+ } else { -+ let _ = reply.send(result); - } - } - -diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs -index 0909579..86d050e 100644 ---- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs -+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs -@@ -1044,17 +1044,13 @@ fn wayland_window_crop(xid: u64) -> anyhow::Result { - window.visible, - ); - } -- if let Some(window) = shell_helper::list_windows(None) -- .and_then(|windows| windows.into_iter().find(|window| window.xid == xid)) -- { -- return attested_wayland_crop( -+ if shell_helper::available() { -+ // GetRects proves geometry, not occlusion. Cropping the Shell stage can -+ // return a covering app (and uses physical pixels with logical bounds). -+ return Err(surface_identity_unproven( - xid, -- window.x, -- window.y, -- window.width, -- window.height, -- window.is_on_screen, -- ); -+ "GNOME stage capture cannot attest an individual window; use desktop capture", -+ )); - } - Err(surface_identity_unproven( - xid, -@@ -1106,6 +1102,14 @@ fn screenshot_dispatch_for_pid(xid: u64, pid: Option) -> anyhow::Result) -> Vec { - } - }; - } -+ // A verified GNOME helper already supplies exact IDs and ownership. -+ // Do not probe unrelated compositor protocols for every preview frame. -+ if let Some(ws) = shell_helper::list_windows(filter_pid) { -+ return listed_windows(ws); -+ } - if let Some(ws) = kwin_helper::list_window_infos() { - return listed_windows(apply_pid_filter(ws, filter_pid)); - } -diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs -index 111b4ed..a36c240 100644 ---- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs -+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs -@@ -18,12 +18,11 @@ - //! - //! Everything here is **best-effort**: if the extension isn't installed/enabled - //! the calls return `None` / no-op and callers keep the prior behaviour (no --//! screen coords, no Wayland cursor). Uses a short-lived `gdbus` subprocess so --//! there's no zbus blocking-feature or async-context coupling — the calls are --//! infrequent (once per `get_window_state`, a few per click). -+//! screen coords, no Wayland cursor). Reuses one session-bus connection on a -+//! dedicated runtime; preview frames do not spawn subprocesses or poll for exit. - - use std::os::unix::fs::{MetadataExt, PermissionsExt}; --use std::process::Command; -+mod transport; - use std::time::Duration; - - use crate::x11::WindowInfo; -@@ -36,6 +35,7 @@ const DBUS_PATH: &str = "/org/freedesktop/DBus"; - const DBUS_IFACE: &str = "org.freedesktop.DBus"; - const BROWSER_HELPER_API_VERSION: u32 = 4; - const SEMANTIC_CURSOR_API_VERSION: u32 = 8; -+const WINDOW_CAPTURE_API_VERSION: u32 = 9; - - #[derive(Debug, Clone)] - struct ShellWindow { -@@ -85,30 +85,7 @@ fn gdbus_call_to( - args: &[String], - timeout: Duration, - ) -> Option { -- let mut cmd = Command::new("gdbus"); -- cmd.arg("call") -- .arg("--session") -- .arg("--dest") -- .arg(destination) -- .arg("--object-path") -- .arg(object_path) -- .arg("--method") -- .arg(method); -- for a in args { -- cmd.arg(a); -- } -- // gdbus is local IPC; cap it so a wedged shell can't stall the caller. -- let child = cmd -- .stdin(std::process::Stdio::null()) -- .stdout(std::process::Stdio::piped()) -- .stderr(std::process::Stdio::null()) -- .spawn() -- .ok()?; -- let out = wait_timeout(child, timeout)?; -- if !out.status.success() { -- return None; -- } -- Some(String::from_utf8_lossy(&out.stdout).into_owned()) -+ transport::call(destination, object_path, method, args, timeout) - } - - /// Resolve the helper's immutable unique bus name and prove that it is hosted -@@ -210,6 +187,62 @@ fn is_trusted_gnome_shell(pid: u32) -> bool { - .is_some_and(|meta| meta.uid() == 0 && meta.permissions().mode() & 0o022 == 0) - } - -+/// Capture the exact compositor actor without activating it. Older helpers -+/// cannot prove window pixels, so callers must not fall back to stage crops. -+pub fn screenshot_window(id: u64, pid: Option) -> Option> { -+ let id = u32::try_from(id).ok()?; -+ let owner = shell_owner_with_min_version(Some(WINDOW_CAPTURE_API_VERSION))?; -+ let raw = gdbus_call_to( -+ &owner, -+ PATH, -+ &format!("{IFACE}.CaptureWindow"), -+ &[id.to_string(), pid.unwrap_or(0).to_string()], -+ Duration::from_secs(5), -+ )?; -+ decode_capture(&raw) -+} -+ -+/// Preview-sized pixels plus the original action-frame dimensions. -+/// API 9 helpers fall back to the existing full-size capture path. -+pub fn screenshot_window_preview(id: u64, pid: u32, max_dimension: u32) -> Option<(Vec, u32, u32)> { -+ let id = u32::try_from(id).ok()?; -+ let owner = shell_owner_with_min_version(Some(10))?; -+ let raw = gdbus_call_to( -+ &owner, PATH, &format!("{IFACE}.CaptureWindowPreview"), -+ &[id.to_string(), pid.to_string(), max_dimension.to_string()], -+ Duration::from_secs(5), -+ )?; -+ let png = strip_preview_timestamp(decode_capture(&raw)?)?; -+ let tail = &raw[raw.rfind('\'')? + 1..]; -+ let dims: Vec = tail.split(',').filter_map(|part| -+ part.trim().trim_end_matches(')').trim().trim_start_matches("uint32 ").parse().ok() -+ ).collect(); -+ match dims.as_slice() { -+ [width, height] if *width > 0 && *height > 0 => Some((png, *width, *height)), -+ _ => None, -+ } -+} -+ -+/// GNOME embeds a wall-clock timestamp even when pixels are identical. Remove -+/// only that ancillary chunk so preview deduplication needs no pixel decoding. -+fn strip_preview_timestamp(mut png: Vec) -> Option> { -+ if !png.starts_with(b"\x89PNG\r\n\x1a\n") { return None; } -+ let mut offset = 8usize; -+ loop { -+ let header = png.get(offset..offset.checked_add(8)?)?; -+ let size = u32::from_be_bytes(header[..4].try_into().ok()?) as usize; -+ let end = offset.checked_add(12)?.checked_add(size)?; -+ let chunk = png.get(offset..end)?; -+ if &chunk[4..8] == b"tEXt" && chunk[8..8 + size].starts_with(b"Creation Time\0") { -+ png.drain(offset..end); -+ } else if &chunk[4..8] == b"IEND" { -+ return (size == 0 && end == png.len()).then_some(png); -+ } else { -+ offset = end; -+ } -+ } -+} -+ - /// Capture the GNOME stage through the compositor helper. - /// - /// Mutter does not expose wlroots screencopy protocols, and its one-shot -@@ -247,53 +280,6 @@ fn decode_capture(raw: &str) -> Option> { - B64.decode(&raw[start..end]).ok() - } - --/// `Child::wait` with a deadline (no extra crates). Kills + reaps on timeout. --fn wait_timeout(mut child: std::process::Child, dur: Duration) -> Option { -- use std::io::Read; -- -- // Drain stdout while the child is running. Capture() returns a base64 PNG -- // that readily exceeds a pipe's ~64 KiB capacity; waiting for exit before -- // reading deadlocks the child on a full pipe and turns a healthy Shell -- // response into a false timeout. -- let stdout = child.stdout.take()?; -- let reader = std::thread::spawn(move || { -- let mut stdout = stdout; -- let mut bytes = Vec::new(); -- stdout.read_to_end(&mut bytes).ok()?; -- Some(bytes) -- }); -- let deadline = std::time::Instant::now() + dur; -- let status = loop { -- match child.try_wait() { -- Ok(Some(status)) => break status, -- Ok(None) => { -- if std::time::Instant::now() >= deadline { -- let _ = child.kill(); -- let status = child.wait().ok()?; -- let _ = reader.join(); -- if !status.success() { -- return None; -- } -- return None; -- } -- std::thread::sleep(Duration::from_millis(15)); -- } -- Err(_) => { -- let _ = child.kill(); -- let _ = child.wait(); -- let _ = reader.join(); -- return None; -- } -- } -- }; -- let stdout = reader.join().ok().flatten()?; -- Some(std::process::Output { -- status, -- stdout, -- stderr: Vec::new(), -- }) --} -- - /// Screen origin of the compositor frame backing `pid`. - /// - /// `screenshot_window_dispatch` crops the Shell stage to this same frame -@@ -585,6 +571,24 @@ pub fn hide_cursor() { - mod tests { - use super::*; - -+ #[test] -+ fn preview_timestamp_does_not_change_pixels_or_frame_identity() { -+ use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; -+ let first = B64.decode("iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAE3RFWHRDcmVhdGlvbiBUaW1lAGZpcnN0muU4QwAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAAAUSURBVHicY5SLqvjPwMDAwMQABQAczgHzr55lzgAAAABJRU5ErkJggg==").unwrap(); -+ let second = B64.decode("iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFHRFWHRDcmVhdGlvbiBUaW1lAHNlY29uZKzNUcIAAAAZdEVYdFNvZnR3YXJlAGdub21lLXNjcmVlbnNob3TvA78+AAAAFElEQVR4nGOUi6r4z8DAwMDEAAUAHM4B86+eZc4AAAAASUVORK5CYII=").unwrap(); -+ let stable = strip_preview_timestamp(first.clone()).unwrap(); -+ assert_eq!(stable, strip_preview_timestamp(second).unwrap()); -+ assert_eq!(image::load_from_memory(&stable).unwrap().to_rgba8(), -+ image::load_from_memory(&first).unwrap().to_rgba8()); -+ assert!(stable.windows(8).any(|bytes| bytes == b"Software")); -+ assert_eq!(Some(stable.clone()), strip_preview_timestamp(stable)); -+ assert!(strip_preview_timestamp(first[..first.len() - 1].to_vec()).is_none()); -+ let mut malformed = first; -+ malformed[8..12].copy_from_slice(&u32::MAX.to_be_bytes()); -+ assert!(strip_preview_timestamp(malformed).is_none()); -+ } -+ -+ - const EXTENSION_SOURCE: &str = - include_str!("../../../../../wayland-helper/winrects@cua/extension.js"); - const EXTENSION_METADATA: &str = -@@ -643,9 +647,9 @@ mod tests { - } - - #[test] -- fn bundled_helper_v8_uses_host_owned_modifier_badge_chips() { -+ fn bundled_helper_uses_host_owned_modifier_badge_chips() { - assert!(EXTENSION_SOURCE.contains("GetVersion()")); -- assert!(EXTENSION_SOURCE.contains("return 8;")); -+ assert!(EXTENSION_SOURCE.contains("return 10;")); - assert!(EXTENSION_SOURCE.contains("SetCursorState")); - assert!(EXTENSION_SOURCE.contains("SetCursorColor")); - assert!(EXTENSION_SOURCE.contains("SetSessionLabel")); -@@ -670,7 +674,7 @@ mod tests { - assert!(!EXTENSION_SOURCE.contains("function drawModifiers")); - let metadata: serde_json::Value = - serde_json::from_str(EXTENSION_METADATA).expect("valid bundled helper metadata"); -- assert_eq!(metadata["version"], 8); -+ assert_eq!(metadata["version"], 10); - - for action in [ - "idle", "observe", "click", "drag", "scroll", "text", "key", "navigate", "app", -diff --git a/libs/cua-driver/wayland-helper/winrects@cua/extension.js b/libs/cua-driver/wayland-helper/winrects@cua/extension.js -index 743acc2..cbc2888 100644 ---- a/libs/cua-driver/wayland-helper/winrects@cua/extension.js -+++ b/libs/cua-driver/wayland-helper/winrects@cua/extension.js -@@ -3,16 +3,20 @@ import GLib from 'gi://GLib'; - import Shell from 'gi://Shell'; - import St from 'gi://St'; - import Clutter from 'gi://Clutter'; -+import Cogl from 'gi://Cogl'; - import Cairo from 'cairo'; - import * as Main from 'resource:///org/gnome/shell/ui/main.js'; - import { Extension } from 'resource:///org/gnome/shell/extensions/extension.js'; - - Gio._promisify(Shell.Screenshot.prototype, 'screenshot'); -+Gio._promisify(Shell.Screenshot, 'composite_to_stream'); - - const IFACE = ` - - - -+ -+ - - - -@@ -680,6 +684,7 @@ function drawBadgeChip(cr, glyph, filled, fillColor) { - - export default class WinRectsExtension extends Extension { - enable() { -+ this._captureClones = new Map(); - this._impl = Gio.DBusExportedObject.wrapJSObject(IFACE, this); - this._impl.export(Gio.DBus.session, '/org/cua/WinRects'); - this._nameId = Gio.bus_own_name( -@@ -822,6 +827,11 @@ export default class WinRectsExtension extends Extension { - }); - } - disable() { -+ for (const entry of this._captureClones.values()) { -+ GLib.source_remove(entry.timeout); -+ entry.clone.destroy(); -+ } -+ this._captureClones.clear(); - if (this._frameId) { - GLib.source_remove(this._frameId); - this._frameId = 0; -@@ -851,7 +861,7 @@ export default class WinRectsExtension extends Extension { - } - } - GetVersion() { -- return 8; -+ return 10; - } - GetRects() { - const actors = global.get_window_actors(); -@@ -891,6 +901,96 @@ export default class WinRectsExtension extends Extension { - } - return JSON.stringify(out); - } -+ CaptureWindowAsync([id, pid], invocation) { -+ return this._captureWindow(id, pid, 0, invocation); -+ } -+ CaptureWindowPreviewAsync([id, pid, maxDimension], invocation) { -+ if (maxDimension < 1 || maxDimension > 4096) { -+ invocation.return_dbus_error('org.cua.WinRects.InvalidSize', 'Invalid preview size'); -+ return; -+ } -+ return this._captureWindow(id, pid, maxDimension, invocation); -+ } -+ async _captureWindow(id, pid, maxDimension, invocation) { -+ let stream; -+ try { -+ if (Main.sessionMode.isLocked || Main.sessionMode.isGreeter) -+ throw new Error('Desktop is unavailable'); -+ const actor = global.get_window_actors().find( -+ candidate => candidate.meta_window?.get_stable_sequence() === id); -+ const window = actor?.meta_window; -+ if (!window || (pid !== 0 && window.get_pid() !== pid) || window.minimized) -+ throw new Error('Target window is unavailable'); -+ // A mapped clone keeps Mutter from suspending a fully covered source. -+ // It paints no pixels and expires when capture requests stop. -+ let entry = this._captureClones.get(id); -+ if (entry) GLib.source_remove(entry.timeout); -+ else { -+ const clone = new Clutter.Clone({source: actor, opacity: 0, width: 1, height: 1, reactive: false}); -+ global.stage.add_child(clone); -+ entry = {clone, timeout: 0}; -+ this._captureClones.set(id, entry); -+ } -+ entry.timeout = GLib.timeout_add(GLib.PRIORITY_DEFAULT, 4000, () => { -+ entry.clone.destroy(); -+ this._captureClones.delete(id); -+ return GLib.SOURCE_REMOVE; -+ }); -+ // Snapshot this actor, never the stage. Covered windows retain their own -+ // content; neither activation nor a desktop crop is needed. -+ const frame = window.get_frame_rect(); -+ const buffer = window.get_buffer_rect(); -+ const scale = actor.get_resource_scale(); -+ const content = actor.paint_to_content(null); -+ const texture = content?.get_texture(); -+ if (!texture || !Number.isFinite(scale) || scale <= 0) -+ throw new Error('Window content is unavailable'); -+ const x = Math.round((frame.x - buffer.x) * scale); -+ const y = Math.round((frame.y - buffer.y) * scale); -+ const width = Math.round(frame.width * scale); -+ const height = Math.round(frame.height * scale); -+ if (x < 0 || y < 0 || width <= 0 || height <= 0 || -+ x + width > texture.get_width() || y + height > texture.get_height()) -+ throw new Error('Window content does not match its frame'); -+ stream = Gio.MemoryOutputStream.new_resizable(); -+ let captureTexture = texture; -+ let captureX = x, captureY = y, captureWidth = width, captureHeight = height; -+ if (maxDimension > 0 && Math.max(width, height) > maxDimension) { -+ const ratio = maxDimension / Math.max(width, height); -+ captureWidth = Math.max(1, Math.round(width * ratio)); -+ captureHeight = Math.max(1, Math.round(height * ratio)); -+ const context = global.stage.get_context().get_backend().get_cogl_context(); -+ captureTexture = Cogl.Texture2D.new_with_size(context, captureWidth, captureHeight); -+ const framebuffer = Cogl.Offscreen.new_with_texture(captureTexture); -+ framebuffer.allocate(); -+ framebuffer.set_viewport(0, 0, captureWidth, captureHeight); -+ framebuffer.orthographic(0, 0, captureWidth, captureHeight, -1, 1); -+ const pipeline = Cogl.Pipeline.new(context); -+ pipeline.set_layer_texture(0, texture); -+ framebuffer.draw_textured_rectangle(pipeline, 0, 0, captureWidth, captureHeight, -+ x / texture.get_width(), y / texture.get_height(), -+ (x + width) / texture.get_width(), (y + height) / texture.get_height()); -+ framebuffer.finish(); -+ captureX = 0; -+ captureY = 0; -+ } -+ await Shell.Screenshot.composite_to_stream( -+ captureTexture, captureX, captureY, captureWidth, captureHeight, 1, null, 0, 0, 1, stream); -+ stream.close(null); -+ if (Main.sessionMode.isLocked || Main.sessionMode.isGreeter) -+ throw new Error('Desktop became unavailable'); -+ if (stream.get_data_size() > 32 * 1024 * 1024) -+ throw new Error('Window capture is too large'); -+ const encoded = GLib.base64_encode(stream.steal_as_bytes().get_data()); -+ invocation.return_value(maxDimension > 0 -+ ? new GLib.Variant('(suu)', [encoded, width, height]) -+ : new GLib.Variant('(s)', [encoded])); -+ } catch (error) { -+ invocation.return_dbus_error('org.cua.WinRects.CaptureFailed', String(error)); -+ } finally { -+ if (stream && !stream.is_closed()) stream.close(null); -+ } -+ } - async CaptureAsync(_params, invocation) { - try { - const shooter = new Shell.Screenshot(); -diff --git a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json -index 3faf86b..a2843ca 100644 ---- a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json -+++ b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json -@@ -3,5 +3,5 @@ - "name": "WinRects", - "description": "Expose window geometry, capture, cursor, and verified activation for cua-driver on Wayland.", - "shell-version": ["45", "46", "47", "48", "49", "50"], -- "version": 8 -+ "version": 10 - } -diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs -new file mode 100644 -index 0000000..a054366 ---- /dev/null -+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper/transport.rs -@@ -0,0 +1,262 @@ -+//! Persistent session-bus transport for synchronous compositor helpers. -+//! A separate runtime also serves callers already inside Tokio without nested block_on. -+use std::sync::{mpsc, OnceLock}; -+use std::time::Duration; -+use tokio::sync::Mutex; -+use zbus::{Connection, Message}; -+ -+static CONNECTION: Mutex> = Mutex::const_new(None); -+ -+fn runtime() -> &'static tokio::runtime::Runtime { -+ static RUNTIME: OnceLock = OnceLock::new(); -+ RUNTIME.get_or_init(|| { -+ tokio::runtime::Builder::new_multi_thread() -+ .worker_threads(1) -+ .enable_all() -+ .build() -+ .expect("compositor D-Bus runtime") -+ }) -+} -+ -+pub(super) fn call( -+ destination: &str, -+ path: &str, -+ method: &str, -+ args: &[String], -+ timeout: Duration, -+) -> Option { -+ let destination = destination.to_owned(); -+ let path = path.to_owned(); -+ let method = method.to_owned(); -+ let args = args.to_vec(); -+ let (tx, rx) = mpsc::sync_channel(1); -+ runtime().spawn(async move { -+ let result = tokio::time::timeout(timeout, async { -+ let connection = { -+ let mut shared = CONNECTION.lock().await; -+ if shared.is_none() { -+ *shared = Some(Connection::session().await.ok()?); -+ } -+ shared.as_ref()?.clone() -+ }; -+ invoke(&connection, &destination, &path, &method, &args).await -+ }) -+ .await -+ .ok() -+ .flatten(); -+ // A failed or timed-out call may mean the session bus disconnected. -+ // Reconnect on the next call; never replay a possibly applied mutation. -+ if result.is_none() { -+ CONNECTION.lock().await.take(); -+ } -+ let _ = tx.send(result); -+ }); -+ rx.recv_timeout(timeout + Duration::from_millis(100)) -+ .ok() -+ .flatten() -+} -+ -+// Preserve the small textual reply contract consumed by shell_helper's existing -+// parsers. Bodies on the wire are typed; strings are never shell or GVariant code. -+async fn invoke( -+ connection: &Connection, -+ destination: &str, -+ path: &str, -+ method: &str, -+ args: &[String], -+) -> Option { -+ let (interface, member) = method.rsplit_once('.')?; -+ macro_rules! send { -+ ($body:expr) => { -+ connection -+ .call_method(Some(destination), path, Some(interface), member, &$body) -+ .await -+ .ok()? -+ }; -+ } -+ let reply: Message = match (member, args) { -+ ( -+ "GetNameOwner" -+ | "GetConnectionUnixProcessID" -+ | "GetConnectionUnixUser" -+ | "SetCursorColor" -+ | "SetSessionLabel", -+ [value], -+ ) => send!((value.as_str(),)), -+ ("GetRects" | "GetVersion" | "Capture" | "HideCursor", []) => send!(()), -+ ("Activate", [id]) => send!((id.parse::().ok()?,)), -+ ("MoveCursor" | "ClickPulse", [x, y]) => { -+ send!((x.parse::().ok()?, y.parse::().ok()?)) -+ } -+ ("CaptureWindow", [id, pid]) => send!((id.parse::().ok()?, pid.parse::().ok()?)), -+ ("CaptureWindowPreview", [id, pid, max]) => send!(( -+ id.parse::().ok()?, -+ pid.parse::().ok()?, -+ max.parse::().ok()? -+ )), -+ ("SetCursorState", [action, delivery, target, active]) => send!(( -+ action.as_str(), -+ delivery.as_str(), -+ target.as_str(), -+ active.parse::().ok()? -+ )), -+ _ => return None, -+ }; -+ let body = reply.body(); -+ match member { -+ "GetRects" => body.deserialize::().ok(), -+ "GetNameOwner" | "Capture" | "CaptureWindow" => { -+ Some(format!("('{}',)", body.deserialize::().ok()?)) -+ } -+ "GetVersion" | "GetConnectionUnixProcessID" | "GetConnectionUnixUser" => { -+ Some(format!("(uint32 {},)", body.deserialize::().ok()?)) -+ } -+ "Activate" => Some(format!("({},)", body.deserialize::().ok()?)), -+ "CaptureWindowPreview" => { -+ let (png, width, height) = body.deserialize::<(String, u32, u32)>().ok()?; -+ Some(format!("('{png}', uint32 {width}, uint32 {height})")) -+ } -+ _ => { -+ body.deserialize::<()>().ok()?; -+ Some("()".into()) -+ } -+ } -+} -+ -+#[cfg(test)] -+mod tests { -+ use super::*; -+ use std::io::BufRead; -+ use std::process::{Child, Command, Stdio}; -+ -+ struct PrivateBus(Child); -+ impl Drop for PrivateBus { -+ fn drop(&mut self) { -+ let _ = self.0.kill(); -+ let _ = self.0.wait(); -+ } -+ } -+ struct Helper; -+ #[zbus::interface(name = "org.cua.WinRects")] -+ impl Helper { -+ fn get_version(&self) -> u32 { -+ 10 -+ } -+ fn get_rects(&self) -> String { -+ r#"[{"title":"Joe's window","pid":42}]"#.into() -+ } -+ fn capture_window_preview( -+ &self, -+ id: u32, -+ pid: u32, -+ max: u32, -+ ) -> zbus::fdo::Result<(String, u32, u32)> { -+ if id != 7 || pid != 42 || max != 800 { -+ return Err(zbus::fdo::Error::InvalidArgs("wrong target".into())); -+ } -+ Ok(("cG5n".into(), 1600, 900)) -+ } -+ fn activate(&self, id: u32) -> bool { -+ id == 7 -+ } -+ fn set_cursor_state( -+ &self, -+ action: &str, -+ delivery: &str, -+ target: &str, -+ active: bool, -+ ) -> zbus::fdo::Result<()> { -+ if (action, delivery, target, active) != ("observe", "", "window", false) { -+ return Err(zbus::fdo::Error::InvalidArgs("wrong state".into())); -+ } -+ Ok(()) -+ } -+ } -+ -+ #[tokio::test] -+ async fn typed_roundtrips_on_one_private_connection() { -+ let mut bus = PrivateBus( -+ Command::new("dbus-daemon") -+ .args(["--session", "--nofork", "--print-address=1"]) -+ .stdout(Stdio::piped()) -+ .spawn() -+ .expect("private test bus"), -+ ); -+ let mut address = String::new(); -+ std::io::BufReader::new(bus.0.stdout.take().unwrap()) -+ .read_line(&mut address) -+ .unwrap(); -+ let service = zbus::connection::Builder::address(address.trim()) -+ .unwrap() -+ .name("org.cua.WinRects") -+ .unwrap() -+ .serve_at("/org/cua/WinRects", Helper) -+ .unwrap() -+ .build() -+ .await -+ .unwrap(); -+ let client = zbus::connection::Builder::address(address.trim()) -+ .unwrap() -+ .build() -+ .await -+ .unwrap(); -+ let call = |method: &'static str, args: Vec| { -+ let client = client.clone(); -+ async move { -+ invoke( -+ &client, -+ "org.cua.WinRects", -+ "/org/cua/WinRects", -+ &format!("org.cua.WinRects.{method}"), -+ &args, -+ ) -+ .await -+ } -+ }; -+ for _ in 0..3 { -+ assert_eq!( -+ call("GetVersion", vec![]).await.as_deref(), -+ Some("(uint32 10,)") -+ ); -+ assert_eq!( -+ call("GetRects", vec![]).await.as_deref(), -+ Some(r#"[{"title":"Joe's window","pid":42}]"#) -+ ); -+ let raw = call( -+ "CaptureWindowPreview", -+ vec!["7".into(), "42".into(), "800".into()], -+ ) -+ .await -+ .unwrap(); -+ assert_eq!(super::super::decode_capture(&raw), Some(b"png".to_vec())); -+ assert_eq!(raw, "('cG5n', uint32 1600, uint32 900)"); -+ } -+ assert_eq!( -+ call("Activate", vec!["7".into()]).await.as_deref(), -+ Some("(true,)") -+ ); -+ assert_eq!( -+ call("Activate", vec!["8".into()]).await.as_deref(), -+ Some("(false,)") -+ ); -+ assert_eq!( -+ call( -+ "SetCursorState", -+ vec!["observe".into(), "".into(), "window".into(), "false".into()] -+ ) -+ .await -+ .as_deref(), -+ Some("()") -+ ); -+ assert_eq!( -+ call( -+ "CaptureWindowPreview", -+ vec!["7".into(), "43".into(), "800".into()] -+ ) -+ .await, -+ None -+ ); -+ assert_eq!(call("Activate", vec!["-1".into()]).await, None); -+ drop(service); -+ } -+} diff --git a/patches/cua-driver-linux/README.md b/patches/cua-driver-linux/README.md deleted file mode 100644 index 0fd6348049df7..0000000000000 --- a/patches/cua-driver-linux/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Patched Linux Cua Driver - -Linux desktop packages and Linux CLI archives build Cua Driver from trycua/cua -commit `b0968e1b12834e485dda68789541a3cc57664a9f` (`cua-driver-rs-v0.34.0`), applying -`0.34.0.patch`. This is a source patch, not a pnpm dependency patch. - -The normal packaging commands invoke `scripts/build-cua-linux.sh`. It builds the -driver, cursor helper, and SDK library together in release mode, with upstream's -`cua-driver/portal-input` feature. Packaging also ships the patched GNOME helper -(API 10) and replaces the SDK npm package's native library. Replacing only the -daemon would leave the embedded host running upstream code. - -Linux CLI archives carry their own driver beside the `t3` executable. They use -that bundle before any previously downloaded version and fail if it is -incomplete, rather than silently downloading an unpatched driver. macOS and -Windows keep their upstream native binaries. Source-run development servers -retain the upstream downloader unless `T3CODE_CUA_DRIVER_PATH` selects a custom -driver; the source patch is applied during Linux release packaging. - -Build on Linux matching the target architecture (`x64` or `arm64`), with Rustup, -Git, a C/C++ toolchain, pkg-config, and the development packages `libx11-dev`, -`libxi-dev`, `libxtst-dev`, `libxext-dev`, `libwayland-dev`, and `libxkbcommon-dev`. -CI installs these dependencies. The upstream checkout pins its Rust toolchain. - -To build just the native bundle from the repository root: - -```sh -bash scripts/build-cua-linux.sh "$PWD" x64 /tmp/t3-cua-linux -``` - -Builds are cached by architecture, source commit, patch, and build script; cached -bundle files are checksum-verified. Desktop and CLI packaging share that cache. -A native build or SDK staging failure stops packaging. - -The patch preserves Wayland host opt-in, compositor-attested window capture, -preview downscaling, persistent D-Bus transport, and display-only screenshots -that do not replace agent snapshots or register action captures. It removes -GNOME's changing PNG `Creation Time` chunk without decoding pixels, allowing -static-frame deduplication. D-Bus calls retain owner attestation and reconnect -after failure without replaying mutations. - -Build 2757's earlier local validation used debug binaries without `portal-input`. -Normal release packaging uses the optimized build described above. diff --git a/scripts/build-cli-archive.ts b/scripts/build-cli-archive.ts index 4f318c938eb5f..3c74b6714ddeb 100644 --- a/scripts/build-cli-archive.ts +++ b/scripts/build-cli-archive.ts @@ -21,14 +21,12 @@ import rootPackageJson from "../package.json" with { type: "json" }; import serverPackageJson from "../apps/server/package.json" with { type: "json" }; import { - stageCuaDriverExecutable, createStagePatchedDependencies, createStageWorkspaceConfig, resolveFffNativeDependencies, STAGE_INSTALL_ARGS, } from "./build-desktop-artifact.ts"; import { assertLinuxArm64PageCompatibility } from "./lib/linux-arm64-sea.ts"; -import { stageCuaLinuxSdk } from "./lib/cua-linux-bundle.ts"; import { selectCliRuntimeExternalDependencies } from "./lib/cli-external-packages.ts"; import { signWithJsign } from "./lib/windows-jsign.ts"; import { windowsSystemTar } from "./lib/windows-tar.ts"; @@ -532,22 +530,6 @@ const buildCliArchive = Effect.fn("buildCliArchive")(function* (input: { version: input.version, }); - if (input.platform === "linux") { - yield* stageCuaDriverExecutable({ - platform: input.platform, - arch: input.arch, - repoRoot, - stageRoot, - stageResourcesDir: contentDir, - verbose: true, - }); - yield* stageCuaLinuxSdk({ - arch: input.arch, - bundleDir: path.join(contentDir, "cua-driver"), - nodeModulesDir: path.join(contentDir, "node_modules"), - }); - } - const executablePath = path.join(contentDir, executableName); if (input.platform === "linux") { yield* bundleLinuxLibatomic({ repoRoot, contentDir, executablePath }); diff --git a/scripts/build-cua-linux.sh b/scripts/build-cua-linux.sh deleted file mode 100755 index 5c7b0f7e6905e..0000000000000 --- a/scripts/build-cua-linux.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Build the Linux driver and SDK from the same patched source. Build hosts need -# Rust/rustup, git, and the X11/Wayland development libraries used by upstream. -set -euo pipefail - -repo_root=$(cd "$1" && pwd) -arch=$2 -output_dir=$3 -version=${4:-0.34.0} -if [[ "$version" != "0.34.0" ]]; then - echo "Update the Linux source pin and patch before shipping Cua Driver $version." >&2 - exit 1 -fi -source_commit=b0968e1b12834e485dda68789541a3cc57664a9f -patch_file="$repo_root/patches/cua-driver-linux/0.34.0.patch" -case "$(uname -s):$(uname -m):$arch" in - Linux:x86_64:x64|Linux:aarch64:arm64) ;; - *) echo 'Patched Cua Driver must be built on a Linux host of the target architecture.' >&2; exit 1 ;; -esac -patch_hash=$(sha256sum "$patch_file" | cut -d ' ' -f 1) -cache_root="$repo_root/node_modules/.cache/t3code/cua-linux" -build_hash=$(cat "$patch_file" "$repo_root/scripts/build-cua-linux.sh" | sha256sum | cut -d ' ' -f 1) -cache_dir="$cache_root/$arch-$source_commit-$build_hash" -mkdir -p "$cache_root" -# Desktop and CLI packaging can share the build without racing a checkout or copy. -exec 9>"$cache_root/build.lock" -flock 9 - -if [[ ! -f "$cache_dir/bundle/SHA256SUMS" ]] || ! (cd "$cache_dir/bundle" && sha256sum --check --status SHA256SUMS); then - mkdir -p "$cache_dir" - source_dir="$cache_dir/source" - if [[ ! -f "$source_dir/.t3-patched" ]]; then - work_dir=$(mktemp -d "$cache_dir/source-XXXXXX") - trap 'rm -rf "$work_dir"' EXIT - git -C "$work_dir" init --quiet - git -C "$work_dir" fetch --quiet --depth=1 https://github.com/trycua/cua.git "$source_commit" - git -C "$work_dir" checkout --quiet --detach FETCH_HEAD - git -C "$work_dir" apply --check "$patch_file" - git -C "$work_dir" apply "$patch_file" - touch "$work_dir/.t3-patched" - mv "$work_dir" "$source_dir" - trap - EXIT - fi - ( - cd "$source_dir/libs/cua-driver/rust" - CARGO_TARGET_DIR="$cache_root/target-$arch" cargo build --locked --release \ - -p cua-driver -p cua-driver-sdk -p cursor-theme-cli --features cua-driver/portal-input - ) - bundle_dir=$(mktemp -d "$cache_dir/bundle-XXXXXX") - trap 'rm -rf "$bundle_dir"' EXIT - chmod 755 "$bundle_dir" - for binary in cua-driver cua-cursor-theme libcua_driver_sdk.so; do - cp "$cache_root/target-$arch/release/$binary" "$bundle_dir/$binary" - done - mkdir -p "$bundle_dir/wayland-helper" - cp -a "$source_dir/libs/cua-driver/wayland-helper/winrects@cua" "$bundle_dir/wayland-helper/" - cp "$source_dir/LICENSE.md" "$bundle_dir/LICENSE" - cp "$source_dir/libs/cua-driver/rust/THIRD_PARTY_NOTICES.md" "$bundle_dir/THIRD_PARTY_NOTICES.md" - (cd "$bundle_dir" && find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS) - # A failed build never publishes a partially updated bundle. - rm -rf "$cache_dir/bundle" - mv "$bundle_dir" "$cache_dir/bundle" - trap - EXIT -fi -mkdir -p "$output_dir" -cp -a "$cache_dir/bundle/." "$output_dir/" -echo "Staged patched Cua Driver 0.34.0 ($arch, patch $patch_hash)." diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index d1f8e81098d2f..8f6e218d1eb15 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -259,7 +259,7 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { ); const error = yield* Effect.flip( stageCuaDriverExecutable({ - platform: "mac", + platform: "linux", arch: "x64", repoRoot: root, stageRoot: root, @@ -276,37 +276,6 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { ), ); - it.effect("does not fall back to upstream when the patched Linux build fails", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cua-build-failure-" }); - const commands: string[] = []; - const error = yield* Effect.flip( - stageCuaDriverExecutable({ - platform: "linux", - arch: "arm64", - repoRoot: root, - stageRoot: root, - stageResourcesDir: `${root}/resources`, - verbose: false, - }).pipe( - Effect.provideService( - ChildProcessSpawner.ChildProcessSpawner, - ChildProcessSpawner.make((command) => { - commands.push((command as unknown as { command: string }).command); - return Effect.succeed(mockProcess(42)); - }), - ), - ), - ); - assert.instanceOf(error, BuildCommandFailedError); - assert.deepStrictEqual(commands, ["bash"]); - assert.isFalse(yield* fs.exists(`${root}/resources/cua-driver`)); - }), - ), - ); - it("bundles the universal Cua Driver on every macOS arch", () => { for (const arch of ["arm64", "x64", "universal"] as const) { assert.include(resolveCuaDriverAsset("mac", arch).archiveName, "darwin-universal-binary"); diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index 489794ab74c15..6e06df50a2935 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -30,7 +30,6 @@ import { releasePackageFiles, updateReleasePackageVersions, } from "./update-release-package-versions.ts"; -import { stageCuaLinuxSdk } from "./lib/cua-linux-bundle.ts"; import { applyWebBrandAssets } from "./apply-web-brand-assets.ts"; import { BRAND_ASSET_PATHS, @@ -3060,19 +3059,6 @@ export const stageCuaDriverExecutable = Effect.fn("stageCuaDriverExecutable")(fu }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - if (input.platform === "linux") { - yield* runCommand( - ChildProcess.make("bash", [ - path.join(input.repoRoot, "scripts/build-cua-linux.sh"), - input.repoRoot, - input.arch, - path.join(input.stageResourcesDir, "cua-driver"), - CUA_DRIVER_VERSION, - ]), - { label: "build patched Linux Cua Driver", verbose: input.verbose }, - ); - return; - } const asset = resolveCuaDriverAsset(input.platform, input.arch); const cacheDir = path.join(input.repoRoot, "node_modules/.cache/t3code/cua-driver"); const archivePath = path.join(cacheDir, asset.archiveName); @@ -4106,13 +4092,6 @@ const buildDesktopArtifactUnlocked = Effect.fn("buildDesktopArtifactUnlocked")(f }), { label: "vp install --prod", verbose: options.verbose }, ); - if (options.platform === "linux") { - yield* stageCuaLinuxSdk({ - arch: options.arch, - bundleDir: path.join(stageProdResourcesDir, "cua-driver"), - nodeModulesDir: path.join(stageAppDir, "node_modules"), - }); - } yield* stageClerkPasskeyNativeBinaries(stageAppDir, options.platform, options.arch); yield* stageKeyringNativeBinaries(stageAppDir, options.platform, options.arch); diff --git a/scripts/lib/cua-linux-bundle.test.ts b/scripts/lib/cua-linux-bundle.test.ts deleted file mode 100644 index 1857688eb1a4b..0000000000000 --- a/scripts/lib/cua-linux-bundle.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { expect, it } from "@effect/vitest"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Path from "effect/Path"; -import { stageCuaLinuxSdk } from "./cua-linux-bundle.ts"; - -it.layer(NodeServices.layer)("stageCuaLinuxSdk", (it) => { - it.effect.each(["hoisted", "pnpm"] as const)( - "replaces the %s SDK without changing a shared store file", - (layout) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "cua-sdk-stage-" }); - const nodeModulesDir = path.join(root, "node_modules"); - const deps = - layout === "pnpm" ? path.join(nodeModulesDir, ".pnpm/sdk/node_modules") : nodeModulesDir; - const sdk = path.join(deps, "@trycua/cua-driver"); - const native = path.join(deps, "@trycua/cua-driver-linux-x64-gnu"); - yield* fs.makeDirectory(sdk, { recursive: true }); - yield* fs.makeDirectory(native, { recursive: true }); - yield* fs.writeFileString(path.join(sdk, "package.json"), '{"name":"@trycua/cua-driver"}'); - yield* fs.writeFileString( - path.join(native, "package.json"), - '{"name":"@trycua/cua-driver-linux-x64-gnu"}', - ); - if (layout === "pnpm") { - yield* fs.makeDirectory(path.join(nodeModulesDir, "@trycua"), { recursive: true }); - yield* fs.symlink(sdk, path.join(nodeModulesDir, "@trycua/cua-driver")); - } - const shared = path.join(root, "store-library"); - yield* fs.writeFileString(shared, "upstream"); - const library = path.join(native, "libcua_driver_sdk.so"); - yield* fs.link(shared, library); - const bundleDir = path.join(root, "bundle"); - yield* fs.makeDirectory(bundleDir); - yield* fs.writeFileString(path.join(bundleDir, "libcua_driver_sdk.so"), "patched"); - yield* stageCuaLinuxSdk({ arch: "x64", bundleDir, nodeModulesDir }); - expect(yield* fs.readFileString(library)).toBe("patched"); - expect(yield* fs.readFileString(shared)).toBe("upstream"); - yield* fs.remove(library); - const error = yield* Effect.flip( - stageCuaLinuxSdk({ arch: "x64", bundleDir, nodeModulesDir }), - ); - expect(error._tag).toBe("CuaLinuxSdkMissingError"); - }).pipe(Effect.scoped), - ); -}); diff --git a/scripts/lib/cua-linux-bundle.ts b/scripts/lib/cua-linux-bundle.ts deleted file mode 100644 index 80419235999da..0000000000000 --- a/scripts/lib/cua-linux-bundle.ts +++ /dev/null @@ -1,35 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off - Resolve the native dependency through its installed SDK package. -import * as NodeModule from "node:module"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Path from "effect/Path"; -import * as Schema from "effect/Schema"; - -export class CuaLinuxSdkMissingError extends Schema.TaggedError()( - "CuaLinuxSdkMissingError", - { filePath: Schema.String }, -) {} - -/** Replace the SDK's native library as well as the daemon; they are one build. */ -export const stageCuaLinuxSdk = Effect.fn("stageCuaLinuxSdk")(function* (input: { - readonly arch: string; - readonly bundleDir: string; - readonly nodeModulesDir: string; -}) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const sdkDir = yield* fs.realPath(path.join(input.nodeModulesDir, "@trycua/cua-driver")); - const nativePackage = `@trycua/cua-driver-linux-${input.arch}-gnu/package.json`; - const nativeManifest = yield* Effect.try({ - try: () => NodeModule.createRequire(path.join(sdkDir, "package.json")).resolve(nativePackage), - catch: () => new CuaLinuxSdkMissingError({ filePath: nativePackage }), - }); - const destination = path.join(path.dirname(nativeManifest), "libcua_driver_sdk.so"); - if (!(yield* fs.exists(destination))) { - return yield* new CuaLinuxSdkMissingError({ filePath: destination }); - } - // pnpm may hardlink this file into its store. Unlink before replacing it so - // packaging cannot modify another checkout's upstream SDK. - yield* fs.remove(destination); - yield* fs.copyFile(path.join(input.bundleDir, "libcua_driver_sdk.so"), destination); -}); From bbb4369c5c5152f06e38c024428379d3f8014172 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 16 Oct 2026 15:24:25 -0700 Subject: [PATCH 7/8] fix(cua): stabilize Windows app identity and preview frames --- .../src/assets/nativeAppIcons/win32.test.ts | 54 ++++++++ .../server/src/assets/nativeAppIcons/win32.ts | 73 ++++++++--- apps/server/src/cua/CuaWindowPreview.test.ts | 21 +++- apps/server/src/cua/CuaWindowPreview.ts | 18 ++- .../preview/ThreadPreviewMiniPlayer.tsx | 60 +++------ .../useComputerPreviewFrame.browser.test.tsx | 84 +++++++++++++ .../preview/useComputerPreviewFrame.ts | 37 ++++++ packages/contracts/src/cua.ts | 2 + packages/contracts/src/providerRuntime.ts | 2 +- .../src/server/cuaToolPresentation.test.ts | 118 ++++++++++++++++++ .../src/server/cuaToolPresentation.ts | 106 +++++++++++++--- 11 files changed, 493 insertions(+), 82 deletions(-) create mode 100644 apps/server/src/assets/nativeAppIcons/win32.test.ts create mode 100644 apps/web/src/components/preview/useComputerPreviewFrame.browser.test.tsx create mode 100644 apps/web/src/components/preview/useComputerPreviewFrame.ts diff --git a/apps/server/src/assets/nativeAppIcons/win32.test.ts b/apps/server/src/assets/nativeAppIcons/win32.test.ts new file mode 100644 index 0000000000000..7fad9ba6e468f --- /dev/null +++ b/apps/server/src/assets/nativeAppIcons/win32.test.ts @@ -0,0 +1,54 @@ +// @effect-diagnostics nodeBuiltinImport:off - Runs the Windows icon script against an isolated package fixture. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { describe, expect, it } from "vite-plus/test"; + +import { WINDOWS_APP_ICON_SCRIPT } from "./win32.ts"; + +// oxlint-disable-next-line t3code/no-global-process-runtime -- This integration test needs Windows PowerShell on the physical test host. +describe.runIf(NodeOS.platform() === "win32")("Windows packaged app icons", () => { + it("uses a package's unplated logo for both its name and AUMID", () => { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-packaged-icon-")); + const fixture = ` +Add-Type -AssemblyName System.Drawing +$image = New-Object System.Drawing.Bitmap 1, 1 +try { + $image.SetPixel(0, 0, [System.Drawing.Color]::Red) + $image.Save((Join-Path $env:T3_TEST_PACKAGE 'App.targetsize-32_altform-unplated.png')) +} finally { $image.Dispose() } +function Get-StartApps { [pscustomobject]@{ Name = 'Fixture Calculator'; AppID = 'Fixture_family!App' } } +function Get-AppxPackage { [pscustomobject]@{ PackageFamilyName = 'Fixture_family'; PackageFullName = 'Fixture'; InstallLocation = $env:T3_TEST_PACKAGE } } +function Get-AppxPackageManifest { [xml]'' } +function Get-Process { [pscustomobject]@{ ProcessName = 'ApplicationFrameHost'; MainWindowTitle = 'Fixture Calculator'; Path = 'C:\\Windows\\System32\\ApplicationFrameHost.exe' } } +`; + try { + for (const name of ["Fixture Calculator", "Fixture_family!App"]) { + const outputPath = NodePath.join(root, "output.png"); + const source = NodeChildProcess.execFileSync( + "powershell.exe", + ["-NoProfile", "-NonInteractive", "-Command", fixture + WINDOWS_APP_ICON_SCRIPT], + { + windowsHide: true, + timeout: 10_000, + encoding: "utf8", + env: { + ...process.env, + T3_TEST_PACKAGE: root, + T3_APP_NAME: name, + T3_ICON_OUT: outputPath, + }, + }, + ).trim(); + expect(source).toBe(NodePath.join(root, "App.targetsize-32_altform-unplated.png")); + expect([...NodeFS.readFileSync(outputPath).subarray(0, 8)]).toEqual([ + 137, 80, 78, 71, 13, 10, 26, 10, + ]); + } + } finally { + NodeFS.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/server/src/assets/nativeAppIcons/win32.ts b/apps/server/src/assets/nativeAppIcons/win32.ts index 6acf1b55c0c45..a603398f837e1 100644 --- a/apps/server/src/assets/nativeAppIcons/win32.ts +++ b/apps/server/src/assets/nativeAppIcons/win32.ts @@ -13,19 +13,46 @@ import { existingFile, type NativeAppIconSource } from "./source.ts"; const COMMAND_TIMEOUT = "10 seconds"; /** - * Writes the app's icon as a PNG. Windows has no bundle ids, so the script tries a running process - * with Cua's app name, then a Start Menu shortcut. Arguments arrive as environment variables so - * the name is never spliced into the script. + * Packaged apps use their manifest logo; a shared ApplicationFrameHost + * executable cannot identify their icon. Arguments travel as environment + * variables so the name is never spliced into the script. */ -const EXTRACT_SCRIPT = ` +export const WINDOWS_APP_ICON_SCRIPT = ` $ErrorActionPreference = 'Stop' Add-Type -AssemblyName System.Drawing $name = $env:T3_APP_NAME $exe = $null +$packaged = Get-StartApps | Where-Object { $_.Name -ieq $name -or $_.AppID -ieq $name } | Select-Object -First 1 +if ($packaged -and $packaged.AppID -match '^([^!]+)!(.+)$') { + $family = $Matches[1] + $applicationId = $Matches[2] + $package = Get-AppxPackage | Where-Object { $_.PackageFamilyName -eq $family } | Select-Object -First 1 + if ($package) { + $manifest = Get-AppxPackageManifest -Package $package.PackageFullName + $application = $manifest.Package.Applications.Application | Where-Object { $_.Id -eq $applicationId } | Select-Object -First 1 + $logo = $application.VisualElements.Square44x44Logo + if ($logo) { + $logoPath = Join-Path $package.InstallLocation $logo + $directory = Split-Path $logoPath + $stem = [IO.Path]::GetFileNameWithoutExtension($logoPath) + $extension = [IO.Path]::GetExtension($logoPath) + $candidates = @("$stem.targetsize-32_altform-unplated$extension", "$stem.targetsize-48_altform-unplated$extension", "$stem.scale-200$extension", "$stem$extension") + foreach ($candidate in $candidates) { + $file = Join-Path $directory $candidate + if (Test-Path -LiteralPath $file -PathType Leaf) { $exe = $file; break } + } + if (-not $exe) { + $exe = Get-ChildItem -LiteralPath $directory -Filter "$stem.*$extension" -ErrorAction SilentlyContinue | + Sort-Object Name | Select-Object -First 1 -ExpandProperty FullName + } + } + } +} +$processName = [IO.Path]::GetFileNameWithoutExtension($name) $process = Get-Process -ErrorAction SilentlyContinue | - Where-Object { $_.Path -and ($_.ProcessName -ieq $name -or $_.MainWindowTitle -ieq $name -or $_.Product -ieq $name) } | + Where-Object { $_.Path -and $_.ProcessName -notin @('ApplicationFrameHost', 'WWAHost') -and ($_.ProcessName -ieq $processName -or $_.MainWindowTitle -ieq $name -or $_.Product -ieq $name) } | Select-Object -First 1 -if ($process) { $exe = $process.Path } +if (-not $exe -and $process) { $exe = $process.Path } if (-not $exe) { $shell = New-Object -ComObject WScript.Shell $roots = @("$env:ProgramData\\Microsoft\\Windows\\Start Menu\\Programs", "$env:AppData\\Microsoft\\Windows\\Start Menu\\Programs") @@ -36,9 +63,20 @@ if (-not $exe) { if (-not $exe -or -not (Test-Path -LiteralPath $exe -PathType Leaf)) { exit 3 } Write-Output $exe if ($env:T3_ICON_OUT) { - $icon = [System.Drawing.Icon]::ExtractAssociatedIcon($exe) - $bitmap = $icon.ToBitmap() - $bitmap.Save($env:T3_ICON_OUT, [System.Drawing.Imaging.ImageFormat]::Png) + $icon = $null + $bitmap = $null + try { + if ([IO.Path]::GetExtension($exe) -ieq '.png') { + $bitmap = [System.Drawing.Image]::FromFile($exe) + } else { + $icon = [System.Drawing.Icon]::ExtractAssociatedIcon($exe) + $bitmap = $icon.ToBitmap() + } + $bitmap.Save($env:T3_ICON_OUT, [System.Drawing.Imaging.ImageFormat]::Png) + } finally { + if ($bitmap) { $bitmap.Dispose() } + if ($icon) { $icon.Dispose() } + } } `; @@ -50,7 +88,14 @@ const runScript = Effect.fn("NativeAppIconResolver.win32.runScript")(function* ( .string( ChildProcess.make( "powershell.exe", - ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", EXTRACT_SCRIPT], + [ + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-Command", + WINDOWS_APP_ICON_SCRIPT, + ], { stdin: "ignore", stderr: "ignore", env, extendEnv: true }, ), ) @@ -66,7 +111,7 @@ const resolveIcon = Effect.fn("NativeAppIconResolver.win32.resolve")(function* ( const name = (app._tag === "display-name" ? app.displayName : app.appId).trim(); if (name.length === 0 || /[\\/:*?"<>|]/u.test(name)) return null; - // The first run only locates the executable; its path and size key the cache. + // The source image or executable and its size key the cache. const executable = (yield* runScript({ T3_APP_NAME: name })).trim(); if (!executable) return null; const info = yield* fileSystem.stat(executable); @@ -92,11 +137,7 @@ const resolveIcon = Effect.fn("NativeAppIconResolver.win32.resolve")(function* ( return yield* existingFile(cachePath); }); -/** - * Cua reports a Windows app by its name. PowerShell finds the executable - * (a running process first, then a Start Menu shortcut) and System.Drawing - * renders its associated icon to a cached PNG. - */ +/** Resolves packaged-app logos and traditional executable icons on the host. */ export const win32Source: NativeAppIconSource = { platform: "win32", resolve: resolveIcon, diff --git a/apps/server/src/cua/CuaWindowPreview.test.ts b/apps/server/src/cua/CuaWindowPreview.test.ts index 22c77f6da4af3..ffd306f54b5eb 100644 --- a/apps/server/src/cua/CuaWindowPreview.test.ts +++ b/apps/server/src/cua/CuaWindowPreview.test.ts @@ -15,6 +15,7 @@ import { describe, expect } from "vite-plus/test"; import { clearCuaToolContext, cuaToolPresentation, + rememberCuaToolResult, } from "@t3tools/provider-core/server/cuaToolPresentation"; import type { McpProviderSessionConfig } from "@t3tools/provider-core/server/mcpSession"; import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions"; @@ -34,7 +35,7 @@ const turnEvent = (type: "turn.started" | "turn.completed") => const makeHarness = Effect.fn(function* ( windowFailure?: "missing" | "throw" | "transient", onWindowCapture?: () => void | Promise, - options: { displayOnly?: boolean; image?: () => string } = {}, + options: { displayOnly?: boolean; image?: () => string; appName?: string } = {}, ) { const events = yield* PubSub.unbounded(); const captures: string[] = []; @@ -80,7 +81,7 @@ const makeHarness = Effect.fn(function* ( return Promise.resolve({ pid: input.pid, windowId: input.windowId, - appName: "Calendar", + appName: options.appName ?? "Calendar", windowTitle: "December 2026", screenshotWidth: 640, screenshotHeight: 400, @@ -316,13 +317,26 @@ describe("CuaWindowPreview", () => { it.live("prefers the window the agent last targeted", () => Effect.gen(function* () { yield* setCuaSession; + rememberCuaToolResult( + threadId, + "launch_app", + { name: "Calendar" }, + { + pid: 42, + bundle_id: "Microsoft.Calendar_8wekyb3d8bbwe!App", + windows: [{ window_id: 7, title: "Calendar" }], + }, + ); cuaToolPresentation({ threadId, rawToolName: "cua-driver/click", args: { pid: 42, window_id: "7", x: 1, y: 1 }, status: "completed", }); - const harness = yield* makeHarness(); + const harness = yield* makeHarness(undefined, undefined, { + displayOnly: true, + appName: "ApplicationFrameHost.exe", + }); const first = yield* harness.service.stream(threadId).pipe( Stream.filter((state) => state.status === "live"), Stream.runHead, @@ -335,6 +349,7 @@ describe("CuaWindowPreview", () => { if (state._tag !== "Some") return; expect(state.value.frame).toMatchObject({ appName: "Calendar", + app: { _tag: "app-id", appId: "Microsoft.Calendar_8wekyb3d8bbwe!App" }, windowTitle: "December 2026", width: 640, height: 400, diff --git a/apps/server/src/cua/CuaWindowPreview.ts b/apps/server/src/cua/CuaWindowPreview.ts index 2f7dd2d6d6f73..bb6199384820f 100644 --- a/apps/server/src/cua/CuaWindowPreview.ts +++ b/apps/server/src/cua/CuaWindowPreview.ts @@ -167,8 +167,19 @@ const captureFrame = Effect.fn("CuaWindowPreview.capture")(function* ( cause: "Window capture is unavailable.", }); } + const appName = + target.app?.name ?? + (/^(ApplicationFrameHost|WWAHost)(\.exe)?$/iu.test(window.appName ?? "") + ? window.windowTitle + : window.appName); + const app = target.app?.bundleId + ? { _tag: "app-id" as const, appId: target.app.bundleId } + : appName + ? { _tag: "display-name" as const, displayName: appName.slice(0, 160) } + : undefined; return { - ...(window.appName ? { appName: window.appName } : {}), + ...(app ? { app } : {}), + ...(appName ? { appName } : {}), ...(window.windowTitle ? { windowTitle: window.windowTitle } : {}), width: window.screenshotWidth ?? 0, height: window.screenshotHeight ?? 0, @@ -293,7 +304,10 @@ export const make = Effect.fn("CuaWindowPreview.make")(function* ( previous.width === frame.width && previous.height === frame.height && previous.appName === frame.appName && - previous.windowTitle === frame.windowTitle; + previous.windowTitle === frame.windowTitle && + previous.app?._tag === frame.app?._tag && + (previous.app?._tag === "app-id" ? previous.app.appId : previous.app?.displayName) === + (frame.app?._tag === "app-id" ? frame.app.appId : frame.app?.displayName); previousTarget = target; unchangedFrames = unchanged ? Math.min(unchangedFrames + 1, 3) : 0; return unchanged ? Effect.void : publish(preview, { status: "live", frame }); diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index bcc438c48f09a..8907b669c245c 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -1,8 +1,4 @@ -import { - type CuaWindowPreviewFrame, - FILL_PREVIEW_VIEWPORT, - type ScopedThreadRef, -} from "@t3tools/contracts"; +import { FILL_PREVIEW_VIEWPORT, type ScopedThreadRef } from "@t3tools/contracts"; import { previewPdfAssetUrl } from "@t3tools/shared/preview"; import { useEnvironmentHttpBaseUrl } from "~/state/environments"; import { PanelRightIcon, PictureInPicture2, XIcon } from "lucide-react"; @@ -65,6 +61,7 @@ import type { PreviewStreamViewport } from "@t3tools/client-runtime/preview/serv import type { ComputerUsePreview } from "./computerUsePreview"; import { previewBridge } from "./previewBridge"; import { showPreviewPopup } from "./showPreviewPopup"; +import { useComputerPreviewFrame } from "./useComputerPreviewFrame"; import { clampPreviewMiniPlayerPosition, PREVIEW_MINI_PLAYER_CORNER_RADIUS, @@ -255,35 +252,10 @@ function ComputerMiniPlayer({ preview, }: Props & { readonly preview: ComputerUsePreview | null }) { const live = useCuaWindowPreview(threadRef); - const [decoded, setDecoded] = useState<{ - frame: CuaWindowPreviewFrame; - src: string; - } | null>(null); const liveFrame = live.status === "live" ? (live.frame ?? null) : null; - if (live.status === "live" && !liveFrame && decoded !== null) setDecoded(null); - useEffect(() => { - if (!liveFrame) return; - let cancelled = false; - const image = new Image(); - const src = `data:${liveFrame.mimeType};base64,${liveFrame.dataBase64}`; - image.src = src; - // Keep the visible frame until the replacement is ready to paint. - void image.decode().then( - () => { - if (!cancelled) setDecoded({ frame: liveFrame, src }); - }, - () => { - // A malformed frame must not erase the last successfully decoded one. - }, - ); - return () => { - cancelled = true; - }; - }, [liveFrame]); - const visible = - live.status === "unavailable" || (live.status === "live" && !liveFrame) ? null : decoded; - const frame = visible?.frame ?? null; - const src = visible?.src ?? null; + const painted = useComputerPreviewFrame(liveFrame); + const frame = + live.status === "unavailable" || (live.status === "live" && !liveFrame) ? null : painted.frame; const sourceSize = frame && frame.width > 0 && frame.height > 0 ? { width: frame.width, height: frame.height } @@ -301,12 +273,12 @@ function ComputerMiniPlayer({ true, ); }, [viewportKey, sourceSize.width, sourceSize.height]); - const label = frame?.appName ?? preview?.appName ?? "Computer"; + const label = frame?.appName ?? "Computer"; const caption = frame?.windowTitle && frame.windowTitle !== label ? `${label} · ${frame.windowTitle}` : label; const appIcon = useAssetUrlState( threadRef.environmentId, - preview?.app ? { _tag: "native-app-icon", app: preview.app } : null, + frame?.app ? { _tag: "native-app-icon", app: frame.app } : null, ); return ( @@ -322,19 +294,17 @@ function ComputerMiniPlayer({ className="pointer-events-auto absolute inset-0 overflow-hidden rounded-[inherit] bg-muted" style={{ zIndex: PREVIEW_MINI_PLAYER_WEBVIEW_Z_INDEX }} > - {src ? ( - {caption} - ) : ( + + {!frame ? (
{live.status === "unavailable" ? "Preview unavailable" : "Waiting for the agent…"}
- )} + ) : null} {live.status === "unavailable" && live.detail ? (
{live.detail} diff --git a/apps/web/src/components/preview/useComputerPreviewFrame.browser.test.tsx b/apps/web/src/components/preview/useComputerPreviewFrame.browser.test.tsx new file mode 100644 index 0000000000000..910612c070931 --- /dev/null +++ b/apps/web/src/components/preview/useComputerPreviewFrame.browser.test.tsx @@ -0,0 +1,84 @@ +import type { CuaWindowPreviewFrame } from "@t3tools/contracts"; +import { expect, it, vi } from "vite-plus/test"; + +import { act, render } from "../../../test/render"; +import { useComputerPreviewFrame } from "./useComputerPreviewFrame"; + +const images: DeferredImage[] = []; +class DeferredImage { + src = ""; + naturalWidth = 480; + naturalHeight = 800; + resolve!: () => void; + ready = new Promise((resolve) => { + this.resolve = resolve; + }); + constructor() { + images.push(this); + } + decode() { + return this.ready; + } +} + +const makeFrame = (data: string): CuaWindowPreviewFrame => ({ + appName: data, + mimeType: "image/png", + dataBase64: data, + width: 480, + height: 800, + capturedAt: "2026-10-16T00:00:00Z", +}); + +function Harness({ frame }: { frame: CuaWindowPreviewFrame | null }) { + const { canvasRef, frame: painted } = useComputerPreviewFrame(frame); + return ( + <> + + {painted?.appName ?? ""} + + ); +} + +// Images decode only when a test resolves them, and canvas drawing is recorded. +function stubFrameDecoding() { + vi.stubGlobal("Image", DeferredImage); + images.length = 0; + const drawImage = vi + .spyOn(CanvasRenderingContext2D.prototype, "drawImage") + .mockImplementation(() => undefined); + return { + drawImage, + [Symbol.dispose]() { + drawImage.mockRestore(); + vi.unstubAllGlobals(); + }, + }; +} + +it("keeps the previous pixels and caption until the replacement finishes decoding", async () => { + using stubs = stubFrameDecoding(); + const { drawImage } = stubs; + await using screen = await render(, { router: false }); + const caption = () => screen.container.querySelector("output")?.textContent; + await act(async () => images[0]!.resolve()); + expect(caption()).toBe("first"); + + await screen.rerender(); + expect(drawImage).toHaveBeenCalledTimes(1); + expect(caption()).toBe("first"); + await act(async () => images[1]!.resolve()); + expect(drawImage).toHaveBeenLastCalledWith(images[1], 0, 0); + expect(caption()).toBe("next"); +}); + +it("ignores a decode that finishes after a newer frame", async () => { + using stubs = stubFrameDecoding(); + const { drawImage } = stubs; + await using screen = await render(, { router: false }); + await screen.rerender(); + await act(async () => images[1]!.resolve()); + await act(async () => images[0]!.resolve()); + expect(drawImage).toHaveBeenCalledTimes(1); + expect(screen.container.querySelector("output")?.textContent).toBe("new"); +}); diff --git a/apps/web/src/components/preview/useComputerPreviewFrame.ts b/apps/web/src/components/preview/useComputerPreviewFrame.ts new file mode 100644 index 0000000000000..94685886be6ce --- /dev/null +++ b/apps/web/src/components/preview/useComputerPreviewFrame.ts @@ -0,0 +1,37 @@ +import type { CuaWindowPreviewFrame } from "@t3tools/contracts"; +import { useEffect, useRef, useState } from "react"; + +export function useComputerPreviewFrame(nextFrame: CuaWindowPreviewFrame | null) { + const canvasRef = useRef(null); + const [frame, setFrame] = useState(null); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- Decodes each streamed frame off-screen and paints it into the canvas only once it can replace the shown one. + useEffect(() => { + if (!nextFrame) return; + let cancelled = false; + const image = new Image(); + image.src = `data:${nextFrame.mimeType};base64,${nextFrame.dataBase64}`; + void image + .decode() + .then(() => { + if (cancelled) return; + const canvas = canvasRef.current; + const context = canvas?.getContext("2d"); + if (!canvas || !context) return; + // Changing an img src can blank its previous frame while decoding. + // Keep the canvas pixels until the replacement can be drawn in one task. + if (canvas.width !== image.naturalWidth || canvas.height !== image.naturalHeight) { + canvas.width = image.naturalWidth; + canvas.height = image.naturalHeight; + } + context.drawImage(image, 0, 0); + setFrame(nextFrame); + }) + .catch(() => undefined); + return () => { + cancelled = true; + }; + }, [nextFrame]); + + return { canvasRef, frame }; +} diff --git a/packages/contracts/src/cua.ts b/packages/contracts/src/cua.ts index 9651e897f3c2d..3d23ae59d3dac 100644 --- a/packages/contracts/src/cua.ts +++ b/packages/contracts/src/cua.ts @@ -1,6 +1,7 @@ import * as Schema from "effect/Schema"; import { ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; +import { ToolActivityNativeAppReference } from "./providerRuntime.ts"; /** Native host descriptor, confined to the desktop/server control pipe. */ export const CuaDriverMcpConfiguration = Schema.Struct({ @@ -14,6 +15,7 @@ export type CuaDriverMcpConfiguration = typeof CuaDriverMcpConfiguration.Type; /** A window the agent is driving, refreshed for the user while a client watches. */ export const CuaWindowPreviewFrame = Schema.Struct({ + app: Schema.optionalKey(ToolActivityNativeAppReference), appName: Schema.optionalKey(Schema.String), windowTitle: Schema.optionalKey(Schema.String), width: Schema.Number, diff --git a/packages/contracts/src/providerRuntime.ts b/packages/contracts/src/providerRuntime.ts index 4740a9fd0ccf7..5efec0f0316d7 100644 --- a/packages/contracts/src/providerRuntime.ts +++ b/packages/contracts/src/providerRuntime.ts @@ -84,7 +84,7 @@ export const ToolActivityNativeAppReference = Schema.Union([ Schema.TaggedStruct("app-id", { appId: TrimmedNonEmptyStringSchema.check( Schema.isMaxLength(512), - Schema.isPattern(/^[A-Za-z0-9._-]+$/u), + Schema.isPattern(/^[A-Za-z0-9._-]+(?:![A-Za-z0-9._-]+)?$/u), ), }), Schema.TaggedStruct("display-name", { diff --git a/packages/provider-core/src/server/cuaToolPresentation.test.ts b/packages/provider-core/src/server/cuaToolPresentation.test.ts index 2fad60dc2fdd3..b080cd2528728 100644 --- a/packages/provider-core/src/server/cuaToolPresentation.test.ts +++ b/packages/provider-core/src/server/cuaToolPresentation.test.ts @@ -37,6 +37,100 @@ describe("parseCuaToolName", () => { }); describe("cuaToolPresentation", () => { + it("retains a discovered app's display name and icon identity in window listings", () => { + clearCuaToolContext(threadId); + rememberCuaToolResult( + threadId, + "list_apps", + {}, + { apps: [{ pid: 12, name: "Helium", bundle_id: "net.imput.helium" }] }, + ); + rememberCuaToolResult( + threadId, + "list_windows", + {}, + { windows: [{ pid: 12, window_id: 34, app_name: "chrome.exe", title: "Releases - Helium" }] }, + ); + const presentation = cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: { pid: 12, window_id: 34 }, + status: "completed", + }); + NodeAssert.equal(presentation?.title, "Looked at Helium"); + NodeAssert.deepEqual(presentation?.toolIcon, { + _tag: "native-app", + app: { _tag: "app-id", appId: "net.imput.helium" }, + }); + }); + + it("keeps identities for Windows apps sharing a host PID separate", () => { + clearCuaToolContext(threadId); + rememberCuaToolResult( + threadId, + "launch_app", + { name: "Calculator" }, + { + pid: 11804, + name: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App", + bundle_id: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App", + windows: [{ window_id: 10, title: "Calculator" }], + }, + ); + rememberCuaToolResult( + threadId, + "list_windows", + {}, + { + windows: [ + { pid: 11804, window_id: 10, app_name: "ApplicationFrameHost.exe", title: "Calculator" }, + { pid: 11804, window_id: 20, app_name: "ApplicationFrameHost.exe", title: "Settings" }, + ], + }, + ); + for (const [windowId, expected] of [ + [10, "Calculator"], + [20, "Settings"], + [10, "Calculator"], + ] as const) { + rememberCuaToolResult( + threadId, + "get_window_state", + { pid: 11804, window_id: windowId }, + { + pid: 11804, + window_id: windowId, + app_name: "ApplicationFrameHost.exe", + window_title: expected, + }, + ); + const viewed = cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args: + windowId === 20 + ? { target: { kind: "window", pid: 11804, window_id: windowId } } + : { pid: 11804, window_id: windowId }, + status: "completed", + }); + NodeAssert.equal(viewed?.title, `Looked at ${expected}`); + const clicked = cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/click", + args: { pid: 11804, element_token: "snapshot:1" }, + status: "completed", + }); + NodeAssert.equal(clicked?.title, `Clicked in ${expected}`); + NodeAssert.deepEqual(clicked?.toolIcon, { + _tag: "native-app", + app: + expected === "Calculator" + ? { _tag: "app-id", appId: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App" } + : { _tag: "display-name", displayName: "Settings" }, + }); + } + }); + it("resolves apps by pid from earlier discovery results", () => { clearCuaToolContext(threadId); rememberCuaToolResult( @@ -165,6 +259,15 @@ describe("cuaToolPresentation", () => { })?.title, "Typed text in Safari", ); + NodeAssert.equal( + cuaToolPresentation({ + threadId, + rawToolName: "mcp__cua-driver__launch_app", + args: { name: "/Applications/Visual Studio Code.app" }, + status: "inProgress", + })?.title, + "Launching Visual Studio Code", + ); }); it("returns nothing for tools outside Cua", () => { @@ -279,6 +382,21 @@ describe("readCuaWindowTarget", () => { clearCuaToolContext(threadId); }); + it("keeps display-only inspections from redirecting the preview", () => { + clearCuaToolContext(threadId); + for (const args of [ + { pid: 42, window_id: 7 }, + { pid: 99, window_id: 8, display_only: true }, + ]) { + cuaToolPresentation({ + threadId, + rawToolName: "cua-driver/get_window_state", + args, + status: "completed", + }); + } + NodeAssert.deepEqual(readCuaWindowTarget(threadId), { pid: 42, windowId: 7n }); + }); it("remembers the last pid and window id a tool call addressed", () => { clearCuaToolContext(threadId); NodeAssert.equal(readCuaWindowTarget(threadId), undefined); diff --git a/packages/provider-core/src/server/cuaToolPresentation.ts b/packages/provider-core/src/server/cuaToolPresentation.ts index 7b5f1c25fab27..7576b50902a51 100644 --- a/packages/provider-core/src/server/cuaToolPresentation.ts +++ b/packages/provider-core/src/server/cuaToolPresentation.ts @@ -10,8 +10,8 @@ import { CUA_MCP_SERVER_NAME } from "./mcpSession.ts"; /** * Shared presentation for Cua Driver tool calls, so every provider renders * them like Codex computer use: a "computer" surface, the target app's icon, - * and a human title. Cua addresses apps by pid, so the directory learns - * pid-to-app mappings from discovery results and resolves later calls. + * and a human title. The directory learns app and window identities from + * discovery results; Windows can host several apps inside one process. */ export interface CuaToolPresentation { readonly title: string; @@ -51,10 +51,12 @@ const MAX_APPS_PER_THREAD = 128; export interface CuaWindowTarget { readonly pid: number; readonly windowId: bigint; + readonly app?: CuaApp; } interface ThreadDirectory { readonly byPid: Map; + readonly byWindow: Map; lastApp: CuaApp | undefined; lastWindow: CuaWindowTarget | undefined; } @@ -74,7 +76,12 @@ function directoryFor(threadId: ThreadId): ThreadDirectory { if (oldest === undefined) break; directories.delete(oldest); } - const created: ThreadDirectory = { byPid: new Map(), lastApp: undefined, lastWindow: undefined }; + const created: ThreadDirectory = { + byPid: new Map(), + byWindow: new Map(), + lastApp: undefined, + lastWindow: undefined, + }; directories.set(threadId, created); return created; } @@ -91,7 +98,11 @@ export function clearCuaWindowTarget(threadId: ThreadId): void { /** Last window the agent targeted in this thread, if any tool call named one. */ export function readCuaWindowTarget(threadId: ThreadId): CuaWindowTarget | undefined { - return directories.get(threadId)?.lastWindow; + const directory = directories.get(threadId); + const target = directory?.lastWindow; + if (!target) return undefined; + const app = directory.byWindow.get(`${target.pid}:${target.windowId}`); + return { ...target, ...(app ? { app } : {}) }; } function asWindowId(value: unknown): bigint | undefined { @@ -124,7 +135,7 @@ function asText(value: unknown, maxLength: number): string | undefined { const asAppName = (value: unknown) => asText(value, 160); const asBundleId = (value: unknown) => { const id = asText(value, 512); - return id && /^[A-Za-z0-9._-]+$/u.test(id) ? id : undefined; + return id && /^[A-Za-z0-9._-]+(?:![A-Za-z0-9._-]+)?$/u.test(id) ? id : undefined; }; /** Accepts tool result text, MCP content arrays, or already-parsed objects. */ @@ -167,13 +178,39 @@ function rememberApp(directory: ThreadDirectory, pid: number | undefined, app: C } function learnFromRecord(directory: ThreadDirectory, record: Record): void { - const name = asAppName(record.app_name) ?? asAppName(record.name); + const pid = asPid(record.pid); + const reportedName = asAppName(record.app_name) ?? asAppName(record.name); + const sharedHost = /^(?:ApplicationFrameHost|WWAHost)(?:\.exe)?$/iu.test(reportedName ?? ""); + const knownApp = !sharedHost && pid !== undefined ? directory.byPid.get(pid) : undefined; + const name = sharedHost + ? (asAppName(record.window_title) ?? asAppName(record.title)) + : /\.exe$/iu.test(reportedName ?? "") && knownApp?.name && !/\.exe$/iu.test(knownApp.name) + ? knownApp.name + : reportedName; const bundleId = asBundleId(record.bundle_id); + if (sharedHost && pid !== undefined) { + rememberApp(directory, pid, {}); + directory.byPid.set(pid, {}); + } if (!name && !bundleId) return; - rememberApp(directory, asPid(record.pid), { + const app = { + ...knownApp, ...(name ? { name } : {}), ...(bundleId ? { bundleId } : {}), - }); + }; + const windowId = asWindowId(record.window_id); + if (pid !== undefined && windowId !== undefined) { + const key = `${pid}:${windowId}`; + const known = directory.byWindow.get(key); + directory.byWindow.delete(key); + directory.byWindow.set(key, { ...known, ...app }); + if (directory.byWindow.size > MAX_APPS_PER_THREAD) { + const oldest = directory.byWindow.keys().next().value; + if (oldest !== undefined) directory.byWindow.delete(oldest); + } + } + // A Windows host PID can own Calculator and Settings at the same time. + if (!sharedHost) rememberApp(directory, pid, app); } /** @@ -206,6 +243,30 @@ export function rememberCuaToolResult( if (requestedBundleId && pid !== undefined && toolName === "launch_app") { rememberApp(directory, pid, { bundleId: requestedBundleId }); } + if (toolName === "launch_app") { + const name = appNameFromLaunchTarget(asRecord(args)?.name); + if (name) { + learnFromRecord(directory, { ...parsed, name }); + if (Array.isArray(parsed.windows)) { + for (const window of parsed.windows.slice(0, MAX_APPS_PER_THREAD)) { + const record = asRecord(window); + if (record) learnFromRecord(directory, { ...parsed, ...record, name }); + } + } + } + } +} + +// `launch_app` also accepts a `.app` bundle or executable path as its name. +function appNameFromLaunchTarget(value: unknown): string | undefined { + const name = asAppName(value); + if (!name || !/[\\/]/u.test(name)) return name; + return asAppName( + name + .split(/[\\/]/u) + .findLast((part) => part.length > 0) + ?.replace(/\.(?:app|exe)$/iu, ""), + ); } function humanizeBundleId(bundleId: string): string | undefined { @@ -219,12 +280,23 @@ function resolveApp( trackWindow: boolean, ): CuaApp | undefined { const directory = directories.get(threadId); - const pid = asPid(args?.pid); + const target = asRecord(args?.target); + const pid = asPid(args?.pid) ?? asPid(target?.pid); const bundleId = asBundleId(args?.bundle_id); - const name = asAppName(args?.app_name) ?? asAppName(args?.app); - const known = pid !== undefined ? directory?.byPid.get(pid) : undefined; - const windowId = asWindowId(args?.window_id); - if (trackWindow && pid !== undefined && windowId !== undefined) { + const name = + asAppName(args?.app_name) ?? asAppName(args?.app) ?? appNameFromLaunchTarget(args?.name); + const windowId = asWindowId(args?.window_id) ?? asWindowId(target?.window_id); + const targetedWindow = + windowId ?? + (pid !== undefined && directory?.lastWindow?.pid === pid + ? directory.lastWindow.windowId + : undefined); + const windowApp = + pid !== undefined && targetedWindow !== undefined + ? directory?.byWindow.get(`${pid}:${targetedWindow}`) + : undefined; + const known = windowApp ?? (pid !== undefined ? directory?.byPid.get(pid) : undefined); + if (trackWindow && pid !== undefined && windowId !== undefined && args?.display_only !== true) { directoryFor(threadId).lastWindow = { pid, windowId }; } const app: CuaApp = { @@ -234,7 +306,7 @@ function resolveApp( }; if (app.name || app.bundleId) { if (directory) { - if (pid !== undefined) rememberApp(directory, pid, app); + if (pid !== undefined && !windowApp) rememberApp(directory, pid, app); directory.lastApp = app; } return app; @@ -475,7 +547,11 @@ export function cuaToolPresentation(input: { // Restoring the user's foreground window must not redirect the live preview. const app = TOOLS_WITHOUT_APP.has(tool) ? undefined - : resolveApp(input.threadId, targetArgs, tool !== "bring_to_front" && input.status !== "failed"); + : resolveApp( + input.threadId, + targetArgs, + tool !== "bring_to_front" && input.status !== "failed", + ); const appName = app?.name ?? (app?.bundleId ? humanizeBundleId(app.bundleId) : undefined); const reference = app ? nativeApp(app) : undefined; const icon = reference ? ({ _tag: "native-app", app: reference } as const) : undefined; From 3f61c220f8c38a0fcd81bad602f572c237891603 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 16 Oct 2026 16:04:50 -0700 Subject: [PATCH 8/8] feat(cua): mention installed applications in the composer --- .../src/features/threads/ThreadFeed.tsx | 2 + apps/server/src/auth/RpcAuthorization.ts | 1 + apps/server/src/cua/CuaApplications.test.ts | 92 ++++++++++++++++++ apps/server/src/cua/CuaApplications.ts | 94 +++++++++++++++++++ .../src/observability/RpcInstrumentation.ts | 1 + apps/server/src/server.ts | 2 + apps/server/src/ws.ts | 5 + apps/web/src/components/ChatMarkdown.tsx | 8 +- .../components/ComposerPromptEditorTiptap.tsx | 32 +++++++ apps/web/src/components/NativeAppChip.tsx | 53 +++++++++++ apps/web/src/components/chat/ChatComposer.tsx | 16 +++- .../components/chat/ComposerCommandMenu.tsx | 20 +++- .../components/chat/useComposerMenuItems.ts | 17 +++- apps/web/src/composer-editor-mentions.test.ts | 13 +++ apps/web/src/composer-editor-mentions.ts | 10 +- apps/web/src/composer-logic.test.ts | 8 ++ apps/web/src/composer-logic.ts | 2 + apps/web/src/composer-rich-text-doc.ts | 4 + apps/web/src/state/cua.ts | 8 +- docs/user/computer-use.md | 7 +- packages/client-runtime/src/state/cua.ts | 11 ++- packages/contracts/src/cua.ts | 17 ++++ packages/contracts/src/rpc.ts | 10 ++ packages/shared/package.json | 4 + .../shared/src/composerContextReferences.ts | 8 +- packages/shared/src/nativeAppMentions.test.ts | 43 +++++++++ packages/shared/src/nativeAppMentions.ts | 63 +++++++++++++ 27 files changed, 535 insertions(+), 16 deletions(-) create mode 100644 apps/server/src/cua/CuaApplications.test.ts create mode 100644 apps/server/src/cua/CuaApplications.ts create mode 100644 apps/web/src/components/NativeAppChip.tsx create mode 100644 packages/shared/src/nativeAppMentions.test.ts create mode 100644 packages/shared/src/nativeAppMentions.ts diff --git a/apps/mobile/src/features/threads/ThreadFeed.tsx b/apps/mobile/src/features/threads/ThreadFeed.tsx index 0c63f78b16401..21a7910c42502 100644 --- a/apps/mobile/src/features/threads/ThreadFeed.tsx +++ b/apps/mobile/src/features/threads/ThreadFeed.tsx @@ -1,5 +1,6 @@ import { Button } from "../../components/ui/Button"; import * as Dialog from "../../components/ui/Dialog"; +import { parseNativeAppMention } from "@t3tools/shared/nativeAppMentions"; import { useOpenPullRequest } from "../../native/useOpenPullRequest"; import { useWorkspaceSettledContentWidth } from "../layout/workspace-content-width"; import { ThreadMarkdownVideo, ThreadMediaVisibleContext } from "./ThreadMarkdownVideo"; @@ -2288,6 +2289,7 @@ function UserMessageContent(props: UserMessageContentProps) { return `[${ref.label}${available ? "" : " (unavailable)"}](t3-context://v1/${ref.kind}/${ref.contextId})`; }); const onLinkPress = (href: string) => { + if (parseNativeAppMention(href)) return; const reference = parseComposerContextHref(href); if (!reference) return props.linkHandlers.onLinkPress?.(href); const record = props.context?.records.find( diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 3b8fb902cd718..d0c96a1cf9bd8 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -62,6 +62,7 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.serverProbe]: AuthOrchestrationReadScope, [WS_METHODS.serverGetConfig]: AuthOrchestrationReadScope, [WS_METHODS.serverCheckComputerUse]: AuthOrchestrationReadScope, + [WS_METHODS.serverListApplications]: AuthOrchestrationReadScope, // Operate: the check reads the host's browser profiles; the action installs a tool. [WS_METHODS.serverCheckBrowserTabs]: AuthOrchestrationOperateScope, [WS_METHODS.serverRunBrowserTabsAction]: AuthOrchestrationOperateScope, diff --git a/apps/server/src/cua/CuaApplications.test.ts b/apps/server/src/cua/CuaApplications.test.ts new file mode 100644 index 0000000000000..94c451166652f --- /dev/null +++ b/apps/server/src/cua/CuaApplications.test.ts @@ -0,0 +1,92 @@ +import { it } from "@effect/vitest"; +import { expect } from "vite-plus/test"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as CuaApplications from "./CuaApplications.ts"; +import * as CuaDriver from "./CuaDriver.ts"; + +it.effect( + "lists and deduplicates native identities across platforms, then releases its client", + () => + Effect.gen(function* () { + let released = 0; + const windows = { + name: "Calculator", + bundleId: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App", + pid: 42, + running: true, + }; + const sdk = { + CuaDriver: { + connect: () => ({ + listApps: async () => ({ + apps: [ + windows, + windows, + { + name: "Helium", + bundleId: "C:\\Apps\\helium.exe", + launchPath: "C:\\Apps\\helium.exe", + pid: 1, + running: true, + }, + { + name: "Terminal", + launchPath: "/usr/share/applications/terminal.desktop", + pid: 0, + running: false, + }, + { name: "", pid: 0, running: false }, + ], + }), + shutdown: async () => {}, + uniffiDestroy: () => { + released++; + }, + }), + }, + ListAppsInput: { new: () => ({}) }, + } as unknown as CuaApplications.Sdk; + const service = yield* CuaApplications.make({ loadSdk: Effect.succeed(sdk) }); + expect(yield* service.list).toEqual([ + { name: "Calculator", bundleId: windows.bundleId, pid: 42 }, + { name: "Helium", launchPath: "C:\\Apps\\helium.exe", pid: 1 }, + { name: "Terminal", launchPath: "/usr/share/applications/terminal.desktop" }, + ]); + expect(released).toBe(1); + }).pipe( + Effect.provideService( + CuaDriver.CuaDriver, + CuaDriver.CuaDriver.of({ + enabled: Effect.succeed(true), + acquire: Effect.succeedSome({ + command: "driver", + args: [], + environment: [], + socketPath: "test", + }), + lastFailure: Effect.succeedNone, + status: Effect.succeed({ platform: "win32", running: true }), + }), + ), + Effect.provide(NodeServices.layer), + ), +); + +it.effect("does not start the driver when Computer Use is disabled", () => + Effect.gen(function* () { + const service = yield* CuaApplications.make({ loadSdk: Effect.die("must not load") }); + expect(yield* service.list).toEqual([]); + }).pipe( + Effect.provideService( + CuaDriver.CuaDriver, + CuaDriver.CuaDriver.of({ + enabled: Effect.succeed(false), + acquire: Effect.die("must not start"), + lastFailure: Effect.succeedNone, + status: Effect.succeed({ platform: "win32", running: false }), + }), + ), + Effect.provide(NodeServices.layer), + ), +); diff --git a/apps/server/src/cua/CuaApplications.ts b/apps/server/src/cua/CuaApplications.ts new file mode 100644 index 0000000000000..d7fe0f8cd94a3 --- /dev/null +++ b/apps/server/src/cua/CuaApplications.ts @@ -0,0 +1,94 @@ +import * as NodeModule from "node:module"; +import { CuaApplication, CuaApplicationsError } from "@t3tools/contracts"; +import type * as CuaSdk from "@trycua/cua-driver"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as CuaDriver from "./CuaDriver.ts"; + +export type Sdk = Pick; +type Client = Pick; +const isApplication = Schema.is(CuaApplication); +const requireSdk = NodeModule.createRequire(import.meta.url); +const loadSdk = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + for (const root of requireSdk.resolve.paths("@trycua/cua-driver") ?? []) { + const dir = path.join(root, "@trycua", "cua-driver"); + if (yield* fs.exists(path.join(dir, "package.json"))) { + return yield* Effect.try(() => requireSdk(path.join(dir, "dist", "index.js")) as Sdk); + } + } + return yield* new CuaApplicationsError({ cause: "Cua Driver SDK is not installed." }); +}).pipe(Effect.mapError((cause) => new CuaApplicationsError({ cause }))); + +export class CuaApplications extends Context.Service< + CuaApplications, + { + readonly list: Effect.Effect, CuaApplicationsError>; + } +>()("t3/cua/CuaApplications") {} + +export const make = Effect.fn("CuaApplications.make")(function* ( + options: { loadSdk?: Effect.Effect } = {}, +) { + const driver = yield* CuaDriver.CuaDriver; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const sdk = + options.loadSdk ?? + loadSdk.pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + ); + const list = Effect.gen(function* () { + if (!(yield* driver.enabled)) return []; + const config = yield* driver.acquire; + if (Option.isNone(config) || !config.value.socketPath) return []; + const module = yield* sdk; + return yield* Effect.scoped( + Effect.gen(function* () { + const client = yield* Effect.acquireRelease( + Effect.try(() => module.CuaDriver.connect(config.value.socketPath!) as unknown as Client), + (client) => + Effect.promise(() => client.shutdown()).pipe( + Effect.ignore, + Effect.andThen(Effect.sync(() => client.uniffiDestroy())), + ), + ); + const result = yield* Effect.tryPromise((signal) => + client.listApps(module.ListAppsInput.new({}), { signal }), + ); + const apps: CuaApplication[] = []; + const seen = new Set(); + for (const app of result.apps) { + const packagedId = /^shell:appsFolder\\(.+)$/iu.exec(app.launchPath ?? "")?.[1]; + const bundleId = + packagedId ?? + (app.bundleId && /^[A-Za-z0-9._-]+(?:![A-Za-z0-9._-]+)?$/u.test(app.bundleId) + ? app.bundleId + : undefined); + const candidate = { + name: app.name, + ...(bundleId ? { bundleId } : {}), + ...(app.launchPath ? { launchPath: app.launchPath } : {}), + ...(app.running && app.pid > 0 ? { pid: app.pid } : {}), + }; + if (!isApplication(candidate)) continue; + const key = bundleId ?? app.launchPath ?? app.name; + if (seen.has(key)) continue; + seen.add(key); + apps.push(candidate); + } + return apps.sort((a, b) => a.name.localeCompare(b.name)); + }), + ); + }).pipe(Effect.mapError((cause) => new CuaApplicationsError({ cause }))); + return CuaApplications.of({ list }); +}); + +export const layer = Layer.effect(CuaApplications, make()); diff --git a/apps/server/src/observability/RpcInstrumentation.ts b/apps/server/src/observability/RpcInstrumentation.ts index 05e07bde288ab..e96b29e9a1958 100644 --- a/apps/server/src/observability/RpcInstrumentation.ts +++ b/apps/server/src/observability/RpcInstrumentation.ts @@ -33,6 +33,7 @@ const RPC_AGGREGATES = { [WS_METHODS.projectsMutate]: "orchestration", [WS_METHODS.serverProbe]: "server", [WS_METHODS.serverCheckComputerUse]: "server", + [WS_METHODS.serverListApplications]: "server", [WS_METHODS.serverCheckBrowserTabs]: "server", [WS_METHODS.serverRunBrowserTabsAction]: "server", [WS_METHODS.serverGetConfig]: "server", diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 27e5000fd7ffd..980b41b983830 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -174,6 +174,7 @@ import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; import * as TriageService from "./triage/TriageService.ts"; import * as DesktopTelemetryReceiver from "./resourceTelemetry/DesktopTelemetryReceiver.ts"; import * as CuaDriver from "./cua/CuaDriver.ts"; +import * as CuaApplications from "./cua/CuaApplications.ts"; import * as BrowserTabs from "./mcp/BrowserTabs.ts"; import * as CuaWindowPreview from "./cua/CuaWindowPreview.ts"; import * as NativeTelemetryClient from "./resourceTelemetry/NativeTelemetryClient.ts"; @@ -593,6 +594,7 @@ const layerProviderInstallationRefresh = Layer.effectDiscard( const layerRuntimeCoreDependenciesBase = Layer.mergeAll( AgentAwarenessRelay.layer, CuaWindowPreview.layer, + CuaApplications.layer, HeldHooksWaker.layer, layerThreadSettlementWorker, layerIdleCompactionWorker, diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index e0956432e3746..bf409ab2e021d 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -188,6 +188,7 @@ import { terminalOutputStream, withTerminalOutputWindow } from "./terminal/Outpu import * as ServerBrowser from "./preview/ServerBrowser.ts"; import * as DeviceService from "./device/DeviceService.ts"; import * as CuaDriver from "./cua/CuaDriver.ts"; +import * as CuaApplications from "./cua/CuaApplications.ts"; import * as BrowserTabs from "./mcp/BrowserTabs.ts"; import * as CuaWindowPreview from "./cua/CuaWindowPreview.ts"; import { remoteSshDeviceHosts } from "./device/localSshDeviceHost.ts"; @@ -2187,6 +2188,10 @@ export const layerWsRpc = ( : Effect.succeed({ platform: "other", running: false } as const), ), ), + [WS_METHODS.serverListApplications]: () => + Effect.flatMap(Effect.serviceOption(CuaApplications.CuaApplications), (service) => + Option.isSome(service) ? service.value.list : Effect.succeed([]), + ), [WS_METHODS.serverCheckBrowserTabs]: (_input) => Effect.serviceOption(BrowserTabs.BrowserTabs).pipe( Effect.flatMap((tabs) => diff --git a/apps/web/src/components/ChatMarkdown.tsx b/apps/web/src/components/ChatMarkdown.tsx index bd13a8bdc96d5..6b1cd1d72c1fb 100644 --- a/apps/web/src/components/ChatMarkdown.tsx +++ b/apps/web/src/components/ChatMarkdown.tsx @@ -92,6 +92,8 @@ import type { Pluggable, PluggableList } from "unified"; import { defaultUrlTransform } from "react-markdown"; import { parseAssistantCitationHref } from "@t3tools/shared/assistantCitations"; import { parseComposerContextHref } from "@t3tools/shared/composerContextReferences"; +import { parseNativeAppMention } from "@t3tools/shared/nativeAppMentions"; +import { NativeAppChip } from "./NativeAppChip"; import { parseThreadLinkHref } from "@t3tools/shared/threadLinks"; import { AssistantCitationChip } from "./chat/AssistantCitationChip"; import { MarkdownThreadLink } from "./chat/MarkdownThreadLink"; @@ -400,7 +402,7 @@ const CHAT_MARKDOWN_LITERAL_HTML_REHYPE_PLUGINS = [ function markdownUrlTransform(href: string) { if (parseAssistantCitationHref(href)) return href; - if (parseComposerContextHref(href)) return href; + if (parseComposerContextHref(href) || parseNativeAppMention(href)) return href; if (parseThreadLinkHref(href)) return href; if (isWindowsDrivePathHref(href)) return href; return rewriteMarkdownFileUriHref(href) ?? defaultUrlTransform(href); @@ -450,7 +452,7 @@ function fileLinkSourcesKey( imageBaseDir: string | undefined, ): string { const hrefs = extractMarkdownLinkHrefs(renderCodexInlineDirectivesAsMarkdown(text)) - .filter((href) => !parseComposerContextHref(href)) + .filter((href) => !parseComposerContextHref(href) && !parseNativeAppMention(href)) .map(normalizeMarkdownLinkHrefKey) .filter((href) => resolveMarkdownFileLinkMeta(href, cwd, imageBaseDir ?? cwd)); const spans = extractInlineCodeSpans(text).filter((span) => @@ -1078,6 +1080,8 @@ const CHAT_MARKDOWN_COMPONENTS = { } = use(ChatMarkdownRendererContext); const text = use(ChatMarkdownTextContext); const citation = href ? parseAssistantCitationHref(href) : null; + const application = href ? parseNativeAppMention(href) : null; + if (application) return ; if (citation) return ; const threadLink = href ? parseThreadLinkHref(href) : null; if (threadLink) { diff --git a/apps/web/src/components/ComposerPromptEditorTiptap.tsx b/apps/web/src/components/ComposerPromptEditorTiptap.tsx index abc9549ee3d00..69d8832e5215f 100644 --- a/apps/web/src/components/ComposerPromptEditorTiptap.tsx +++ b/apps/web/src/components/ComposerPromptEditorTiptap.tsx @@ -1,6 +1,8 @@ import { documentChipKindForFileName } from "~/pierre-icons"; import { isProviderSkillUserInvocable } from "@t3tools/client-runtime/providerSkills"; import { formatProviderSkillDisplayName } from "@t3tools/shared/inlineSkills"; +import { parseNativeAppMention } from "@t3tools/shared/nativeAppMentions"; +import { NativeAppChip } from "./NativeAppChip"; import { Extension, findChildren, @@ -510,6 +512,35 @@ const ComposerContextReferenceExtension = Node.create({ }, }); +const ComposerApplicationExtension = Node.create({ + name: "composer-application", + group: "inline", + inline: true, + atom: true, + selectable: true, + addAttributes() { + return { href: { default: "" }, source: { default: "" } }; + }, + parseHTML() { + return [{ tag: "span[data-composer-application]" }]; + }, + renderHTML({ HTMLAttributes }) { + return ["span", { "data-composer-application": "", ...HTMLAttributes }]; + }, + addNodeView() { + return ReactNodeViewRenderer(ComposerApplicationNodeView); + }, +}); + +function ComposerApplicationNodeView({ node }: NodeViewProps) { + const mention = parseNativeAppMention(String(node.attrs.href)); + return ( + + {mention ? : String(node.attrs.source)} + + ); +} + function ComposerContextReferenceNodeView({ node }: NodeViewProps) { return ( @@ -836,6 +867,7 @@ function createComposerExtensions(richText: boolean) { ComposerUndoGroupingExtension, ComposerTextDirectionExtension, ComposerMentionExtension, + ComposerApplicationExtension, ComposerSkillExtension, ComposerCitationExtension, ComposerContextReferenceExtension, diff --git a/apps/web/src/components/NativeAppChip.tsx b/apps/web/src/components/NativeAppChip.tsx new file mode 100644 index 0000000000000..98001448f7b3c --- /dev/null +++ b/apps/web/src/components/NativeAppChip.tsx @@ -0,0 +1,53 @@ +import { EnvironmentId, type CuaApplication } from "@t3tools/contracts"; +import { AppWindowIcon } from "lucide-react"; +import { useState } from "react"; +import { useAssetUrlState } from "~/assets/assetUrls"; +import { ContextChip, ContextChipLabel } from "./ContextChip"; + +export function NativeAppIcon({ + app, + environmentId, +}: { + app: CuaApplication; + environmentId: string; +}) { + const reference = + app.bundleId && /^[A-Za-z0-9._-]+(?:![A-Za-z0-9._-]+)?$/u.test(app.bundleId) + ? { _tag: "app-id" as const, appId: app.bundleId } + : { _tag: "display-name" as const, displayName: app.name }; + const icon = useAssetUrlState(EnvironmentId.make(environmentId), { + _tag: "native-app-icon", + app: reference, + }); + const [loadedUrl, setLoadedUrl] = useState(null); + const [failedUrl, setFailedUrl] = useState(null); + const url = icon._tag === "Success" ? icon.url : null; + const loaded = url !== null && loadedUrl === url && failedUrl !== url; + return ( + + ); +} + +export function NativeAppChip(props: { app: CuaApplication; environmentId: string }) { + return ( + + + {props.app.name} + + ); +} diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index ae0bb19336e0e..70471856a1c5a 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -1,3 +1,4 @@ +import { formatNativeAppMention } from "@t3tools/shared/nativeAppMentions"; import { useComposerTypingGuard } from "./useComposerTypingGuard"; import { insertAttachmentReferencesForDraft } from "./composerAttachmentReferences"; import { DESKTOP_PASTE_AS_TEXT_EVENT } from "../../lib/desktopPasteAsText"; @@ -1841,6 +1842,19 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }); const { snapshot, trigger } = resolveActiveComposerTrigger(); if (!trigger) return; + if (item.type === "application") { + const replacement = `${formatNativeAppMention(item.application, item.environmentId)} `; + const replacementRangeEnd = extendReplacementRangeForTrailingSpace( + snapshot.value, + trigger.rangeEnd, + replacement, + ); + const applied = applyPromptReplacement(trigger.rangeStart, replacementRangeEnd, replacement, { + expectedText: snapshot.value.slice(trigger.rangeStart, replacementRangeEnd), + }); + if (applied) setComposerHighlightedItemId(null); + return; + } if (item.type === "path") { if (!isPathSuggestionCurrent(item.searchTarget, trigger, environmentId, gitCwd)) return; const replacement = `${serializeComposerFileLink(item.path)} `; @@ -3286,7 +3300,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) ? isComposerMenuLoading ? composerTriggerKind === "pull-request" ? "Finding pull request..." - : "Searching workspace files..." + : "Searching files, threads, and applications..." : composerMenuEmptyState : ""}
diff --git a/apps/web/src/components/chat/ComposerCommandMenu.tsx b/apps/web/src/components/chat/ComposerCommandMenu.tsx index afa058587b553..d3f15fc3b9d9c 100644 --- a/apps/web/src/components/chat/ComposerCommandMenu.tsx +++ b/apps/web/src/components/chat/ComposerCommandMenu.tsx @@ -8,6 +8,7 @@ import { type EnvironmentId, type EnvironmentMachineKind, type ProjectEntry, + type CuaApplication, type ProviderDriverKind, type PullRequestContextMetadata, type ScopedThreadRef, @@ -33,8 +34,17 @@ import { EnvironmentMachineIcon } from "../EnvironmentMachineIcon"; import { PierreEntryIcon } from "./PierreEntryIcon"; import { ComposerBanner } from "./ComposerBanner"; import { resolvePullRequestState } from "../pullRequest/pullRequestPresentation"; +import { NativeAppIcon } from "../NativeAppChip"; export type ComposerCommandItem = + | { + id: string; + type: "application"; + application: CuaApplication; + environmentId: string; + label: string; + description: string; + } | { id: string; type: "path"; @@ -156,12 +166,12 @@ export const ComposerCommandMenu = memo(function ComposerCommandMenu(props: { ? "Searching workspace skills..." : props.triggerKind === "pull-request" ? "Finding pull request..." - : "Searching workspace files..." + : "Searching files, threads, and applications..." : (props.emptyStateText ?? (props.triggerKind === "skill" ? "No skills found. Try / to browse provider commands." : props.triggerKind === "path" - ? "No matching files or folders." + ? "No matching files, threads, or applications." : "No matching command."))}

@@ -214,7 +224,9 @@ const ComposerCommandMenuItem = memo(function ComposerCommandMenuItem(props: { {props.item.type === "thread" ? (