Skip to content
Draft
3 changes: 3 additions & 0 deletions apps/desktop/src/cua/DesktopCuaDriver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ const fixture = Effect.fn(function* (
}),
loadEmbedded: async () => ({
EmbeddedCuaDriverHost: class {
static withOptions() {
return new this();
}
async start() {
starts++;
Queue.offerUnsafe(starting, undefined);
Expand Down
41 changes: 34 additions & 7 deletions apps/desktop/src/cua/DesktopCuaDriver.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
import { CuaDriverMcpConfiguration, type DesktopCuaDriverRequest } from "@t3tools/contracts";
import type { EmbeddedCuaDriverHost, EmbeddedDriverConnection } from "@trycua/cua-driver/embedded";
import { cuaDriverHostOptions } from "@t3tools/shared/cuaDriverHostOptions";
import type {
EmbeddedCuaDriverHost,
EmbeddedDriverConnection,
EmbeddedDriverHostOptions,
} from "@trycua/cua-driver/embedded";
import * as Cause from "effect/Cause";
import * as Exit from "effect/Exit";
import * as Context from "effect/Context";
Expand All @@ -16,10 +21,11 @@ import * as DesktopTelemetryPublisher from "../telemetry/DesktopTelemetryPublish

export interface DesktopCuaDriverDependencies {
readonly loadEmbedded: () => Promise<{
readonly EmbeddedCuaDriverHost: new (
path: string,
hostBundleId: string,
) => Pick<EmbeddedCuaDriverHost, "start" | "stop" | "waitForExit" | "uniffiDestroy">;
readonly EmbeddedCuaDriverHost: {
readonly withOptions: (
options: EmbeddedDriverHostOptions,
) => Pick<EmbeddedCuaDriverHost, "start" | "stop" | "waitForExit" | "uniffiDestroy">;
};
}>;
readonly loadElectron: () => Promise<
Pick<
Expand All @@ -33,7 +39,21 @@ const decodeMcpConfiguration = Schema.decodeEffect(CuaDriverMcpConfiguration);
// The SDK loads its native libraries from the archive's .unpacked sibling
// through our @ubjs/node patch, so a packaged app imports it in place.
const defaultDependencies: DesktopCuaDriverDependencies = {
loadEmbedded: () => import("@trycua/cua-driver/embedded"),
loadEmbedded: async () => {
const module = await import("@trycua/cua-driver/embedded");
return {
EmbeddedCuaDriverHost: {
withOptions: (options) => {
const host = module.EmbeddedCuaDriverHost.withOptions(options);
// The generated factory's interface omits the native object's disposal method.
if (!(host instanceof module.EmbeddedCuaDriverHost)) {
throw new Error("Cua Driver returned an unexpected host object.");
}
return host;
},
},
};
},
loadElectron: () => import("@trycua/cua-driver/electron"),
};

Expand Down Expand Up @@ -217,7 +237,14 @@ export const make = Effect.fn("desktop.cuaDriver.make")(function* (
const module = yield* Effect.tryPromise(dependencies.loadEmbedded);
const owned = yield* Effect.try(() => {
const owned: OwnedHost = {
host: new module.EmbeddedCuaDriverHost(binaryPath.value, environment.appUserModelId),
host: module.EmbeddedCuaDriverHost.withOptions(
cuaDriverHostOptions(
binaryPath.value,
environment.appUserModelId,
environment.platform,
process.env,
),
),
abort: new AbortController(),
};
hosts.add(owned);
Expand Down
2 changes: 2 additions & 0 deletions apps/mobile/src/features/threads/ThreadFeed.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { Button } from "../../components/ui/Button";
import * as Dialog from "../../components/ui/Dialog";
import { parseNativeAppMention } from "@t3tools/shared/nativeAppMentions";
import { useOpenPullRequest } from "../../native/useOpenPullRequest";
import { useWorkspaceSettledContentWidth } from "../layout/workspace-content-width";
import { ThreadMarkdownVideo, ThreadMediaVisibleContext } from "./ThreadMarkdownVideo";
Expand Down Expand Up @@ -1939,6 +1940,7 @@ function UserMessageContent(props: UserMessageContentProps) {
return `[${ref.label}${available ? "" : " (unavailable)"}](t3-context://v1/${ref.kind}/${ref.contextId})`;
});
const onLinkPress = (href: string) => {
if (parseNativeAppMention(href)) return;
const reference = parseComposerContextHref(href);
if (!reference) return props.linkHandlers.onLinkPress?.(href);
const record = props.context?.records.find(
Expand Down
54 changes: 54 additions & 0 deletions apps/server/src/assets/nativeAppIcons/win32.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
// @effect-diagnostics nodeBuiltinImport:off - Runs the Windows icon script against an isolated package fixture.
import * as NodeChildProcess from "node:child_process";
import * as NodeFS from "node:fs";
import * as NodeOS from "node:os";
import * as NodePath from "node:path";

import { describe, expect, it } from "vite-plus/test";

import { WINDOWS_APP_ICON_SCRIPT } from "./win32.ts";

// oxlint-disable-next-line t3code/no-global-process-runtime -- This integration test needs Windows PowerShell on the physical test host.
describe.runIf(NodeOS.platform() === "win32")("Windows packaged app icons", () => {
it("uses a package's unplated logo for both its name and AUMID", () => {
const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-packaged-icon-"));
const fixture = `
Add-Type -AssemblyName System.Drawing
$image = New-Object System.Drawing.Bitmap 1, 1
try {
$image.SetPixel(0, 0, [System.Drawing.Color]::Red)
$image.Save((Join-Path $env:T3_TEST_PACKAGE 'App.targetsize-32_altform-unplated.png'))
} finally { $image.Dispose() }
function Get-StartApps { [pscustomobject]@{ Name = 'Fixture Calculator'; AppID = 'Fixture_family!App' } }
function Get-AppxPackage { [pscustomobject]@{ PackageFamilyName = 'Fixture_family'; PackageFullName = 'Fixture'; InstallLocation = $env:T3_TEST_PACKAGE } }
function Get-AppxPackageManifest { [xml]'<Package><Applications><Application Id="App"><VisualElements Square44x44Logo="App.png" /></Application></Applications></Package>' }
function Get-Process { [pscustomobject]@{ ProcessName = 'ApplicationFrameHost'; MainWindowTitle = 'Fixture Calculator'; Path = 'C:\\Windows\\System32\\ApplicationFrameHost.exe' } }
`;
try {
for (const name of ["Fixture Calculator", "Fixture_family!App"]) {
const outputPath = NodePath.join(root, "output.png");
const source = NodeChildProcess.execFileSync(
"powershell.exe",
["-NoProfile", "-NonInteractive", "-Command", fixture + WINDOWS_APP_ICON_SCRIPT],
{
windowsHide: true,
timeout: 10_000,
encoding: "utf8",
env: {
...process.env,
T3_TEST_PACKAGE: root,
T3_APP_NAME: name,
T3_ICON_OUT: outputPath,
},
},
).trim();
expect(source).toBe(NodePath.join(root, "App.targetsize-32_altform-unplated.png"));
expect([...NodeFS.readFileSync(outputPath).subarray(0, 8)]).toEqual([
137, 80, 78, 71, 13, 10, 26, 10,
]);
}
} finally {
NodeFS.rmSync(root, { recursive: true, force: true });
}
});
});
73 changes: 57 additions & 16 deletions apps/server/src/assets/nativeAppIcons/win32.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,46 @@ import { existingFile, type NativeAppIconSource } from "./source.ts";
const COMMAND_TIMEOUT = "10 seconds";

/**
* Writes the app's icon as a PNG. Windows has no bundle ids, so the script tries a running process
* with Cua's app name, then a Start Menu shortcut. Arguments arrive as environment variables so
* the name is never spliced into the script.
* Packaged apps use their manifest logo; a shared ApplicationFrameHost
* executable cannot identify their icon. Arguments travel as environment
* variables so the name is never spliced into the script.
*/
const EXTRACT_SCRIPT = `
export const WINDOWS_APP_ICON_SCRIPT = `
$ErrorActionPreference = 'Stop'
Add-Type -AssemblyName System.Drawing
$name = $env:T3_APP_NAME
$exe = $null
$packaged = Get-StartApps | Where-Object { $_.Name -ieq $name -or $_.AppID -ieq $name } | Select-Object -First 1
if ($packaged -and $packaged.AppID -match '^([^!]+)!(.+)$') {
$family = $Matches[1]
$applicationId = $Matches[2]
$package = Get-AppxPackage | Where-Object { $_.PackageFamilyName -eq $family } | Select-Object -First 1
if ($package) {
$manifest = Get-AppxPackageManifest -Package $package.PackageFullName
$application = $manifest.Package.Applications.Application | Where-Object { $_.Id -eq $applicationId } | Select-Object -First 1
$logo = $application.VisualElements.Square44x44Logo
if ($logo) {
$logoPath = Join-Path $package.InstallLocation $logo
$directory = Split-Path $logoPath
$stem = [IO.Path]::GetFileNameWithoutExtension($logoPath)
$extension = [IO.Path]::GetExtension($logoPath)
$candidates = @("$stem.targetsize-32_altform-unplated$extension", "$stem.targetsize-48_altform-unplated$extension", "$stem.scale-200$extension", "$stem$extension")
foreach ($candidate in $candidates) {
$file = Join-Path $directory $candidate
if (Test-Path -LiteralPath $file -PathType Leaf) { $exe = $file; break }
}
if (-not $exe) {
$exe = Get-ChildItem -LiteralPath $directory -Filter "$stem.*$extension" -ErrorAction SilentlyContinue |
Sort-Object Name | Select-Object -First 1 -ExpandProperty FullName
}
}
}
}
$processName = [IO.Path]::GetFileNameWithoutExtension($name)
$process = Get-Process -ErrorAction SilentlyContinue |
Where-Object { $_.Path -and ($_.ProcessName -ieq $name -or $_.MainWindowTitle -ieq $name -or $_.Product -ieq $name) } |
Where-Object { $_.Path -and $_.ProcessName -notin @('ApplicationFrameHost', 'WWAHost') -and ($_.ProcessName -ieq $processName -or $_.MainWindowTitle -ieq $name -or $_.Product -ieq $name) } |
Select-Object -First 1
if ($process) { $exe = $process.Path }
if (-not $exe -and $process) { $exe = $process.Path }
if (-not $exe) {
$shell = New-Object -ComObject WScript.Shell
$roots = @("$env:ProgramData\\Microsoft\\Windows\\Start Menu\\Programs", "$env:AppData\\Microsoft\\Windows\\Start Menu\\Programs")
Expand All @@ -36,9 +63,20 @@ if (-not $exe) {
if (-not $exe -or -not (Test-Path -LiteralPath $exe -PathType Leaf)) { exit 3 }
Write-Output $exe
if ($env:T3_ICON_OUT) {
$icon = [System.Drawing.Icon]::ExtractAssociatedIcon($exe)
$bitmap = $icon.ToBitmap()
$bitmap.Save($env:T3_ICON_OUT, [System.Drawing.Imaging.ImageFormat]::Png)
$icon = $null
$bitmap = $null
try {
if ([IO.Path]::GetExtension($exe) -ieq '.png') {
$bitmap = [System.Drawing.Image]::FromFile($exe)
} else {
$icon = [System.Drawing.Icon]::ExtractAssociatedIcon($exe)
$bitmap = $icon.ToBitmap()
}
$bitmap.Save($env:T3_ICON_OUT, [System.Drawing.Imaging.ImageFormat]::Png)
} finally {
if ($bitmap) { $bitmap.Dispose() }
if ($icon) { $icon.Dispose() }
}
}
`;

Expand All @@ -50,7 +88,14 @@ const runScript = Effect.fn("NativeAppIconResolver.win32.runScript")(function* (
.string(
ChildProcess.make(
"powershell.exe",
["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command", EXTRACT_SCRIPT],
[
"-NoProfile",
"-NonInteractive",
"-ExecutionPolicy",
"Bypass",
"-Command",
WINDOWS_APP_ICON_SCRIPT,
],
{ stdin: "ignore", stderr: "ignore", env, extendEnv: true },
),
)
Expand All @@ -66,7 +111,7 @@ const resolveIcon = Effect.fn("NativeAppIconResolver.win32.resolve")(function* (
const name = (app._tag === "display-name" ? app.displayName : app.appId).trim();
if (name.length === 0 || /[\\/:*?"<>|]/u.test(name)) return null;

// The first run only locates the executable; its path and size key the cache.
// The source image or executable and its size key the cache.
const executable = (yield* runScript({ T3_APP_NAME: name })).trim();
if (!executable) return null;
const info = yield* fileSystem.stat(executable);
Expand All @@ -92,11 +137,7 @@ const resolveIcon = Effect.fn("NativeAppIconResolver.win32.resolve")(function* (
return yield* existingFile(cachePath);
});

/**
* Cua reports a Windows app by its name. PowerShell finds the executable
* (a running process first, then a Start Menu shortcut) and System.Drawing
* renders its associated icon to a cached PNG.
*/
/** Resolves packaged-app logos and traditional executable icons on the host. */
export const win32Source: NativeAppIconSource = {
platform: "win32",
resolve: resolveIcon,
Expand Down
1 change: 1 addition & 0 deletions apps/server/src/auth/RpcAuthorization.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ export const RPC_REQUIRED_SCOPES = {
[WS_METHODS.serverProbe]: AuthOrchestrationReadScope,
[WS_METHODS.serverGetConfig]: AuthOrchestrationReadScope,
[WS_METHODS.serverCheckComputerUse]: AuthOrchestrationReadScope,
[WS_METHODS.serverListApplications]: AuthOrchestrationReadScope,
// Operate: the check reads the host's browser profiles; the action installs a tool.
[WS_METHODS.serverCheckBrowserTabs]: AuthOrchestrationOperateScope,
[WS_METHODS.serverRunBrowserTabsAction]: AuthOrchestrationOperateScope,
Expand Down
92 changes: 92 additions & 0 deletions apps/server/src/cua/CuaApplications.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
import { it } from "@effect/vitest";
import { expect } from "vite-plus/test";
import * as NodeServices from "@effect/platform-node/NodeServices";
import * as Effect from "effect/Effect";
import * as CuaApplications from "./CuaApplications.ts";
import * as CuaDriver from "./CuaDriver.ts";

it.effect(
"lists and deduplicates native identities across platforms, then releases its client",
() =>
Effect.gen(function* () {
let released = 0;
const windows = {
name: "Calculator",
bundleId: "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App",
pid: 42,
running: true,
};
const sdk = {
CuaDriver: {
connect: () => ({
listApps: async () => ({
apps: [
windows,
windows,
{
name: "Helium",
bundleId: "C:\\Apps\\helium.exe",
launchPath: "C:\\Apps\\helium.exe",
pid: 1,
running: true,
},
{
name: "Terminal",
launchPath: "/usr/share/applications/terminal.desktop",
pid: 0,
running: false,
},
{ name: "", pid: 0, running: false },
],
}),
shutdown: async () => {},
uniffiDestroy: () => {
released++;
},
}),
},
ListAppsInput: { new: () => ({}) },
} as unknown as CuaApplications.Sdk;
const service = yield* CuaApplications.make({ loadSdk: Effect.succeed(sdk) });
expect(yield* service.list).toEqual([
{ name: "Calculator", bundleId: windows.bundleId, pid: 42 },
{ name: "Helium", launchPath: "C:\\Apps\\helium.exe", pid: 1 },
{ name: "Terminal", launchPath: "/usr/share/applications/terminal.desktop" },
]);
expect(released).toBe(1);
}).pipe(
Effect.provideService(
CuaDriver.CuaDriver,
CuaDriver.CuaDriver.of({
enabled: Effect.succeed(true),
acquire: Effect.succeedSome({
command: "driver",
args: [],
environment: [],
socketPath: "test",
}),
lastFailure: Effect.succeedNone,
status: Effect.succeed({ platform: "win32", running: true }),
}),
),
Effect.provide(NodeServices.layer),
),
);

it.effect("does not start the driver when Computer Use is disabled", () =>
Effect.gen(function* () {
const service = yield* CuaApplications.make({ loadSdk: Effect.die("must not load") });
expect(yield* service.list).toEqual([]);
}).pipe(
Effect.provideService(
CuaDriver.CuaDriver,
CuaDriver.CuaDriver.of({
enabled: Effect.succeed(false),
acquire: Effect.die("must not start"),
lastFailure: Effect.succeedNone,
status: Effect.succeed({ platform: "win32", running: false }),
}),
),
Effect.provide(NodeServices.layer),
),
);
Loading
Loading