diff --git a/apps/desktop/README.md b/apps/desktop/README.md index 11eecdcbb00..d3b421602fc 100644 --- a/apps/desktop/README.md +++ b/apps/desktop/README.md @@ -239,9 +239,12 @@ Mothership's `computer` tool is available on macOS 14+ once the user opts in und group with the **Computer Use** restriction. Older desktop builds omit this optional bridge and continue to work. -The user grants Accessibility and Screen Recording through macOS, then approves each app for the -current task or persistently. Persistent app approvals are listed in Settings and can be revoked. -Sign-out and deployment changes clear computer-use approval and opt-in. The conversation shows the +The user grants Accessibility and Screen Recording through macOS; every app is then available except +Sim, System Settings, password managers and terminals (shell access has its own Terminal control), +which the helper refuses. With **Ask before using a new +app** on, each app's first use is approved for the task or persistently, and persistent approvals are +listed in Settings and can be revoked. Sign-out and deployment changes clear opt-in, the ask choice +and approvals. The conversation shows the active app/action and a Stop button; **⌘⇧Esc** also stops active work when the shortcut is available. Stop cancels queued actions, in-flight authorization, approval prompts and the native helper. Already completed input cannot be undone. diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 67a01aae22e..c3da7d8ff31 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -142,10 +142,26 @@ func elementRect(_ element: AXUIElement) -> CGRect? { AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil } return CGRect(origin: position, size: size) } +/// Apps computer use never operates, lowercased: macOS security settings, password managers, and +/// terminals, whose shell access has its own Terminal control. +let PROTECTED_APPS: Set = [ + "com.apple.systempreferences", "com.apple.keychainaccess", "com.apple.passwords", + "com.1password.1password", "com.agilebits.onepassword7", "com.agilebits.onepassword-osx", "com.bitwarden.desktop", + "org.keepassxc.keepassxc", "com.lastpass.lastpass", "in.sinew.enpass-desktop", "me.proton.pass.electron", + "com.markmcguill.strongbox", "com.markmcguill.strongbox.mac", + "com.apple.terminal", "com.googlecode.iterm2", "dev.warp.warp-stable", "com.mitchellh.ghostty", + "net.kovidgoyal.kitty", "org.alacritty", "com.github.wez.wezterm", "co.zeit.hyper", "org.tabby", +] +/// Whether computer use refuses an app: Sim's own apps and `PROTECTED_APPS`. Bundle identifiers match +/// case-insensitively in Launch Services. +func isProtectedApp(_ bundle: String) -> Bool { + let key = bundle.lowercased() + return key == Bundle.main.bundleIdentifier?.lowercased() || key.hasPrefix("com.simstudio.") || key.hasPrefix("ai.sim.desktop") || PROTECTED_APPS.contains(key) +} func appFor(_ bundleId: String?, launch: Bool = false) async throws -> NSRunningApplication { let bundle = try required(bundleId, "bundleId") - guard bundle != Bundle.main.bundleIdentifier, !bundle.hasPrefix("com.simstudio."), !bundle.hasPrefix("ai.sim.desktop"), bundle != "com.apple.systempreferences" else { - throw ComputerError("protected_app", "Computer use cannot control its own app or macOS security settings.") + guard !isProtectedApp(bundle) else { + throw ComputerError("protected_app", "Computer use does not operate Sim, System Settings, password managers, or terminals; use the terminal tool for shell work.") } var apps = NSRunningApplication.runningApplications(withBundleIdentifier: bundle).filter { !$0.isTerminated } if apps.isEmpty && launch { diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index c3971b54d70..c4a22e3a704 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -10,7 +10,7 @@ Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` return The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized. The menu bar and menus are read in place, so an open menu's items and their submenus come first; items of closed menus are read last, capped at 80 menu nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. An Accessibility failure counts as sent unless Accessibility rejected the request before the app saw it (unsupported attribute or action, illegal argument). Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise. -Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected, and an element target uses its own window; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps and System Settings are protected targets. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`. +Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected, and an element target uses its own window; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps, System Settings, password managers and terminals are protected targets (`PROTECTED_APPS`, matched case-insensitively); shell work belongs to the separate Terminal control. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`. AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A hit on a window's own background counts as that window. Background tabs and minimized windows share bounds with the window in front, so they take no coordinate input. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves the item's own menu open with the app highlighting that item (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 198911e3ff6..ec5875bbfef 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -26,6 +26,10 @@ import AppKit expectError("unsupported_key") { _ = try parseKey("F99") } expectError("invalid_arguments") { _ = try parseKey("Hyper+A") } expectError("invalid_arguments") { _ = try required(Optional.none, "example") } + for bundle in ["com.apple.systempreferences", "com.apple.Terminal", "COM.GOOGLECODE.ITERM2", "net.kovidgoyal.kitty", "com.1password.1password", "com.apple.Passwords", "com.bitwarden.desktop", "ai.sim.desktop", "com.simstudio.computer-use"] { + precondition(isProtectedApp(bundle), bundle) + } + for bundle in ["com.apple.TextEdit", "com.apple.finder", "com.apple.Safari"] { precondition(!isProtectedApp(bundle), bundle) } var queue = AXTraversalQueue(limit: 1000) queue.append(AXTraversalEntry(element: "menu", parent: nil, depth: 0, menu: true, priority: false)) queue.append(AXTraversalEntry(element: "content", parent: nil, depth: 0, menu: false, priority: false)) diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index 67d81dff4f4..7a66b868db1 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -12,11 +12,13 @@ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) }) -function setup(enabled = true) { +/** `askForApps` defaults on so the approval tests exercise the prompt; the default device mode asks for nothing. */ +function setup(enabled = true, askForApps = true) { const root = mkdtempSync(join(tmpdir(), 'sim-computer-use-test-')) roots.push(root) const config = createConfigStore(join(root, 'settings.json')) config.set('computerUseEnabled', enabled) + config.set('computerUseAskForApps', askForApps) let sequence = 0 const request = vi.fn(async (method: string): Promise => { if (method === 'status' || method === 'request_permission') @@ -412,6 +414,25 @@ describe('native computer use authority and lifecycle', () => { await expect(click('s1')).rejects.toThrow('stale') }) + it('operates any unprotected app without asking or saving an approval by default', async () => { + const { state, click, service, native, approveApp, onActivity } = setup(true, false) + await state() + await click('s1') + expect(approveApp).not.toHaveBeenCalled() + expect(service.listAppPermissions()).toEqual([]) + expect(onActivity).toHaveBeenCalledWith(expect.objectContaining({ appName: 'Fixture' })) + expect(native.request.mock.calls.filter(([method]) => method === 'list_apps')).toHaveLength(1) + }) + + it('asks for new apps only once the user turns asking on, and clears that choice on reset', async () => { + const { state, service, approveApp } = setup(true, false) + await expect(service.setAskForApps(true)).resolves.toMatchObject({ askForApps: true }) + approveApp.mockResolvedValueOnce('deny') + await expect(state()).rejects.toThrow('denied') + service.reset() + expect((await service.getStatus()).askForApps).toBe(false) + }) + it('clears grants and opt-in on account reset', async () => { const { state, service, native } = setup() await state() diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index 3f9cfd3fd02..afacf4696f3 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -51,6 +51,8 @@ export class ComputerUseService { private snapshots = new Map() private activity: ComputerUseActivity | null = null private taskGrants = new Map>() + /** App names from the helper's app list, so an unprompted action can name its app without listing apps again. */ + private appNames = new Map() constructor(private readonly deps: ComputerUseServiceDeps) {} @@ -58,6 +60,10 @@ export class ComputerUseService { return this.deps.supported && this.deps.config.get('computerUseEnabled') === true } + private asksForApps(): boolean { + return this.deps.config.get('computerUseAskForApps') === true + } + getActivity(): ComputerUseActivity | null { return this.activity } @@ -68,6 +74,7 @@ export class ComputerUseService { enabled: this.isEnabled(), permissions: { accessibility: false, screenCapture: false }, activeAction: this.activity, + askForApps: this.asksForApps(), } if (!status.enabled) return status const native = await this.deps.native.request('status', {}) @@ -94,6 +101,16 @@ export class ComputerUseService { return this.getStatus() } + async setAskForApps(ask: boolean): Promise { + const previous = this.asksForApps() + this.deps.config.set('computerUseAskForApps', ask) + if (!this.deps.config.flush()) { + this.deps.config.set('computerUseAskForApps', previous) + throw new Error('Could not save Computer Use settings.') + } + return this.getStatus() + } + async requestPermission( permission: 'accessibility' | 'screenCapture' ): Promise { @@ -127,7 +144,9 @@ export class ComputerUseService { reset(): void { this.cancel() + this.appNames.clear() this.deps.config.set('computerUseEnabled', false) + this.deps.config.set('computerUseAskForApps', false) this.deps.config.set('computerUseAllowedApps', []) if (!this.deps.config.flush()) throw new Error('Could not clear saved Computer Use permissions.') @@ -338,16 +357,14 @@ export class ComputerUseService { }) } + /** Every unprotected app is allowed unless the user chose to approve each app's first use. */ private async authorizeApp(bundleId: string, work: ToolWork): Promise { const approved = this.listAppPermissions().find((app) => app.bundleId === bundleId) if (approved) return approved.displayName const taskName = this.taskGrants.get(work.scopeId)?.get(bundleId) if (taskName) return taskName - const apps = await this.deps.native.request('list_apps', {}) - this.check(work) - const app = - apps.kind === 'apps' ? apps.apps.find((app) => app.bundleId === bundleId) : undefined - const target = { bundleId, displayName: app?.name ?? bundleId } + const target = { bundleId, displayName: await this.appName(bundleId, work) } + if (!this.asksForApps()) return target.displayName const answer = await this.deps.approveApp(target, work.controller.signal) this.check(work) if (answer === 'deny') @@ -371,4 +388,18 @@ export class ComputerUseService { } return target.displayName } + + private async appName(bundleId: string, work: ToolWork): Promise { + const known = this.appNames.get(bundleId) + if (known) return known + const apps = await this.deps.native.request('list_apps', {}) + this.check(work) + if (apps.kind === 'apps') { + this.appNames.clear() + for (const app of apps.apps) this.appNames.set(app.bundleId, app.name) + } + /** An app outside the scanned folders keeps its bundle ID, so it is not rescanned on every action. */ + if (!this.appNames.has(bundleId)) this.appNames.set(bundleId, bundleId) + return this.appNames.get(bundleId) ?? bundleId + } } diff --git a/apps/desktop/src/main/config.ts b/apps/desktop/src/main/config.ts index 28ca953f16f..e9657914f57 100644 --- a/apps/desktop/src/main/config.ts +++ b/apps/desktop/src/main/config.ts @@ -111,6 +111,8 @@ export interface DesktopSettings { preventSleepWhileRunning?: boolean /** Native app access is opt-in and cleared when the signed-in account changes. */ computerUseEnabled?: boolean + /** Ask before operating an app for the first time; by default every unprotected app is allowed. */ + computerUseAskForApps?: boolean computerUseAllowedApps?: Array<{ bundleId: string; displayName: string }> /** Device-wide browser page appearance; `app` follows Sim. */ browserTheme?: 'app' | 'light' | 'dark' diff --git a/apps/desktop/src/main/ipc.test.ts b/apps/desktop/src/main/ipc.test.ts index b5580f76393..030c9620362 100644 --- a/apps/desktop/src/main/ipc.test.ts +++ b/apps/desktop/src/main/ipc.test.ts @@ -583,6 +583,7 @@ describe('registerIpcHandlers', () => { computerRoots.push(root) const config = createConfigStore(join(root, 'settings.json')) config.set('computerUseEnabled', true) + config.set('computerUseAskForApps', true) const status: ComputerUseResult = { kind: 'status', platform: 'darwin', diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index b88167462ac..068b315b7e7 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -751,6 +751,18 @@ export function registerIpcHandlers(deps: IpcDeps): void { return deps.computerUse?.requestPermission(permission) }, }, + 'computer-use:set-ask-for-apps': { + kind: 'invoke', + gate: 'app-origin', + requiresAccountData: true, + needsUserActivation: true, + denied: null, + handler: (ask) => { + if (typeof ask !== 'boolean') + throw new Error('A boolean Computer Use preference is required.') + return deps.computerUse?.setAskForApps(ask) + }, + }, 'computer-use:list-permissions': { kind: 'invoke', gate: 'app-origin', diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index b59f2e933fa..7f06f2b5180 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -131,6 +131,7 @@ const api: SimDesktopApi = { setEnabled: (enabled: boolean) => ipcRenderer.invoke('computer-use:set-enabled', enabled), requestPermission: (permission: 'accessibility' | 'screenCapture') => ipcRenderer.invoke('computer-use:request-permission', permission), + setAskForApps: (ask: boolean) => ipcRenderer.invoke('computer-use:set-ask-for-apps', ask), listAppPermissions: () => ipcRenderer.invoke('computer-use:list-permissions'), revokeApp: (bundleId: string) => ipcRenderer.invoke('computer-use:revoke-app', bundleId), executeTool: (toolCallId: string, params: ComputerUseInput) => diff --git a/apps/docs/content/docs/desktop/index.mdx b/apps/docs/content/docs/desktop/index.mdx index 393f553975f..6bc787d7a29 100644 --- a/apps/docs/content/docs/desktop/index.mdx +++ b/apps/docs/content/docs/desktop/index.mdx @@ -91,19 +91,20 @@ By default, local file tools ask for folder access when they need an unapproved ### Native Mac-app control -**Computer Use** appears under **Settings → Account → Desktop** only when the deployment and installed app provide it. It requires **macOS 14 or later** and is off by default on each Mac. Browser, Terminal, and local file permissions are separate controls. +**Computer Use** appears under **Settings → Account → Desktop** when the installed app provides it. It requires **macOS 14 or later** and is off by default on each Mac. Organization admins can withhold it with the **Computer Use** permission-group restriction. Browser, Terminal, and local file permissions are separate controls. To enable it when available: -1. Open **Computer Use** and turn **Allow Mothership to use Mac apps** to **On** to allow Sim to use native apps. -2. Use **Open System Settings** to grant **Accessibility** for app interaction and **Screen Recording** for screenshots. Grant these permissions yourself, then return to Sim to refresh their status. -3. When Sim requests an app, choose **Allow for This Task**, **Always Allow**, or **Cancel**. **Always Allow** saves that app's permission on this Mac. +1. Turn **Use apps on this Mac** to **On**. +2. Select **Allow** next to **Accessibility** (app interaction) and **Screen Recording** (screenshots), grant them in System Settings, then return to Sim. -Sim can read approved app windows and interact through clicks, text, keyboard shortcuts, scrolling, and supported accessibility controls. Screenshots and returned app content can be sent to your Sim deployment and model. Some actions require bringing the app to the foreground; app compatibility and macOS permissions can still limit what succeeds. +Sim can then use any app except terminals, password managers, and macOS System Settings (and Sim itself); shell work goes through **Terminal** access instead. To approve each app the first time Sim uses it, turn on **Ask before using a new app**; Sim then offers **Allow for This Task**, **Always Allow**, or **Cancel**. + +Sim can read app windows and interact through clicks, text, keyboard shortcuts, scrolling, and supported accessibility controls. Screenshots and returned app content can be sent to your Sim deployment and model. Some actions require bringing the app to the foreground; app compatibility and macOS permissions can still limit what succeeds. The conversation shows the active app and action. Select **Stop**, or press **Cmd + Shift + Escape** when that shortcut is available, to cancel active and queued computer actions. Stopping cannot undo input already delivered to an app. Verify the app's current state before repeating an interrupted action. -Use **Revoke** under **Approved apps** to remove a saved permission, or switch Computer Use off. Signing out or changing deployments clears the device opt-in and app approvals. Computer Use cannot operate Sim's own permission controls or macOS System Settings. +With **Ask before using a new app** on, use **Revoke** under **Approved apps** to remove a saved approval. Signing out or changing deployments clears the device opt-in and app approvals. Computer Use never operates Sim, macOS System Settings, password managers, or terminals. ### Terminal access diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.test.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.test.tsx index 6c03a168096..4e323e494e5 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.test.tsx @@ -110,6 +110,26 @@ function appRow(bundleId: string) { return container.querySelector(`[title="${bundleId}"]`)?.parentElement } +it('lists approved apps only while first use of an app needs approval', async () => { + const status = (askForApps: boolean): ComputerUseStatus => ({ + supported: true, + enabled: true, + permissions: { accessibility: true, screenCapture: true }, + activeAction: null, + askForApps, + }) + libDesktopMockFns.mockGetDesktopBridge.mockReturnValue({ + computerUse: { ...native, setAskForApps: vi.fn(async (ask: boolean) => status(ask)) }, + }) + native.getStatus.mockResolvedValue(status(false)) + await act(async () => renderSettings()) + expect(container.querySelector('[aria-label="Ask before using a new app"]')).not.toBeNull() + expect(appRow('com.example.First')).toBeUndefined() + native.getStatus.mockResolvedValue(status(true)) + await act(async () => window.dispatchEvent(new Event('focus'))) + expect(appRow('com.example.First')).toBeDefined() +}) + it('does not restore a revoked app when an earlier activity refresh settles late', async () => { await act(async () => renderSettings()) const stale = createDeferred() diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx index af632d25f50..eb7e63eff6d 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx @@ -11,6 +11,11 @@ import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/componen import { useUserPermissionConfig } from '@/ee/access-control/hooks/permission-groups' import { useComputerUseStatus } from '@/hooks/use-computer-use-status' +const ON_OFF_OPTIONS = [ + { value: 'off', label: 'Off' }, + { value: 'on', label: 'On' }, +] + /** The activity stays visible so an in-flight action can be stopped; the device controls need the permission group's consent. */ export function ComputerUseSettings() { const { workspaceId } = useParams<{ workspaceId: string }>() @@ -60,19 +65,18 @@ function ComputerUseSettingsControls() { setPending(false) } } + const askForApps = status?.askForApps + const setAskForApps = bridge.setAskForApps return (
@@ -81,38 +85,62 @@ function ComputerUseSettingsControls() { />

- Off by default. Each app requires your approval. You can stop an action from the - conversation or revoke an app below. + {askForApps === undefined + ? 'Each app needs your approval before Sim can use it.' + : askForApps + ? 'Sim asks before using each new app, and never uses terminals, password managers, System Settings, or Sim itself.' + : 'Sim can use any app except terminals, password managers, System Settings, and Sim itself.'}

{error && (
- Could not connect to the computer helper. + Could not reach the Computer Use helper. void refresh()}>Retry
)} - {(['accessibility', 'screenCapture'] as const).map((permission) => ( -
- - {status?.permissions[permission] ? ( - Allowed - ) : ( - - void update(async () => setStatus(await bridge.requestPermission(permission))) - } - > - Open System Settings - + {status?.enabled && ( + <> + {(['accessibility', 'screenCapture'] as const).map((permission) => { + const label = permission === 'accessibility' ? 'Accessibility' : 'Screen Recording' + return ( +
+ + {status.permissions[permission] ? ( + Allowed + ) : ( + + void update(async () => + setStatus(await bridge.requestPermission(permission)) + ) + } + > + Allow + + )} +
+ ) + })} + {setAskForApps && askForApps !== undefined && ( +
+ + + void update(async () => setStatus(await setAskForApps(value === 'on'))) + } + /> +
)} -
- ))} -

- Accessibility allows interaction with approved apps. Screen Recording allows screenshots. - Return here after granting access to refresh the status. -

- {apps.length > 0 && ( + + )} + {askForApps !== false && apps.length > 0 && (
{apps.map((app) => ( diff --git a/packages/desktop-bridge/src/index.ts b/packages/desktop-bridge/src/index.ts index 43c36eb48e0..526273b05b3 100644 --- a/packages/desktop-bridge/src/index.ts +++ b/packages/desktop-bridge/src/index.ts @@ -60,6 +60,8 @@ export interface ComputerUseStatus { enabled: boolean permissions: { accessibility: boolean; screenCapture: boolean } activeAction: ComputerUseActivity | null + /** Whether a first use of each app needs approval; absent in older shells, which always ask. */ + askForApps?: boolean } export interface ComputerUseAppPermission { @@ -88,6 +90,8 @@ interface SimDesktopComputerUseApi { getStatus(): Promise setEnabled(enabled: boolean): Promise requestPermission(permission: 'accessibility' | 'screenCapture'): Promise + /** Absent in older shells, which always ask before a first use of each app. */ + setAskForApps?(ask: boolean): Promise listAppPermissions(): Promise revokeApp(bundleId: string): Promise executeTool(