Skip to content

Commit dd01573

Browse files
committed
improvement(computer-use): allow every app by default, block password managers, make per-app approval optional, and trim the settings
Turning Computer Use on and granting Accessibility and Screen Recording now covers every app except Sim, System Settings and password managers. Approving each app's first use becomes an opt-in setting, and the settings section keeps only the switch, the two macOS permissions and the optional approvals.
1 parent d8d7c8c commit dd01573

12 files changed

Lines changed: 168 additions & 48 deletions

File tree

‎apps/desktop/README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -239,9 +239,11 @@ Mothership's `computer` tool is available on macOS 14+ once the user opts in und
239239
group with the **Computer Use** restriction. Older desktop builds omit this optional bridge and
240240
continue to work.
241241

242-
The user grants Accessibility and Screen Recording through macOS, then approves each app for the
243-
current task or persistently. Persistent app approvals are listed in Settings and can be revoked.
244-
Sign-out and deployment changes clear computer-use approval and opt-in. The conversation shows the
242+
The user grants Accessibility and Screen Recording through macOS; every app is then available except
243+
Sim, System Settings and password managers, which the helper refuses. With **Ask before using a new
244+
app** on, each app's first use is approved for the task or persistently, and persistent approvals are
245+
listed in Settings and can be revoked. Sign-out and deployment changes clear opt-in, the ask choice
246+
and approvals. The conversation shows the
245247
active app/action and a Stop button; **⌘⇧Esc** also stops active work when the shortcut is available.
246248
Stop cancels queued actions, in-flight authorization, approval prompts and the native helper.
247249
Already completed input cannot be undone.

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,10 +142,15 @@ func elementRect(_ element: AXUIElement) -> CGRect? {
142142
AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil }
143143
return CGRect(origin: position, size: size)
144144
}
145+
/// Apps computer use never operates: macOS security settings and password managers.
146+
let PROTECTED_APPS: Set<String> = [
147+
"com.apple.systempreferences", "com.apple.keychainaccess", "com.apple.Passwords",
148+
"com.1password.1password", "com.agilebits.onepassword7", "com.bitwarden.desktop", "org.keepassxc.keepassxc",
149+
]
145150
func appFor(_ bundleId: String?, launch: Bool = false) async throws -> NSRunningApplication {
146151
let bundle = try required(bundleId, "bundleId")
147-
guard bundle != Bundle.main.bundleIdentifier, !bundle.hasPrefix("com.simstudio."), !bundle.hasPrefix("ai.sim.desktop"), bundle != "com.apple.systempreferences" else {
148-
throw ComputerError("protected_app", "Computer use cannot control its own app or macOS security settings.")
152+
guard bundle != Bundle.main.bundleIdentifier, !bundle.hasPrefix("com.simstudio."), !bundle.hasPrefix("ai.sim.desktop"), !PROTECTED_APPS.contains(bundle) else {
153+
throw ComputerError("protected_app", "Computer use does not operate Sim, System Settings, or password managers.")
149154
}
150155
var apps = NSRunningApplication.runningApplications(withBundleIdentifier: bundle).filter { !$0.isTerminated }
151156
if apps.isEmpty && launch {

‎apps/desktop/src/main/computer-use/service.test.ts‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@ afterEach(() => {
1212
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true })
1313
})
1414

15-
function setup(enabled = true) {
15+
/** `askForApps` defaults on so the approval tests exercise the prompt; the default device mode asks for nothing. */
16+
function setup(enabled = true, askForApps = true) {
1617
const root = mkdtempSync(join(tmpdir(), 'sim-computer-use-test-'))
1718
roots.push(root)
1819
const config = createConfigStore(join(root, 'settings.json'))
1920
config.set('computerUseEnabled', enabled)
21+
config.set('computerUseAskForApps', askForApps)
2022
let sequence = 0
2123
const request = vi.fn(async (method: string): Promise<ComputerUseResult> => {
2224
if (method === 'status' || method === 'request_permission')
@@ -412,6 +414,25 @@ describe('native computer use authority and lifecycle', () => {
412414
await expect(click('s1')).rejects.toThrow('stale')
413415
})
414416

417+
it('operates any unprotected app without asking or saving an approval by default', async () => {
418+
const { state, click, service, native, approveApp, onActivity } = setup(true, false)
419+
await state()
420+
await click('s1')
421+
expect(approveApp).not.toHaveBeenCalled()
422+
expect(service.listAppPermissions()).toEqual([])
423+
expect(onActivity).toHaveBeenCalledWith(expect.objectContaining({ appName: 'Fixture' }))
424+
expect(native.request.mock.calls.filter(([method]) => method === 'list_apps')).toHaveLength(1)
425+
})
426+
427+
it('asks for new apps only once the user turns asking on, and clears that choice on reset', async () => {
428+
const { state, service, approveApp } = setup(true, false)
429+
await expect(service.setAskForApps(true)).resolves.toMatchObject({ askForApps: true })
430+
approveApp.mockResolvedValueOnce('deny')
431+
await expect(state()).rejects.toThrow('denied')
432+
service.reset()
433+
expect((await service.getStatus()).askForApps).toBe(false)
434+
})
435+
415436
it('clears grants and opt-in on account reset', async () => {
416437
const { state, service, native } = setup()
417438
await state()

‎apps/desktop/src/main/computer-use/service.ts‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,19 @@ export class ComputerUseService {
5151
private snapshots = new Map<string, SnapshotOwnership>()
5252
private activity: ComputerUseActivity | null = null
5353
private taskGrants = new Map<string, Map<string, string>>()
54+
/** App names from the helper's app list, so an unprompted action can name its app without listing apps again. */
55+
private appNames = new Map<string, string>()
5456

5557
constructor(private readonly deps: ComputerUseServiceDeps) {}
5658

5759
isEnabled(): boolean {
5860
return this.deps.supported && this.deps.config.get('computerUseEnabled') === true
5961
}
6062

63+
private asksForApps(): boolean {
64+
return this.deps.config.get('computerUseAskForApps') === true
65+
}
66+
6167
getActivity(): ComputerUseActivity | null {
6268
return this.activity
6369
}
@@ -68,6 +74,7 @@ export class ComputerUseService {
6874
enabled: this.isEnabled(),
6975
permissions: { accessibility: false, screenCapture: false },
7076
activeAction: this.activity,
77+
askForApps: this.asksForApps(),
7178
}
7279
if (!status.enabled) return status
7380
const native = await this.deps.native.request('status', {})
@@ -94,6 +101,16 @@ export class ComputerUseService {
94101
return this.getStatus()
95102
}
96103

104+
async setAskForApps(ask: boolean): Promise<ComputerUseStatus> {
105+
const previous = this.asksForApps()
106+
this.deps.config.set('computerUseAskForApps', ask)
107+
if (!this.deps.config.flush()) {
108+
this.deps.config.set('computerUseAskForApps', previous)
109+
throw new Error('Could not save Computer Use settings.')
110+
}
111+
return this.getStatus()
112+
}
113+
97114
async requestPermission(
98115
permission: 'accessibility' | 'screenCapture'
99116
): Promise<ComputerUseStatus> {
@@ -128,6 +145,7 @@ export class ComputerUseService {
128145
reset(): void {
129146
this.cancel()
130147
this.deps.config.set('computerUseEnabled', false)
148+
this.deps.config.set('computerUseAskForApps', false)
131149
this.deps.config.set('computerUseAllowedApps', [])
132150
if (!this.deps.config.flush())
133151
throw new Error('Could not clear saved Computer Use permissions.')
@@ -338,16 +356,14 @@ export class ComputerUseService {
338356
})
339357
}
340358

359+
/** Every unprotected app is allowed unless the user chose to approve each app's first use. */
341360
private async authorizeApp(bundleId: string, work: ToolWork): Promise<string> {
342361
const approved = this.listAppPermissions().find((app) => app.bundleId === bundleId)
343362
if (approved) return approved.displayName
344363
const taskName = this.taskGrants.get(work.scopeId)?.get(bundleId)
345364
if (taskName) return taskName
346-
const apps = await this.deps.native.request('list_apps', {})
347-
this.check(work)
348-
const app =
349-
apps.kind === 'apps' ? apps.apps.find((app) => app.bundleId === bundleId) : undefined
350-
const target = { bundleId, displayName: app?.name ?? bundleId }
365+
const target = { bundleId, displayName: await this.appName(bundleId, work) }
366+
if (!this.asksForApps()) return target.displayName
351367
const answer = await this.deps.approveApp(target, work.controller.signal)
352368
this.check(work)
353369
if (answer === 'deny')
@@ -371,4 +387,16 @@ export class ComputerUseService {
371387
}
372388
return target.displayName
373389
}
390+
391+
private async appName(bundleId: string, work: ToolWork): Promise<string> {
392+
const known = this.appNames.get(bundleId)
393+
if (known) return known
394+
const apps = await this.deps.native.request('list_apps', {})
395+
this.check(work)
396+
if (apps.kind === 'apps') {
397+
this.appNames.clear()
398+
for (const app of apps.apps) this.appNames.set(app.bundleId, app.name)
399+
}
400+
return this.appNames.get(bundleId) ?? bundleId
401+
}
374402
}

‎apps/desktop/src/main/config.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ export interface DesktopSettings {
111111
preventSleepWhileRunning?: boolean
112112
/** Native app access is opt-in and cleared when the signed-in account changes. */
113113
computerUseEnabled?: boolean
114+
/** Ask before operating an app for the first time; by default every unprotected app is allowed. */
115+
computerUseAskForApps?: boolean
114116
computerUseAllowedApps?: Array<{ bundleId: string; displayName: string }>
115117
/** Device-wide browser page appearance; `app` follows Sim. */
116118
browserTheme?: 'app' | 'light' | 'dark'

‎apps/desktop/src/main/ipc.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -583,6 +583,7 @@ describe('registerIpcHandlers', () => {
583583
computerRoots.push(root)
584584
const config = createConfigStore(join(root, 'settings.json'))
585585
config.set('computerUseEnabled', true)
586+
config.set('computerUseAskForApps', true)
586587
const status: ComputerUseResult = {
587588
kind: 'status',
588589
platform: 'darwin',

‎apps/desktop/src/main/ipc.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -751,6 +751,18 @@ export function registerIpcHandlers(deps: IpcDeps): void {
751751
return deps.computerUse?.requestPermission(permission)
752752
},
753753
},
754+
'computer-use:set-ask-for-apps': {
755+
kind: 'invoke',
756+
gate: 'app-origin',
757+
requiresAccountData: true,
758+
needsUserActivation: true,
759+
denied: null,
760+
handler: (ask) => {
761+
if (typeof ask !== 'boolean')
762+
throw new Error('A boolean Computer Use preference is required.')
763+
return deps.computerUse?.setAskForApps(ask)
764+
},
765+
},
754766
'computer-use:list-permissions': {
755767
kind: 'invoke',
756768
gate: 'app-origin',

‎apps/desktop/src/preload/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ const api: SimDesktopApi = {
131131
setEnabled: (enabled: boolean) => ipcRenderer.invoke('computer-use:set-enabled', enabled),
132132
requestPermission: (permission: 'accessibility' | 'screenCapture') =>
133133
ipcRenderer.invoke('computer-use:request-permission', permission),
134+
setAskForApps: (ask: boolean) => ipcRenderer.invoke('computer-use:set-ask-for-apps', ask),
134135
listAppPermissions: () => ipcRenderer.invoke('computer-use:list-permissions'),
135136
revokeApp: (bundleId: string) => ipcRenderer.invoke('computer-use:revoke-app', bundleId),
136137
executeTool: (toolCallId: string, params: ComputerUseInput) =>

‎apps/docs/content/docs/desktop/index.mdx‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -91,19 +91,20 @@ By default, local file tools ask for folder access when they need an unapproved
9191

9292
### Native Mac-app control
9393

94-
**Computer Use** appears under **Settings → Account → Desktop** only when the deployment and installed app provide it. It requires **macOS 14 or later** and is off by default on each Mac. Browser, Terminal, and local file permissions are separate controls.
94+
**Computer Use** appears under **Settings → Account → Desktop** when the installed app provides it. It requires **macOS 14 or later** and is off by default on each Mac. Organization admins can withhold it with the **Computer Use** permission-group restriction. Browser, Terminal, and local file permissions are separate controls.
9595

9696
To enable it when available:
9797

98-
1. Open **Computer Use** and turn **Allow Mothership to use Mac apps** to **On** to allow Sim to use native apps.
99-
2. Use **Open System Settings** to grant **Accessibility** for app interaction and **Screen Recording** for screenshots. Grant these permissions yourself, then return to Sim to refresh their status.
100-
3. When Sim requests an app, choose **Allow for This Task**, **Always Allow**, or **Cancel**. **Always Allow** saves that app's permission on this Mac.
98+
1. Turn **Use apps on this Mac** to **On**.
99+
2. Select **Allow** next to **Accessibility** (app interaction) and **Screen Recording** (screenshots), grant them in System Settings, then return to Sim.
101100

102-
Sim can read approved app windows and interact through clicks, text, keyboard shortcuts, scrolling, and supported accessibility controls. Screenshots and returned app content can be sent to your Sim deployment and model. Some actions require bringing the app to the foreground; app compatibility and macOS permissions can still limit what succeeds.
101+
Sim can then use any app except password managers and macOS System Settings. To approve each app the first time Sim uses it, turn on **Ask before using a new app**; Sim then offers **Allow for This Task**, **Always Allow**, or **Cancel**.
102+
103+
Sim can read app windows and interact through clicks, text, keyboard shortcuts, scrolling, and supported accessibility controls. Screenshots and returned app content can be sent to your Sim deployment and model. Some actions require bringing the app to the foreground; app compatibility and macOS permissions can still limit what succeeds.
103104

104105
The conversation shows the active app and action. Select **Stop**, or press **Cmd + Shift + Escape** when that shortcut is available, to cancel active and queued computer actions. Stopping cannot undo input already delivered to an app. Verify the app's current state before repeating an interrupted action.
105106

106-
Use **Revoke** under **Approved apps** to remove a saved permission, or switch Computer Use off. Signing out or changing deployments clears the device opt-in and app approvals. Computer Use cannot operate Sim's own permission controls or macOS System Settings.
107+
With **Ask before using a new app** on, use **Revoke** under **Approved apps** to remove a saved approval. Signing out or changing deployments clears the device opt-in and app approvals. Computer Use never operates Sim, macOS System Settings, or password managers.
107108

108109
### Terminal access
109110

‎apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.test.tsx‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,26 @@ function appRow(bundleId: string) {
110110
return container.querySelector<HTMLElement>(`[title="${bundleId}"]`)?.parentElement
111111
}
112112

113+
it('lists approved apps only while first use of an app needs approval', async () => {
114+
const status = (askForApps: boolean): ComputerUseStatus => ({
115+
supported: true,
116+
enabled: true,
117+
permissions: { accessibility: true, screenCapture: true },
118+
activeAction: null,
119+
askForApps,
120+
})
121+
libDesktopMockFns.mockGetDesktopBridge.mockReturnValue({
122+
computerUse: { ...native, setAskForApps: vi.fn(async (ask: boolean) => status(ask)) },
123+
})
124+
native.getStatus.mockResolvedValue(status(false))
125+
await act(async () => renderSettings())
126+
expect(container.querySelector('[aria-label="Ask before using a new app"]')).not.toBeNull()
127+
expect(appRow('com.example.First')).toBeUndefined()
128+
native.getStatus.mockResolvedValue(status(true))
129+
await act(async () => window.dispatchEvent(new Event('focus')))
130+
expect(appRow('com.example.First')).toBeDefined()
131+
})
132+
113133
it('does not restore a revoked app when an earlier activity refresh settles late', async () => {
114134
await act(async () => renderSettings())
115135
const stale = createDeferred<ComputerUseAppPermission[]>()

0 commit comments

Comments
 (0)