Skip to content

Commit a327762

Browse files
committed
fix(computer-use): protect more terminals, test the protected-app guard, and keep the approval copy for older shells
1 parent 855a607 commit a327762

4 files changed

Lines changed: 16 additions & 5 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,12 +150,17 @@ let PROTECTED_APPS: Set<String> = [
150150
"org.keepassxc.keepassxc", "com.lastpass.lastpass", "in.sinew.enpass-desktop", "me.proton.pass.electron",
151151
"com.markmcguill.strongbox", "com.markmcguill.strongbox.mac",
152152
"com.apple.terminal", "com.googlecode.iterm2", "dev.warp.warp-stable", "com.mitchellh.ghostty",
153+
"net.kovidgoyal.kitty", "org.alacritty", "com.github.wez.wezterm", "co.zeit.hyper", "org.tabby",
153154
]
155+
/// Whether computer use refuses an app: Sim's own apps and `PROTECTED_APPS`. Bundle identifiers match
156+
/// case-insensitively in Launch Services.
157+
func isProtectedApp(_ bundle: String) -> Bool {
158+
let key = bundle.lowercased()
159+
return key == Bundle.main.bundleIdentifier?.lowercased() || key.hasPrefix("com.simstudio.") || key.hasPrefix("ai.sim.desktop") || PROTECTED_APPS.contains(key)
160+
}
154161
func appFor(_ bundleId: String?, launch: Bool = false) async throws -> NSRunningApplication {
155162
let bundle = try required(bundleId, "bundleId")
156-
// Bundle identifiers match case-insensitively in Launch Services.
157-
let key = bundle.lowercased()
158-
guard key != Bundle.main.bundleIdentifier?.lowercased(), !key.hasPrefix("com.simstudio."), !key.hasPrefix("ai.sim.desktop"), !PROTECTED_APPS.contains(key) else {
163+
guard !isProtectedApp(bundle) else {
159164
throw ComputerError("protected_app", "Computer use does not operate Sim, System Settings, password managers, or terminals; use the terminal tool for shell work.")
160165
}
161166
var apps = NSRunningApplication.runningApplications(withBundleIdentifier: bundle).filter { !$0.isTerminated }

‎apps/desktop/native/computer-use/tests/NativeTests.swift‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ import AppKit
2626
expectError("unsupported_key") { _ = try parseKey("F99") }
2727
expectError("invalid_arguments") { _ = try parseKey("Hyper+A") }
2828
expectError("invalid_arguments") { _ = try required(Optional<String>.none, "example") }
29+
for bundle in ["com.apple.systempreferences", "com.apple.Terminal", "COM.GOOGLECODE.ITERM2", "net.kovidgoyal.kitty", "com.1password.1password", "com.apple.Passwords", "com.bitwarden.desktop", "ai.sim.desktop", "com.simstudio.computer-use"] {
30+
precondition(isProtectedApp(bundle), bundle)
31+
}
32+
for bundle in ["com.apple.TextEdit", "com.apple.finder", "com.apple.Safari"] { precondition(!isProtectedApp(bundle), bundle) }
2933
var queue = AXTraversalQueue<String>(limit: 1000)
3034
queue.append(AXTraversalEntry(element: "menu", parent: nil, depth: 0, menu: true, priority: false))
3135
queue.append(AXTraversalEntry(element: "content", parent: nil, depth: 0, menu: false, priority: false))

‎apps/docs/content/docs/desktop/index.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ To enable it when available:
9898
1. Turn **Use apps on this Mac** to **On**.
9999
2. Select **Allow** next to **Accessibility** (app interaction) and **Screen Recording** (screenshots), grant them in System Settings, then return to Sim.
100100

101-
Sim can then use any app except terminals, password managers, and macOS System Settings; shell work goes through **Terminal** access instead. To approve each app the first time Sim uses it, turn on **Ask before using a new app**; Sim then offers **Allow for This Task**, **Always Allow**, or **Cancel**.
101+
Sim can then use any app except terminals, password managers, and macOS System Settings (and Sim itself); shell work goes through **Terminal** access instead. To approve each app the first time Sim uses it, turn on **Ask before using a new app**; Sim then offers **Allow for This Task**, **Always Allow**, or **Cancel**.
102102

103103
Sim can read app windows and interact through clicks, text, keyboard shortcuts, scrolling, and supported accessibility controls. Screenshots and returned app content can be sent to your Sim deployment and model. Some actions require bringing the app to the foreground; app compatibility and macOS permissions can still limit what succeeds.
104104

‎apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,9 @@ function ComputerUseSettingsControls() {
8585
/>
8686
</div>
8787
<p className='text-[var(--text-muted)] text-sm'>
88-
Sim can use any app except terminals, password managers, and System Settings.
88+
{askForApps === undefined
89+
? 'Each app needs your approval before Sim can use it.'
90+
: 'Sim can use any app except terminals, password managers, System Settings, and Sim itself.'}
8991
</p>
9092
{error && (
9193
<div className='flex items-center justify-between text-sm'>

0 commit comments

Comments
 (0)