Skip to content

Commit 87fefdf

Browse files
authored
v0.9.56: PII masking for long texts, Agent block on the shared model fallback, Computer Use opens to every app with sensitive apps protected, AWS Marketplace container license check, slimmer realtime and migrations images, desktop browser rendering waits
2 parents 833c3af + b97449a commit 87fefdf

164 files changed

Lines changed: 3998 additions & 1562 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/image/action.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ inputs:
2323
description: Newline-separated Docker build arguments.
2424
required: false
2525
default: ''
26+
target:
27+
description: Dockerfile stage to build. Empty builds the final stage.
28+
required: false
29+
default: ''
2630
max-cache-size-mb:
2731
description: >-
2832
Layer cache to retain after this action prunes, in MB. Must stay above one
@@ -77,6 +81,7 @@ runs:
7781
push: true
7882
tags: ${{ inputs.tags }}
7983
build-args: ${{ inputs.build-args }}
84+
target: ${{ inputs.target }}
8085
provenance: false
8186
sbom: false
8287

@@ -183,5 +188,6 @@ runs:
183188
push: true
184189
tags: ${{ inputs.tags }}
185190
build-args: ${{ inputs.build-args }}
191+
target: ${{ inputs.target }}
186192
provenance: false
187193
sbom: false

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,8 +209,11 @@ jobs:
209209
ecr_repo_secret: ECR_APP
210210
gh_runner: linux-x64-8-core
211211
bs_runner: blacksmith-16vcpu-ubuntu-2404
212+
# Dev runs `db:push`, which needs drizzle-kit; every other environment
213+
# gets the final stage, which carries runtime dependencies only.
212214
- id: db
213215
dockerfile: ./docker/db.Dockerfile
216+
target: dev
214217
ecr_repo_secret: ECR_MIGRATIONS
215218
gh_runner: ubuntu-latest
216219
bs_runner: blacksmith-2vcpu-ubuntu-2404
@@ -259,6 +262,7 @@ jobs:
259262
tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ github.sha }}-dev
260263
build-args: |
261264
MSHIP_PLAN_MODE_DEFAULT=true
265+
target: ${{ matrix.target }}
262266
max-cache-size-mb: ${{ matrix.cache_mb }}
263267

264268
# Staging/production coordinate task releases with app traffic cutover.

‎apps/desktop/README.md‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -239,9 +239,13 @@ Mothership's `computer` tool is available on macOS 14+ once the user opts in und
239239
group with the **Computer Use** restriction. Older desktop builds omit this optional bridge and
240240
continue to work.
241241

242-
The user grants Accessibility and Screen Recording through macOS, then approves each app for the
243-
current task or persistently. Persistent app approvals are listed in Settings and can be revoked.
244-
Sign-out and deployment changes clear computer-use approval and opt-in. The conversation shows the
242+
The user grants Accessibility and Screen Recording through macOS; every app is then available except
243+
Sim, System Settings, password managers, terminals and scripting apps (shell access has its own
244+
Terminal control),
245+
which the helper refuses. With **Ask before using a new
246+
app** on, each app's first use is approved for the task or persistently, and persistent approvals are
247+
listed in Settings and can be revoked. Sign-out and deployment changes clear opt-in, the ask choice
248+
and approvals. The conversation shows the
245249
active app/action and a Stop button; **⌘⇧Esc** also stops active work when the shortcut is available.
246250
Stop cancels queued actions, in-flight authorization, approval prompts and the native helper.
247251
Already completed input cannot be undone.

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,10 +142,29 @@ func elementRect(_ element: AXUIElement) -> CGRect? {
142142
AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil }
143143
return CGRect(origin: position, size: size)
144144
}
145+
/// Apps computer use never operates, lowercased: macOS security settings, password managers, and apps
146+
/// whose primary purpose is running shell commands or scripts (terminals, Script Editor, Automator,
147+
/// Shortcuts), since shell access goes through the Terminal tool's own opt-in. Apps that merely embed a
148+
/// terminal, such as editors, stay usable.
149+
let PROTECTED_APPS: Set<String> = [
150+
"com.apple.systempreferences", "com.apple.keychainaccess", "com.apple.passwords",
151+
"com.1password.1password", "com.agilebits.onepassword7", "com.agilebits.onepassword-osx", "com.bitwarden.desktop",
152+
"org.keepassxc.keepassxc", "com.lastpass.lastpass", "in.sinew.enpass-desktop", "me.proton.pass.electron",
153+
"com.markmcguill.strongbox", "com.markmcguill.strongbox.mac",
154+
"com.apple.terminal", "com.googlecode.iterm2", "dev.warp.warp-stable", "com.mitchellh.ghostty",
155+
"net.kovidgoyal.kitty", "org.alacritty", "com.github.wez.wezterm", "co.zeit.hyper", "org.tabby",
156+
"com.apple.scripteditor2", "com.apple.automator", "com.apple.shortcuts",
157+
]
158+
/// Whether computer use refuses an app: Sim's own apps and `PROTECTED_APPS`. Bundle identifiers match
159+
/// case-insensitively in Launch Services.
160+
func isProtectedApp(_ bundle: String) -> Bool {
161+
let key = bundle.lowercased()
162+
return key == Bundle.main.bundleIdentifier?.lowercased() || key.hasPrefix("com.simstudio.") || key.hasPrefix("ai.sim.desktop") || PROTECTED_APPS.contains(key)
163+
}
145164
func appFor(_ bundleId: String?, launch: Bool = false) async throws -> NSRunningApplication {
146165
let bundle = try required(bundleId, "bundleId")
147-
guard bundle != Bundle.main.bundleIdentifier, !bundle.hasPrefix("com.simstudio."), !bundle.hasPrefix("ai.sim.desktop"), bundle != "com.apple.systempreferences" else {
148-
throw ComputerError("protected_app", "Computer use cannot control its own app or macOS security settings.")
166+
guard !isProtectedApp(bundle) else {
167+
throw ComputerError("protected_app", "Computer use does not operate Sim, System Settings, password managers, terminals, or scripting apps; use the terminal tool for shell work.")
149168
}
150169
var apps = NSRunningApplication.runningApplications(withBundleIdentifier: bundle).filter { !$0.isTerminated }
151170
if apps.isEmpty && launch {

‎apps/desktop/native/computer-use/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` return
1010

1111
The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized. The menu bar and menus are read in place, so an open menu's items and their submenus come first; items of closed menus are read last, capped at 80 menu nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. An Accessibility failure counts as sent unless Accessibility rejected the request before the app saw it (unsupported attribute or action, illegal argument). Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise.
1212

13-
Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected, and an element target uses its own window; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps and System Settings are protected targets. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`.
13+
Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected, and an element target uses its own window; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps, System Settings, password managers, terminals and scripting apps (Script Editor, Automator, Shortcuts) are protected targets (`PROTECTED_APPS`, matched case-insensitively); shell work belongs to the separate Terminal control. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`.
1414

1515
AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A hit on a window's own background counts as that window. Background tabs and minimized windows share bounds with the window in front, so they take no coordinate input. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves the item's own menu open with the app highlighting that item (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly.
1616

‎apps/desktop/native/computer-use/tests/NativeTests.swift‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ import AppKit
2626
expectError("unsupported_key") { _ = try parseKey("F99") }
2727
expectError("invalid_arguments") { _ = try parseKey("Hyper+A") }
2828
expectError("invalid_arguments") { _ = try required(Optional<String>.none, "example") }
29+
for bundle in ["com.apple.systempreferences", "com.apple.Terminal", "COM.GOOGLECODE.ITERM2", "net.kovidgoyal.kitty", "com.1password.1password", "com.apple.Passwords", "com.bitwarden.desktop", "ai.sim.desktop", "com.simstudio.computer-use", "com.apple.ScriptEditor2", "com.apple.Automator", "com.apple.shortcuts"] {
30+
precondition(isProtectedApp(bundle), bundle)
31+
}
32+
for bundle in ["com.apple.TextEdit", "com.apple.finder", "com.apple.Safari"] { precondition(!isProtectedApp(bundle), bundle) }
2933
var queue = AXTraversalQueue<String>(limit: 1000)
3034
queue.append(AXTraversalEntry(element: "menu", parent: nil, depth: 0, menu: true, priority: false))
3135
queue.append(AXTraversalEntry(element: "content", parent: nil, depth: 0, menu: false, priority: false))

‎apps/desktop/scripts/build-native.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ function compileComputerUse(): void {
8989
try {
9090
execFileSync('xcrun', ['lipo', binary, '-verify_arch', 'arm64', 'x86_64'])
9191
execFileSync('codesign', ['--verify', '--strict', bundle])
92+
assertWeakScreenCaptureKit(binary)
9293
return
9394
} catch {
9495
rmSync(bundle, { recursive: true, force: true })
@@ -108,6 +109,11 @@ function compileComputerUse(): void {
108109
'-O',
109110
'-target',
110111
`${arch}-apple-macosx14.0`,
112+
// The macOS 26 screenshot classes do not exist on 14/15; a strong reference stops the helper loading there.
113+
'-Xlinker',
114+
'-weak_framework',
115+
'-Xlinker',
116+
'ScreenCaptureKit',
111117
'native/computer-use/ComputerUse.swift',
112118
'-o',
113119
output,
@@ -118,9 +124,27 @@ function compileComputerUse(): void {
118124
}
119125
execFileSync('xcrun', ['lipo', '-create', ...parts, '-output', binary], { stdio: 'inherit' })
120126
for (const part of parts) rmSync(part)
127+
assertWeakScreenCaptureKit(binary)
121128
execFileSync('codesign', ['--force', '--sign', '-', bundle], { stdio: 'inherit' })
122129
recordNativeBuild('computer-use', outputs)
123130
logger.info('Compiled native macOS Computer Use helper')
124131
}
125132

133+
/**
134+
* The helper targets macOS 14 but uses ScreenCaptureKit classes added later behind `#available`.
135+
* Runners build on the newest macOS, where a strong reference still loads, so this is the only
136+
* place a helper that dyld would refuse on older macOS is caught before it ships.
137+
*/
138+
function assertWeakScreenCaptureKit(binary: string) {
139+
const strong = execFileSync('nm', ['-m', binary], { encoding: 'utf8' })
140+
.split('\n')
141+
.filter(
142+
(line) => line.includes('(undefined) external') && line.includes('(from ScreenCaptureKit)')
143+
)
144+
if (strong.length > 0)
145+
throw new Error(
146+
`The Computer Use helper links ScreenCaptureKit strongly and would not load on macOS 14/15:\n${strong.join('\n')}`
147+
)
148+
}
149+
126150
if (args.includes('--computer-use')) compileComputerUse()

‎apps/desktop/src/main/browser-agent/driver.test.ts‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1894,6 +1894,7 @@ describe('credential protection', () => {
18941894
activeElementSecrecy: 'safe',
18951895
readActiveElementState: {},
18961896
readPageActionState: {},
1897+
readDomQuiet: { quietMs: 1_000, loading: false },
18971898
})
18981899
const pageCall = vi.mocked(contents.executeJavaScript).getMockImplementation()
18991900
let releaseObservation: (value: unknown) => void = () => {}
@@ -3212,6 +3213,113 @@ describe('credential protection', () => {
32123213
})
32133214
})
32143215

3216+
it.each([
3217+
['still rendering', [0, 40, 90, 200], 4],
3218+
['already quiet', [400], 1],
3219+
['never settling', Array.from({ length: 200 }, () => 0), undefined],
3220+
] as const)(
3221+
'observes after an action only once a page %s has gone quiet, within the settle cap',
3222+
async (_label, quietReadings, expectedProbes) => {
3223+
const contents = await openPage()
3224+
const calls: string[] = []
3225+
let probe = 0
3226+
vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
3227+
if (isPageCall(expression, 'readDomQuiet')) {
3228+
calls.push('probe')
3229+
const quietMs = quietReadings[Math.min(probe++, quietReadings.length - 1)]
3230+
return Promise.resolve({ quietMs, loading: false })
3231+
}
3232+
if (isPageCall(expression, 'collectSnapshot')) {
3233+
calls.push('snapshot')
3234+
return Promise.resolve({
3235+
url: 'https://example.com/',
3236+
title: 'Example',
3237+
outline: '- heading "Results"',
3238+
truncated: false,
3239+
refIds: [],
3240+
refLineIndexes: {},
3241+
nextElementId: 1_000,
3242+
})
3243+
}
3244+
if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET)
3245+
return Promise.resolve(undefined)
3246+
})
3247+
vi.useFakeTimers()
3248+
try {
3249+
const clicking = driver.executeTool('chat-test', 'browser_click', {
3250+
elementId: 0,
3251+
observe: {},
3252+
})
3253+
await vi.advanceTimersByTimeAsync(5_000)
3254+
3255+
expect(await clicking).toMatchObject({ ok: true, result: { observation: { ok: true } } })
3256+
const probesBeforeSnapshot = calls.indexOf('snapshot')
3257+
expect(probesBeforeSnapshot).toBeGreaterThan(0)
3258+
if (expectedProbes !== undefined) expect(probesBeforeSnapshot).toBe(expectedProbes)
3259+
} finally {
3260+
vi.useRealTimers()
3261+
}
3262+
}
3263+
)
3264+
3265+
it('keeps settling through failed quiet probes, and never observes once cancelled', async () => {
3266+
const contents = await openPage()
3267+
const calls: string[] = []
3268+
let failing = 2
3269+
vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
3270+
if (isPageCall(expression, 'readDomQuiet')) {
3271+
calls.push('probe')
3272+
if (failing > 0) {
3273+
failing--
3274+
vi.mocked(contents.isLoading).mockReturnValue(true)
3275+
return Promise.reject(new Error('Execution context was destroyed'))
3276+
}
3277+
vi.mocked(contents.isLoading).mockReturnValue(false)
3278+
return Promise.resolve({ quietMs: failing === -1 ? 0 : 400, loading: false })
3279+
}
3280+
if (isPageCall(expression, 'collectSnapshot')) {
3281+
calls.push('snapshot')
3282+
return Promise.resolve({
3283+
url: 'https://example.com/',
3284+
title: 'Example',
3285+
outline: '- heading "Results"',
3286+
truncated: false,
3287+
refIds: [],
3288+
refLineIndexes: {},
3289+
nextElementId: 1_000,
3290+
})
3291+
}
3292+
if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET)
3293+
return Promise.resolve(undefined)
3294+
})
3295+
vi.useFakeTimers()
3296+
try {
3297+
const settled = driver.executeTool('chat-test', 'browser_click', {
3298+
elementId: 0,
3299+
observe: {},
3300+
})
3301+
await vi.advanceTimersByTimeAsync(5_000)
3302+
expect(await settled).toMatchObject({ ok: true, result: { observation: { ok: true } } })
3303+
expect(calls.indexOf('snapshot')).toBe(3)
3304+
3305+
calls.length = 0
3306+
failing = -1
3307+
const cancelled = driver.executeTool(
3308+
'chat-test',
3309+
'browser_click',
3310+
{ elementId: 0, observe: { query: 'Results' } },
3311+
'click-cancelled'
3312+
)
3313+
await vi.advanceTimersByTimeAsync(300)
3314+
expect(driver.cancelTool('chat-test', 'click-cancelled')).toBe(true)
3315+
await vi.advanceTimersByTimeAsync(5_000)
3316+
await cancelled
3317+
expect(calls).not.toContain('snapshot')
3318+
} finally {
3319+
vi.useRealTimers()
3320+
}
3321+
})
3322+
32153323
it('acts on a ref an earlier snapshot of the same page issued', async () => {
32163324
const contents = await openPage()
32173325
respondWith(contents, {

0 commit comments

Comments
 (0)