From 940cfa9ac174c88280624acacc4b580229109df7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 13:19:58 -0800 Subject: [PATCH 1/3] fix(computer-use): tighten batch and click-to-focus edge cases from the staging audit Validate every batch step before the first runs and name a refused first step; observe the window a batch targets; wait 250 ms for Accessibility focus before clicking, treat a refused click as no click and put the caret at the end; never report nothing-dispatched after a click; skip per-attribute fallback reads on a hung app; guard a self-parent in the projection; fix legend and README wording. Syncs the contract's activateFirst wording. --- .../native/computer-use/ComputerUse.swift | 69 ++++++++++++++----- apps/desktop/native/computer-use/README.md | 4 +- .../computer-use/tests/NativeTests.swift | 4 ++ .../desktop/native/computer-use/tests/live.py | 8 ++- .../src/main/computer-use/service.test.ts | 41 +++++++++++ apps/desktop/src/main/computer-use/service.ts | 23 +++++-- .../mothership-chat/rollout-controls.test.tsx | 6 +- .../tools/client/computer-tool-result.test.ts | 18 +++++ .../tools/client/computer-tool-result.ts | 12 +++- .../src/computer-use.generated.ts | 2 +- 10 files changed, 154 insertions(+), 33 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 45f68e5af81..03720a73685 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -91,8 +91,11 @@ func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? { /// the element lacks or fails to return is omitted, as `attribute` would return nil for it. func attributes(_ element: AXUIElement, _ names: [String]) -> [String: CFTypeRef] { var raw: CFArray? - guard AXUIElementCopyMultipleAttributeValues(element, names as CFArray, AXCopyMultipleAttributeOptions(rawValue: 0), &raw) == .success, - let values = raw as [AnyObject]?, values.count == names.count else { + let status = AXUIElementCopyMultipleAttributeValues(element, names as CFArray, AXCopyMultipleAttributeOptions(rawValue: 0), &raw) + guard status == .success, let values = raw as [AnyObject]?, values.count == names.count else { + // Only an element that cannot answer a batched read is asked one attribute at a time; a + // timeout or failure would only repeat once per attribute. + guard [.attributeUnsupported, .illegalArgument, .notImplemented].contains(status) else { return [:] } return names.reduce(into: [:]) { result, name in result[name] = attribute(element, name) } } var result: [String: CFTypeRef] = [:] @@ -467,9 +470,19 @@ func validateBatchSteps(_ steps: [Parameters]) throws { guard (2...8).contains(steps.count) else { throw ComputerError("invalid_arguments", "A batch requires 2 to 8 steps.") } var units = 0 for step in steps { - guard let action = step.action, BATCH_ACTIONS.contains(action), step.bundleId == nil, step.snapshotId == nil, step.steps == nil else { + guard let action = step.action, BATCH_ACTIONS.contains(action), step.bundleId == nil, step.snapshotId == nil else { throw ComputerError("invalid_arguments", "Batch steps are click, type_text, set_value, press_key, scroll or perform_action, without their own app or snapshot.") } + // Malformed steps are refused before step 1 runs, so a typo never leaves a half-applied batch. + switch action { + case "type_text": _ = try required(step.elementId, "elementId"); _ = try required(step.text, "text") + case "set_value": _ = try required(step.elementId, "elementId"); _ = try required(step.value, "value") + case "perform_action": _ = try required(step.elementId, "elementId"); _ = try required(step.accessibilityAction, "accessibilityAction") + case "press_key": _ = try parseKey(required(step.key, "key")) + case "click": + guard ["left", "right"].contains(step.button ?? "left"), (1...3).contains(step.clickCount ?? 1) else { throw ComputerError("invalid_arguments", "Invalid button or clickCount.") } + default: break + } units += (step.text?.utf16.count ?? 0) + (step.value?.utf16.count ?? 0) } guard units <= 32000 else { throw ComputerError("invalid_arguments", "Combined batch text exceeds 32000 UTF-16 units.") } @@ -599,15 +612,33 @@ func focusByClicking(_ element: AXUIElement, pid: pid_t, snapshot: Snapshot) asy let center = CGPoint(x: rect.midX, y: rect.midY) // The point must land on the editor or a text control inside it, never a button or row nested in a container. guard takesText(element), frame.contains(center), let hit = elementAtPoint(center) else { return } - let insideEditor = takesText(hit) ? try ancestorPath(hit, to: element) != nil : false + let insideEditor = try takesText(hit) && ancestorPath(hit, to: element) != nil // Secure inputs stay untargetable, so a password field is never clicked into focus either. guard CFEqual(hit, element) || insideEditor, (try? requireNonSecure(hit)) != nil else { return } - try await coordinateAction { dispatch in - try checkCancellation() - try postMouse(pid: pid, type: .leftMouseDown, point: center, button: .left, windowID: windowID, window: window, frame: frame, dispatch: dispatch) - try postMouse(pid: pid, type: .leftMouseUp, point: center, button: .left, windowID: windowID, window: window, frame: frame, release: true, dispatch: dispatch) + do { + try await coordinateAction { dispatch in + try checkCancellation() + try postMouse(pid: pid, type: .leftMouseDown, point: center, button: .left, windowID: windowID, window: window, frame: frame, dispatch: dispatch) + try postMouse(pid: pid, type: .leftMouseUp, point: center, button: .left, windowID: windowID, window: window, frame: frame, release: true, dispatch: dispatch) + } + } catch let error as ComputerError where error.dispatchState == "not_started" { + // A mouse guard refused the point before anything was sent; the focus check reports the outcome. + return + } + // A click leaves the caret where it landed; typing inserts at the end, as after an Accessibility focus. + let root = AXUIElementCreateApplication(pid) + for _ in 0..<5 { + try await Task.sleep(for: .milliseconds(50)) + guard let focused = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(focused, element) else { continue } + let editor = unsafeBitCast(focused, to: AXUIElement.self) + var settable: DarwinBoolean = false + if AXUIElementIsAttributeSettable(editor, kAXSelectedTextRangeAttribute as CFString, &settable) == .success, settable.boolValue, + let value = attribute(editor, kAXValueAttribute) as? String { + var range = CFRange(location: value.utf16.count, length: 0) + if let end = AXValueCreate(.cfRange, &range) { _ = AXUIElementSetAttributeValue(editor, kAXSelectedTextRangeAttribute as CFString, end) } + } + return } - try await Task.sleep(for: .milliseconds(100)) } func discoverApps() throws -> [[String: Any]] { var entries: [String: [String: Any]] = [:] @@ -767,14 +798,16 @@ final class Driver { "sequence": ["completedSteps": completed, "totalSteps": steps.count, "error": String((message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch.").prefix(2000))]] } for (index, step) in steps.enumerated() { - // Stop can arrive during the previous step's settle; no later step may start after it. - do { try checkCancellation() } catch { if completed == 0 { throw error }; return stopped("Stopped before step \(index + 1).") } - var params = step; params.bundleId = p.bundleId; params.snapshotId = p.snapshotId + // Stop can arrive during the previous step's render wait; no later step may start after it. + if index > 0 { do { try checkCancellation() } catch { return stopped("Stopped before step \(index + 1).") } } let before = uiStructure(pid: pid) - do { _ = try await act(Request(id: request.id, method: try required(step.action, "action"), params: params), app: app, bundle: bundle, snapshot: snapshot) } + do { _ = try await act(Request(id: request.id, method: try required(step.action, "action"), params: step), app: app, bundle: bundle, snapshot: snapshot) } catch { // A first step that sent nothing fails like the single action, keeping the snapshot valid. - if !inputDispatched { throw error } + if completed == 0 && !inputDispatched { + guard let error = error as? ComputerError else { throw error } + throw ComputerError(error.code, "Batch step 1 failed: " + error.message, dispatchState: error.dispatchState) + } return stopped("Step \(index + 1) failed: " + ((error as? ComputerError)?.message ?? "its outcome is unknown.")) } completed += 1 @@ -832,7 +865,11 @@ final class Driver { // Activation can move focus, so whether the editor holds it is read after activating. if attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) != true { try axCheck(AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)) - try await Task.sleep(for: .milliseconds(50)) + // Focus can settle asynchronously; slow apps get 250 ms before any click is considered. + for _ in 0..<5 { + try await Task.sleep(for: .milliseconds(50)) + if attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) == true { break } + } // Some apps (SwiftUI sheets such as Clock's city search) ignore an Accessibility focus // request; in a foreground app the field is focused the way a user would, by clicking it. if app.isActive, attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) != true { @@ -861,7 +898,7 @@ final class Driver { do { try validateFocus(); break } catch let error as ComputerError where error.code == "focus_changed" { if attempt < 9 { try await Task.sleep(for: .milliseconds(50)) } - else if !app.isActive { throw ComputerError("activation_required", "No input was dispatched. This background app does not expose the exact keyboard focus target. Call activate_app for this bundle, then get_app_state and retry using the fresh editor reference.", dispatchState: "not_started") } + else if !app.isActive && !inputDispatched { throw ComputerError("activation_required", "No input was dispatched. This background app does not expose the exact keyboard focus target. Call activate_app for this bundle, then get_app_state and retry using the fresh editor reference.", dispatchState: "not_started") } else { throw error } } } diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index de05138d588..aa66f01bd5e 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -14,7 +14,7 @@ Coordinates in action inputs are window-local points; node frames are global scr AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A hit on a window's own background counts as that window. Background tabs and minimized windows share bounds with the window in front, so they take no coordinate input. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves the item's own menu open with the app highlighting that item (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. -`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date and time pickers, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. When an app ignores the Accessibility focus request (SwiftUI sheets such as Clock's city search), a foreground app's editor is clicked at its center to take focus, only if that point hit-tests to the editor inside its observed window; the usual focus checks then apply. `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. `batch` runs 2–8 click, type_text, set_value, press_key, scroll or perform_action steps planned from one snapshot, each through its own action's checks; it stops at the first step that fails and after any step that opens or closes a window, sheet, menu or popover, reports progress the same way, and a first step that sent nothing is refused with `dispatchState: "not_started"`. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. +`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date and time pickers, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. When an app ignores the Accessibility focus request (SwiftUI sheets such as Clock's city search), a foreground app's editor is clicked at its center to take focus after 250 ms without it, only if that point hit-tests to the editor (or a non-secure text control inside it) within its observed window; a click refused by the mouse guards sends nothing, the caret is moved to the end so typing inserts there, and the usual focus checks then apply. `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the sequence before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. `batch` runs 2–8 click, type_text, set_value, press_key, scroll or perform_action steps planned from one snapshot, each through its own action's checks; it stops at the first step that fails and after any step that changes the app's window structure (on-screen windows, which include sheets, menus and popovers, or the focused window's direct children), reports progress the same way, and a first step that sent nothing is refused with `dispatchState: "not_started"`. Every step is validated before the first one runs. The structure check is a heuristic: it misses modals inside web content and UI that appears after the step's render wait, and stops conservatively on tooltips and autocomplete popovers. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. ## Cancellation @@ -22,7 +22,7 @@ SIGTERM/SIGINT sets a cancellation flag. Traversal, typing and drag loops check ## Tests and limitations -Compile pure native tests with `swiftc -parse-as-library -D COMPUTER_USE_TEST ComputerUse.swift tests/NativeTests.swift -o native-tests`. They cover multi-display negative origins, coordinate bounds/nonfinite input, shortcut modifiers, invalid key handling and cooperative cancellation. Real status and app discovery outputs were validated against the generated worker Zod schema (156 apps, including 147 installed-only apps). +Compile pure native tests with `swiftc -parse-as-library -D COMPUTER_USE_TEST ComputerUse.swift tests/NativeTests.swift -o native-tests`. They cover multi-display negative origins, coordinate bounds/nonfinite input, shortcut modifiers, invalid key handling, input-sequence and batch validation, protected apps and cooperative cancellation. The live fixture also covers batches and click-to-focus for an editor that ignores Accessibility focus. Real status and app discovery outputs were validated against the generated worker Zod schema (156 apps, including 147 installed-only apps). `python3 tests/protocol.py ` checks persistent framing, bounded-request recovery, Unicode decoding, no-prompt status, invalid methods/permissions, permission gates and clean EOF. `tests/Fixture.swift` compiles into a disposable AppKit app with a text field, secure field, increment button/counter, slider, scroll area and a visual-only marker. `python3 tests/live.py ` requires both grants and verifies actual outcomes. `--allow-missing-screen` explicitly reports screenshots blocked rather than passing them; `--existing-fixture` leaves an existing fixture running for coordinated model trials. Give the fixture bundle ID `com.mothership.computer-use-fixture` when packaging it for integration tests. diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 39969e61287..ef436a613f3 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -56,6 +56,10 @@ import AppKit expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", elementId: "e1")]) } expectError("invalid_arguments") { try validateBatchSteps(Array(repeating: Parameters(action: "press_key", key: "Tab"), count: 9)) } expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "drag"), Parameters(action: "press_key", key: "Enter")]) } + expectError("unsupported_key") { try validateBatchSteps([Parameters(action: "click", elementId: "e1"), Parameters(action: "press_key", key: "F99")]) } + expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", elementId: "e1"), Parameters(action: "type_text", elementId: "e2")]) } + expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "set_value", value: "x"), Parameters(action: "click", elementId: "e1")]) } + expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", elementId: "e1", clickCount: 4), Parameters(action: "press_key", key: "Enter")]) } expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", bundleId: "com.example", elementId: "e1"), Parameters(action: "press_key", key: "Enter")]) } expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "set_value", elementId: "e1", value: String(repeating: "🌍", count: 8001)), Parameters(action: "type_text", elementId: "e2", text: String(repeating: "🌍", count: 8000))]) } let tabFocus = try keyMayChangeFocus("Tab"); precondition(tabFocus) diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index febc63144f6..2e8d936bf4b 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -177,6 +177,12 @@ def batch(snapshot, steps): result = call('type_text', bundleId=bundle, snapshotId=snapshot['snapshotId'], elementId=click_field['elementId'], text='clicked') assert result['dispatched'], result assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'clicked', 'Click-to-focus did not reach the editor' + # With text already in it and focus elsewhere, the clicked editor inserts at the end, not where the click landed. + snapshot = state() + mutate('click', snapshot, **coordinate(snapshot, find(snapshot, role='AXTextField', label='Fixture editable text'))) + snapshot = state() + call('type_text', bundleId=bundle, snapshotId=snapshot['snapshotId'], elementId=find(snapshot, role='AXTextField', label='Fixture click-focus field')['elementId'], text=' twice') + assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'clicked twice', 'Click-to-focus typed at the click point' # Move focus to the other editor, then put another app in front: the click-only editor cannot take focus. snapshot = state() mutate('click', snapshot, **coordinate(snapshot, find(snapshot, role='AXTextField', label='Fixture editable text'))) @@ -185,7 +191,7 @@ def batch(snapshot, steps): snapshot = state() error = refused('type_text', snapshot, elementId=find(snapshot, role='AXTextField', label='Fixture click-focus field')['elementId'], text='x') assert error['dispatchState'] == 'not_started', error - assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'clicked', 'A background app was clicked or typed into' + assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'clicked twice', 'A background app was clicked or typed into' call('activate_app', bundleId=bundle) print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index 7a66b868db1..c5d463b75d1 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -233,6 +233,47 @@ describe('native computer use authority and lifecycle', () => { await expect(click('s2', 'chat', 'next')).resolves.toMatchObject({ dispatched: true }) }) + it.each([ + { + steps: [ + { action: 'click', elementId: 'e1' }, + { action: 'set_value', elementId: 'e1', value: 'x' }, + ], + windowId: '1', + }, + { + steps: [ + { action: 'click', elementId: 'e1' }, + { action: 'press_key', key: 'Enter' }, + ], + windowId: undefined, + }, + ])( + 'observes the one window a batch targets, else the focused window ($windowId)', + async ({ steps, windowId }) => { + const { state, service, native } = setup() + await state() + native.request.mockResolvedValueOnce({ + kind: 'action', + action: 'batch', + bundleId: 'com.example.Fixture', + dispatched: true, + verified: false, + sequence: { completedSteps: 2, totalSteps: 2 }, + }) + await service.execute('batch', 'chat', { + action: 'batch', + bundleId: 'com.example.Fixture', + snapshotId: 's1', + steps, + }) + expect(native.request).toHaveBeenLastCalledWith('get_app_state', { + bundleId: 'com.example.Fixture', + ...(windowId ? { windowId } : {}), + }) + } + ) + it('observes a partial sequence without retrying any input', async () => { const { state, service, native } = setup() await state() diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index afacf4696f3..397f5041589 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -258,18 +258,29 @@ export class ComputerUseService { } this.check(work) const owner = 'snapshotId' in input ? this.snapshots.get(input.snapshotId) : undefined - const elementId = 'elementId' in input ? input.elementId : undefined /** * Element and coordinate calls observe the window they acted on. An untargeted key keeps the * focused-window default, so a key that closes the window still returns fresh state. */ - const windowId = - ('windowId' in input ? input.windowId : undefined) ?? - (elementId - ? (owner?.elementWindows.get(elementId) ?? owner?.windowId) - : 'x' in input + const targetWindow = (target: { windowId?: string; elementId?: string; x?: number }) => + target.windowId ?? + (target.elementId + ? (owner?.elementWindows.get(target.elementId) ?? owner?.windowId) + : target.x !== undefined ? owner?.windowId : undefined) + /** A batch observes the one window all of its steps target; any untargeted step keeps the focused default. */ + const batchWindows = + input.action === 'batch' ? new Set(input.steps.map(targetWindow)) : undefined + const windowId = batchWindows + ? batchWindows.size === 1 + ? [...batchWindows][0] + : undefined + : targetWindow({ + windowId: 'windowId' in input ? input.windowId : undefined, + elementId: 'elementId' in input ? input.elementId : undefined, + x: 'x' in input ? input.x : undefined, + }) if ('snapshotId' in input) this.snapshots.delete(input.snapshotId) /** Mutations accept includeScreenshot as shorthand for observeAfter.includeScreenshot. */ const isMutation = input.action !== 'get_app_state' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/rollout-controls.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/rollout-controls.test.tsx index 894652185aa..2ad9197109e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/rollout-controls.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/rollout-controls.test.tsx @@ -1,6 +1,6 @@ /** @vitest-environment jsdom */ import { act } from 'react' -import { libDesktopMock, libDesktopMockFns } from '@sim/testing/mocks/lib-desktop.mock' +import { libDesktopMock } from '@sim/testing/mocks/lib-desktop.mock' import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, expect, it, vi } from 'vitest' @@ -30,7 +30,6 @@ beforeEach(() => { useMothershipEffortStore.getState().reset() useMothershipPlanEffortStore.getState().reset() client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) - libDesktopMockFns.mockGetDesktopBridge.mockReturnValue(undefined) container = document.createElement('div') document.body.append(container) root = createRoot(container) @@ -40,7 +39,7 @@ afterEach(async () => { client.clear() container.remove() }) -async function render(advanced: boolean, workspaceId?: string) { +async function render(advanced: boolean) { await act(async () => root.render( @@ -54,7 +53,6 @@ async function render(advanced: boolean, workspaceId?: string) { }} > { expect(result.omittedNodeCount).toBe(0) }) + it('never names a node as its own parent through a wrapper cycle', () => { + const state = snapshot() + state.nodes = [ + { + elementId: 'button', + parentId: 'wrap', + role: 'AXButton', + label: 'Go', + actions: ['AXPress'], + }, + { elementId: 'wrap', parentId: 'button', role: 'AXGroup', actions: [] }, + ] + const result = computerToolResultForModel(state) + if (!('accessibilityTree' in result)) throw new Error('Missing projected tree') + expect(result.accessibilityTree).toContain('button AXButton') + expect(result.accessibilityTree).not.toContain('parent=button') + }) + it('keeps an empty layout group that reports itself disabled', () => { const state = snapshot() state.nodes = [ diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts index 0f53a975af9..d366836c9f6 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts @@ -98,7 +98,7 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { windows, omittedWindowCount: allWindows.length - windows.length, treeFormat: - 'Rows: elementId role [parent=id] [label|value|placeholder] [focused] [editable] [disabled] [actions=JSON array of exact names] [window=id, omitted for windowId] [screenRect=x,y,w,h in global points]. Empty layout groups are folded into their parent.', + 'Rows: elementId role [parent=id, omitted when the parent is not shown] [label=] [value=] [placeholder=] (JSON strings) [focused] [editable] [disabled] [actions=JSON array of exact names] [window=id, omitted for windowId] [screenRect=x,y,w,h in global points]. Empty layout groups are folded into their parent.', accessibilityTree: '', omittedNodeCount: nodes.filter((node) => !isWrapper(node)).length, nodeTextLimit: 512, @@ -126,7 +126,12 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { seen.add(parent.elementId) parent = parent.parentId ? byId.get(parent.parentId) : undefined } - shownParent.set(node.elementId, parent && !isWrapper(parent) ? parent.elementId : undefined) + shownParent.set( + node.elementId, + parent && !isWrapper(parent) && parent.elementId !== node.elementId + ? parent.elementId + : undefined + ) } const rows = new Map( shown.map((node) => [ @@ -173,7 +178,8 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { let bytes = jsonBytes(model) + envelopeBytes const add = (ids: string[]) => { const additional = ids.filter((id) => !selected.has(id)) - const cost = additional.reduce((total, id) => total + jsonBytes(rows.get(id)) - 2 + 2, 0) + /** A row's JSON length less its two quotes, plus the newline that joins it. */ + const cost = additional.reduce((total, id) => total + jsonBytes(rows.get(id)) - 1, 0) if (bytes + cost > MAX_STATE_BYTES) return false for (const id of additional) selected.add(id) bytes += cost diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index c31d33cbe92..feef762b713 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -187,7 +187,7 @@ export const ComputerUseSchema = z.union([ .boolean() .optional() .describe( - "Explicitly bring this app to the foreground before the batch. Defaults to false; leave omitted for background input. An inactive app alone does not require activation. Do not set true when the user requires background operation. Editor, window, and geometry checks still apply.", + "Explicitly bring this app to the foreground before the sequence. Defaults to false; leave omitted for background input. An inactive app alone does not require activation. Do not set true when the user requires background operation. Editor, window, and geometry checks still apply.", ), steps: z .array(InputSequenceStep) From 8ec2773fdcbb7864497d32646782ee28744c99f8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 13:36:39 -0800 Subject: [PATCH 2/3] fix(computer-use): check Stop before every batch step, validate click and scroll targets up front, and keep the escaped-newline row cost --- apps/desktop/native/computer-use/ComputerUse.swift | 10 +++++++++- .../native/computer-use/tests/NativeTests.swift | 2 ++ .../mothership/tools/client/computer-tool-result.ts | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 03720a73685..41b3b07c83c 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -481,6 +481,10 @@ func validateBatchSteps(_ steps: [Parameters]) throws { case "press_key": _ = try parseKey(required(step.key, "key")) case "click": guard ["left", "right"].contains(step.button ?? "left"), (1...3).contains(step.clickCount ?? 1) else { throw ComputerError("invalid_arguments", "Invalid button or clickCount.") } + guard step.elementId != nil || (step.x != nil && step.y != nil) else { throw ComputerError("invalid_arguments", "A click step needs an elementId or x and y.") } + case "scroll": + guard step.elementId != nil || (step.x != nil && step.y != nil) else { throw ComputerError("invalid_arguments", "A scroll step needs an elementId or x and y.") } + guard step.deltaX != nil || step.deltaY != nil else { throw ComputerError("invalid_arguments", "A scroll step needs deltaX or deltaY.") } default: break } units += (step.text?.utf16.count ?? 0) + (step.value?.utf16.count ?? 0) @@ -799,7 +803,11 @@ final class Driver { } for (index, step) in steps.enumerated() { // Stop can arrive during the previous step's render wait; no later step may start after it. - if index > 0 { do { try checkCancellation() } catch { return stopped("Stopped before step \(index + 1).") } } + do { try checkCancellation() } catch { + // Before any step ran nothing was sent, so Stop fails the batch as cancelled. + if completed == 0 { throw error } + return stopped("Stopped before step \(index + 1).") + } let before = uiStructure(pid: pid) do { _ = try await act(Request(id: request.id, method: try required(step.action, "action"), params: step), app: app, bundle: bundle, snapshot: snapshot) } catch { diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index ef436a613f3..c5a6d238c2c 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -57,6 +57,8 @@ import AppKit expectError("invalid_arguments") { try validateBatchSteps(Array(repeating: Parameters(action: "press_key", key: "Tab"), count: 9)) } expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "drag"), Parameters(action: "press_key", key: "Enter")]) } expectError("unsupported_key") { try validateBatchSteps([Parameters(action: "click", elementId: "e1"), Parameters(action: "press_key", key: "F99")]) } + expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", elementId: "e1"), Parameters(action: "click")]) } + expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", elementId: "e1"), Parameters(action: "scroll", elementId: "e2")]) } expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", elementId: "e1"), Parameters(action: "type_text", elementId: "e2")]) } expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "set_value", value: "x"), Parameters(action: "click", elementId: "e1")]) } expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", elementId: "e1", clickCount: 4), Parameters(action: "press_key", key: "Enter")]) } diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts index d366836c9f6..72db13ab9cf 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts @@ -178,8 +178,8 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { let bytes = jsonBytes(model) + envelopeBytes const add = (ids: string[]) => { const additional = ids.filter((id) => !selected.has(id)) - /** A row's JSON length less its two quotes, plus the newline that joins it. */ - const cost = additional.reduce((total, id) => total + jsonBytes(rows.get(id)) - 1, 0) + /** A row's JSON length: its two quotes come off, and the escaped joining newline (`\\n`) adds two bytes back. */ + const cost = additional.reduce((total, id) => total + jsonBytes(rows.get(id)), 0) if (bytes + cost > MAX_STATE_BYTES) return false for (const id of additional) selected.add(id) bytes += cost From b3da66485b03c526de1d5184540aadb4d4e7aaf6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 13:47:40 -0800 Subject: [PATCH 3/3] fix(computer-use): send no text when the caret cannot be moved to the end after a focus click --- apps/desktop/native/computer-use/ComputerUse.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 41b3b07c83c..8cb917a8210 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -639,7 +639,9 @@ func focusByClicking(_ element: AXUIElement, pid: pid_t, snapshot: Snapshot) asy if AXUIElementIsAttributeSettable(editor, kAXSelectedTextRangeAttribute as CFString, &settable) == .success, settable.boolValue, let value = attribute(editor, kAXValueAttribute) as? String { var range = CFRange(location: value.utf16.count, length: 0) - if let end = AXValueCreate(.cfRange, &range) { _ = AXUIElementSetAttributeValue(editor, kAXSelectedTextRangeAttribute as CFString, end) } + guard let end = AXValueCreate(.cfRange, &range), AXUIElementSetAttributeValue(editor, kAXSelectedTextRangeAttribute as CFString, end) == .success else { + throw ComputerError("caret_unconfirmed", "The editor was clicked into focus, but its caret could not be moved to the end, so no text was sent. Observe the app, then use set_value or move the caret with press_key.") + } } return }