From ffbcd94846cb5a4276d6c896ec94aec5e44bbee3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 12 Nov 2026 12:03:32 -0800 Subject: [PATCH 1/4] improvement(computer-use): read each node's attributes in one Accessibility round trip Index observed elements by CFHash so membership and parent lookups stay linear, and query value settability only for text roles. --- .../native/computer-use/ComputerUse.swift | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index bb5b604d746..ed20e149ace 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -86,6 +86,30 @@ func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? { var value: CFTypeRef? return AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success ? value : nil } +/// Reads several attributes in one Accessibility round trip instead of one per attribute; an attribute +/// the element lacks or fails to return is omitted, as `attribute` would return nil for it. +func attributes(_ element: AXUIElement, _ names: [String]) -> [String: CFTypeRef] { + var raw: CFArray? + guard AXUIElementCopyMultipleAttributeValues(element, names as CFArray, AXCopyMultipleAttributeOptions(rawValue: 0), &raw) == .success, + let values = raw as [AnyObject]?, values.count == names.count else { + return names.reduce(into: [:]) { result, name in result[name] = attribute(element, name) } + } + var result: [String: CFTypeRef] = [:] + for (name, value) in zip(names, values) { + if value is NSNull { continue } + if CFGetTypeID(value) == AXValueGetTypeID(), AXValueGetType(unsafeBitCast(value, to: AXValue.self)) == .axError { continue } + result[name] = value + } + return result +} +/// Elements bucketed by CFHash, so a walk's membership and parent lookups stay linear in its size. +struct ElementIndex { + private var buckets: [CFHashCode: [(element: AXUIElement, id: String)]] = [:] + func id(of element: AXUIElement) -> String? { buckets[CFHash(element)]?.first { CFEqual($0.element, element) }?.id } + mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)) } +} +let NODE_ATTRIBUTES = [kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXDescriptionAttribute, kAXPlaceholderValueAttribute, + kAXHelpAttribute, kAXValueAttribute, kAXFocusedAttribute, kAXEnabledAttribute, kAXPositionAttribute, kAXSizeAttribute] func secure(_ element: AXUIElement) -> Bool { attribute(element, kAXSubroleAttribute) as? String == kAXSecureTextFieldSubrole } @@ -135,8 +159,11 @@ func axActions(_ element: AXUIElement) -> [String] { return AXUIElementCopyActionNames(element, &names) == .success ? names as? [String] ?? [] : [] } func elementRect(_ element: AXUIElement) -> CGRect? { - guard let rawPosition = attribute(element, kAXPositionAttribute), CFGetTypeID(rawPosition) == AXValueGetTypeID(), - let rawSize = attribute(element, kAXSizeAttribute), CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil } + axRect(position: attribute(element, kAXPositionAttribute), size: attribute(element, kAXSizeAttribute)) +} +func axRect(position rawPosition: CFTypeRef?, size rawSize: CFTypeRef?) -> CGRect? { + guard let rawPosition, CFGetTypeID(rawPosition) == AXValueGetTypeID(), + let rawSize, CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil } var position = CGPoint.zero; var size = CGSize.zero guard AXValueGetValue(unsafeBitCast(rawPosition, to: AXValue.self), .cgPoint, &position), AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil } @@ -309,12 +336,14 @@ final class Snapshot { queue.append(AXTraversalEntry(element: unsafeBitCast(raw, to: AXUIElement.self), parent: root, depth: 1, menu: false, priority: true)) } var menuCount = 0 + var index = ElementIndex() while nodes.count < 2000 && Date().timeIntervalSince(created) < 8, let entry = queue.next() { try checkCancellation() let element = entry.element AXUIElementSetMessagingTimeout(element, 0.25) - if elements.values.contains(where: { CFEqual($0, element) }) { continue } - let role = attribute(element, kAXRoleAttribute) as? String ?? "AXUnknown" + if index.id(of: element) != nil { continue } + let values = attributes(element, NODE_ATTRIBUTES) + let role = values[kAXRoleAttribute] as? String ?? "AXUnknown" let isMenu = entry.menu || [kAXMenuBarRole, kAXMenuRole, kAXMenuItemRole, kAXMenuBarItemRole].contains(role) if role == kAXMenuItemRole && !entry.menu && !entry.priority { queue.append(AXTraversalEntry(element: element, parent: entry.parent, depth: entry.depth, menu: true, priority: false)); continue @@ -323,27 +352,28 @@ final class Snapshot { guard menuCount < 80 else { truncated = true; continue } menuCount += 1 } - let id = "e\(nodes.count)"; elements[id] = element + let id = "e\(nodes.count)"; elements[id] = element; index.insert(element, id: id) var node: [String: Any] = ["elementId": id, "role": role, "actions": [String]()] - if let parent = entry.parent, let parentID = elements.first(where: { CFEqual($0.value, parent) })?.key { node["parentId"] = parentID } - let isSecure = secure(element) - var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { attribute(element, $0) as? String }.filter { !$0.isEmpty } - if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { attribute(element, $0) as? String }.filter { !$0.isEmpty } } + if let parent = entry.parent, let parentID = index.id(of: parent) { node["parentId"] = parentID } + let isSecure = values[kAXSubroleAttribute] as? String == kAXSecureTextFieldSubrole + var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } + if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } } if role == kAXMenuItemRole, let shortcut = menuShortcut(element) { labels.append(shortcut) } if !isSecure, !labels.isEmpty { node["label"] = String(labels.joined(separator: " ").prefix(1024)) } - if !isSecure, let raw = attribute(element, kAXValueAttribute) { + if !isSecure, let raw = values[kAXValueAttribute] { if let value = raw as? String { node["value"] = String(value.prefix(2048)) } else if let value = raw as? NSNumber { node["value"] = value.stringValue } } if !isSecure { - if let focused = attribute(element, kAXFocusedAttribute) as? Bool { node["focused"] = focused } + if let focused = values[kAXFocusedAttribute] as? Bool { node["focused"] = focused } var writable: DarwinBoolean = false - let writableValue = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &writable) == .success && writable.boolValue - node["editable"] = [kAXTextAreaRole, kAXTextFieldRole].contains(role) && writableValue - if let placeholder = attribute(element, kAXPlaceholderValueAttribute) as? String { node["placeholder"] = String(placeholder.prefix(1024)) } + let writableValue = [kAXTextAreaRole, kAXTextFieldRole].contains(role) + && AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &writable) == .success && writable.boolValue + node["editable"] = writableValue + if let placeholder = values[kAXPlaceholderValueAttribute] as? String { node["placeholder"] = String(placeholder.prefix(1024)) } } - if let enabled = attribute(element, kAXEnabledAttribute) as? Bool { node["enabled"] = enabled } - if let rect = elementRect(element) { node.merge(rectJSON(rect)) { _, new in new } } + if let enabled = values[kAXEnabledAttribute] as? Bool { node["enabled"] = enabled } + if let rect = axRect(position: values[kAXPositionAttribute], size: values[kAXSizeAttribute]) { node.merge(rectJSON(rect)) { _, new in new } } var actions: CFArray? if AXUIElementCopyActionNames(element, &actions) == .success { node["actions"] = Array((actions as? [String] ?? []).prefix(128)).map { String($0.prefix(128)) } } nodes.append(node) From cac68d6cf860649cad7f8bbc9f42ced0d2642a41 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 12 Nov 2026 13:17:44 -0800 Subject: [PATCH 2/4] feat(computer-use): batch 2-8 actions planned from one snapshot Each step runs through its single action's checks; the batch stops at the first failure and after any step that changes the app's window structure, and returns one fresh observation. --- .../native/computer-use/ComputerUse.swift | 62 +++++++++++- apps/desktop/native/computer-use/README.md | 2 +- .../native/computer-use/tests/Fixture.swift | 4 +- .../computer-use/tests/NativeTests.swift | 18 ++-- .../desktop/native/computer-use/tests/live.py | 40 +++++++- apps/sim/components/computer-use/activity.tsx | 1 + .../src/computer-use.generated.ts | 96 ++++++++++++++++--- 7 files changed, 197 insertions(+), 26 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index ed20e149ace..082b430b1ba 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -16,9 +16,10 @@ struct ComputerError: Error { let dispatchState: String? init(_ code: String, _ message: String, dispatchState: String? = nil) { self.code = code; self.message = message; self.dispatchState = dispatchState } } -struct InputStep: Decodable { let action: String; let text: String?; let key: String? } +/// A request's arguments. An input sequence or batch step is itself a request's arguments, with its action. struct Parameters: Decodable { - var steps: [InputStep]? + var action: String? + var steps: [Parameters]? var activateFirst: Bool? var permission: String? var bundleId: String? @@ -460,7 +461,20 @@ func parseKey(_ key: String) throws -> (CGKeyCode, CGEventFlags) { let aliases = ["esc": "escape", "arrowleft": "left", "arrowright": "right", "arrowup": "up", "arrowdown": "down", "forwarddelete": "delete", "pgup": "pageup", "pgdn": "pagedown", "pgdown": "pagedown", "minus": "-", "equal": "=", "equals": "=", "comma": ",", "period": ".", "slash": "/", "semicolon": ";", "quote": "'", "backslash": "\\", "backtick": "`", "grave": "`", "leftbracket": "[", "rightbracket": "]"] guard let code = keys[aliases[name] ?? name] else { throw ComputerError("unsupported_key", "Unsupported key name. Use names such as Enter, Tab, Escape, Backspace, Delete (forward delete), Left, PageUp, F5, or a single character, with Cmd/Shift/Alt/Ctrl modifiers.") }; return (code, flags) } -func validateInputSteps(_ steps: [InputStep]) throws { +/// The actions a batch may run; each is validated again by its own action path when it runs. +let BATCH_ACTIONS: Set = ["click", "type_text", "set_value", "press_key", "scroll", "perform_action"] +func validateBatchSteps(_ steps: [Parameters]) throws { + guard (2...8).contains(steps.count) else { throw ComputerError("invalid_arguments", "A batch requires 2 to 8 steps.") } + var units = 0 + for step in steps { + guard let action = step.action, BATCH_ACTIONS.contains(action), step.bundleId == nil, step.snapshotId == nil, step.steps == nil else { + throw ComputerError("invalid_arguments", "Batch steps are click, type_text, set_value, press_key, scroll or perform_action, without their own app or snapshot.") + } + units += (step.text?.utf16.count ?? 0) + (step.value?.utf16.count ?? 0) + } + guard units <= 32000 else { throw ComputerError("invalid_arguments", "Combined batch text exceeds 32000 UTF-16 units.") } +} +func validateInputSteps(_ steps: [Parameters]) throws { guard (1...32).contains(steps.count) else { throw ComputerError("invalid_arguments", "Input sequence requires 1 to 32 steps.") } var units = 0 for step in steps { @@ -700,7 +714,7 @@ final class Driver { } result["isActive"] = app.isActive return result - case "click", "type_text", "input_sequence", "press_key", "scroll", "drag", "set_value", "perform_action": break + case "click", "type_text", "input_sequence", "press_key", "scroll", "drag", "set_value", "perform_action", "batch": break default: throw ComputerError("unknown_method", "Unknown computer-use method.") } try requireAccessibility() @@ -725,6 +739,29 @@ final class Driver { let p = request.params let pid = app.processIdentifier switch request.method { + case "batch": + let steps = try required(p.steps, "steps") + try validateBatchSteps(steps) + var completed = 0 + func stopped(_ message: String) -> [String: Any] { + ["kind": "action", "action": "batch", "bundleId": bundle, "dispatched": true, "verified": false, + "sequence": ["completedSteps": completed, "totalSteps": steps.count, "error": String((message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch.").prefix(2000))]] + } + for (index, step) in steps.enumerated() { + var params = step; params.bundleId = p.bundleId; params.snapshotId = p.snapshotId + let before = uiStructure(pid: pid) + do { _ = try await act(Request(id: request.id, method: try required(step.action, "action"), params: params), app: app, bundle: bundle, snapshot: snapshot) } + catch { + // A first step that sent nothing fails like the single action, keeping the snapshot valid. + if !inputDispatched { throw error } + return stopped("Step \(index + 1) failed: " + ((error as? ComputerError)?.message ?? "its outcome is unknown.")) + } + completed += 1 + if completed < steps.count, uiStructure(pid: pid) != before { + return stopped("Step \(index + 1) opened or closed a window, sheet, menu or popover.") + } + } + return ["kind": "action", "action": "batch", "bundleId": bundle, "dispatched": true, "verified": false, "sequence": ["completedSteps": completed, "totalSteps": steps.count]] case "perform_action": let element = try snapshot.element(p.elementId); let action = try required(p.accessibilityAction, "accessibilityAction") var names: CFArray?; try axCheck(AXUIElementCopyActionNames(element, &names)) @@ -761,7 +798,7 @@ final class Driver { } case "type_text", "input_sequence": let element = try snapshot.element(p.elementId) - let steps = request.method == "type_text" ? [InputStep(action: "type_text", text: try required(p.text, "text"), key: nil)] : try required(p.steps, "steps") + let steps = request.method == "type_text" ? [Parameters(action: "type_text", text: try required(p.text, "text"))] : try required(p.steps, "steps") try validateInputSteps(steps) let root = AXUIElementCreateApplication(pid) // Keystrokes go to whatever holds keyboard focus: a text control (terminals included, read-only @@ -909,6 +946,21 @@ final class Driver { return ["kind": "action", "action": request.method, "bundleId": bundle, "dispatched": true, "verified": false] } } +/// The app's window structure: its windows, which one has focus and its direct content (a sheet is +/// a child of its window), and on-screen surfaces such as menus and popovers. +func uiStructure(pid: pid_t) -> [Int] { + let root = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(root, 0.1) + var structure = [(attribute(root, kAXWindowsAttribute) as? [AXUIElement])?.count ?? -1] + if let raw = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(raw) == AXUIElementGetTypeID() { + var count = 0 + structure.append(Int(bitPattern: UInt(CFHash(raw)))) + structure.append(AXUIElementGetAttributeValueCount(unsafeBitCast(raw, to: AXUIElement.self), kAXChildrenAttribute as CFString, &count) == .success ? count : -1) + } + structure.append((CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] ?? []) + .filter { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid }.count) + return structure +} func nextLine() throws -> String? { var bytes = [UInt8](); bytes.reserveCapacity(4096) var oversized = false diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index c615ec6649e..c9a4287f719 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -14,7 +14,7 @@ Coordinates in action inputs are window-local points; node frames are global scr AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A hit on a window's own background counts as that window. Background tabs and minimized windows share bounds with the window in front, so they take no coordinate input. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves the item's own menu open with the app highlighting that item (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. -`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date and time pickers, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. +`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date and time pickers, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. `batch` runs 2–8 click, type_text, set_value, press_key, scroll or perform_action steps planned from one snapshot, each through its own action's checks; it stops at the first step that fails and after any step that opens or closes a window, sheet, menu or popover, reports progress the same way, and a first step that sent nothing is refused with `dispatchState: "not_started"`. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. ## Cancellation diff --git a/apps/desktop/native/computer-use/tests/Fixture.swift b/apps/desktop/native/computer-use/tests/Fixture.swift index 6445f5bbbfb..06ffccf17b5 100644 --- a/apps/desktop/native/computer-use/tests/Fixture.swift +++ b/apps/desktop/native/computer-use/tests/Fixture.swift @@ -31,6 +31,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { let field = NSTextField(string: "Fixture text"); field.setAccessibilityIdentifier("fixture-text"); field.setAccessibilityLabel("Fixture editable text") let secure = NSSecureTextField(string: "fixture-only-secret"); secure.setAccessibilityIdentifier("fixture-password"); secure.setAccessibilityLabel("Fixture password") let button = NSButton(title: "Increment", target: self, action: #selector(increment)); button.setAccessibilityIdentifier("fixture-increment") + let sheetButton = NSButton(title: "Show sheet", target: self, action: #selector(showSheet)); sheetButton.setAccessibilityIdentifier("fixture-show-sheet") let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider") let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time") time.dateValue = Calendar.current.date(bySettingHour: 10, minute: 15, second: 0, of: Date()) ?? Date(); time.target = self; time.action = #selector(timeChanged(_:)) @@ -43,7 +44,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { scroll.documentView = document scroll.contentView.postsBoundsChangedNotifications = true NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView) - for view in [field, secure, button, counter, slider, time, timeLabel, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) } + for view in [field, secure, button, counter, sheetButton, slider, time, timeLabel, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) } window.contentView!.addSubview(stack) NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)]) window.makeKeyAndOrderFront(nil) @@ -52,6 +53,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { print("fixture-ready"); fflush(stdout) } @objc func increment() { count += 1; counter.stringValue = "Count: \(count)" } + @objc func showSheet() { let alert = NSAlert(); alert.messageText = "Fixture sheet"; alert.addButton(withTitle: "Dismiss"); alert.beginSheetModal(for: window) } @objc func timeChanged(_ picker: NSDatePicker) { let time = Calendar.current.dateComponents([.hour, .minute], from: picker.dateValue) timeLabel.stringValue = "Time: \(time.hour ?? 0):\(String(format: "%02d", time.minute ?? 0))" diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 3de6f79b94a..39969e61287 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -45,13 +45,19 @@ import AppKit precondition(bounded.next()?.element == 1) precondition(bounded.next()?.element == 2) precondition(bounded.next() == nil) - try validateInputSteps([InputStep(action: "press_key", text: nil, key: "Cmd+A"), InputStep(action: "type_text", text: "hello 🌍", key: nil)]) - try validateInputSteps((1...5).flatMap { [InputStep(action: "type_text", text: String($0), key: nil), InputStep(action: "press_key", text: nil, key: "Enter")] }) + try validateInputSteps([Parameters(action: "press_key", key: "Cmd+A"), Parameters(action: "type_text", text: "hello 🌍")]) + try validateInputSteps((1...5).flatMap { [Parameters(action: "type_text", text: String($0)), Parameters(action: "press_key", key: "Enter")] }) expectError("invalid_arguments") { try validateInputSteps([]) } - expectError("invalid_arguments") { try validateInputSteps(Array(repeating: InputStep(action: "type_text", text: "x", key: nil), count: 33)) } - expectError("invalid_arguments") { try validateInputSteps([InputStep(action: "type_text", text: String(repeating: "🌍", count: 16001), key: nil)]) } - expectError("invalid_arguments") { try validateInputSteps([InputStep(action: "click", text: nil, key: nil)]) } - expectError("unsupported_key") { try validateInputSteps([InputStep(action: "press_key", text: nil, key: "F99")]) } + expectError("invalid_arguments") { try validateInputSteps(Array(repeating: Parameters(action: "type_text", text: "x"), count: 33)) } + expectError("invalid_arguments") { try validateInputSteps([Parameters(action: "type_text", text: String(repeating: "🌍", count: 16001))]) } + expectError("invalid_arguments") { try validateInputSteps([Parameters(action: "click")]) } + expectError("unsupported_key") { try validateInputSteps([Parameters(action: "press_key", key: "F99")]) } + try validateBatchSteps([Parameters(action: "set_value", elementId: "e1", value: "Name"), Parameters(action: "click", elementId: "e2")]) + expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", elementId: "e1")]) } + expectError("invalid_arguments") { try validateBatchSteps(Array(repeating: Parameters(action: "press_key", key: "Tab"), count: 9)) } + expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "drag"), Parameters(action: "press_key", key: "Enter")]) } + expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "click", bundleId: "com.example", elementId: "e1"), Parameters(action: "press_key", key: "Enter")]) } + expectError("invalid_arguments") { try validateBatchSteps([Parameters(action: "set_value", elementId: "e1", value: String(repeating: "🌍", count: 8001)), Parameters(action: "type_text", elementId: "e2", text: String(repeating: "🌍", count: 8000))]) } let tabFocus = try keyMayChangeFocus("Tab"); precondition(tabFocus) let submitFocus = try keyMayChangeFocus("Enter"); precondition(!submitFocus) let searchFocus = try keyMayChangeFocus("Cmd+K"); precondition(searchFocus) diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index f6ddf624952..9c34cd0621b 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -130,7 +130,45 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): button = find(snapshot, role='AXButton', label='Increment') snapshot = mutate('perform_action', snapshot, elementId=button['elementId'], accessibilityAction='AXPress') assert any(f'Count: {baseline + 2}' in node.get('value', '') or f'Count: {baseline + 2}' in node.get('label', '') for node in snapshot['nodes']), 'Explicit AX action did not increment' - print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) + def count(snapshot): + return int(next(node.get('value', node.get('label', '')) for node in snapshot['nodes'] if node.get('value', node.get('label', '')).startswith('Count: ')).split(': ')[1]) + + def batch(snapshot, steps): + result = call('batch', bundleId=bundle, snapshotId=snapshot['snapshotId'], steps=steps) + assert result['action'] == 'batch' and result['dispatched'] and not result['verified'], result + return result['sequence'], state() + + snapshot = state() + before = count(snapshot) + field = find(snapshot, role='AXTextField', label='Fixture editable text') + button = find(snapshot, role='AXButton', label='Increment') + progress, snapshot = batch(snapshot, [ + {'action': 'set_value', 'elementId': field['elementId'], 'value': 'Batched'}, + {'action': 'click', 'elementId': button['elementId']}, + {'action': 'perform_action', 'elementId': button['elementId'], 'accessibilityAction': 'AXPress'}, + ]) + assert progress == {'completedSteps': 3, 'totalSteps': 3}, progress + assert find(snapshot, role='AXTextField', label='Fixture editable text')['value'] == 'Batched' and count(snapshot) == before + 2, 'Batch steps did not all apply' + slider = find(snapshot, role='AXSlider') + progress, snapshot = batch(snapshot, [ + {'action': 'click', 'elementId': find(snapshot, role='AXButton', label='Increment')['elementId']}, + {'action': 'type_text', 'elementId': slider['elementId'], 'text': 'x'}, + {'action': 'click', 'elementId': find(snapshot, role='AXButton', label='Increment')['elementId']}, + ]) + assert progress['completedSteps'] == 1 and 'Step 2 failed' in progress['error'] and count(snapshot) == before + 3, progress + error = refused('batch', snapshot, steps=[ + {'action': 'type_text', 'elementId': find(snapshot, role='AXSlider')['elementId'], 'text': 'x'}, + {'action': 'click', 'elementId': find(snapshot, role='AXButton', label='Increment')['elementId']}, + ]) + assert error['dispatchState'] == 'not_started' and count(state()) == before + 3, error + snapshot = state() + progress, snapshot = batch(snapshot, [ + {'action': 'click', 'elementId': find(snapshot, role='AXButton', label='Show sheet')['elementId']}, + {'action': 'click', 'elementId': find(snapshot, role='AXButton', label='Increment')['elementId']}, + ]) + assert progress['completedSteps'] == 1 and 'opened or closed' in progress['error'] and count(snapshot) == before + 3, progress + snapshot = mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Dismiss')['elementId']) + print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: if fixture: fixture.terminate() diff --git a/apps/sim/components/computer-use/activity.tsx b/apps/sim/components/computer-use/activity.tsx index 384d8d0764f..f365b9bddee 100644 --- a/apps/sim/components/computer-use/activity.tsx +++ b/apps/sim/components/computer-use/activity.tsx @@ -17,6 +17,7 @@ const ACTION_LABELS: Record = { drag: 'Dragging', set_value: 'Editing', perform_action: 'Interacting', + batch: 'Running steps', } /** Remains visible while a native action is active, even if the permission group or device access revokes it. */ diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index 6cd48b45022..7444781564a 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -74,6 +74,56 @@ const ScrollOffsets = { const hasScrollDistance = (input: { deltaX?: number | undefined; deltaY?: number | undefined }) => input.deltaX !== undefined || input.deltaY !== undefined; +const TypedText = z.string().max(32_000).describe("Literal text to type into the observed editable element."); +const ReplacementValue = z + .string() + .max(32_000) + .describe("Replacement value for an accessibility-writable element."); +const KeyChord = z + .string() + .min(1) + .max(128) + .describe( + "One key or chord, such as Enter, Tab, Escape, or Cmd+A. With elementId, focuses that element first.", + ); +const AccessibilityAction = z.string().min(1).max(128).describe("Exact action advertised by this element."); + +/** A batch step is the matching single action without its app, snapshot and observation fields, which the batch carries. */ +const BatchStep = z.union([ + z.strictObject({ action: z.literal("click"), elementId: ElementId, ...ClickOptions }), + z.strictObject({ + action: z.literal("click"), + windowId: TargetWindowId, + x: Coordinate, + y: Coordinate, + ...ClickOptions, + }), + z.strictObject({ action: z.literal("type_text"), elementId: ElementId, text: TypedText }), + z.strictObject({ action: z.literal("set_value"), elementId: ElementId, value: ReplacementValue }), + z.strictObject({ + action: z.literal("press_key"), + windowId: TargetWindowId, + elementId: ElementId.optional(), + key: KeyChord, + }), + z + .strictObject({ action: z.literal("scroll"), elementId: ElementId, ...ScrollOffsets }) + .refine(hasScrollDistance, "Scroll needs deltaX or deltaY."), + z + .strictObject({ + action: z.literal("scroll"), + windowId: TargetWindowId, + x: Coordinate, + y: Coordinate, + ...ScrollOffsets, + }) + .refine(hasScrollDistance, "Scroll needs deltaX or deltaY."), + z.strictObject({ + action: z.literal("perform_action"), + elementId: ElementId, + accessibilityAction: AccessibilityAction, + }), +]); /** Strict branches require an observed, unambiguous target before native input. */ export const ComputerUseSchema = z.union([ @@ -92,7 +142,7 @@ export const ComputerUseSchema = z.union([ action: z.literal("type_text"), ...ObserveAfter, ...ElementTarget, - text: z.string().max(32_000).describe("Literal text to type into the observed editable element."), + text: TypedText, }), z.strictObject({ action: z.literal("press_key"), @@ -101,13 +151,7 @@ export const ComputerUseSchema = z.union([ snapshotId: SnapshotId, windowId: TargetWindowId, elementId: ElementId.optional(), - key: z - .string() - .min(1) - .max(128) - .describe( - "One key or chord, such as Enter, Tab, Escape, or Cmd+A. With elementId, focuses that element first.", - ), + key: KeyChord, }), z .strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }) @@ -126,7 +170,7 @@ export const ComputerUseSchema = z.union([ action: z.literal("set_value"), ...ObserveAfter, ...ElementTarget, - value: z.string().max(32_000).describe("Replacement value for an accessibility-writable element."), + value: ReplacementValue, }), z.strictObject({ action: z.literal("input_sequence"), @@ -156,7 +200,34 @@ export const ComputerUseSchema = z.union([ action: z.literal("perform_action"), ...ObserveAfter, ...ElementTarget, - accessibilityAction: z.string().min(1).max(128).describe("Exact action advertised by this element."), + accessibilityAction: AccessibilityAction, + }), + z.strictObject({ + action: z.literal("batch"), + bundleId: BundleId, + snapshotId: SnapshotId, + ...ObserveAfter, + steps: z + .array(BatchStep) + .min(2) + .max(8) + .refine( + (steps) => + steps.reduce( + (total, step) => + total + + (step.action === "type_text" + ? step.text.length + : step.action === "set_value" + ? step.value.length + : 0), + 0, + ) <= 32_000, + "Combined batch text exceeds 32000 UTF-16 code units", + ) + .describe( + "2–8 actions on this app planned from the same snapshot, run in order with that snapshot's references. The batch stops at the first step that fails and after any step that opens or closes a window, sheet, menu or popover; later steps do not run. Read sequence.completedSteps before continuing, and never automatically retry a partial batch.", + ), }), ]); export type ComputerUseInput = z.infer; @@ -250,6 +321,7 @@ export const ComputerUseResultSchema = z.discriminatedUnion("kind", [ "set_value", "perform_action", "input_sequence", + "batch", ]), bundleId: BundleId, dispatched: z.literal(true), @@ -265,11 +337,11 @@ export const ComputerUseResultSchema = z.discriminatedUnion("kind", [ observationError: z.string().min(1).max(2000).optional(), }) .superRefine((result, ctx) => { - if (result.action === "input_sequence" && !result.sequence) + if ((result.action === "input_sequence" || result.action === "batch") && !result.sequence) ctx.addIssue({ code: "custom", path: ["sequence"], - message: "Input sequences require dispatch progress", + message: "Input sequences and batches require dispatch progress", }); if (result.sequence && result.sequence.completedSteps > result.sequence.totalSteps) ctx.addIssue({ From afaf1398d502ad171b08135d4732a7f295415aa3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 12 Nov 2026 16:20:22 -0800 Subject: [PATCH 3/4] test(computer-use): check the sheet batch's skipped step after dismissing the sheet --- apps/desktop/native/computer-use/tests/live.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 9c34cd0621b..9310373ba98 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -166,8 +166,10 @@ def batch(snapshot, steps): {'action': 'click', 'elementId': find(snapshot, role='AXButton', label='Show sheet')['elementId']}, {'action': 'click', 'elementId': find(snapshot, role='AXButton', label='Increment')['elementId']}, ]) - assert progress['completedSteps'] == 1 and 'opened or closed' in progress['error'] and count(snapshot) == before + 3, progress + assert progress['completedSteps'] == 1 and 'opened or closed' in progress['error'], progress + # While the sheet is open only the sheet is observed; the step after it must not have run. snapshot = mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Dismiss')['elementId']) + assert count(snapshot) == before + 3, 'A batch step ran after a sheet opened' print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: if fixture: From d6b81a25c5a26d271442de95550a5d26c28b7499 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 01:04:25 -0800 Subject: [PATCH 4/4] fix(computer-use): stop a batch on menu or sheet switches and before any step after Stop The structure guard compares on-screen window and focused-window child identities instead of counts, and every step checks cancellation first. Synced the contract, which now bounds batch progress to 2-8 steps. --- .../native/computer-use/ComputerUse.swift | 18 +++++++----- .../src/computer-use.generated.ts | 29 ++++++++++++------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 082b430b1ba..8ca12493ef4 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -748,6 +748,8 @@ final class Driver { "sequence": ["completedSteps": completed, "totalSteps": steps.count, "error": String((message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch.").prefix(2000))]] } for (index, step) in steps.enumerated() { + // Stop can arrive during the previous step's settle; no later step may start after it. + do { try checkCancellation() } catch { if completed == 0 { throw error }; return stopped("Stopped before step \(index + 1).") } var params = step; params.bundleId = p.bundleId; params.snapshotId = p.snapshotId let before = uiStructure(pid: pid) do { _ = try await act(Request(id: request.id, method: try required(step.action, "action"), params: params), app: app, bundle: bundle, snapshot: snapshot) } @@ -946,19 +948,21 @@ final class Driver { return ["kind": "action", "action": request.method, "bundleId": bundle, "dispatched": true, "verified": false] } } -/// The app's window structure: its windows, which one has focus and its direct content (a sheet is -/// a child of its window), and on-screen surfaces such as menus and popovers. +/// The app's window structure by identity: its on-screen windows (sheets, menus and popovers are +/// windows of their own), the focused window, and that window's direct content, where a sheet sits. +/// Identities rather than counts, so switching one open menu or sheet for another still registers. func uiStructure(pid: pid_t) -> [Int] { let root = AXUIElementCreateApplication(pid) AXUIElementSetMessagingTimeout(root, 0.1) - var structure = [(attribute(root, kAXWindowsAttribute) as? [AXUIElement])?.count ?? -1] + var structure = (CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] ?? []) + .filter { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid } + .compactMap { ($0[kCGWindowNumber as String] as? NSNumber)?.intValue } + .sorted() if let raw = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(raw) == AXUIElementGetTypeID() { - var count = 0 structure.append(Int(bitPattern: UInt(CFHash(raw)))) - structure.append(AXUIElementGetAttributeValueCount(unsafeBitCast(raw, to: AXUIElement.self), kAXChildrenAttribute as CFString, &count) == .success ? count : -1) + let (children, _) = pagedElements(unsafeBitCast(raw, to: AXUIElement.self), name: kAXChildrenAttribute, limit: 64) + structure.append(contentsOf: children.map { Int(bitPattern: UInt(CFHash($0))) }) } - structure.append((CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] ?? []) - .filter { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid }.count) return structure } func nextLine() throws -> String? { diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index 7444781564a..c31d33cbe92 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -88,6 +88,13 @@ const KeyChord = z ); const AccessibilityAction = z.string().min(1).max(128).describe("Exact action advertised by this element."); +/** The text a batch step sends, counted against the batch's 32000 UTF-16 code units. */ +function batchStepTextLength(step: { action: string; text?: string; value?: string }): number { + if (step.action === "type_text") return step.text?.length ?? 0; + if (step.action === "set_value") return step.value?.length ?? 0; + return 0; +} + /** A batch step is the matching single action without its app, snapshot and observation fields, which the batch carries. */ const BatchStep = z.union([ z.strictObject({ action: z.literal("click"), elementId: ElementId, ...ClickOptions }), @@ -212,17 +219,7 @@ export const ComputerUseSchema = z.union([ .min(2) .max(8) .refine( - (steps) => - steps.reduce( - (total, step) => - total + - (step.action === "type_text" - ? step.text.length - : step.action === "set_value" - ? step.value.length - : 0), - 0, - ) <= 32_000, + (steps) => steps.reduce((total, step) => total + batchStepTextLength(step), 0) <= 32_000, "Combined batch text exceeds 32000 UTF-16 code units", ) .describe( @@ -343,6 +340,16 @@ export const ComputerUseResultSchema = z.discriminatedUnion("kind", [ path: ["sequence"], message: "Input sequences and batches require dispatch progress", }); + if ( + result.action === "batch" && + result.sequence && + (result.sequence.totalSteps < 2 || result.sequence.totalSteps > 8) + ) + ctx.addIssue({ + code: "custom", + path: ["sequence", "totalSteps"], + message: "A batch has 2 to 8 steps", + }); if (result.sequence && result.sequence.completedSteps > result.sequence.totalSteps) ctx.addIssue({ code: "custom",