feat(computer-use): batch 2-8 actions per call and read app state in one round trip per element

apps/desktop/native/computer-use/ComputerUse.swift

@@ -16,9 +16,10 @@ struct ComputerError: Error {
     let dispatchState: String?
     init(_ code: String, _ message: String, dispatchState: String? = nil) { self.code = code; self.message = message; self.dispatchState = dispatchState }
 }
-struct InputStep: Decodable { let action: String; let text: String?; let key: String? }
+/// A request's arguments. An input sequence or batch step is itself a request's arguments, with its action.
 struct Parameters: Decodable {
-    var steps: [InputStep]?
+    var action: String?
+    var steps: [Parameters]?
     var activateFirst: Bool?
     var permission: String?
     var bundleId: String?
@@ -86,6 +87,30 @@ func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
     var value: CFTypeRef?
     return AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success ? value : nil
 }
+/// Reads several attributes in one Accessibility round trip instead of one per attribute; an attribute
+/// the element lacks or fails to return is omitted, as `attribute` would return nil for it.
+func attributes(_ element: AXUIElement, _ names: [String]) -> [String: CFTypeRef] {
+    var raw: CFArray?
+    guard AXUIElementCopyMultipleAttributeValues(element, names as CFArray, AXCopyMultipleAttributeOptions(rawValue: 0), &raw) == .success,
+          let values = raw as [AnyObject]?, values.count == names.count else {
+        return names.reduce(into: [:]) { result, name in result[name] = attribute(element, name) }
+    }
+    var result: [String: CFTypeRef] = [:]
+    for (name, value) in zip(names, values) {
+        if value is NSNull { continue }
+        if CFGetTypeID(value) == AXValueGetTypeID(), AXValueGetType(unsafeBitCast(value, to: AXValue.self)) == .axError { continue }
+        result[name] = value
+    }
+    return result
+}
+/// Elements bucketed by CFHash, so a walk's membership and parent lookups stay linear in its size.
+struct ElementIndex {
+    private var buckets: [CFHashCode: [(element: AXUIElement, id: String)]] = [:]
+    func id(of element: AXUIElement) -> String? { buckets[CFHash(element)]?.first { CFEqual($0.element, element) }?.id }
+    mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)) }
+}
+let NODE_ATTRIBUTES = [kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXDescriptionAttribute, kAXPlaceholderValueAttribute,
+                       kAXHelpAttribute, kAXValueAttribute, kAXFocusedAttribute, kAXEnabledAttribute, kAXPositionAttribute, kAXSizeAttribute]
 func secure(_ element: AXUIElement) -> Bool {
     attribute(element, kAXSubroleAttribute) as? String == kAXSecureTextFieldSubrole
 }
@@ -135,8 +160,11 @@ func axActions(_ element: AXUIElement) -> [String] {
     return AXUIElementCopyActionNames(element, &names) == .success ? names as? [String] ?? [] : []
 }
 func elementRect(_ element: AXUIElement) -> CGRect? {
-    guard let rawPosition = attribute(element, kAXPositionAttribute), CFGetTypeID(rawPosition) == AXValueGetTypeID(),
-          let rawSize = attribute(element, kAXSizeAttribute), CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil }
+    axRect(position: attribute(element, kAXPositionAttribute), size: attribute(element, kAXSizeAttribute))
+}
+func axRect(position rawPosition: CFTypeRef?, size rawSize: CFTypeRef?) -> CGRect? {
+    guard let rawPosition, CFGetTypeID(rawPosition) == AXValueGetTypeID(),
+          let rawSize, CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil }
     var position = CGPoint.zero; var size = CGSize.zero
     guard AXValueGetValue(unsafeBitCast(rawPosition, to: AXValue.self), .cgPoint, &position),
           AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil }
@@ -309,12 +337,14 @@ final class Snapshot {
             queue.append(AXTraversalEntry(element: unsafeBitCast(raw, to: AXUIElement.self), parent: root, depth: 1, menu: false, priority: true))
         }
         var menuCount = 0
+        var index = ElementIndex()
         while nodes.count < 2000 && Date().timeIntervalSince(created) < 8, let entry = queue.next() {
             try checkCancellation()
             let element = entry.element
             AXUIElementSetMessagingTimeout(element, 0.25)
-            if elements.values.contains(where: { CFEqual($0, element) }) { continue }
-            let role = attribute(element, kAXRoleAttribute) as? String ?? "AXUnknown"
+            if index.id(of: element) != nil { continue }
+            let values = attributes(element, NODE_ATTRIBUTES)
+            let role = values[kAXRoleAttribute] as? String ?? "AXUnknown"
             let isMenu = entry.menu || [kAXMenuBarRole, kAXMenuRole, kAXMenuItemRole, kAXMenuBarItemRole].contains(role)
             if role == kAXMenuItemRole && !entry.menu && !entry.priority {
                 queue.append(AXTraversalEntry(element: element, parent: entry.parent, depth: entry.depth, menu: true, priority: false)); continue
@@ -323,27 +353,28 @@ final class Snapshot {
                 guard menuCount < 80 else { truncated = true; continue }
                 menuCount += 1
             }
-            let id = "e\(nodes.count)"; elements[id] = element
+            let id = "e\(nodes.count)"; elements[id] = element; index.insert(element, id: id)
             var node: [String: Any] = ["elementId": id, "role": role, "actions": [String]()]
-            if let parent = entry.parent, let parentID = elements.first(where: { CFEqual($0.value, parent) })?.key { node["parentId"] = parentID }
-            let isSecure = secure(element)
-            var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { attribute(element, $0) as? String }.filter { !$0.isEmpty }
-            if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { attribute(element, $0) as? String }.filter { !$0.isEmpty } }
+            if let parent = entry.parent, let parentID = index.id(of: parent) { node["parentId"] = parentID }
+            let isSecure = values[kAXSubroleAttribute] as? String == kAXSecureTextFieldSubrole
+            var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty }
+            if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } }
             if role == kAXMenuItemRole, let shortcut = menuShortcut(element) { labels.append(shortcut) }
             if !isSecure, !labels.isEmpty { node["label"] = String(labels.joined(separator: " ").prefix(1024)) }
-            if !isSecure, let raw = attribute(element, kAXValueAttribute) {
+            if !isSecure, let raw = values[kAXValueAttribute] {
                 if let value = raw as? String { node["value"] = String(value.prefix(2048)) }
                 else if let value = raw as? NSNumber { node["value"] = value.stringValue }
             }
             if !isSecure {
-                if let focused = attribute(element, kAXFocusedAttribute) as? Bool { node["focused"] = focused }
+                if let focused = values[kAXFocusedAttribute] as? Bool { node["focused"] = focused }
                 var writable: DarwinBoolean = false
-                let writableValue = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &writable) == .success && writable.boolValue
-                node["editable"] = [kAXTextAreaRole, kAXTextFieldRole].contains(role) && writableValue
-                if let placeholder = attribute(element, kAXPlaceholderValueAttribute) as? String { node["placeholder"] = String(placeholder.prefix(1024)) }
+                let writableValue = [kAXTextAreaRole, kAXTextFieldRole].contains(role)
+                    && AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &writable) == .success && writable.boolValue
+                node["editable"] = writableValue
+                if let placeholder = values[kAXPlaceholderValueAttribute] as? String { node["placeholder"] = String(placeholder.prefix(1024)) }
             }
-            if let enabled = attribute(element, kAXEnabledAttribute) as? Bool { node["enabled"] = enabled }
-            if let rect = elementRect(element) { node.merge(rectJSON(rect)) { _, new in new } }
+            if let enabled = values[kAXEnabledAttribute] as? Bool { node["enabled"] = enabled }
+            if let rect = axRect(position: values[kAXPositionAttribute], size: values[kAXSizeAttribute]) { node.merge(rectJSON(rect)) { _, new in new } }
             var actions: CFArray?
             if AXUIElementCopyActionNames(element, &actions) == .success { node["actions"] = Array((actions as? [String] ?? []).prefix(128)).map { String($0.prefix(128)) } }
             nodes.append(node)
@@ -430,7 +461,20 @@ func parseKey(_ key: String) throws -> (CGKeyCode, CGEventFlags) {
             let aliases = ["esc": "escape", "arrowleft": "left", "arrowright": "right", "arrowup": "up", "arrowdown": "down", "forwarddelete": "delete", "pgup": "pageup", "pgdn": "pagedown", "pgdown": "pagedown", "minus": "-", "equal": "=", "equals": "=", "comma": ",", "period": ".", "slash": "/", "semicolon": ";", "quote": "'", "backslash": "\\", "backtick": "`", "grave": "`", "leftbracket": "[", "rightbracket": "]"]
             guard let code = keys[aliases[name] ?? name] else { throw ComputerError("unsupported_key", "Unsupported key name. Use names such as Enter, Tab, Escape, Backspace, Delete (forward delete), Left, PageUp, F5, or a single character, with Cmd/Shift/Alt/Ctrl modifiers.") }; return (code, flags)
 }
-func validateInputSteps(_ steps: [InputStep]) throws {
+/// The actions a batch may run; each is validated again by its own action path when it runs.
+let BATCH_ACTIONS: Set<String> = ["click", "type_text", "set_value", "press_key", "scroll", "perform_action"]
+func validateBatchSteps(_ steps: [Parameters]) throws {
+    guard (2...8).contains(steps.count) else { throw ComputerError("invalid_arguments", "A batch requires 2 to 8 steps.") }
+    var units = 0
+    for step in steps {
+        guard let action = step.action, BATCH_ACTIONS.contains(action), step.bundleId == nil, step.snapshotId == nil, step.steps == nil else {
+            throw ComputerError("invalid_arguments", "Batch steps are click, type_text, set_value, press_key, scroll or perform_action, without their own app or snapshot.")
+        }
+        units += (step.text?.utf16.count ?? 0) + (step.value?.utf16.count ?? 0)
+    }
+    guard units <= 32000 else { throw ComputerError("invalid_arguments", "Combined batch text exceeds 32000 UTF-16 units.") }
+}
+func validateInputSteps(_ steps: [Parameters]) throws {
     guard (1...32).contains(steps.count) else { throw ComputerError("invalid_arguments", "Input sequence requires 1 to 32 steps.") }
     var units = 0
     for step in steps {
@@ -670,7 +714,7 @@ final class Driver {
             }
             result["isActive"] = app.isActive
             return result
-        case "click", "type_text", "input_sequence", "press_key", "scroll", "drag", "set_value", "perform_action": break
+        case "click", "type_text", "input_sequence", "press_key", "scroll", "drag", "set_value", "perform_action", "batch": break
         default: throw ComputerError("unknown_method", "Unknown computer-use method.")
         }
         try requireAccessibility()
@@ -695,6 +739,31 @@ final class Driver {
         let p = request.params
         let pid = app.processIdentifier
         switch request.method {
+        case "batch":
+            let steps = try required(p.steps, "steps")
+            try validateBatchSteps(steps)
+            var completed = 0
+            func stopped(_ message: String) -> [String: Any] {
+                ["kind": "action", "action": "batch", "bundleId": bundle, "dispatched": true, "verified": false,
+                 "sequence": ["completedSteps": completed, "totalSteps": steps.count, "error": String((message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch.").prefix(2000))]]
+            }
+            for (index, step) in steps.enumerated() {
+                // Stop can arrive during the previous step's settle; no later step may start after it.
+                do { try checkCancellation() } catch { if completed == 0 { throw error }; return stopped("Stopped before step \(index + 1).") }
+                var params = step; params.bundleId = p.bundleId; params.snapshotId = p.snapshotId
+                let before = uiStructure(pid: pid)
+                do { _ = try await act(Request(id: request.id, method: try required(step.action, "action"), params: params), app: app, bundle: bundle, snapshot: snapshot) }
+                catch {
+                    // A first step that sent nothing fails like the single action, keeping the snapshot valid.
+                    if !inputDispatched { throw error }
+                    return stopped("Step \(index + 1) failed: " + ((error as? ComputerError)?.message ?? "its outcome is unknown."))
+                }
+                completed += 1
+                if completed < steps.count, uiStructure(pid: pid) != before {
+                    return stopped("Step \(index + 1) opened or closed a window, sheet, menu or popover.")
+                }
+            }
+            return ["kind": "action", "action": "batch", "bundleId": bundle, "dispatched": true, "verified": false, "sequence": ["completedSteps": completed, "totalSteps": steps.count]]
         case "perform_action":
             let element = try snapshot.element(p.elementId); let action = try required(p.accessibilityAction, "accessibilityAction")
             var names: CFArray?; try axCheck(AXUIElementCopyActionNames(element, &names))
@@ -731,7 +800,7 @@ final class Driver {
             }
         case "type_text", "input_sequence":
             let element = try snapshot.element(p.elementId)
-            let steps = request.method == "type_text" ? [InputStep(action: "type_text", text: try required(p.text, "text"), key: nil)] : try required(p.steps, "steps")
+            let steps = request.method == "type_text" ? [Parameters(action: "type_text", text: try required(p.text, "text"))] : try required(p.steps, "steps")
             try validateInputSteps(steps)
             let root = AXUIElementCreateApplication(pid)
             // Keystrokes go to whatever holds keyboard focus: a text control (terminals included, read-only
@@ -879,6 +948,23 @@ final class Driver {
         return ["kind": "action", "action": request.method, "bundleId": bundle, "dispatched": true, "verified": false]
     }
 }
+/// The app's window structure by identity: its on-screen windows (sheets, menus and popovers are
+/// windows of their own), the focused window, and that window's direct content, where a sheet sits.
+/// Identities rather than counts, so switching one open menu or sheet for another still registers.
+func uiStructure(pid: pid_t) -> [Int] {
+    let root = AXUIElementCreateApplication(pid)
+    AXUIElementSetMessagingTimeout(root, 0.1)
+    var structure = (CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] ?? [])
+        .filter { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid }
+        .compactMap { ($0[kCGWindowNumber as String] as? NSNumber)?.intValue }
+        .sorted()
+    if let raw = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(raw) == AXUIElementGetTypeID() {
+        structure.append(Int(bitPattern: UInt(CFHash(raw))))
+        let (children, _) = pagedElements(unsafeBitCast(raw, to: AXUIElement.self), name: kAXChildrenAttribute, limit: 64)
+        structure.append(contentsOf: children.map { Int(bitPattern: UInt(CFHash($0))) })
+    }
+    return structure
+}
 func nextLine() throws -> String? {
     var bytes = [UInt8](); bytes.reserveCapacity(4096)
     var oversized = false