Skip to content

Commit cd82e4f

Browse files
authored
feat(computer-use): batch 2-8 actions per call and read app state in one round trip per element (#9995)
* improvement(computer-use): read each node's attributes in one Accessibility round trip Index observed elements by CFHash so membership and parent lookups stay linear, and query value settability only for text roles. * feat(computer-use): batch 2-8 actions planned from one snapshot Each step runs through its single action's checks; the batch stops at the first failure and after any step that changes the app's window structure, and returns one fresh observation. * test(computer-use): check the sheet batch's skipped step after dismissing the sheet * fix(computer-use): stop a batch on menu or sheet switches and before any step after Stop The structure guard compares on-screen window and focused-window child identities instead of counts, and every step checks cancellation first. Synced the contract, which now bounds batch progress to 2-8 steps.
1 parent 96c4b0b commit cd82e4f

7 files changed

Lines changed: 256 additions & 42 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 107 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,10 @@ struct ComputerError: Error {
1616
let dispatchState: String?
1717
init(_ code: String, _ message: String, dispatchState: String? = nil) { self.code = code; self.message = message; self.dispatchState = dispatchState }
1818
}
19-
struct InputStep: Decodable { let action: String; let text: String?; let key: String? }
19+
/// A request's arguments. An input sequence or batch step is itself a request's arguments, with its action.
2020
struct Parameters: Decodable {
21-
var steps: [InputStep]?
21+
var action: String?
22+
var steps: [Parameters]?
2223
var activateFirst: Bool?
2324
var permission: String?
2425
var bundleId: String?
@@ -86,6 +87,30 @@ func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
8687
var value: CFTypeRef?
8788
return AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success ? value : nil
8889
}
90+
/// Reads several attributes in one Accessibility round trip instead of one per attribute; an attribute
91+
/// the element lacks or fails to return is omitted, as `attribute` would return nil for it.
92+
func attributes(_ element: AXUIElement, _ names: [String]) -> [String: CFTypeRef] {
93+
var raw: CFArray?
94+
guard AXUIElementCopyMultipleAttributeValues(element, names as CFArray, AXCopyMultipleAttributeOptions(rawValue: 0), &raw) == .success,
95+
let values = raw as [AnyObject]?, values.count == names.count else {
96+
return names.reduce(into: [:]) { result, name in result[name] = attribute(element, name) }
97+
}
98+
var result: [String: CFTypeRef] = [:]
99+
for (name, value) in zip(names, values) {
100+
if value is NSNull { continue }
101+
if CFGetTypeID(value) == AXValueGetTypeID(), AXValueGetType(unsafeBitCast(value, to: AXValue.self)) == .axError { continue }
102+
result[name] = value
103+
}
104+
return result
105+
}
106+
/// Elements bucketed by CFHash, so a walk's membership and parent lookups stay linear in its size.
107+
struct ElementIndex {
108+
private var buckets: [CFHashCode: [(element: AXUIElement, id: String)]] = [:]
109+
func id(of element: AXUIElement) -> String? { buckets[CFHash(element)]?.first { CFEqual($0.element, element) }?.id }
110+
mutating func insert(_ element: AXUIElement, id: String) { buckets[CFHash(element), default: []].append((element, id)) }
111+
}
112+
let NODE_ATTRIBUTES = [kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXDescriptionAttribute, kAXPlaceholderValueAttribute,
113+
kAXHelpAttribute, kAXValueAttribute, kAXFocusedAttribute, kAXEnabledAttribute, kAXPositionAttribute, kAXSizeAttribute]
89114
func secure(_ element: AXUIElement) -> Bool {
90115
attribute(element, kAXSubroleAttribute) as? String == kAXSecureTextFieldSubrole
91116
}
@@ -135,8 +160,11 @@ func axActions(_ element: AXUIElement) -> [String] {
135160
return AXUIElementCopyActionNames(element, &names) == .success ? names as? [String] ?? [] : []
136161
}
137162
func elementRect(_ element: AXUIElement) -> CGRect? {
138-
guard let rawPosition = attribute(element, kAXPositionAttribute), CFGetTypeID(rawPosition) == AXValueGetTypeID(),
139-
let rawSize = attribute(element, kAXSizeAttribute), CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil }
163+
axRect(position: attribute(element, kAXPositionAttribute), size: attribute(element, kAXSizeAttribute))
164+
}
165+
func axRect(position rawPosition: CFTypeRef?, size rawSize: CFTypeRef?) -> CGRect? {
166+
guard let rawPosition, CFGetTypeID(rawPosition) == AXValueGetTypeID(),
167+
let rawSize, CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil }
140168
var position = CGPoint.zero; var size = CGSize.zero
141169
guard AXValueGetValue(unsafeBitCast(rawPosition, to: AXValue.self), .cgPoint, &position),
142170
AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil }
@@ -309,12 +337,14 @@ final class Snapshot {
309337
queue.append(AXTraversalEntry(element: unsafeBitCast(raw, to: AXUIElement.self), parent: root, depth: 1, menu: false, priority: true))
310338
}
311339
var menuCount = 0
340+
var index = ElementIndex()
312341
while nodes.count < 2000 && Date().timeIntervalSince(created) < 8, let entry = queue.next() {
313342
try checkCancellation()
314343
let element = entry.element
315344
AXUIElementSetMessagingTimeout(element, 0.25)
316-
if elements.values.contains(where: { CFEqual($0, element) }) { continue }
317-
let role = attribute(element, kAXRoleAttribute) as? String ?? "AXUnknown"
345+
if index.id(of: element) != nil { continue }
346+
let values = attributes(element, NODE_ATTRIBUTES)
347+
let role = values[kAXRoleAttribute] as? String ?? "AXUnknown"
318348
let isMenu = entry.menu || [kAXMenuBarRole, kAXMenuRole, kAXMenuItemRole, kAXMenuBarItemRole].contains(role)
319349
if role == kAXMenuItemRole && !entry.menu && !entry.priority {
320350
queue.append(AXTraversalEntry(element: element, parent: entry.parent, depth: entry.depth, menu: true, priority: false)); continue
@@ -323,27 +353,28 @@ final class Snapshot {
323353
guard menuCount < 80 else { truncated = true; continue }
324354
menuCount += 1
325355
}
326-
let id = "e\(nodes.count)"; elements[id] = element
356+
let id = "e\(nodes.count)"; elements[id] = element; index.insert(element, id: id)
327357
var node: [String: Any] = ["elementId": id, "role": role, "actions": [String]()]
328-
if let parent = entry.parent, let parentID = elements.first(where: { CFEqual($0.value, parent) })?.key { node["parentId"] = parentID }
329-
let isSecure = secure(element)
330-
var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { attribute(element, $0) as? String }.filter { !$0.isEmpty }
331-
if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { attribute(element, $0) as? String }.filter { !$0.isEmpty } }
358+
if let parent = entry.parent, let parentID = index.id(of: parent) { node["parentId"] = parentID }
359+
let isSecure = values[kAXSubroleAttribute] as? String == kAXSecureTextFieldSubrole
360+
var labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty }
361+
if labels.isEmpty { labels = [kAXPlaceholderValueAttribute, kAXHelpAttribute].compactMap { values[$0] as? String }.filter { !$0.isEmpty } }
332362
if role == kAXMenuItemRole, let shortcut = menuShortcut(element) { labels.append(shortcut) }
333363
if !isSecure, !labels.isEmpty { node["label"] = String(labels.joined(separator: " ").prefix(1024)) }
334-
if !isSecure, let raw = attribute(element, kAXValueAttribute) {
364+
if !isSecure, let raw = values[kAXValueAttribute] {
335365
if let value = raw as? String { node["value"] = String(value.prefix(2048)) }
336366
else if let value = raw as? NSNumber { node["value"] = value.stringValue }
337367
}
338368
if !isSecure {
339-
if let focused = attribute(element, kAXFocusedAttribute) as? Bool { node["focused"] = focused }
369+
if let focused = values[kAXFocusedAttribute] as? Bool { node["focused"] = focused }
340370
var writable: DarwinBoolean = false
341-
let writableValue = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &writable) == .success && writable.boolValue
342-
node["editable"] = [kAXTextAreaRole, kAXTextFieldRole].contains(role) && writableValue
343-
if let placeholder = attribute(element, kAXPlaceholderValueAttribute) as? String { node["placeholder"] = String(placeholder.prefix(1024)) }
371+
let writableValue = [kAXTextAreaRole, kAXTextFieldRole].contains(role)
372+
&& AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &writable) == .success && writable.boolValue
373+
node["editable"] = writableValue
374+
if let placeholder = values[kAXPlaceholderValueAttribute] as? String { node["placeholder"] = String(placeholder.prefix(1024)) }
344375
}
345-
if let enabled = attribute(element, kAXEnabledAttribute) as? Bool { node["enabled"] = enabled }
346-
if let rect = elementRect(element) { node.merge(rectJSON(rect)) { _, new in new } }
376+
if let enabled = values[kAXEnabledAttribute] as? Bool { node["enabled"] = enabled }
377+
if let rect = axRect(position: values[kAXPositionAttribute], size: values[kAXSizeAttribute]) { node.merge(rectJSON(rect)) { _, new in new } }
347378
var actions: CFArray?
348379
if AXUIElementCopyActionNames(element, &actions) == .success { node["actions"] = Array((actions as? [String] ?? []).prefix(128)).map { String($0.prefix(128)) } }
349380
nodes.append(node)
@@ -430,7 +461,20 @@ func parseKey(_ key: String) throws -> (CGKeyCode, CGEventFlags) {
430461
let aliases = ["esc": "escape", "arrowleft": "left", "arrowright": "right", "arrowup": "up", "arrowdown": "down", "forwarddelete": "delete", "pgup": "pageup", "pgdn": "pagedown", "pgdown": "pagedown", "minus": "-", "equal": "=", "equals": "=", "comma": ",", "period": ".", "slash": "/", "semicolon": ";", "quote": "'", "backslash": "\\", "backtick": "`", "grave": "`", "leftbracket": "[", "rightbracket": "]"]
431462
guard let code = keys[aliases[name] ?? name] else { throw ComputerError("unsupported_key", "Unsupported key name. Use names such as Enter, Tab, Escape, Backspace, Delete (forward delete), Left, PageUp, F5, or a single character, with Cmd/Shift/Alt/Ctrl modifiers.") }; return (code, flags)
432463
}
433-
func validateInputSteps(_ steps: [InputStep]) throws {
464+
/// The actions a batch may run; each is validated again by its own action path when it runs.
465+
let BATCH_ACTIONS: Set<String> = ["click", "type_text", "set_value", "press_key", "scroll", "perform_action"]
466+
func validateBatchSteps(_ steps: [Parameters]) throws {
467+
guard (2...8).contains(steps.count) else { throw ComputerError("invalid_arguments", "A batch requires 2 to 8 steps.") }
468+
var units = 0
469+
for step in steps {
470+
guard let action = step.action, BATCH_ACTIONS.contains(action), step.bundleId == nil, step.snapshotId == nil, step.steps == nil else {
471+
throw ComputerError("invalid_arguments", "Batch steps are click, type_text, set_value, press_key, scroll or perform_action, without their own app or snapshot.")
472+
}
473+
units += (step.text?.utf16.count ?? 0) + (step.value?.utf16.count ?? 0)
474+
}
475+
guard units <= 32000 else { throw ComputerError("invalid_arguments", "Combined batch text exceeds 32000 UTF-16 units.") }
476+
}
477+
func validateInputSteps(_ steps: [Parameters]) throws {
434478
guard (1...32).contains(steps.count) else { throw ComputerError("invalid_arguments", "Input sequence requires 1 to 32 steps.") }
435479
var units = 0
436480
for step in steps {
@@ -670,7 +714,7 @@ final class Driver {
670714
}
671715
result["isActive"] = app.isActive
672716
return result
673-
case "click", "type_text", "input_sequence", "press_key", "scroll", "drag", "set_value", "perform_action": break
717+
case "click", "type_text", "input_sequence", "press_key", "scroll", "drag", "set_value", "perform_action", "batch": break
674718
default: throw ComputerError("unknown_method", "Unknown computer-use method.")
675719
}
676720
try requireAccessibility()
@@ -695,6 +739,31 @@ final class Driver {
695739
let p = request.params
696740
let pid = app.processIdentifier
697741
switch request.method {
742+
case "batch":
743+
let steps = try required(p.steps, "steps")
744+
try validateBatchSteps(steps)
745+
var completed = 0
746+
func stopped(_ message: String) -> [String: Any] {
747+
["kind": "action", "action": "batch", "bundleId": bundle, "dispatched": true, "verified": false,
748+
"sequence": ["completedSteps": completed, "totalSteps": steps.count, "error": String((message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch.").prefix(2000))]]
749+
}
750+
for (index, step) in steps.enumerated() {
751+
// Stop can arrive during the previous step's settle; no later step may start after it.
752+
do { try checkCancellation() } catch { if completed == 0 { throw error }; return stopped("Stopped before step \(index + 1).") }
753+
var params = step; params.bundleId = p.bundleId; params.snapshotId = p.snapshotId
754+
let before = uiStructure(pid: pid)
755+
do { _ = try await act(Request(id: request.id, method: try required(step.action, "action"), params: params), app: app, bundle: bundle, snapshot: snapshot) }
756+
catch {
757+
// A first step that sent nothing fails like the single action, keeping the snapshot valid.
758+
if !inputDispatched { throw error }
759+
return stopped("Step \(index + 1) failed: " + ((error as? ComputerError)?.message ?? "its outcome is unknown."))
760+
}
761+
completed += 1
762+
if completed < steps.count, uiStructure(pid: pid) != before {
763+
return stopped("Step \(index + 1) opened or closed a window, sheet, menu or popover.")
764+
}
765+
}
766+
return ["kind": "action", "action": "batch", "bundleId": bundle, "dispatched": true, "verified": false, "sequence": ["completedSteps": completed, "totalSteps": steps.count]]
698767
case "perform_action":
699768
let element = try snapshot.element(p.elementId); let action = try required(p.accessibilityAction, "accessibilityAction")
700769
var names: CFArray?; try axCheck(AXUIElementCopyActionNames(element, &names))
@@ -731,7 +800,7 @@ final class Driver {
731800
}
732801
case "type_text", "input_sequence":
733802
let element = try snapshot.element(p.elementId)
734-
let steps = request.method == "type_text" ? [InputStep(action: "type_text", text: try required(p.text, "text"), key: nil)] : try required(p.steps, "steps")
803+
let steps = request.method == "type_text" ? [Parameters(action: "type_text", text: try required(p.text, "text"))] : try required(p.steps, "steps")
735804
try validateInputSteps(steps)
736805
let root = AXUIElementCreateApplication(pid)
737806
// Keystrokes go to whatever holds keyboard focus: a text control (terminals included, read-only
@@ -879,6 +948,23 @@ final class Driver {
879948
return ["kind": "action", "action": request.method, "bundleId": bundle, "dispatched": true, "verified": false]
880949
}
881950
}
951+
/// The app's window structure by identity: its on-screen windows (sheets, menus and popovers are
952+
/// windows of their own), the focused window, and that window's direct content, where a sheet sits.
953+
/// Identities rather than counts, so switching one open menu or sheet for another still registers.
954+
func uiStructure(pid: pid_t) -> [Int] {
955+
let root = AXUIElementCreateApplication(pid)
956+
AXUIElementSetMessagingTimeout(root, 0.1)
957+
var structure = (CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] ?? [])
958+
.filter { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid }
959+
.compactMap { ($0[kCGWindowNumber as String] as? NSNumber)?.intValue }
960+
.sorted()
961+
if let raw = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(raw) == AXUIElementGetTypeID() {
962+
structure.append(Int(bitPattern: UInt(CFHash(raw))))
963+
let (children, _) = pagedElements(unsafeBitCast(raw, to: AXUIElement.self), name: kAXChildrenAttribute, limit: 64)
964+
structure.append(contentsOf: children.map { Int(bitPattern: UInt(CFHash($0))) })
965+
}
966+
return structure
967+
}
882968
func nextLine() throws -> String? {
883969
var bytes = [UInt8](); bytes.reserveCapacity(4096)
884970
var oversized = false

0 commit comments

Comments
 (0)