@@ -16,9 +16,10 @@ struct ComputerError: Error {
1616 let dispatchState : String ?
1717 init ( _ code : String , _ message : String , dispatchState : String ? = nil ) { self . code = code ; self . message = message ; self . dispatchState = dispatchState }
1818}
19- struct InputStep : Decodable { let action : String ; let text : String ? ; let key : String ? }
19+ /// A request's arguments. An input sequence or batch step is itself a request's arguments, with its action.
2020struct Parameters : Decodable {
21- var steps : [ InputStep ] ?
21+ var action : String ?
22+ var steps : [ Parameters ] ?
2223 var activateFirst : Bool ?
2324 var permission : String ?
2425 var bundleId : String ?
@@ -86,6 +87,30 @@ func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
8687 var value : CFTypeRef ?
8788 return AXUIElementCopyAttributeValue ( element , name as CFString , & value ) == . success ? value : nil
8889}
90+ /// Reads several attributes in one Accessibility round trip instead of one per attribute; an attribute
91+ /// the element lacks or fails to return is omitted, as `attribute` would return nil for it.
92+ func attributes ( _ element : AXUIElement , _ names : [ String ] ) -> [ String : CFTypeRef ] {
93+ var raw : CFArray ?
94+ guard AXUIElementCopyMultipleAttributeValues ( element , names as CFArray , AXCopyMultipleAttributeOptions ( rawValue : 0 ) , & raw ) == . success ,
95+ let values = raw as [ AnyObject ] ? , values . count == names . count else {
96+ return names . reduce ( into : [ : ] ) { result , name in result [ name ] = attribute ( element , name ) }
97+ }
98+ var result : [ String : CFTypeRef ] = [ : ]
99+ for ( name , value ) in zip ( names , values ) {
100+ if value is NSNull { continue }
101+ if CFGetTypeID ( value ) == AXValueGetTypeID ( ) , AXValueGetType ( unsafeBitCast ( value , to : AXValue . self ) ) == . axError { continue }
102+ result [ name ] = value
103+ }
104+ return result
105+ }
106+ /// Elements bucketed by CFHash, so a walk's membership and parent lookups stay linear in its size.
107+ struct ElementIndex {
108+ private var buckets : [ CFHashCode : [ ( element : AXUIElement , id : String ) ] ] = [ : ]
109+ func id ( of element : AXUIElement ) -> String ? { buckets [ CFHash ( element ) ] ? . first { CFEqual ( $0 . element , element ) } ? . id }
110+ mutating func insert ( _ element : AXUIElement , id : String ) { buckets [ CFHash ( element ) , default : [ ] ] . append ( ( element , id ) ) }
111+ }
112+ let NODE_ATTRIBUTES = [ kAXRoleAttribute , kAXSubroleAttribute , kAXTitleAttribute , kAXDescriptionAttribute , kAXPlaceholderValueAttribute ,
113+ kAXHelpAttribute , kAXValueAttribute , kAXFocusedAttribute , kAXEnabledAttribute , kAXPositionAttribute , kAXSizeAttribute ]
89114func secure ( _ element : AXUIElement ) -> Bool {
90115 attribute ( element , kAXSubroleAttribute ) as? String == kAXSecureTextFieldSubrole
91116}
@@ -135,8 +160,11 @@ func axActions(_ element: AXUIElement) -> [String] {
135160 return AXUIElementCopyActionNames ( element , & names ) == . success ? names as? [ String ] ?? [ ] : [ ]
136161}
137162func elementRect ( _ element : AXUIElement ) -> CGRect ? {
138- guard let rawPosition = attribute ( element , kAXPositionAttribute ) , CFGetTypeID ( rawPosition ) == AXValueGetTypeID ( ) ,
139- let rawSize = attribute ( element , kAXSizeAttribute ) , CFGetTypeID ( rawSize ) == AXValueGetTypeID ( ) else { return nil }
163+ axRect ( position : attribute ( element , kAXPositionAttribute ) , size : attribute ( element , kAXSizeAttribute ) )
164+ }
165+ func axRect ( position rawPosition : CFTypeRef ? , size rawSize : CFTypeRef ? ) -> CGRect ? {
166+ guard let rawPosition , CFGetTypeID ( rawPosition ) == AXValueGetTypeID ( ) ,
167+ let rawSize , CFGetTypeID ( rawSize ) == AXValueGetTypeID ( ) else { return nil }
140168 var position = CGPoint . zero ; var size = CGSize . zero
141169 guard AXValueGetValue ( unsafeBitCast ( rawPosition , to : AXValue . self ) , . cgPoint , & position ) ,
142170 AXValueGetValue ( unsafeBitCast ( rawSize , to : AXValue . self ) , . cgSize , & size ) else { return nil }
@@ -309,12 +337,14 @@ final class Snapshot {
309337 queue . append ( AXTraversalEntry ( element : unsafeBitCast ( raw , to : AXUIElement . self ) , parent : root , depth : 1 , menu : false , priority : true ) )
310338 }
311339 var menuCount = 0
340+ var index = ElementIndex ( )
312341 while nodes . count < 2000 && Date ( ) . timeIntervalSince ( created ) < 8 , let entry = queue . next ( ) {
313342 try checkCancellation ( )
314343 let element = entry . element
315344 AXUIElementSetMessagingTimeout ( element , 0.25 )
316- if elements . values . contains ( where : { CFEqual ( $0 , element ) } ) { continue }
317- let role = attribute ( element , kAXRoleAttribute ) as? String ?? " AXUnknown "
345+ if index . id ( of : element ) != nil { continue }
346+ let values = attributes ( element , NODE_ATTRIBUTES )
347+ let role = values [ kAXRoleAttribute ] as? String ?? " AXUnknown "
318348 let isMenu = entry . menu || [ kAXMenuBarRole , kAXMenuRole , kAXMenuItemRole , kAXMenuBarItemRole ] . contains ( role )
319349 if role == kAXMenuItemRole && ! entry . menu && ! entry . priority {
320350 queue . append ( AXTraversalEntry ( element : element , parent : entry . parent , depth : entry . depth , menu : true , priority : false ) ) ; continue
@@ -323,27 +353,28 @@ final class Snapshot {
323353 guard menuCount < 80 else { truncated = true ; continue }
324354 menuCount += 1
325355 }
326- let id = " e \( nodes . count ) " ; elements [ id ] = element
356+ let id = " e \( nodes . count ) " ; elements [ id ] = element ; index . insert ( element , id : id )
327357 var node : [ String : Any ] = [ " elementId " : id , " role " : role , " actions " : [ String ] ( ) ]
328- if let parent = entry . parent , let parentID = elements . first ( where : { CFEqual ( $0 . value , parent ) } ) ? . key { node [ " parentId " ] = parentID }
329- let isSecure = secure ( element )
330- var labels = [ kAXTitleAttribute , kAXDescriptionAttribute ] . compactMap { attribute ( element , $0 ) as? String } . filter { ! $0 . isEmpty }
331- if labels . isEmpty { labels = [ kAXPlaceholderValueAttribute , kAXHelpAttribute ] . compactMap { attribute ( element , $0 ) as? String } . filter { ! $0 . isEmpty } }
358+ if let parent = entry . parent , let parentID = index . id ( of : parent ) { node [ " parentId " ] = parentID }
359+ let isSecure = values [ kAXSubroleAttribute ] as? String == kAXSecureTextFieldSubrole
360+ var labels = [ kAXTitleAttribute , kAXDescriptionAttribute ] . compactMap { values [ $0 ] as? String } . filter { ! $0 . isEmpty }
361+ if labels . isEmpty { labels = [ kAXPlaceholderValueAttribute , kAXHelpAttribute ] . compactMap { values [ $0 ] as? String } . filter { ! $0 . isEmpty } }
332362 if role == kAXMenuItemRole , let shortcut = menuShortcut ( element ) { labels . append ( shortcut ) }
333363 if ! isSecure , ! labels . isEmpty { node [ " label " ] = String ( labels . joined ( separator : " " ) . prefix ( 1024 ) ) }
334- if ! isSecure , let raw = attribute ( element , kAXValueAttribute ) {
364+ if ! isSecure , let raw = values [ kAXValueAttribute ] {
335365 if let value = raw as? String { node [ " value " ] = String ( value . prefix ( 2048 ) ) }
336366 else if let value = raw as? NSNumber { node [ " value " ] = value . stringValue }
337367 }
338368 if ! isSecure {
339- if let focused = attribute ( element , kAXFocusedAttribute ) as? Bool { node [ " focused " ] = focused }
369+ if let focused = values [ kAXFocusedAttribute ] as? Bool { node [ " focused " ] = focused }
340370 var writable : DarwinBoolean = false
341- let writableValue = AXUIElementIsAttributeSettable ( element , kAXValueAttribute as CFString , & writable ) == . success && writable . boolValue
342- node [ " editable " ] = [ kAXTextAreaRole , kAXTextFieldRole ] . contains ( role ) && writableValue
343- if let placeholder = attribute ( element , kAXPlaceholderValueAttribute ) as? String { node [ " placeholder " ] = String ( placeholder . prefix ( 1024 ) ) }
371+ let writableValue = [ kAXTextAreaRole , kAXTextFieldRole ] . contains ( role )
372+ && AXUIElementIsAttributeSettable ( element , kAXValueAttribute as CFString , & writable ) == . success && writable . boolValue
373+ node [ " editable " ] = writableValue
374+ if let placeholder = values [ kAXPlaceholderValueAttribute ] as? String { node [ " placeholder " ] = String ( placeholder . prefix ( 1024 ) ) }
344375 }
345- if let enabled = attribute ( element , kAXEnabledAttribute ) as? Bool { node [ " enabled " ] = enabled }
346- if let rect = elementRect ( element ) { node . merge ( rectJSON ( rect ) ) { _ , new in new } }
376+ if let enabled = values [ kAXEnabledAttribute ] as? Bool { node [ " enabled " ] = enabled }
377+ if let rect = axRect ( position : values [ kAXPositionAttribute ] , size : values [ kAXSizeAttribute ] ) { node . merge ( rectJSON ( rect ) ) { _ , new in new } }
347378 var actions : CFArray ?
348379 if AXUIElementCopyActionNames ( element , & actions ) == . success { node [ " actions " ] = Array ( ( actions as? [ String ] ?? [ ] ) . prefix ( 128 ) ) . map { String ( $0 . prefix ( 128 ) ) } }
349380 nodes . append ( node )
@@ -430,7 +461,20 @@ func parseKey(_ key: String) throws -> (CGKeyCode, CGEventFlags) {
430461 let aliases = [ " esc " : " escape " , " arrowleft " : " left " , " arrowright " : " right " , " arrowup " : " up " , " arrowdown " : " down " , " forwarddelete " : " delete " , " pgup " : " pageup " , " pgdn " : " pagedown " , " pgdown " : " pagedown " , " minus " : " - " , " equal " : " = " , " equals " : " = " , " comma " : " , " , " period " : " . " , " slash " : " / " , " semicolon " : " ; " , " quote " : " ' " , " backslash " : " \\ " , " backtick " : " ` " , " grave " : " ` " , " leftbracket " : " [ " , " rightbracket " : " ] " ]
431462 guard let code = keys [ aliases [ name ] ?? name ] else { throw ComputerError ( " unsupported_key " , " Unsupported key name. Use names such as Enter, Tab, Escape, Backspace, Delete (forward delete), Left, PageUp, F5, or a single character, with Cmd/Shift/Alt/Ctrl modifiers. " ) } ; return ( code , flags )
432463}
433- func validateInputSteps ( _ steps : [ InputStep ] ) throws {
464+ /// The actions a batch may run; each is validated again by its own action path when it runs.
465+ let BATCH_ACTIONS : Set < String > = [ " click " , " type_text " , " set_value " , " press_key " , " scroll " , " perform_action " ]
466+ func validateBatchSteps ( _ steps : [ Parameters ] ) throws {
467+ guard ( 2 ... 8 ) . contains ( steps . count ) else { throw ComputerError ( " invalid_arguments " , " A batch requires 2 to 8 steps. " ) }
468+ var units = 0
469+ for step in steps {
470+ guard let action = step . action , BATCH_ACTIONS . contains ( action ) , step . bundleId == nil , step . snapshotId == nil , step . steps == nil else {
471+ throw ComputerError ( " invalid_arguments " , " Batch steps are click, type_text, set_value, press_key, scroll or perform_action, without their own app or snapshot. " )
472+ }
473+ units += ( step . text ? . utf16 . count ?? 0 ) + ( step . value ? . utf16 . count ?? 0 )
474+ }
475+ guard units <= 32000 else { throw ComputerError ( " invalid_arguments " , " Combined batch text exceeds 32000 UTF-16 units. " ) }
476+ }
477+ func validateInputSteps ( _ steps : [ Parameters ] ) throws {
434478 guard ( 1 ... 32 ) . contains ( steps . count ) else { throw ComputerError ( " invalid_arguments " , " Input sequence requires 1 to 32 steps. " ) }
435479 var units = 0
436480 for step in steps {
@@ -670,7 +714,7 @@ final class Driver {
670714 }
671715 result [ " isActive " ] = app . isActive
672716 return result
673- case " click " , " type_text " , " input_sequence " , " press_key " , " scroll " , " drag " , " set_value " , " perform_action " : break
717+ case " click " , " type_text " , " input_sequence " , " press_key " , " scroll " , " drag " , " set_value " , " perform_action " , " batch " : break
674718 default : throw ComputerError ( " unknown_method " , " Unknown computer-use method. " )
675719 }
676720 try requireAccessibility ( )
@@ -695,6 +739,31 @@ final class Driver {
695739 let p = request . params
696740 let pid = app . processIdentifier
697741 switch request . method {
742+ case " batch " :
743+ let steps = try required ( p . steps , " steps " )
744+ try validateBatchSteps ( steps )
745+ var completed = 0
746+ func stopped ( _ message : String ) -> [ String : Any ] {
747+ [ " kind " : " action " , " action " : " batch " , " bundleId " : bundle , " dispatched " : true , " verified " : false ,
748+ " sequence " : [ " completedSteps " : completed , " totalSteps " : steps . count , " error " : String ( ( message + " The remaining steps were not run; observe the app before continuing, and do not retry the batch. " ) . prefix ( 2000 ) ) ] ]
749+ }
750+ for ( index , step ) in steps . enumerated ( ) {
751+ // Stop can arrive during the previous step's settle; no later step may start after it.
752+ do { try checkCancellation ( ) } catch { if completed == 0 { throw error } ; return stopped ( " Stopped before step \( index + 1 ) . " ) }
753+ var params = step ; params . bundleId = p . bundleId ; params . snapshotId = p . snapshotId
754+ let before = uiStructure ( pid : pid )
755+ do { _ = try await act ( Request ( id : request . id , method : try required ( step . action , " action " ) , params : params ) , app : app , bundle : bundle , snapshot : snapshot ) }
756+ catch {
757+ // A first step that sent nothing fails like the single action, keeping the snapshot valid.
758+ if ! inputDispatched { throw error }
759+ return stopped ( " Step \( index + 1 ) failed: " + ( ( error as? ComputerError ) ? . message ?? " its outcome is unknown. " ) )
760+ }
761+ completed += 1
762+ if completed < steps . count , uiStructure ( pid : pid ) != before {
763+ return stopped ( " Step \( index + 1 ) opened or closed a window, sheet, menu or popover. " )
764+ }
765+ }
766+ return [ " kind " : " action " , " action " : " batch " , " bundleId " : bundle , " dispatched " : true , " verified " : false , " sequence " : [ " completedSteps " : completed , " totalSteps " : steps . count ] ]
698767 case " perform_action " :
699768 let element = try snapshot . element ( p . elementId ) ; let action = try required ( p . accessibilityAction , " accessibilityAction " )
700769 var names : CFArray ? ; try axCheck ( AXUIElementCopyActionNames ( element , & names ) )
@@ -731,7 +800,7 @@ final class Driver {
731800 }
732801 case " type_text " , " input_sequence " :
733802 let element = try snapshot . element ( p . elementId )
734- let steps = request . method == " type_text " ? [ InputStep ( action : " type_text " , text : try required ( p . text , " text " ) , key : nil ) ] : try required ( p . steps , " steps " )
803+ let steps = request . method == " type_text " ? [ Parameters ( action : " type_text " , text : try required ( p . text , " text " ) ) ] : try required ( p . steps , " steps " )
735804 try validateInputSteps ( steps )
736805 let root = AXUIElementCreateApplication ( pid )
737806 // Keystrokes go to whatever holds keyboard focus: a text control (terminals included, read-only
@@ -879,6 +948,23 @@ final class Driver {
879948 return [ " kind " : " action " , " action " : request . method , " bundleId " : bundle , " dispatched " : true , " verified " : false ]
880949 }
881950}
951+ /// The app's window structure by identity: its on-screen windows (sheets, menus and popovers are
952+ /// windows of their own), the focused window, and that window's direct content, where a sheet sits.
953+ /// Identities rather than counts, so switching one open menu or sheet for another still registers.
954+ func uiStructure ( pid : pid_t ) -> [ Int ] {
955+ let root = AXUIElementCreateApplication ( pid )
956+ AXUIElementSetMessagingTimeout ( root , 0.1 )
957+ var structure = ( CGWindowListCopyWindowInfo ( [ . optionOnScreenOnly ] , kCGNullWindowID ) as? [ [ String : Any ] ] ?? [ ] )
958+ . filter { ( $0 [ kCGWindowOwnerPID as String ] as? NSNumber ) ? . int32Value == pid }
959+ . compactMap { ( $0 [ kCGWindowNumber as String ] as? NSNumber ) ? . intValue }
960+ . sorted ( )
961+ if let raw = attribute ( root , kAXFocusedWindowAttribute ) , CFGetTypeID ( raw ) == AXUIElementGetTypeID ( ) {
962+ structure . append ( Int ( bitPattern : UInt ( CFHash ( raw ) ) ) )
963+ let ( children , _) = pagedElements ( unsafeBitCast ( raw , to : AXUIElement . self ) , name : kAXChildrenAttribute , limit : 64 )
964+ structure . append ( contentsOf : children . map { Int ( bitPattern : UInt ( CFHash ( $0 ) ) ) } )
965+ }
966+ return structure
967+ }
882968func nextLine ( ) throws -> String ? {
883969 var bytes = [ UInt8 ] ( ) ; bytes . reserveCapacity ( 4096 )
884970 var oversized = false
0 commit comments