From 5b8403d6ddab90bb926be467bb25e3e486b82cb9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 11:20:34 -0800 Subject: [PATCH 1/6] fix(computer-use): click an editor into focus when Accessibility cannot focus it SwiftUI sheets such as Clock's city search ignore an Accessibility focus request, so type_text refused with 'the exact editor is no longer focused'. In a foreground app the helper now clicks the editor's center, only when that point hit-tests to the editor inside its observed window, then applies the usual focus checks. --- .../native/computer-use/ComputerUse.swift | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 8ca12493ef4..aef65d0ff69 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -590,6 +590,20 @@ func postMouse(pid: pid_t, type: CGEventType, point: CGPoint, button: CGMouseBut event.post(tap: .cghidEventTap) } +/// Clicks the center of an observed editor so it takes keyboard focus, only when that point lands on +/// the editor itself inside its observed window; otherwise nothing is sent and the focus check decides. +@MainActor +func focusByClicking(_ element: AXUIElement, pid: pid_t, snapshot: Snapshot) async throws { + guard let rect = elementRect(element), let windowID = snapshot.windowId(holding: element), + let frame = snapshot.windowFrames[windowID], let window = snapshot.windowElements[windowID] else { return } + let center = CGPoint(x: rect.midX, y: rect.midY) + guard frame.contains(center), let hit = elementAtPoint(center), try ancestorPath(hit, to: element) != nil else { return } + try await coordinateAction { dispatch in + try postMouse(pid: pid, type: .leftMouseDown, point: center, button: .left, windowID: windowID, window: window, frame: frame, dispatch: dispatch) + try postMouse(pid: pid, type: .leftMouseUp, point: center, button: .left, windowID: windowID, window: window, frame: frame, release: true, dispatch: dispatch) + } + try await Task.sleep(for: .milliseconds(100)) +} func discoverApps() throws -> [[String: Any]] { var entries: [String: [String: Any]] = [:] let roots = [URL(fileURLWithPath: "/Applications"), URL(fileURLWithPath: "/System/Applications"), FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Applications")] @@ -814,6 +828,11 @@ final class Driver { if attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) != true { try axCheck(AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)) try await Task.sleep(for: .milliseconds(50)) + // Some apps (SwiftUI sheets such as Clock's city search) ignore an Accessibility focus + // request; in a foreground app the field is focused the way a user would, by clicking it. + if app.isActive, attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) != true { + try await focusByClicking(element, pid: pid, snapshot: snapshot) + } } let observedWindows = snapshot.observedWindows() // The first editor found holding focus receives the whole input, so text never follows focus elsewhere. From 8851e5eaa283c7e8857f6e7016d8ce0dda675058 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 11:25:33 -0800 Subject: [PATCH 2/6] docs(computer-use): describe the click-to-focus fallback --- apps/desktop/native/computer-use/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index c9a4287f719..de05138d588 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -14,7 +14,7 @@ Coordinates in action inputs are window-local points; node frames are global scr AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A hit on a window's own background counts as that window. Background tabs and minimized windows share bounds with the window in front, so they take no coordinate input. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves the item's own menu open with the app highlighting that item (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. -`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date and time pickers, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. `batch` runs 2–8 click, type_text, set_value, press_key, scroll or perform_action steps planned from one snapshot, each through its own action's checks; it stops at the first step that fails and after any step that opens or closes a window, sheet, menu or popover, reports progress the same way, and a first step that sent nothing is refused with `dispatchState: "not_started"`. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. +`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date and time pickers, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. When an app ignores the Accessibility focus request (SwiftUI sheets such as Clock's city search), a foreground app's editor is clicked at its center to take focus, only if that point hit-tests to the editor inside its observed window; the usual focus checks then apply. `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. `batch` runs 2–8 click, type_text, set_value, press_key, scroll or perform_action steps planned from one snapshot, each through its own action's checks; it stops at the first step that fails and after any step that opens or closes a window, sheet, menu or popover, reports progress the same way, and a first step that sent nothing is refused with `dispatchState: "not_started"`. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. ## Cancellation From 4479dca208c6e0fdb389a81e7d150138eed658e0 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 11:34:50 -0800 Subject: [PATCH 3/6] fix(computer-use): click only onto the editor or a text control inside it, and cover click-to-focus in the live fixture --- .../native/computer-use/ComputerUse.swift | 9 ++++++--- .../native/computer-use/tests/Fixture.swift | 9 +++++++-- apps/desktop/native/computer-use/tests/live.py | 16 +++++++++++++++- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index aef65d0ff69..0a477f67e8e 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -590,14 +590,17 @@ func postMouse(pid: pid_t, type: CGEventType, point: CGPoint, button: CGMouseBut event.post(tap: .cghidEventTap) } -/// Clicks the center of an observed editor so it takes keyboard focus, only when that point lands on -/// the editor itself inside its observed window; otherwise nothing is sent and the focus check decides. +/// Clicks the center of an observed text control so it takes keyboard focus, only when that point lands on +/// the control (or a text control inside it) within its observed window; otherwise nothing is sent and the focus check decides. @MainActor func focusByClicking(_ element: AXUIElement, pid: pid_t, snapshot: Snapshot) async throws { guard let rect = elementRect(element), let windowID = snapshot.windowId(holding: element), let frame = snapshot.windowFrames[windowID], let window = snapshot.windowElements[windowID] else { return } let center = CGPoint(x: rect.midX, y: rect.midY) - guard frame.contains(center), let hit = elementAtPoint(center), try ancestorPath(hit, to: element) != nil else { return } + // The point must land on the editor or a text control inside it, never a button or row nested in a container. + guard takesText(element), frame.contains(center), let hit = elementAtPoint(center) else { return } + let insideEditor = takesText(hit) ? try ancestorPath(hit, to: element) != nil : false + guard CFEqual(hit, element) || insideEditor else { return } try await coordinateAction { dispatch in try postMouse(pid: pid, type: .leftMouseDown, point: center, button: .left, windowID: windowID, window: window, frame: frame, dispatch: dispatch) try postMouse(pid: pid, type: .leftMouseUp, point: center, button: .left, windowID: windowID, window: window, frame: frame, release: true, dispatch: dispatch) diff --git a/apps/desktop/native/computer-use/tests/Fixture.swift b/apps/desktop/native/computer-use/tests/Fixture.swift index 06ffccf17b5..14d336d0a47 100644 --- a/apps/desktop/native/computer-use/tests/Fixture.swift +++ b/apps/desktop/native/computer-use/tests/Fixture.swift @@ -8,6 +8,10 @@ final class MarkerView: NSView { } } +/// A field that ignores Accessibility focus requests but takes focus from a click, as SwiftUI sheets do. +final class ClickFocusField: NSTextField { + override func setAccessibilityFocused(_ accessibilityFocused: Bool) {} +} final class FixtureDelegate: NSObject, NSApplicationDelegate { var window: NSWindow! var count = 0 @@ -31,6 +35,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { let field = NSTextField(string: "Fixture text"); field.setAccessibilityIdentifier("fixture-text"); field.setAccessibilityLabel("Fixture editable text") let secure = NSSecureTextField(string: "fixture-only-secret"); secure.setAccessibilityIdentifier("fixture-password"); secure.setAccessibilityLabel("Fixture password") let button = NSButton(title: "Increment", target: self, action: #selector(increment)); button.setAccessibilityIdentifier("fixture-increment") + let clickField = ClickFocusField(string: ""); clickField.setAccessibilityLabel("Fixture click-focus field") let sheetButton = NSButton(title: "Show sheet", target: self, action: #selector(showSheet)); sheetButton.setAccessibilityIdentifier("fixture-show-sheet") let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider") let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time") @@ -44,9 +49,9 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { scroll.documentView = document scroll.contentView.postsBoundsChangedNotifications = true NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView) - for view in [field, secure, button, counter, sheetButton, slider, time, timeLabel, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) } + for view in [field, secure, button, counter, clickField, sheetButton, slider, time, timeLabel, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) } window.contentView!.addSubview(stack) - NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)]) + NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), clickField.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)]) window.makeKeyAndOrderFront(nil) window.makeFirstResponder(field) NSApplication.shared.activate() diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 9310373ba98..8b9478f46cd 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -170,7 +170,21 @@ def batch(snapshot, steps): # While the sheet is open only the sheet is observed; the step after it must not have run. snapshot = mutate('click', snapshot, elementId=find(snapshot, role='AXButton', label='Dismiss')['elementId']) assert count(snapshot) == before + 3, 'A batch step ran after a sheet opened' - print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) + # An editor that ignores Accessibility focus takes focus from a click in a foreground app only. + call('activate_app', bundleId=bundle) + snapshot = state() + click_field = find(snapshot, role='AXTextField', label='Fixture click-focus field') + result = call('type_text', bundleId=bundle, snapshotId=snapshot['snapshotId'], elementId=click_field['elementId'], text='clicked') + assert result['dispatched'], result + assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'clicked', 'Click-to-focus did not reach the editor' + call('activate_app', bundleId='com.apple.finder') + time.sleep(0.5) + snapshot = state() + error = refused('type_text', snapshot, elementId=find(snapshot, role='AXTextField', label='Fixture click-focus field')['elementId'], text='x') + assert error['dispatchState'] == 'not_started', error + assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'clicked', 'A background app was clicked or typed into' + call('activate_app', bundleId=bundle) + print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: if fixture: fixture.terminate() From b6ed6769d30798722ec605702d9c05e12ea1db1c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 11:35:53 -0800 Subject: [PATCH 4/6] test(computer-use): move focus away before checking the background click-to-focus refusal --- apps/desktop/native/computer-use/tests/live.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 8b9478f46cd..febc63144f6 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -177,6 +177,9 @@ def batch(snapshot, steps): result = call('type_text', bundleId=bundle, snapshotId=snapshot['snapshotId'], elementId=click_field['elementId'], text='clicked') assert result['dispatched'], result assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'clicked', 'Click-to-focus did not reach the editor' + # Move focus to the other editor, then put another app in front: the click-only editor cannot take focus. + snapshot = state() + mutate('click', snapshot, **coordinate(snapshot, find(snapshot, role='AXTextField', label='Fixture editable text'))) call('activate_app', bundleId='com.apple.finder') time.sleep(0.5) snapshot = state() From b96170a17df58fba9eba0e3798de1b8fb1d5c543 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 11:45:33 -0800 Subject: [PATCH 5/6] fix(computer-use): check for Stop before the click-to-focus mouse-down --- apps/desktop/native/computer-use/ComputerUse.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 0a477f67e8e..1de2a0f2070 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -602,6 +602,7 @@ func focusByClicking(_ element: AXUIElement, pid: pid_t, snapshot: Snapshot) asy let insideEditor = takesText(hit) ? try ancestorPath(hit, to: element) != nil : false guard CFEqual(hit, element) || insideEditor else { return } try await coordinateAction { dispatch in + try checkCancellation() try postMouse(pid: pid, type: .leftMouseDown, point: center, button: .left, windowID: windowID, window: window, frame: frame, dispatch: dispatch) try postMouse(pid: pid, type: .leftMouseUp, point: center, button: .left, windowID: windowID, window: window, frame: frame, release: true, dispatch: dispatch) } From a739a8a259b58ccaf2e5fbaf510f57adc25594cc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 11:54:20 -0800 Subject: [PATCH 6/6] fix(computer-use): never click a secure field into focus --- apps/desktop/native/computer-use/ComputerUse.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 1de2a0f2070..45f68e5af81 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -600,7 +600,8 @@ func focusByClicking(_ element: AXUIElement, pid: pid_t, snapshot: Snapshot) asy // The point must land on the editor or a text control inside it, never a button or row nested in a container. guard takesText(element), frame.contains(center), let hit = elementAtPoint(center) else { return } let insideEditor = takesText(hit) ? try ancestorPath(hit, to: element) != nil : false - guard CFEqual(hit, element) || insideEditor else { return } + // Secure inputs stay untargetable, so a password field is never clicked into focus either. + guard CFEqual(hit, element) || insideEditor, (try? requireNonSecure(hit)) != nil else { return } try await coordinateAction { dispatch in try checkCancellation() try postMouse(pid: pid, type: .leftMouseDown, point: center, button: .left, windowID: windowID, window: window, frame: frame, dispatch: dispatch)