From aed691598bd91533557722b0d73dfcf333c989a9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 18:25:59 -0800 Subject: [PATCH] fix(computer-use): read elements reported at infinity without a frame instead of crashing the helper --- .../native/computer-use/ComputerUse.swift | 18 +++++++++++++----- apps/desktop/native/computer-use/README.md | 2 +- .../native/computer-use/tests/Fixture.swift | 10 +++++++++- .../computer-use/tests/NativeTests.swift | 6 ++++++ apps/desktop/native/computer-use/tests/live.py | 4 +++- 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 9f287c330a7..a723bb77532 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -170,7 +170,9 @@ func axRect(position rawPosition: CFTypeRef?, size rawSize: CFTypeRef?) -> CGRec let rawSize, CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil } var position = CGPoint.zero; var size = CGSize.zero guard AXValueGetValue(unsafeBitCast(rawPosition, to: AXValue.self), .cgPoint, &position), - AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil } + AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size), + // Some apps report hidden elements at infinity (Maps does once its search field has focus); such an element has no usable frame. + [position.x, position.y, size.width, size.height].allSatisfy(\.isFinite) else { return nil } return CGRect(origin: position, size: size) } /// Apps computer use never operates, lowercased: macOS security settings, password managers, and apps @@ -1067,7 +1069,11 @@ func nextLine() throws -> String? { func encodeReply(_ response: [String: Any]) throws -> Data { let limit = 16 * 1024 * 1024 func encode(_ value: [String: Any]) throws -> Data { - try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]) + // JSONSerialization raises an Objective-C exception, which Swift cannot catch, on a NaN or infinite number. + guard JSONSerialization.isValidJSONObject(value) else { + throw ComputerError("reply_unencodable", "The reply held a value JSON cannot represent.") + } + return try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]) } let original = try encode(response) if original.count <= limit { return original } @@ -1138,9 +1144,11 @@ struct Main { response = ["id": id, "error": details] } catch { response = ["id": id, "error": ["code": "request_failed", "message": String(String(describing: error).prefix(2000))]] } - if let data = try? encodeReply(response) { - FileHandle.standardOutput.write(data); FileHandle.standardOutput.write(Data([10])) - } + let data: Data? + do { data = try encodeReply(response) } catch let error as ComputerError { + data = try? encodeReply(["id": id, "error": ["code": error.code, "message": error.message]]) + } catch { data = nil } + if let data { FileHandle.standardOutput.write(data); FileHandle.standardOutput.write(Data([10])) } } } } diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index 31acd652422..1603675f8e1 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -22,7 +22,7 @@ SIGTERM/SIGINT sets a cancellation flag. Traversal, typing and drag loops check ## Tests and limitations -Compile pure native tests with `swiftc -parse-as-library -D COMPUTER_USE_TEST ComputerUse.swift tests/NativeTests.swift -o native-tests`. They cover multi-display negative origins, coordinate bounds/nonfinite input, shortcut modifiers, invalid key handling, input-sequence and batch validation, protected apps and cooperative cancellation. The live fixture also covers batches and click-to-focus for an editor that ignores Accessibility focus. Real status and app discovery outputs were validated against the generated worker Zod schema (156 apps, including 147 installed-only apps). +Compile pure native tests with `swiftc -parse-as-library -D COMPUTER_USE_TEST ComputerUse.swift tests/NativeTests.swift -o native-tests`. They cover multi-display negative origins, coordinate bounds/nonfinite input, elements reported at infinity and unencodable replies, shortcut modifiers, invalid key handling, input-sequence and batch validation, protected apps and cooperative cancellation. The live fixture also covers batches and click-to-focus for an editor that ignores Accessibility focus. Real status and app discovery outputs were validated against the generated worker Zod schema (156 apps, including 147 installed-only apps). `python3 tests/protocol.py ` checks persistent framing, bounded-request recovery, Unicode decoding, no-prompt status, invalid methods/permissions, permission gates and clean EOF. `tests/Fixture.swift` compiles into a disposable AppKit app with a text field, secure field, increment button/counter, slider, scroll area and a visual-only marker. `python3 tests/live.py ` requires both grants and verifies actual outcomes. `--allow-missing-screen` explicitly reports screenshots blocked rather than passing them; `--existing-fixture` leaves an existing fixture running for coordinated model trials. Give the fixture bundle ID `com.mothership.computer-use-fixture` when packaging it for integration tests. diff --git a/apps/desktop/native/computer-use/tests/Fixture.swift b/apps/desktop/native/computer-use/tests/Fixture.swift index bae9f2a4ceb..69aab0234e6 100644 --- a/apps/desktop/native/computer-use/tests/Fixture.swift +++ b/apps/desktop/native/computer-use/tests/Fixture.swift @@ -16,6 +16,13 @@ class ClickFocusField: NSTextField { final class FixedCaretField: ClickFocusField { override func setAccessibilitySelectedTextRange(_ accessibilitySelectedTextRange: NSRange) {} } +/// An element that reports its frame at infinity, as some apps (Maps) report hidden elements. +final class InfiniteFrameView: NSView { + override func isAccessibilityElement() -> Bool { true } + override func accessibilityRole() -> NSAccessibility.Role? { .group } + override func accessibilityLabel() -> String? { "Fixture infinite frame" } + override func accessibilityFrame() -> NSRect { NSRect(x: CGFloat.infinity, y: -CGFloat.infinity, width: 0, height: 0) } +} final class FixtureDelegate: NSObject, NSApplicationDelegate { var window: NSWindow! var count = 0 @@ -45,6 +52,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider") let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time") time.dateValue = Calendar.current.date(bySettingHour: 10, minute: 15, second: 0, of: Date()) ?? Date(); time.target = self; time.action = #selector(timeChanged(_:)) + let infinite = InfiniteFrameView() let marker = MarkerView(); marker.setAccessibilityElement(false) marker.translatesAutoresizingMaskIntoConstraints = false let scroll = NSScrollView(); scroll.hasVerticalScroller = true @@ -54,7 +62,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { scroll.documentView = document scroll.contentView.postsBoundsChangedNotifications = true NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView) - for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) } + for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) } window.contentView!.addSubview(stack) NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), clickField.widthAnchor.constraint(equalTo: stack.widthAnchor), caretField.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)]) window.makeKeyAndOrderFront(nil) diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index c5a6d238c2c..dccb74ed7ef 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -26,6 +26,12 @@ import AppKit expectError("unsupported_key") { _ = try parseKey("F99") } expectError("invalid_arguments") { _ = try parseKey("Hyper+A") } expectError("invalid_arguments") { _ = try required(Optional.none, "example") } + var finitePoint = CGPoint(x: 10, y: 20), infinitePoint = CGPoint(x: CGFloat.infinity, y: -CGFloat.infinity) + var size = CGSize(width: 30, height: 40) + let sizeValue = AXValueCreate(.cgSize, &size) + precondition(axRect(position: AXValueCreate(.cgPoint, &finitePoint), size: sizeValue) == CGRect(x: 10, y: 20, width: 30, height: 40)) + precondition(axRect(position: AXValueCreate(.cgPoint, &infinitePoint), size: sizeValue) == nil) + expectError("reply_unencodable") { _ = try encodeReply(["id": "nonfinite", "result": ["x": Double.infinity]]) } for bundle in ["com.apple.systempreferences", "com.apple.Terminal", "COM.GOOGLECODE.ITERM2", "net.kovidgoyal.kitty", "com.1password.1password", "com.apple.Passwords", "com.bitwarden.desktop", "ai.sim.desktop", "com.simstudio.computer-use", "com.apple.ScriptEditor2", "com.apple.Automator", "com.apple.shortcuts"] { precondition(isProtectedApp(bundle), bundle) } diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index dbd97b93650..8a4ac1a2fb1 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -81,6 +81,8 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): time.sleep(1) snapshot = state(screenshot=True) assert 'fixture-only-secret' not in json.dumps(snapshot), 'Secure value leaked' + infinite = find(snapshot, label='Fixture infinite frame') + assert not {'x', 'y', 'width', 'height'} & infinite.keys(), f'An infinite frame reached the reply: {infinite}' image = snapshot.get('screenshot') if permissions['screenRecording']: assert image and base64.b64decode(image['base64']).startswith(b'\x89PNG\r\n\x1a\n'), snapshot.get('screenshotError') @@ -209,7 +211,7 @@ def batch(snapshot, steps): assert error['dispatchState'] == 'not_started', error assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'replaced', 'A background app was clicked or typed into' call('activate_app', bundleId=bundle) - print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) + print('PASS: real fixture state, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: if fixture: fixture.terminate()