fix(computer-use): read elements reported at infinity without a frame instead of crashing the helper

apps/desktop/native/computer-use/ComputerUse.swift

@@ -170,7 +170,9 @@ func axRect(position rawPosition: CFTypeRef?, size rawSize: CFTypeRef?) -> CGRec
           let rawSize, CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil }
     var position = CGPoint.zero; var size = CGSize.zero
     guard AXValueGetValue(unsafeBitCast(rawPosition, to: AXValue.self), .cgPoint, &position),
-          AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil }
+          AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size),
+          // Some apps report hidden elements at infinity (Maps does once its search field has focus); such an element has no usable frame.
+          [position.x, position.y, size.width, size.height].allSatisfy(\.isFinite) else { return nil }
     return CGRect(origin: position, size: size)
 }
 /// Apps computer use never operates, lowercased: macOS security settings, password managers, and apps
@@ -1067,7 +1069,11 @@ func nextLine() throws -> String? {
 func encodeReply(_ response: [String: Any]) throws -> Data {
     let limit = 16 * 1024 * 1024
     func encode(_ value: [String: Any]) throws -> Data {
-        try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])
+        // JSONSerialization raises an Objective-C exception, which Swift cannot catch, on a NaN or infinite number.
+        guard JSONSerialization.isValidJSONObject(value) else {
+            throw ComputerError("reply_unencodable", "The reply held a value JSON cannot represent.")
+        }
+        return try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])
     }
     let original = try encode(response)
     if original.count <= limit { return original }
@@ -1138,9 +1144,11 @@ struct Main {
                 response = ["id": id, "error": details]
             }
             catch { response = ["id": id, "error": ["code": "request_failed", "message": String(String(describing: error).prefix(2000))]] }
-            if let data = try? encodeReply(response) {
-                FileHandle.standardOutput.write(data); FileHandle.standardOutput.write(Data([10]))
-            }
+            let data: Data?
+            do { data = try encodeReply(response) } catch let error as ComputerError {
+                data = try? encodeReply(["id": id, "error": ["code": error.code, "message": error.message]])
+            } catch { data = nil }
+            if let data { FileHandle.standardOutput.write(data); FileHandle.standardOutput.write(Data([10])) }
         }
     }
 }

apps/desktop/native/computer-use/README.md

@@ -22,7 +22,7 @@ SIGTERM/SIGINT sets a cancellation flag. Traversal, typing and drag loops check
 
 ## Tests and limitations
 
-Compile pure native tests with `swiftc -parse-as-library -D COMPUTER_USE_TEST ComputerUse.swift tests/NativeTests.swift -o native-tests`. They cover multi-display negative origins, coordinate bounds/nonfinite input, shortcut modifiers, invalid key handling, input-sequence and batch validation, protected apps and cooperative cancellation. The live fixture also covers batches and click-to-focus for an editor that ignores Accessibility focus. Real status and app discovery outputs were validated against the generated worker Zod schema (156 apps, including 147 installed-only apps).
+Compile pure native tests with `swiftc -parse-as-library -D COMPUTER_USE_TEST ComputerUse.swift tests/NativeTests.swift -o native-tests`. They cover multi-display negative origins, coordinate bounds/nonfinite input, elements reported at infinity and unencodable replies, shortcut modifiers, invalid key handling, input-sequence and batch validation, protected apps and cooperative cancellation. The live fixture also covers batches and click-to-focus for an editor that ignores Accessibility focus. Real status and app discovery outputs were validated against the generated worker Zod schema (156 apps, including 147 installed-only apps).
 
 `python3 tests/protocol.py <compiled-helper>` checks persistent framing, bounded-request recovery, Unicode decoding, no-prompt status, invalid methods/permissions, permission gates and clean EOF. `tests/Fixture.swift` compiles into a disposable AppKit app with a text field, secure field, increment button/counter, slider, scroll area and a visual-only marker. `python3 tests/live.py <helper>` requires both grants and verifies actual outcomes. `--allow-missing-screen` explicitly reports screenshots blocked rather than passing them; `--existing-fixture` leaves an existing fixture running for coordinated model trials. Give the fixture bundle ID `com.mothership.computer-use-fixture` when packaging it for integration tests.
 

apps/desktop/native/computer-use/tests/Fixture.swift

@@ -16,6 +16,13 @@ class ClickFocusField: NSTextField {
 final class FixedCaretField: ClickFocusField {
     override func setAccessibilitySelectedTextRange(_ accessibilitySelectedTextRange: NSRange) {}
 }
+/// An element that reports its frame at infinity, as some apps (Maps) report hidden elements.
+final class InfiniteFrameView: NSView {
+    override func isAccessibilityElement() -> Bool { true }
+    override func accessibilityRole() -> NSAccessibility.Role? { .group }
+    override func accessibilityLabel() -> String? { "Fixture infinite frame" }
+    override func accessibilityFrame() -> NSRect { NSRect(x: CGFloat.infinity, y: -CGFloat.infinity, width: 0, height: 0) }
+}
 final class FixtureDelegate: NSObject, NSApplicationDelegate {
     var window: NSWindow!
     var count = 0
@@ -45,6 +52,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate {
         let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider")
         let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time")
         time.dateValue = Calendar.current.date(bySettingHour: 10, minute: 15, second: 0, of: Date()) ?? Date(); time.target = self; time.action = #selector(timeChanged(_:))
+        let infinite = InfiniteFrameView()
         let marker = MarkerView(); marker.setAccessibilityElement(false)
         marker.translatesAutoresizingMaskIntoConstraints = false
         let scroll = NSScrollView(); scroll.hasVerticalScroller = true
@@ -54,7 +62,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate {
         scroll.documentView = document
         scroll.contentView.postsBoundsChangedNotifications = true
         NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView)
-        for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) }
+        for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) }
         window.contentView!.addSubview(stack)
         NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), clickField.widthAnchor.constraint(equalTo: stack.widthAnchor), caretField.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)])
         window.makeKeyAndOrderFront(nil)

apps/desktop/native/computer-use/tests/NativeTests.swift

@@ -26,6 +26,12 @@ import AppKit
         expectError("unsupported_key") { _ = try parseKey("F99") }
         expectError("invalid_arguments") { _ = try parseKey("Hyper+A") }
         expectError("invalid_arguments") { _ = try required(Optional<String>.none, "example") }
+        var finitePoint = CGPoint(x: 10, y: 20), infinitePoint = CGPoint(x: CGFloat.infinity, y: -CGFloat.infinity)
+        var size = CGSize(width: 30, height: 40)
+        let sizeValue = AXValueCreate(.cgSize, &size)
+        precondition(axRect(position: AXValueCreate(.cgPoint, &finitePoint), size: sizeValue) == CGRect(x: 10, y: 20, width: 30, height: 40))
+        precondition(axRect(position: AXValueCreate(.cgPoint, &infinitePoint), size: sizeValue) == nil)
+        expectError("reply_unencodable") { _ = try encodeReply(["id": "nonfinite", "result": ["x": Double.infinity]]) }
         for bundle in ["com.apple.systempreferences", "com.apple.Terminal", "COM.GOOGLECODE.ITERM2", "net.kovidgoyal.kitty", "com.1password.1password", "com.apple.Passwords", "com.bitwarden.desktop", "ai.sim.desktop", "com.simstudio.computer-use", "com.apple.ScriptEditor2", "com.apple.Automator", "com.apple.shortcuts"] {
             precondition(isProtectedApp(bundle), bundle)
         }

apps/desktop/native/computer-use/tests/live.py

@@ -81,6 +81,8 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5):
             time.sleep(1)
         snapshot = state(screenshot=True)
         assert 'fixture-only-secret' not in json.dumps(snapshot), 'Secure value leaked'
+        infinite = find(snapshot, label='Fixture infinite frame')
+        assert not {'x', 'y', 'width', 'height'} & infinite.keys(), f'An infinite frame reached the reply: {infinite}'
         image = snapshot.get('screenshot')
         if permissions['screenRecording']:
             assert image and base64.b64decode(image['base64']).startswith(b'\x89PNG\r\n\x1a\n'), snapshot.get('screenshotError')
@@ -209,7 +211,7 @@ def batch(snapshot, steps):
         assert error['dispatchState'] == 'not_started', error
         assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'replaced', 'A background app was clicked or typed into'
         call('activate_app', bundleId=bundle)
-        print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
+        print('PASS: real fixture state, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
 finally:
     if fixture:
         fixture.terminate()