Skip to content

Commit aed6915

Browse files
committed
fix(computer-use): read elements reported at infinity without a frame instead of crashing the helper
1 parent 57d0968 commit aed6915

5 files changed

Lines changed: 32 additions & 8 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,9 @@ func axRect(position rawPosition: CFTypeRef?, size rawSize: CFTypeRef?) -> CGRec
170170
let rawSize, CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil }
171171
var position = CGPoint.zero; var size = CGSize.zero
172172
guard AXValueGetValue(unsafeBitCast(rawPosition, to: AXValue.self), .cgPoint, &position),
173-
AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil }
173+
AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size),
174+
// Some apps report hidden elements at infinity (Maps does once its search field has focus); such an element has no usable frame.
175+
[position.x, position.y, size.width, size.height].allSatisfy(\.isFinite) else { return nil }
174176
return CGRect(origin: position, size: size)
175177
}
176178
/// Apps computer use never operates, lowercased: macOS security settings, password managers, and apps
@@ -1067,7 +1069,11 @@ func nextLine() throws -> String? {
10671069
func encodeReply(_ response: [String: Any]) throws -> Data {
10681070
let limit = 16 * 1024 * 1024
10691071
func encode(_ value: [String: Any]) throws -> Data {
1070-
try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])
1072+
// JSONSerialization raises an Objective-C exception, which Swift cannot catch, on a NaN or infinite number.
1073+
guard JSONSerialization.isValidJSONObject(value) else {
1074+
throw ComputerError("reply_unencodable", "The reply held a value JSON cannot represent.")
1075+
}
1076+
return try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys])
10711077
}
10721078
let original = try encode(response)
10731079
if original.count <= limit { return original }
@@ -1138,9 +1144,11 @@ struct Main {
11381144
response = ["id": id, "error": details]
11391145
}
11401146
catch { response = ["id": id, "error": ["code": "request_failed", "message": String(String(describing: error).prefix(2000))]] }
1141-
if let data = try? encodeReply(response) {
1142-
FileHandle.standardOutput.write(data); FileHandle.standardOutput.write(Data([10]))
1143-
}
1147+
let data: Data?
1148+
do { data = try encodeReply(response) } catch let error as ComputerError {
1149+
data = try? encodeReply(["id": id, "error": ["code": error.code, "message": error.message]])
1150+
} catch { data = nil }
1151+
if let data { FileHandle.standardOutput.write(data); FileHandle.standardOutput.write(Data([10])) }
11441152
}
11451153
}
11461154
}

‎apps/desktop/native/computer-use/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ SIGTERM/SIGINT sets a cancellation flag. Traversal, typing and drag loops check
2222

2323
## Tests and limitations
2424

25-
Compile pure native tests with `swiftc -parse-as-library -D COMPUTER_USE_TEST ComputerUse.swift tests/NativeTests.swift -o native-tests`. They cover multi-display negative origins, coordinate bounds/nonfinite input, shortcut modifiers, invalid key handling, input-sequence and batch validation, protected apps and cooperative cancellation. The live fixture also covers batches and click-to-focus for an editor that ignores Accessibility focus. Real status and app discovery outputs were validated against the generated worker Zod schema (156 apps, including 147 installed-only apps).
25+
Compile pure native tests with `swiftc -parse-as-library -D COMPUTER_USE_TEST ComputerUse.swift tests/NativeTests.swift -o native-tests`. They cover multi-display negative origins, coordinate bounds/nonfinite input, elements reported at infinity and unencodable replies, shortcut modifiers, invalid key handling, input-sequence and batch validation, protected apps and cooperative cancellation. The live fixture also covers batches and click-to-focus for an editor that ignores Accessibility focus. Real status and app discovery outputs were validated against the generated worker Zod schema (156 apps, including 147 installed-only apps).
2626

2727
`python3 tests/protocol.py <compiled-helper>` checks persistent framing, bounded-request recovery, Unicode decoding, no-prompt status, invalid methods/permissions, permission gates and clean EOF. `tests/Fixture.swift` compiles into a disposable AppKit app with a text field, secure field, increment button/counter, slider, scroll area and a visual-only marker. `python3 tests/live.py <helper>` requires both grants and verifies actual outcomes. `--allow-missing-screen` explicitly reports screenshots blocked rather than passing them; `--existing-fixture` leaves an existing fixture running for coordinated model trials. Give the fixture bundle ID `com.mothership.computer-use-fixture` when packaging it for integration tests.
2828

‎apps/desktop/native/computer-use/tests/Fixture.swift‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,13 @@ class ClickFocusField: NSTextField {
1616
final class FixedCaretField: ClickFocusField {
1717
override func setAccessibilitySelectedTextRange(_ accessibilitySelectedTextRange: NSRange) {}
1818
}
19+
/// An element that reports its frame at infinity, as some apps (Maps) report hidden elements.
20+
final class InfiniteFrameView: NSView {
21+
override func isAccessibilityElement() -> Bool { true }
22+
override func accessibilityRole() -> NSAccessibility.Role? { .group }
23+
override func accessibilityLabel() -> String? { "Fixture infinite frame" }
24+
override func accessibilityFrame() -> NSRect { NSRect(x: CGFloat.infinity, y: -CGFloat.infinity, width: 0, height: 0) }
25+
}
1926
final class FixtureDelegate: NSObject, NSApplicationDelegate {
2027
var window: NSWindow!
2128
var count = 0
@@ -45,6 +52,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate {
4552
let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider")
4653
let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time")
4754
time.dateValue = Calendar.current.date(bySettingHour: 10, minute: 15, second: 0, of: Date()) ?? Date(); time.target = self; time.action = #selector(timeChanged(_:))
55+
let infinite = InfiniteFrameView()
4856
let marker = MarkerView(); marker.setAccessibilityElement(false)
4957
marker.translatesAutoresizingMaskIntoConstraints = false
5058
let scroll = NSScrollView(); scroll.hasVerticalScroller = true
@@ -54,7 +62,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate {
5462
scroll.documentView = document
5563
scroll.contentView.postsBoundsChangedNotifications = true
5664
NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView)
57-
for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) }
65+
for view in [field, secure, button, counter, clickField, caretField, sheetButton, slider, time, timeLabel, marker, infinite, scrollCounter, scroll] { stack.addArrangedSubview(view) }
5866
window.contentView!.addSubview(stack)
5967
NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), clickField.widthAnchor.constraint(equalTo: stack.widthAnchor), caretField.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)])
6068
window.makeKeyAndOrderFront(nil)

‎apps/desktop/native/computer-use/tests/NativeTests.swift‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,12 @@ import AppKit
2626
expectError("unsupported_key") { _ = try parseKey("F99") }
2727
expectError("invalid_arguments") { _ = try parseKey("Hyper+A") }
2828
expectError("invalid_arguments") { _ = try required(Optional<String>.none, "example") }
29+
var finitePoint = CGPoint(x: 10, y: 20), infinitePoint = CGPoint(x: CGFloat.infinity, y: -CGFloat.infinity)
30+
var size = CGSize(width: 30, height: 40)
31+
let sizeValue = AXValueCreate(.cgSize, &size)
32+
precondition(axRect(position: AXValueCreate(.cgPoint, &finitePoint), size: sizeValue) == CGRect(x: 10, y: 20, width: 30, height: 40))
33+
precondition(axRect(position: AXValueCreate(.cgPoint, &infinitePoint), size: sizeValue) == nil)
34+
expectError("reply_unencodable") { _ = try encodeReply(["id": "nonfinite", "result": ["x": Double.infinity]]) }
2935
for bundle in ["com.apple.systempreferences", "com.apple.Terminal", "COM.GOOGLECODE.ITERM2", "net.kovidgoyal.kitty", "com.1password.1password", "com.apple.Passwords", "com.bitwarden.desktop", "ai.sim.desktop", "com.simstudio.computer-use", "com.apple.ScriptEditor2", "com.apple.Automator", "com.apple.shortcuts"] {
3036
precondition(isProtectedApp(bundle), bundle)
3137
}

‎apps/desktop/native/computer-use/tests/live.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5):
8181
time.sleep(1)
8282
snapshot = state(screenshot=True)
8383
assert 'fixture-only-secret' not in json.dumps(snapshot), 'Secure value leaked'
84+
infinite = find(snapshot, label='Fixture infinite frame')
85+
assert not {'x', 'y', 'width', 'height'} & infinite.keys(), f'An infinite frame reached the reply: {infinite}'
8486
image = snapshot.get('screenshot')
8587
if permissions['screenRecording']:
8688
assert image and base64.b64decode(image['base64']).startswith(b'\x89PNG\r\n\x1a\n'), snapshot.get('screenshotError')
@@ -209,7 +211,7 @@ def batch(snapshot, steps):
209211
assert error['dispatchState'] == 'not_started', error
210212
assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'replaced', 'A background app was clicked or typed into'
211213
call('activate_app', bundleId=bundle)
212-
print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
214+
print('PASS: real fixture state, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
213215
finally:
214216
if fixture:
215217
fixture.terminate()

0 commit comments

Comments
 (0)