From 43c033f16e34fa493492c84fb88417f623afd65f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 06:10:40 +0200 Subject: [PATCH 1/4] fix(ios): synthesize long presses so a deep tree cannot end the runner A long press used XCTest's coordinate press, which resolves the app frame through the accessibility tree. On a tree XCTest cannot serialize (the heavy-feed fixture, Bluesky's feed) that read retried for ~16 s, then the press raised "point.x != INFINITY", which ended the runner test. The next command found the runner gone: its readiness preflight failed and the runner restarted, or its reply was lost while the runner exited. On iOS the longPress command and press-series longPress steps now synthesize one stationary contact held for the duration, through the same coordinate context as a synthesized tap, including the sole-screen route. The XCTest press stays as the fallback only while the tree answers. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+CommandExecution.swift | 39 +++++++- .../RunnerTests+SequenceExecution.swift | 38 +++++++- ...RunnerTests+SynthesizedGesturePolicy.swift | 23 ++++- .../RunnerTests+SynthesizedInteraction.swift | 57 ++++++++++++ ...rTests+SynthesizedGesturePolicyTests.swift | 23 ++++- ...nerTests+SynthesizedInteractionTests.swift | 89 +++++++++++++++++++ 6 files changed, 260 insertions(+), 9 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index f8205b0305..90789cb259 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -223,15 +223,48 @@ extension RunnerTests { guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "longPress requires x and y")) } - let duration = (command.durationMs ?? 800) / 1000.0 + let durationMs = command.durationMs ?? 800 + let fallback: GestureFallback? +#if os(iOS) + let policyKind = SynthesizedGesturePolicyKind.coordinateLongPress + let context = synthesizedCoordinateContext( + app: activeApp, + policy: synthesizedGesturePolicy(policyKind) + ) + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { + synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context) + }) { + case .performed(let timing): + let touchFrame = context.map { + TouchVisualizationFrame( + x: Double($0.referenceFrame.minX) + x, + y: Double($0.referenceFrame.minY) + y, + referenceWidth: Double($0.referenceFrame.width), + referenceHeight: Double($0.referenceFrame.height) + ) + } + return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame)) + case .xctestFallback(let message, let hint): + fallback = GestureFallback(strategy: "xctest-coordinate-long-press", message: message, hint: hint) + case .refused(_, let message, let hint): + return unsupportedResponse(message: message, hint: hint) + } +#else + fallback = nil +#endif let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) let (timing, outcome) = performGesture(activeApp) { - longPressAt(app: activeApp, x: x, y: y, duration: duration) + longPressAt(app: activeApp, x: x, y: y, duration: durationMs / 1000.0) } if let response = unsupportedResponse(for: outcome) { return response } - return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame)) + return gestureResponse( + message: "long pressed", + timing: timing, + frame: .touch(touchFrame), + fallback: fallback + ) case .drag: guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else { return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2")) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 9445cd814a..b0bc2d403c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -145,9 +145,17 @@ extension RunnerTests { ) -> SequenceStepOutcome { let x = step.x ?? 0 let y = step.y ?? 0 + let longPressDurationMs = min(max(step.durationMs ?? 800, 16), 10000) if let policyKind = synthesizedPolicyKind(forSequenceStep: step) { switch performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: { - synthesizedTapAt(app: activeApp, x: x, y: y, context: synthesizedContext) + synthesizedSequenceStep( + policyKind, + app: activeApp, + x: x, + y: y, + longPressDurationMs: longPressDurationMs, + context: synthesizedContext + ) }) { case .performed(let timing): return finishedSequenceStep(step, timing: timing, outcome: .performed) @@ -167,8 +175,7 @@ extension RunnerTests { // doubleTapAt per step, matching the behavior of the retired tapSeries doubleTap path. return doubleTapAt(app: activeApp, x: x, y: y) case "longPress": - let duration = min(max(step.durationMs ?? 800, 16), 10000) / 1000.0 - return longPressAt(app: activeApp, x: x, y: y, duration: duration) + return longPressAt(app: activeApp, x: x, y: y, duration: longPressDurationMs / 1000.0) default: return tapAt(app: activeApp, x: x, y: y) } @@ -176,6 +183,31 @@ extension RunnerTests { return finishedSequenceStep(step, timing: timing, outcome: outcome) } + @MainActor + private func synthesizedSequenceStep( + _ policyKind: SynthesizedGesturePolicyKind, + app: XCUIApplication, + x: Double, + y: Double, + longPressDurationMs: Double, + context: SynthesizedCoordinateContext? + ) -> RunnerInteractionOutcome? { + switch policyKind { +#if os(iOS) + case .coordinateLongPress: + return synthesizedLongPressAt( + app: app, + x: x, + y: y, + durationMs: longPressDurationMs, + context: context + ) +#endif + default: + return synthesizedTapAt(app: app, x: x, y: y, context: context) + } + } + private func finishedSequenceStep( _ step: SequenceStep, timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index 0c948da086..3956c01cff 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -45,6 +45,7 @@ enum SynthesizedFallbackPolicy: String, Equatable, Hashable { enum SynthesizedGesturePolicyKind: String, Equatable, Hashable { case coordinateTap + case coordinateLongPress case scroll case synthesizedDrag } @@ -85,6 +86,14 @@ func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> Synthesiz keyboardPolicy: .never, fallbackPolicy: .xctestCoordinateAllowed ) + case .coordinateLongPress: + // XCTest's coordinate press resolves the app frame through the tree. On a tree XCTest cannot + // serialize, that read fails after its retries and the press raises an exception that ends the + // runner test, so the fallback needs a tree that answers. + return SynthesizedGesturePolicy( + keyboardPolicy: .never, + fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable + ) case .scroll: // Scroll places a viewport-center-symmetric swipe, so it cannot tell a keyboard-struck swipe // from a scroll that reached the edge without reading the live keyboard frame (#2500). The @@ -108,9 +117,19 @@ func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool { !xCTestChannelPenalized } -/// A synthesized `tap` step in a `sequence` is a standalone coordinate tap and follows its policy. +/// A synthesized `tap` step in a `sequence` is a standalone coordinate tap and follows its policy. On +/// iOS a `longPress` step is a standalone long press, which is always synthesized. func synthesizedPolicyKind(forSequenceStep step: SequenceStep) -> SynthesizedGesturePolicyKind? { - step.synthesized == true && step.kind == "tap" ? .coordinateTap : nil + switch step.kind { + case "tap": + return step.synthesized == true ? .coordinateTap : nil +#if os(iOS) + case "longPress": + return .coordinateLongPress +#endif + default: + return nil + } } /// A synthesized gesture's result for its call site, which owns the XCTest coordinate gesture. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index fcc40df438..0b8b662d92 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -165,6 +165,63 @@ extension RunnerTests { #endif } +#if os(iOS) + /// A long press is one stationary contact held for `durationMs`, synthesized like a tap: it needs + /// no app tree, so on a tree XCTest cannot serialize it lands through the sole-screen route. + @MainActor + func synthesizedLongPressAt( + app: XCUIApplication, + x: Double, + y: Double, + durationMs: Double, + context: SynthesizedCoordinateContext? + ) -> RunnerInteractionOutcome { + guard x.isFinite, y.isFinite, durationMs.isFinite, durationMs > 0 else { + return .unsupported( + message: "synthesized coordinate long press requires finite coordinates and a positive duration", + hint: "Retry with finite x and y values and a positive durationMs." + ) + } + guard let context else { + return .unsupported( + message: "synthesized coordinate long press could not resolve an app window with a finite screen frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ) + } + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let point = CoordinateSpaceRotation.native( + point: CGPoint(x: x, y: y), + in: context.referenceFrame, + interfaceOrientation: orientation + ) + logSynthesizedDispatch( + kind: "longPress", + start: point, + end: nil, + referenceFrame: context.referenceFrame, + orientation: orientation + ) + let samples: [[String: NSNumber]] = [0, durationMs].map { offsetMs in + [ + "x": NSNumber(value: Double(point.x)), + "y": NSNumber(value: Double(point.y)), + "offsetMs": NSNumber(value: offsetMs), + ] + } + if let message = RunnerSynthesizedGesture.synthesizeGesture( + withApplication: app, + resolvedWindow: context.resolvedWindow, + pointerSamples: [samples] + ) { + return .unsupported( + message: message, + hint: "Private XCTest event synthesis is required for an AX-free long press on iOS; update Xcode if this persists." + ) + } + return .performed + } +#endif + func keyboardAvoidingDragPoints( app: XCUIApplication, x: Double, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift index aa18645434..56462d71de 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -78,6 +78,13 @@ extension RunnerTests { fallbackPolicy: .xctestCoordinateAllowed ) ) + XCTAssertEqual( + synthesizedGesturePolicy(.coordinateLongPress), + SynthesizedGesturePolicy( + keyboardPolicy: .never, + fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable + ) + ) XCTAssertEqual( synthesizedGesturePolicy(.scroll), SynthesizedGesturePolicy( @@ -103,7 +110,19 @@ extension RunnerTests { XCTAssertEqual(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: true)), .coordinateTap) XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: nil))) XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("doubleTap", synthesized: true))) - XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: true))) + } + + func testSequenceLongPressStepsTakeTheLongPressPolicyWithoutARequestFlag() { + for synthesized in [nil, true] as [Bool?] { +#if os(iOS) + XCTAssertEqual( + synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: synthesized)), + .coordinateLongPress + ) +#else + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: synthesized))) +#endif + } } @MainActor @@ -144,6 +163,8 @@ extension RunnerTests { (.synthesizedDrag, unavailable, "refused"), (.synthesizedDrag, unknown, "xctestFallback"), (.coordinateTap, unavailable, "xctestFallback"), + (.coordinateLongPress, unavailable, "refused"), + (.coordinateLongPress, unknown, "xctestFallback"), ] for (kind, context, expected) in cases { let attempt = performSynthesizedGesture(app, kind: kind, context: context, synthesize: failedSynthesis) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index c2ecb1279a..b5aab8e183 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -1,6 +1,95 @@ import XCTest import AgentDeviceSnapshotPresentation +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +/// Stands in for XCTest on an app tree it cannot serialize (a React Native feed at accessibility +/// depth 62 or more): the windows query finds no window and every element frame, the application's +/// included, reads as null. +private final class RunnerUnserializableTreeStub: NSObject { + @objc var allElementsBoundByIndex: [XCUIElement] { [] } + @objc var frame: CGRect { .null } +} + +/// Counts XCTest coordinate presses without performing them. +private final class RunnerCoordinatePressCounter: NSObject { + static var presses = 0 + @objc(pressForDuration:) + func press(forDuration duration: TimeInterval) { + Self.presses += 1 + } +} + +extension RunnerTests { + /// Installs `RunnerUnserializableTreeStub` until the returned closure restores XCTest. + private func simulateUnserializableAppTree() throws -> () -> Void { + let swaps: [(Method?, Method?)] = [ + (class_getInstanceMethod(XCUIElementQuery.self, #selector(getter: XCUIElementQuery.allElementsBoundByIndex)), + class_getInstanceMethod(RunnerUnserializableTreeStub.self, #selector(getter: RunnerUnserializableTreeStub.allElementsBoundByIndex))), + (class_getInstanceMethod(XCUIElement.self, #selector(getter: XCUIElement.frame)), + class_getInstanceMethod(RunnerUnserializableTreeStub.self, #selector(getter: RunnerUnserializableTreeStub.frame))), + (class_getInstanceMethod(XCUICoordinate.self, #selector(XCUICoordinate.press(forDuration:))), + class_getInstanceMethod(RunnerCoordinatePressCounter.self, #selector(RunnerCoordinatePressCounter.press(forDuration:)))), + ] + var originals: [(Method, IMP)] = [] + for (target, stub) in swaps { + let target = try XCTUnwrap(target) + let stub = try XCTUnwrap(stub) + originals.append((target, method_setImplementation(target, method_getImplementation(stub)))) + } + RunnerCoordinatePressCounter.presses = 0 + return { + for (method, original) in originals.reversed() { method_setImplementation(method, original) } + } + } + + private var showsAppsOnlyFullScreen: Bool { + UIDevice.current.userInterfaceIdiom == .phone && XCUIScreen.screens.count == 1 + } + + /// On a tree XCTest cannot serialize, XCTest's coordinate press reads an infinite app frame and + /// raises an exception that ends the runner test. A long press needs no tree: on an iPhone with one + /// screen it is synthesized on that screen, and anything else keeps the XCTest press. + @MainActor + func testLongPressOnAnUnserializableTreeIsSynthesizedOnTheSoleScreen() throws { + app.launch() + mainOwned.app = app + mainOwned.accessibilityHealth = .healthy + let restoreXCTest = try simulateUnserializableAppTree() + defer { + restoreXCTest() + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let longPress = try runnerCommandFixture( + #"{"command":"longPress","commandId":"unserializable-long-press","x":40,"y":40,"durationMs":600}"# + ) + let pressSeries = try runnerCommandFixture( + #"{"command":"sequence","commandId":"unserializable-press-series","steps":[{"kind":"longPress","x":40,"y":40,"durationMs":600,"pauseMs":50},{"kind":"longPress","x":40,"y":40,"durationMs":600}]}"# + ) + + let longPressResponse = try executeOnMainPrepared(command: longPress, activeApp: app) + let longPressCoordinatePresses = RunnerCoordinatePressCounter.presses + let pressSeriesResponse = try executeOnMainPrepared(command: pressSeries, activeApp: app) + + XCTAssertTrue(longPressResponse.ok, String(describing: longPressResponse.error)) + XCTAssertEqual(longPressResponse.data?.message, "long pressed") + XCTAssertTrue(pressSeriesResponse.ok, String(describing: pressSeriesResponse.error)) + if showsAppsOnlyFullScreen { + XCTAssertEqual(longPressCoordinatePresses, 0) + XCTAssertNil(longPressResponse.data?.gestureFallback) + XCTAssertEqual(pressSeriesResponse.data?.completedSteps, 2) + XCTAssertEqual(RunnerCoordinatePressCounter.presses, 0) + } else { + XCTAssertEqual(longPressCoordinatePresses, 1) + XCTAssertEqual(longPressResponse.data?.gestureFallback, "xctest-coordinate-long-press") + XCTAssertEqual(RunnerCoordinatePressCounter.presses, 3) + } + } +} +#endif + #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests { func testSoleScreenReferenceFrameLaysTheScreenInTheAppsOrientation() { From 844842171b0f569d504f73dcc4eda140014df33a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 06:10:42 +0200 Subject: [PATCH 2/4] fix(ios): answer the gesture viewport from the sole screen on deep trees A swipe asks the runner for the gesture viewport before it plans its path. The viewport read the app window or app.frame, and on a tree XCTest cannot serialize both fail, so every swipe failed with "Active app interaction viewport is unavailable" even though the planned gesture itself is synthesized without the tree. On iOS the viewport now follows the synthesized coordinate context: the resolved window, or, when none resolves on an iPhone with one screen, that screen in the app's orientation. iPad and multi-screen devices still require a window. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+CommandExecution.swift | 2 +- .../RunnerTests+SynthesizedInteraction.swift | 19 +++++++++++ ...nerTests+SynthesizedInteractionTests.swift | 32 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 90789cb259..e282e3ccad 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -677,7 +677,7 @@ extension RunnerTests { return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) } case .gestureViewport: - let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame) + let frame = gestureViewportFrame(app: activeApp) guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else { return Response(ok: false, error: ErrorPayload(code: "COMMAND_FAILED", message: "Active app interaction viewport is unavailable")) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 0b8b662d92..edc7c037e0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -326,6 +326,25 @@ extension RunnerTests { return CGRect(x: 0, y: 0, width: 0, height: 0) } + /// The viewport the portable gesture planner lays pointer paths in. A planned gesture's record is + /// routed to the window this frame was read from, so when no window resolves on an iPhone with one + /// screen, the viewport is that screen: the same sole-screen route a synthesized tap or scroll takes. + @MainActor + func gestureViewportFrame(app: XCUIApplication) -> CGRect { +#if os(iOS) + if let resolved = resolveRunnerWindowElement(app: app) { + return frameAvoidingKeyboard(app: app, frame: resolved.frame) + } + // The windows query just failed on this tree, so a keyboard probe would fail on it too. + if let soleScreenFrame = soleScreenReferenceFrame(app: app) { + return soleScreenFrame + } + return frameAvoidingKeyboard(app: app, frame: app.frame) +#else + return resolvedTouchReferenceFrame(app: app, appFrame: app.frame) +#endif + } + private func frameAvoidingKeyboard(app: XCUIApplication, frame: CGRect) -> CGRect { #if os(iOS) guard let keyboardFrame = visibleKeyboardFrame(app: app), !frame.isEmpty else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index b5aab8e183..387a1d631f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -87,6 +87,38 @@ extension RunnerTests { XCTAssertEqual(RunnerCoordinatePressCounter.presses, 3) } } + + /// A swipe asks for the gesture viewport before it plans its path. On a tree XCTest cannot + /// serialize, an iPhone with one screen answers with that screen, the frame the planned gesture's + /// record is routed to; anything else keeps requiring a window. + @MainActor + func testGestureViewportOnAnUnserializableTreeIsTheSoleScreen() throws { + app.launch() + mainOwned.app = app + let restoreXCTest = try simulateUnserializableAppTree() + defer { + restoreXCTest() + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + #"{"command":"gestureViewport","commandId":"unserializable-gesture-viewport"}"# + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + if showsAppsOnlyFullScreen { + XCTAssertTrue(response.ok, String(describing: response.error)) + let screen = UIScreen.main.bounds + XCTAssertEqual(response.data?.x, 0) + XCTAssertEqual(response.data?.y, 0) + XCTAssertEqual(response.data?.x2, Double(screen.width)) + XCTAssertEqual(response.data?.y2, Double(screen.height)) + } else { + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.message, "Active app interaction viewport is unavailable") + } + } } #endif From 521e511b5d2e75538f40dadd7ad6bff3a1ad6573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 06:21:37 +0200 Subject: [PATCH 3/4] perf(ios): take a synthesized press series' touch frame from its context A press series read the app frame and the windows query for its first step's touch frame before any step ran. On a tree XCTest cannot serialize each of those reads costs about 2 s of XCTest retries, and now that held presses run as synthesized sequence steps that cost came before every hold. A sequence with a synthesized context reads the frame from that context, as the single longPress command does. The long-press test now also pins zero frame reads and the held duration of each synthesized hold. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+CommandExecution.swift | 9 +------ .../RunnerTests+SequenceExecution.swift | 10 ++++---- .../RunnerTests+SynthesizedInteraction.swift | 15 ++++++++++++ ...nerTests+SynthesizedInteractionTests.swift | 24 ++++++++++++++----- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index e282e3ccad..ef32de15d4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -235,14 +235,7 @@ extension RunnerTests { synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context) }) { case .performed(let timing): - let touchFrame = context.map { - TouchVisualizationFrame( - x: Double($0.referenceFrame.minX) + x, - y: Double($0.referenceFrame.minY) + y, - referenceWidth: Double($0.referenceFrame.width), - referenceHeight: Double($0.referenceFrame.height) - ) - } + let touchFrame = context.map { touchVisualizationFrame(x: x, y: y, context: $0) } return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame)) case .xctestFallback(let message, let hint): fallback = GestureFallback(strategy: "xctest-coordinate-long-press", message: message, hint: hint) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index b0bc2d403c..586cdc038c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -35,14 +35,16 @@ extension RunnerTests { } } - // Touch frame resolves from the first step's coords so recording-gestures works unchanged. + let synthesizedContext = synthesizedSequenceCoordinateContext(steps: steps, app: activeApp) + + // Touch frame resolves from the first step's coords so recording-gestures works unchanged. A + // synthesized context already names the frame, so the app tree is not read for it again. let firstStep = steps[0] let firstFrame = (firstStep.x != nil && firstStep.y != nil) - ? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) + ? synthesizedContext.map { touchVisualizationFrame(x: firstStep.x!, y: firstStep.y!, context: $0) } + ?? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) : nil - let synthesizedContext = synthesizedSequenceCoordinateContext(steps: steps, app: activeApp) - let execution = assembleSequenceExecution(steps: steps) { _, step in performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index edc7c037e0..229726efce 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -296,6 +296,21 @@ extension RunnerTests { ) } + /// The touch visualization frame of a synthesized gesture, read from its context instead of the + /// app tree. + func touchVisualizationFrame( + x: Double, + y: Double, + context: SynthesizedCoordinateContext + ) -> TouchVisualizationFrame { + TouchVisualizationFrame( + x: Double(context.referenceFrame.minX) + x, + y: Double(context.referenceFrame.minY) + y, + referenceWidth: Double(context.referenceFrame.width), + referenceHeight: Double(context.referenceFrame.height) + ) + } + func resolvedDragVisualizationFrame( app: XCUIApplication, x: Double, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index 387a1d631f..c056141e8c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -8,8 +8,13 @@ import ObjectiveC.runtime /// depth 62 or more): the windows query finds no window and every element frame, the application's /// included, reads as null. private final class RunnerUnserializableTreeStub: NSObject { + /// Each element frame read costs XCTest's retries on such a tree, about 2 s each. + static var frameReads = 0 @objc var allElementsBoundByIndex: [XCUIElement] { [] } - @objc var frame: CGRect { .null } + @objc var frame: CGRect { + Self.frameReads += 1 + return .null + } } /// Counts XCTest coordinate presses without performing them. @@ -32,13 +37,12 @@ extension RunnerTests { (class_getInstanceMethod(XCUICoordinate.self, #selector(XCUICoordinate.press(forDuration:))), class_getInstanceMethod(RunnerCoordinatePressCounter.self, #selector(RunnerCoordinatePressCounter.press(forDuration:)))), ] - var originals: [(Method, IMP)] = [] - for (target, stub) in swaps { - let target = try XCTUnwrap(target) - let stub = try XCTUnwrap(stub) - originals.append((target, method_setImplementation(target, method_getImplementation(stub)))) + let resolved = try swaps.map { (try XCTUnwrap($0.0), try XCTUnwrap($0.1)) } + let originals = resolved.map { target, stub in + (target, method_setImplementation(target, method_getImplementation(stub))) } RunnerCoordinatePressCounter.presses = 0 + RunnerUnserializableTreeStub.frameReads = 0 return { for (method, original) in originals.reversed() { method_setImplementation(method, original) } } @@ -81,6 +85,14 @@ extension RunnerTests { XCTAssertNil(longPressResponse.data?.gestureFallback) XCTAssertEqual(pressSeriesResponse.data?.completedSteps, 2) XCTAssertEqual(RunnerCoordinatePressCounter.presses, 0) + XCTAssertEqual(RunnerUnserializableTreeStub.frameReads, 0) + let steps = pressSeriesResponse.data?.sequenceResults ?? [] + XCTAssertEqual(steps.count, 2) + let holds = [(longPressResponse.data?.gestureStartUptimeMs, longPressResponse.data?.gestureEndUptimeMs)] + + steps.map { ($0.gestureStartUptimeMs, $0.gestureEndUptimeMs) } + for (start, end) in holds { + XCTAssertGreaterThanOrEqual(try XCTUnwrap(end) - XCTUnwrap(start), 600) + } } else { XCTAssertEqual(longPressCoordinatePresses, 1) XCTAssertEqual(longPressResponse.data?.gestureFallback, "xctest-coordinate-long-press") From 9f5cbc9ebdb50e3daf078ba7e3b36cedc55e389b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 10:40:05 +0200 Subject: [PATCH 4/4] refactor(ios): derive the gesture viewport from the synthesized context The window, then sole screen, then nothing ladder lives in synthesizedCoordinateContext; the viewport now reads its frame instead of repeating the ladder. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+SynthesizedInteraction.swift | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index 229726efce..c7ad0d3aa8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -341,20 +341,22 @@ extension RunnerTests { return CGRect(x: 0, y: 0, width: 0, height: 0) } - /// The viewport the portable gesture planner lays pointer paths in. A planned gesture's record is - /// routed to the window this frame was read from, so when no window resolves on an iPhone with one - /// screen, the viewport is that screen: the same sole-screen route a synthesized tap or scroll takes. + /// The viewport the portable gesture planner lays pointer paths in: the frame of the synthesized + /// context its record is routed through, so a tree with no window on an iPhone with one screen + /// plans on that screen, the same route a synthesized tap or scroll takes. @MainActor func gestureViewportFrame(app: XCUIApplication) -> CGRect { #if os(iOS) - if let resolved = resolveRunnerWindowElement(app: app) { - return frameAvoidingKeyboard(app: app, frame: resolved.frame) - } - // The windows query just failed on this tree, so a keyboard probe would fail on it too. - if let soleScreenFrame = soleScreenReferenceFrame(app: app) { - return soleScreenFrame + guard let context = synthesizedCoordinateContext( + app: app, + policy: synthesizedGesturePolicy(.synthesizedDrag) + ) else { + return frameAvoidingKeyboard(app: app, frame: app.frame) } - return frameAvoidingKeyboard(app: app, frame: app.frame) + // The sole-screen route means the windows query just failed, so a keyboard probe would too. + return context.resolvedWindow == nil + ? context.referenceFrame + : frameAvoidingKeyboard(app: app, frame: context.referenceFrame) #else return resolvedTouchReferenceFrame(app: app, appFrame: app.frame) #endif