diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index f8205b0305..ef32de15d4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -223,15 +223,41 @@ extension RunnerTests { guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "longPress requires x and y")) } - let duration = (command.durationMs ?? 800) / 1000.0 + let durationMs = command.durationMs ?? 800 + let fallback: GestureFallback? +#if os(iOS) + let policyKind = SynthesizedGesturePolicyKind.coordinateLongPress + let context = synthesizedCoordinateContext( + app: activeApp, + policy: synthesizedGesturePolicy(policyKind) + ) + switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: { + synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context) + }) { + case .performed(let timing): + let touchFrame = context.map { touchVisualizationFrame(x: x, y: y, context: $0) } + return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame)) + case .xctestFallback(let message, let hint): + fallback = GestureFallback(strategy: "xctest-coordinate-long-press", message: message, hint: hint) + case .refused(_, let message, let hint): + return unsupportedResponse(message: message, hint: hint) + } +#else + fallback = nil +#endif let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y) let (timing, outcome) = performGesture(activeApp) { - longPressAt(app: activeApp, x: x, y: y, duration: duration) + longPressAt(app: activeApp, x: x, y: y, duration: durationMs / 1000.0) } if let response = unsupportedResponse(for: outcome) { return response } - return gestureResponse(message: "long pressed", timing: timing, frame: .touch(touchFrame)) + return gestureResponse( + message: "long pressed", + timing: timing, + frame: .touch(touchFrame), + fallback: fallback + ) case .drag: guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else { return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2")) @@ -644,7 +670,7 @@ extension RunnerTests { return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome) } case .gestureViewport: - let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame) + let frame = gestureViewportFrame(app: activeApp) guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else { return Response(ok: false, error: ErrorPayload(code: "COMMAND_FAILED", message: "Active app interaction viewport is unavailable")) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 9445cd814a..586cdc038c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -35,14 +35,16 @@ extension RunnerTests { } } - // Touch frame resolves from the first step's coords so recording-gestures works unchanged. + let synthesizedContext = synthesizedSequenceCoordinateContext(steps: steps, app: activeApp) + + // Touch frame resolves from the first step's coords so recording-gestures works unchanged. A + // synthesized context already names the frame, so the app tree is not read for it again. let firstStep = steps[0] let firstFrame = (firstStep.x != nil && firstStep.y != nil) - ? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) + ? synthesizedContext.map { touchVisualizationFrame(x: firstStep.x!, y: firstStep.y!, context: $0) } + ?? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!) : nil - let synthesizedContext = synthesizedSequenceCoordinateContext(steps: steps, app: activeApp) - let execution = assembleSequenceExecution(steps: steps) { _, step in performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext) } @@ -145,9 +147,17 @@ extension RunnerTests { ) -> SequenceStepOutcome { let x = step.x ?? 0 let y = step.y ?? 0 + let longPressDurationMs = min(max(step.durationMs ?? 800, 16), 10000) if let policyKind = synthesizedPolicyKind(forSequenceStep: step) { switch performSynthesizedGesture(activeApp, kind: policyKind, context: synthesizedContext, synthesize: { - synthesizedTapAt(app: activeApp, x: x, y: y, context: synthesizedContext) + synthesizedSequenceStep( + policyKind, + app: activeApp, + x: x, + y: y, + longPressDurationMs: longPressDurationMs, + context: synthesizedContext + ) }) { case .performed(let timing): return finishedSequenceStep(step, timing: timing, outcome: .performed) @@ -167,8 +177,7 @@ extension RunnerTests { // doubleTapAt per step, matching the behavior of the retired tapSeries doubleTap path. return doubleTapAt(app: activeApp, x: x, y: y) case "longPress": - let duration = min(max(step.durationMs ?? 800, 16), 10000) / 1000.0 - return longPressAt(app: activeApp, x: x, y: y, duration: duration) + return longPressAt(app: activeApp, x: x, y: y, duration: longPressDurationMs / 1000.0) default: return tapAt(app: activeApp, x: x, y: y) } @@ -176,6 +185,31 @@ extension RunnerTests { return finishedSequenceStep(step, timing: timing, outcome: outcome) } + @MainActor + private func synthesizedSequenceStep( + _ policyKind: SynthesizedGesturePolicyKind, + app: XCUIApplication, + x: Double, + y: Double, + longPressDurationMs: Double, + context: SynthesizedCoordinateContext? + ) -> RunnerInteractionOutcome? { + switch policyKind { +#if os(iOS) + case .coordinateLongPress: + return synthesizedLongPressAt( + app: app, + x: x, + y: y, + durationMs: longPressDurationMs, + context: context + ) +#endif + default: + return synthesizedTapAt(app: app, x: x, y: y, context: context) + } + } + private func finishedSequenceStep( _ step: SequenceStep, timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index 0c948da086..3956c01cff 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -45,6 +45,7 @@ enum SynthesizedFallbackPolicy: String, Equatable, Hashable { enum SynthesizedGesturePolicyKind: String, Equatable, Hashable { case coordinateTap + case coordinateLongPress case scroll case synthesizedDrag } @@ -85,6 +86,14 @@ func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> Synthesiz keyboardPolicy: .never, fallbackPolicy: .xctestCoordinateAllowed ) + case .coordinateLongPress: + // XCTest's coordinate press resolves the app frame through the tree. On a tree XCTest cannot + // serialize, that read fails after its retries and the press raises an exception that ends the + // runner test, so the fallback needs a tree that answers. + return SynthesizedGesturePolicy( + keyboardPolicy: .never, + fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable + ) case .scroll: // Scroll places a viewport-center-symmetric swipe, so it cannot tell a keyboard-struck swipe // from a scroll that reached the edge without reading the live keyboard frame (#2500). The @@ -108,9 +117,19 @@ func shouldProbeCoordinateTapTextInput(xCTestChannelPenalized: Bool) -> Bool { !xCTestChannelPenalized } -/// A synthesized `tap` step in a `sequence` is a standalone coordinate tap and follows its policy. +/// A synthesized `tap` step in a `sequence` is a standalone coordinate tap and follows its policy. On +/// iOS a `longPress` step is a standalone long press, which is always synthesized. func synthesizedPolicyKind(forSequenceStep step: SequenceStep) -> SynthesizedGesturePolicyKind? { - step.synthesized == true && step.kind == "tap" ? .coordinateTap : nil + switch step.kind { + case "tap": + return step.synthesized == true ? .coordinateTap : nil +#if os(iOS) + case "longPress": + return .coordinateLongPress +#endif + default: + return nil + } } /// A synthesized gesture's result for its call site, which owns the XCTest coordinate gesture. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index fcc40df438..c7ad0d3aa8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -165,6 +165,63 @@ extension RunnerTests { #endif } +#if os(iOS) + /// A long press is one stationary contact held for `durationMs`, synthesized like a tap: it needs + /// no app tree, so on a tree XCTest cannot serialize it lands through the sole-screen route. + @MainActor + func synthesizedLongPressAt( + app: XCUIApplication, + x: Double, + y: Double, + durationMs: Double, + context: SynthesizedCoordinateContext? + ) -> RunnerInteractionOutcome { + guard x.isFinite, y.isFinite, durationMs.isFinite, durationMs > 0 else { + return .unsupported( + message: "synthesized coordinate long press requires finite coordinates and a positive duration", + hint: "Retry with finite x and y values and a positive durationMs." + ) + } + guard let context else { + return .unsupported( + message: "synthesized coordinate long press could not resolve an app window with a finite screen frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ) + } + let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) + let point = CoordinateSpaceRotation.native( + point: CGPoint(x: x, y: y), + in: context.referenceFrame, + interfaceOrientation: orientation + ) + logSynthesizedDispatch( + kind: "longPress", + start: point, + end: nil, + referenceFrame: context.referenceFrame, + orientation: orientation + ) + let samples: [[String: NSNumber]] = [0, durationMs].map { offsetMs in + [ + "x": NSNumber(value: Double(point.x)), + "y": NSNumber(value: Double(point.y)), + "offsetMs": NSNumber(value: offsetMs), + ] + } + if let message = RunnerSynthesizedGesture.synthesizeGesture( + withApplication: app, + resolvedWindow: context.resolvedWindow, + pointerSamples: [samples] + ) { + return .unsupported( + message: message, + hint: "Private XCTest event synthesis is required for an AX-free long press on iOS; update Xcode if this persists." + ) + } + return .performed + } +#endif + func keyboardAvoidingDragPoints( app: XCUIApplication, x: Double, @@ -239,6 +296,21 @@ extension RunnerTests { ) } + /// The touch visualization frame of a synthesized gesture, read from its context instead of the + /// app tree. + func touchVisualizationFrame( + x: Double, + y: Double, + context: SynthesizedCoordinateContext + ) -> TouchVisualizationFrame { + TouchVisualizationFrame( + x: Double(context.referenceFrame.minX) + x, + y: Double(context.referenceFrame.minY) + y, + referenceWidth: Double(context.referenceFrame.width), + referenceHeight: Double(context.referenceFrame.height) + ) + } + func resolvedDragVisualizationFrame( app: XCUIApplication, x: Double, @@ -269,6 +341,27 @@ extension RunnerTests { return CGRect(x: 0, y: 0, width: 0, height: 0) } + /// The viewport the portable gesture planner lays pointer paths in: the frame of the synthesized + /// context its record is routed through, so a tree with no window on an iPhone with one screen + /// plans on that screen, the same route a synthesized tap or scroll takes. + @MainActor + func gestureViewportFrame(app: XCUIApplication) -> CGRect { +#if os(iOS) + guard let context = synthesizedCoordinateContext( + app: app, + policy: synthesizedGesturePolicy(.synthesizedDrag) + ) else { + return frameAvoidingKeyboard(app: app, frame: app.frame) + } + // The sole-screen route means the windows query just failed, so a keyboard probe would too. + return context.resolvedWindow == nil + ? context.referenceFrame + : frameAvoidingKeyboard(app: app, frame: context.referenceFrame) +#else + return resolvedTouchReferenceFrame(app: app, appFrame: app.frame) +#endif + } + private func frameAvoidingKeyboard(app: XCUIApplication, frame: CGRect) -> CGRect { #if os(iOS) guard let keyboardFrame = visibleKeyboardFrame(app: app), !frame.isEmpty else { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift index aa18645434..56462d71de 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedGesturePolicyTests.swift @@ -78,6 +78,13 @@ extension RunnerTests { fallbackPolicy: .xctestCoordinateAllowed ) ) + XCTAssertEqual( + synthesizedGesturePolicy(.coordinateLongPress), + SynthesizedGesturePolicy( + keyboardPolicy: .never, + fallbackPolicy: .xctestCoordinateWhenAccessibilityAvailable + ) + ) XCTAssertEqual( synthesizedGesturePolicy(.scroll), SynthesizedGesturePolicy( @@ -103,7 +110,19 @@ extension RunnerTests { XCTAssertEqual(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: true)), .coordinateTap) XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("tap", synthesized: nil))) XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("doubleTap", synthesized: true))) - XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: true))) + } + + func testSequenceLongPressStepsTakeTheLongPressPolicyWithoutARequestFlag() { + for synthesized in [nil, true] as [Bool?] { +#if os(iOS) + XCTAssertEqual( + synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: synthesized)), + .coordinateLongPress + ) +#else + XCTAssertNil(synthesizedPolicyKind(forSequenceStep: sequenceStep("longPress", synthesized: synthesized))) +#endif + } } @MainActor @@ -144,6 +163,8 @@ extension RunnerTests { (.synthesizedDrag, unavailable, "refused"), (.synthesizedDrag, unknown, "xctestFallback"), (.coordinateTap, unavailable, "xctestFallback"), + (.coordinateLongPress, unavailable, "refused"), + (.coordinateLongPress, unknown, "xctestFallback"), ] for (kind, context, expected) in cases { let attempt = performSynthesizedGesture(app, kind: kind, context: context, synthesize: failedSynthesis) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift index c2ecb1279a..c056141e8c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedInteractionTests.swift @@ -1,6 +1,139 @@ import XCTest import AgentDeviceSnapshotPresentation +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) +import ObjectiveC.runtime + +/// Stands in for XCTest on an app tree it cannot serialize (a React Native feed at accessibility +/// depth 62 or more): the windows query finds no window and every element frame, the application's +/// included, reads as null. +private final class RunnerUnserializableTreeStub: NSObject { + /// Each element frame read costs XCTest's retries on such a tree, about 2 s each. + static var frameReads = 0 + @objc var allElementsBoundByIndex: [XCUIElement] { [] } + @objc var frame: CGRect { + Self.frameReads += 1 + return .null + } +} + +/// Counts XCTest coordinate presses without performing them. +private final class RunnerCoordinatePressCounter: NSObject { + static var presses = 0 + @objc(pressForDuration:) + func press(forDuration duration: TimeInterval) { + Self.presses += 1 + } +} + +extension RunnerTests { + /// Installs `RunnerUnserializableTreeStub` until the returned closure restores XCTest. + private func simulateUnserializableAppTree() throws -> () -> Void { + let swaps: [(Method?, Method?)] = [ + (class_getInstanceMethod(XCUIElementQuery.self, #selector(getter: XCUIElementQuery.allElementsBoundByIndex)), + class_getInstanceMethod(RunnerUnserializableTreeStub.self, #selector(getter: RunnerUnserializableTreeStub.allElementsBoundByIndex))), + (class_getInstanceMethod(XCUIElement.self, #selector(getter: XCUIElement.frame)), + class_getInstanceMethod(RunnerUnserializableTreeStub.self, #selector(getter: RunnerUnserializableTreeStub.frame))), + (class_getInstanceMethod(XCUICoordinate.self, #selector(XCUICoordinate.press(forDuration:))), + class_getInstanceMethod(RunnerCoordinatePressCounter.self, #selector(RunnerCoordinatePressCounter.press(forDuration:)))), + ] + let resolved = try swaps.map { (try XCTUnwrap($0.0), try XCTUnwrap($0.1)) } + let originals = resolved.map { target, stub in + (target, method_setImplementation(target, method_getImplementation(stub))) + } + RunnerCoordinatePressCounter.presses = 0 + RunnerUnserializableTreeStub.frameReads = 0 + return { + for (method, original) in originals.reversed() { method_setImplementation(method, original) } + } + } + + private var showsAppsOnlyFullScreen: Bool { + UIDevice.current.userInterfaceIdiom == .phone && XCUIScreen.screens.count == 1 + } + + /// On a tree XCTest cannot serialize, XCTest's coordinate press reads an infinite app frame and + /// raises an exception that ends the runner test. A long press needs no tree: on an iPhone with one + /// screen it is synthesized on that screen, and anything else keeps the XCTest press. + @MainActor + func testLongPressOnAnUnserializableTreeIsSynthesizedOnTheSoleScreen() throws { + app.launch() + mainOwned.app = app + mainOwned.accessibilityHealth = .healthy + let restoreXCTest = try simulateUnserializableAppTree() + defer { + restoreXCTest() + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let longPress = try runnerCommandFixture( + #"{"command":"longPress","commandId":"unserializable-long-press","x":40,"y":40,"durationMs":600}"# + ) + let pressSeries = try runnerCommandFixture( + #"{"command":"sequence","commandId":"unserializable-press-series","steps":[{"kind":"longPress","x":40,"y":40,"durationMs":600,"pauseMs":50},{"kind":"longPress","x":40,"y":40,"durationMs":600}]}"# + ) + + let longPressResponse = try executeOnMainPrepared(command: longPress, activeApp: app) + let longPressCoordinatePresses = RunnerCoordinatePressCounter.presses + let pressSeriesResponse = try executeOnMainPrepared(command: pressSeries, activeApp: app) + + XCTAssertTrue(longPressResponse.ok, String(describing: longPressResponse.error)) + XCTAssertEqual(longPressResponse.data?.message, "long pressed") + XCTAssertTrue(pressSeriesResponse.ok, String(describing: pressSeriesResponse.error)) + if showsAppsOnlyFullScreen { + XCTAssertEqual(longPressCoordinatePresses, 0) + XCTAssertNil(longPressResponse.data?.gestureFallback) + XCTAssertEqual(pressSeriesResponse.data?.completedSteps, 2) + XCTAssertEqual(RunnerCoordinatePressCounter.presses, 0) + XCTAssertEqual(RunnerUnserializableTreeStub.frameReads, 0) + let steps = pressSeriesResponse.data?.sequenceResults ?? [] + XCTAssertEqual(steps.count, 2) + let holds = [(longPressResponse.data?.gestureStartUptimeMs, longPressResponse.data?.gestureEndUptimeMs)] + + steps.map { ($0.gestureStartUptimeMs, $0.gestureEndUptimeMs) } + for (start, end) in holds { + XCTAssertGreaterThanOrEqual(try XCTUnwrap(end) - XCTUnwrap(start), 600) + } + } else { + XCTAssertEqual(longPressCoordinatePresses, 1) + XCTAssertEqual(longPressResponse.data?.gestureFallback, "xctest-coordinate-long-press") + XCTAssertEqual(RunnerCoordinatePressCounter.presses, 3) + } + } + + /// A swipe asks for the gesture viewport before it plans its path. On a tree XCTest cannot + /// serialize, an iPhone with one screen answers with that screen, the frame the planned gesture's + /// record is routed to; anything else keeps requiring a window. + @MainActor + func testGestureViewportOnAnUnserializableTreeIsTheSoleScreen() throws { + app.launch() + mainOwned.app = app + let restoreXCTest = try simulateUnserializableAppTree() + defer { + restoreXCTest() + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + let command = try runnerCommandFixture( + #"{"command":"gestureViewport","commandId":"unserializable-gesture-viewport"}"# + ) + + let response = try executeOnMainPrepared(command: command, activeApp: app) + + if showsAppsOnlyFullScreen { + XCTAssertTrue(response.ok, String(describing: response.error)) + let screen = UIScreen.main.bounds + XCTAssertEqual(response.data?.x, 0) + XCTAssertEqual(response.data?.y, 0) + XCTAssertEqual(response.data?.x2, Double(screen.width)) + XCTAssertEqual(response.data?.y2, Double(screen.height)) + } else { + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.message, "Active app interaction viewport is unavailable") + } + } +} +#endif + #if AGENT_DEVICE_RUNNER_UNIT_TESTS extension RunnerTests { func testSoleScreenReferenceFrameLaysTheScreenInTheAppsOrientation() {