diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift index e44934777f..952a934d2b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift @@ -129,6 +129,26 @@ extension RunnerTests { else { return .failure(RunnerAppScreenCaptureFailure(resolution)) } + return capturedScreen(captureScreen, displayID: displayID) + } + + /// Samples the same sole-screen route that synthesized gestures use when the app tree refuses + /// its window. This route cannot name a screen on an iPad or a multi-screen iPhone. + func captureSoleAppScreen() -> Result { + var screen: AnyObject? + var displayID: UInt = 0 + guard RunnerResolveSoleScreen(&screen, &displayID) == nil, + let captureScreen = screen as? XCUIScreen + else { + return .failure(.unresolvedScreen) + } + return Self.capturedScreen(captureScreen, displayID: displayID) + } + + private static func capturedScreen( + _ captureScreen: XCUIScreen, + displayID: UInt + ) -> Result { let image = captureScreen.screenshot().image guard let upright = runnerUprightCaptureImage(image) else { return .failure(.unrenderableImage) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 60de088694..a0d888c33f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -58,6 +58,7 @@ extension RunnerTests { _ = pressTvRemote(.menu) return .performed #else + let rejectedQueriesBefore = navigationRejectedQueryCount() let buttons = app.navigationBars.buttons.allElementsBoundByIndex if let back = buttons.first(where: { $0.isHittable }) { // `isHittable` promises a hittable point, not a hittable center: a bar button partly under an @@ -65,12 +66,17 @@ extension RunnerTests { back.tap() return .performed } - if isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) { + if navigationRejectedQueryCount() != rejectedQueriesBefore + || isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) + { NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", mainOwned.bundleId ?? "") } else if let frame = topNavigationBackFrame(app: app) { return tapResolvedNavigationBackControl(app: app, frame: frame) } - return tapTopLeadingNavigationFallback(app: app) + return tapTopLeadingNavigationFallback( + app: app, + windowQueryRejected: navigationRejectedQueryCount() != rejectedQueriesBefore + ) #endif } @@ -179,28 +185,31 @@ extension RunnerTests { } // Aim at the standard leading navigation slot, bounded for compact and tablet widths. let xOffset = min(max(frame.width * 0.08, 28), 44) - // Sit below the status/dynamic-island region and inside common custom RN search headers. - let yOffset = min(max(frame.height * 0.155, 56), 132) + // Sit below the status/dynamic-island region, inside the bar's leading control: on a 402x874 + // iPhone the bar spans y 62-116 and its back control 62-106. + let yOffset = min(max(frame.height * 0.1, 56), 96) return CGPoint(x: frame.minX + xOffset, y: frame.minY + yOffset) } @MainActor - private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> InAppBackOutcome { + private func tapTopLeadingNavigationFallback( + app: XCUIApplication, + windowQueryRejected: Bool + ) -> InAppBackOutcome { #if os(iOS) - let frame = onScreenWindowFrame(app: app) - guard let point = Self.topLeadingNavigationFallbackPoint(in: frame) else { + guard let context = synthesizedCoordinateContext( + app: app, + policy: synthesizedGesturePolicy(.coordinateTap), + windowQueryRejected: windowQueryRejected + ), let point = Self.topLeadingNavigationFallbackPoint(in: context.referenceFrame) else { return .unavailable } - let before = captureNavigationFallbackVisualState(app: app) - let context = synthesizedCoordinateContext( - app: app, - policy: synthesizedGesturePolicy(.coordinateTap) - )?.withReferenceFrame(frame) + let before = captureNavigationFallbackVisualState(app: app, context: context) switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: { synthesizedTapAt(app: app, x: point.x, y: point.y, context: context) }) { case .performed: - return verifyNavigationFallbackOutcome(app: app, before: before) + return verifyNavigationFallbackOutcome(app: app, before: before, context: context) case .refused: return .unavailable case .xctestFallback: @@ -208,17 +217,30 @@ extension RunnerTests { tapAt(app: app, x: point.x, y: point.y) } if case .performed = fallback.outcome { - return verifyNavigationFallbackOutcome(app: app, before: before) + return verifyNavigationFallbackOutcome(app: app, before: before, context: context) } } #endif return .unavailable } - private func captureNavigationFallbackVisualState(app: XCUIApplication) -> NavigationVisualSample { + private func navigationRejectedQueryCount() -> Int { + suppressedIssueLock.lock() + defer { suppressedIssueLock.unlock() } + return suppressedAxSnapshotIssueCount + } + + private func captureNavigationFallbackVisualState( + app: XCUIApplication, + context: SynthesizedCoordinateContext + ) -> NavigationVisualSample { #if os(iOS) return Self.navigationFallbackSample( - resolvingApp: { self.captureResolvedAppScreen(app: app) }, + resolvingApp: { + context.resolvedWindow == nil + ? self.captureSoleAppScreen() + : self.captureResolvedAppScreen(app: app) + }, systemSurface: { self.captureResolvedAppScreen(app: self.springboard) }, encoding: { runnerPngData(for: $0.image) } ) @@ -273,10 +295,11 @@ extension RunnerTests { private func verifyNavigationFallbackOutcome( app: XCUIApplication, - before: NavigationVisualSample + before: NavigationVisualSample, + context: SynthesizedCoordinateContext ) -> InAppBackOutcome { sleepFor(Self.navigationFallbackVerificationDelay) - let after = captureNavigationFallbackVisualState(app: app) + let after = captureNavigationFallbackVisualState(app: app, context: context) let observation = Self.navigationVisualObservation(before: before.data, after: after.data) // The sample sizes and the observation name together tell a refused capture apart from an // unchanged screen, and the fallback is rare enough that saying so every time costs nothing. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift index fcc40df438..6985f92eae 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift @@ -384,16 +384,20 @@ extension RunnerTests { @MainActor func synthesizedCoordinateContext( app: XCUIApplication, - policy: SynthesizedGesturePolicy + policy: SynthesizedGesturePolicy, + windowQueryRejected: Bool = false ) -> SynthesizedCoordinateContext? { #if os(iOS) var health = mainOwned.accessibilityHealth let window: XCUIElement? let referenceFrame: CGRect - if let resolved = resolveRunnerWindowElement(app: app) { + // A tree that just rejected a query skips the windows query only where the sole screen answers + // without it; anywhere else that query is still the only route. + let rejectedTreeSoleScreenFrame = windowQueryRejected ? soleScreenReferenceFrame(app: app) : nil + if rejectedTreeSoleScreenFrame == nil, let resolved = resolveRunnerWindowElement(app: app) { window = resolved.window referenceFrame = resolved.frame - } else if let soleScreenFrame = soleScreenReferenceFrame(app: app) { + } else if let soleScreenFrame = rejectedTreeSoleScreenFrame ?? soleScreenReferenceFrame(app: app) { window = nil referenceFrame = soleScreenFrame // The windows query just failed on this tree, so this gesture's keyboard probes and XCTest diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h index daf5da4616..23649dea38 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h @@ -84,6 +84,12 @@ FOUNDATION_EXPORT NSString * _Nullable RunnerResolveWindowDisplayID( // screen, and only a window can say which panel lights a foldable's app. FOUNDATION_EXPORT NSString * _Nullable RunnerResolveSoleScreenDisplayID(NSUInteger *displayID); +// Resolves the screen and display identity under the same sole-screen rule used by synthesis. +FOUNDATION_EXPORT NSString * _Nullable RunnerResolveSoleScreen( + id _Nullable *screen, + NSUInteger *displayID +); + FOUNDATION_EXPORT NSString * _Nullable RunnerRequireClass( Class cls, NSString *className, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m index 6d2c34e7c3..cb7fa39516 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m @@ -219,6 +219,12 @@ BOOL RunnerResolveApplicationScreen( // its frame is read before its screen, and a window that answers empty or raises fails closed with // the same reasons the application walk gives. NSString * _Nullable RunnerResolveSoleScreenDisplayID(NSUInteger *displayID) { + id screen = nil; + return RunnerResolveSoleScreen(&screen, displayID); +} + +NSString * _Nullable RunnerResolveSoleScreen(id _Nullable *screen, NSUInteger *displayID) { + *screen = nil; #if TARGET_OS_IOS // Only an iPhone keeps every app full-screen; an iPad can show one in part of its only screen. if (UIDevice.currentDevice.userInterfaceIdiom != UIUserInterfaceIdiomPhone) { @@ -229,6 +235,7 @@ BOOL RunnerResolveApplicationScreen( #endif __block NSUInteger screenCount = 0; __block NSUInteger readDisplayID = 0; + __block XCUIScreen *soleScreen = nil; NSString *exception = [RunnerObjCExceptionCatcher catchException:^{ NSArray *screens = XCUIScreen.screens; screenCount = screens.count; @@ -236,6 +243,7 @@ BOOL RunnerResolveApplicationScreen( NSNumber *identifier = [screens.firstObject valueForKey:@"displayID"]; if ([identifier isKindOfClass:NSNumber.class]) { readDisplayID = identifier.unsignedIntegerValue; + soleScreen = screens.firstObject; } }]; if (exception != nil || screenCount != 1 || readDisplayID == 0) { @@ -246,6 +254,7 @@ BOOL RunnerResolveApplicationScreen( ); return @"private XCTest event synthesis unavailable: no resolved application window and no sole screen to route to"; } + *screen = soleScreen; *displayID = readDisplayID; return nil; } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 438dab5e6e..08e8722ed7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -17,6 +17,124 @@ extension RunnerTests { try assertInAppBackDispatch(navigationBar: true) } + @MainActor + func testInAppBackVerifiesSoleScreenWhenXCTestCannotReadFramesOrEnumerate() throws { + let status = try launchBackFixture(["--agent-device-back-regression"]) + defer { endBackFixture() } + + let outcome = try withRejectedAppTree(framesReadable: false) { tapInAppBackControl(app: app) } + + guard showsAppsOnlyFullScreen else { + guard case .unavailable = outcome else { + return XCTFail("without a window or a sole screen there is no route to tap") + } + return + } + guard case .performed = outcome else { + return XCTFail("a window-less back must synthesize and verify against the sole display") + } + XCTAssertEqual(status.label, "Back performed") + } + + /// The sole-screen route compares real captures: a tap that lands on nothing leaves the screen + /// unchanged, so back is unavailable rather than performed. + @MainActor + func testInAppBackOnTheSoleScreenReportsATapThatChangedNothingAsUnavailable() throws { + _ = try launchBackFixture([]) + defer { endBackFixture() } + + let outcome = try withRejectedAppTree(framesReadable: false) { tapInAppBackControl(app: app) } + + guard case .unavailable = outcome else { + return XCTFail("a top-leading tap on an app without a back control must not read as performed") + } + } + + /// Only an iPhone with one screen can skip the windows query after the tree rejected one; anywhere + /// else the windows query is still asked, and its window carries the tap. + @MainActor + func testInAppBackStillAsksForWindowsWhereNoSoleScreenAnswers() throws { + let status = try launchBackFixture(["--agent-device-back-regression"]) + defer { endBackFixture() } + + let outcome = try withRejectedAppTree(framesReadable: true, rejections: 1) { + tapInAppBackControl(app: app) + } + + guard case .performed = outcome else { + return XCTFail("back must reach the fixture's top-leading control through a window or the sole screen") + } + XCTAssertEqual(status.label, "Back performed") + } + + private var showsAppsOnlyFullScreen: Bool { + UIDevice.current.userInterfaceIdiom == .phone && XCUIScreen.screens.count == 1 + } + + /// Launches the runner app; with `--agent-device-back-regression` it shows a top-leading back + /// control whose press sets the returned status to "Back performed", and without it nothing there. + @MainActor + private func launchBackFixture(_ arguments: [String]) throws -> XCUIElement { + app.launchArguments = arguments + app.launch() + mainOwned.app = app + mainOwned.bundleId = "com.callstack.agentdevice.runner" + clearSnapshotXCTestChannelPenalty(reason: "navigation-test") + let status = arguments.contains("--agent-device-back-regression") + ? app.staticTexts["agent-device-back-status"] + : app.staticTexts.element(matching: NSPredicate(format: "label == %@", "Agent Device Runner")) + XCTAssertTrue(status.waitForFixtureExistence(timeout: appExistenceTimeout)) + return status + } + + @MainActor + private func endBackFixture() { + invalidateCachedTarget(reason: "unit_test_cleanup") + app.terminate() + } + + /// Stands in for XCTest on a tree it cannot serialize while `body` runs: the first `rejections` + /// queries record the rejection the runner suppresses and find nothing, and unless + /// `framesReadable`, every element frame reads empty. + @MainActor + private func withRejectedAppTree( + framesReadable: Bool, + rejections: Int = .max, + _ body: () -> T + ) throws -> T { + var restorations: [() -> Void] = [] + defer { restorations.reversed().forEach { $0() } } + let selector = NSSelectorFromString("allElementsBoundByIndex") + let query = try XCTUnwrap(class_getInstanceMethod(XCUIElementQuery.self, selector)) + let originalQuery = method_getImplementation(query) + var remainingRejections = rejections + let rejectedQuery = imp_implementationWithBlock({ (target: AnyObject) -> NSArray in + guard remainingRejections > 0 else { + typealias Query = @convention(c) (AnyObject, Selector) -> NSArray + return unsafeBitCast(originalQuery, to: Query.self)(target, selector) + } + remainingRejections -= 1 + self.record(XCTIssue(type: .assertionFailure, compactDescription: "Failed to resolve query: kAXErrorIllegalArgument")) + return [] + } as @convention(block) (AnyObject) -> NSArray) + method_setImplementation(query, rejectedQuery) + restorations.append { + method_setImplementation(query, originalQuery) + imp_removeBlock(rejectedQuery) + } + if !framesReadable { + let frame = try XCTUnwrap(class_getInstanceMethod(XCUIElement.self, NSSelectorFromString("frame"))) + let originalFrame = method_getImplementation(frame) + let unavailableFrame = imp_implementationWithBlock({ (_: AnyObject) -> CGRect in .zero } as @convention(block) (AnyObject) -> CGRect) + method_setImplementation(frame, unavailableFrame) + restorations.append { + method_setImplementation(frame, originalFrame) + imp_removeBlock(unavailableFrame) + } + } + return body() + } + @MainActor private func assertInAppBackDispatch(navigationBar: Bool) throws { app.launchArguments = ["--agent-device-back-regression"] + (navigationBar ? ["--navigation-bar"] : []) @@ -62,15 +180,16 @@ extension RunnerTests { } #endif + /// A 402x874 iPhone's navigation bar spans y 62-116 and its back control 62-106. func testTopLeadingNavigationFallbackPointTargetsHeaderControlBand() throws { let point = try XCTUnwrap( Self.topLeadingNavigationFallbackPoint( - in: CGRect(x: 0, y: 0, width: 430, height: 932) + in: CGRect(x: 0, y: 0, width: 402, height: 874) ) ) - XCTAssertEqual(point.x, 34.4, accuracy: 0.01) - XCTAssertEqual(point.y, 132, accuracy: 0.01) + XCTAssertEqual(point.x, 32.16, accuracy: 0.01) + XCTAssertEqual(point.y, 87.4, accuracy: 0.01) } func testTopLeadingNavigationFallbackPointRejectsInvalidFrame() { diff --git a/docs/adr/0025-foldable-apple-panels.md b/docs/adr/0025-foldable-apple-panels.md index 2854d28b77..6543c705ef 100644 --- a/docs/adr/0025-foldable-apple-panels.md +++ b/docs/adr/0025-foldable-apple-panels.md @@ -152,8 +152,11 @@ A visual check of the app's own transition requires different evidence. The `back` fallback compares before/after captures to determine whether this app changed after receiving a leading tap. Two SpringBoard home-screen captures are byte-identical regardless of the app's behavior. System-surface fallback would therefore report false "unchanged" evidence for the failure under test. -The check observes only the app's own resolved window. -If none resolves, typed refusal produces an unknown outcome instead of false no-change evidence from another process (#2728). +The check observes the display of the app's resolved window. +Exception: if XCTest cannot resolve a window on an iPhone with exactly one screen, the check uses the same sole screen as synthesized dispatch. +An iPhone app occupies that screen, so this exception needs no app-tree query. +iPad and multi-screen devices still require a resolved window. +If neither route can name the display, typed refusal produces an unknown outcome instead of false no-change evidence from another process (#2728). Required and optional helper consumers differ in their frame requirements. `screenshot` and the `record start` bootstrap frame require a capture. diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index e5588f6e4d..909fe39c17 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -287,7 +287,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - 'Navigate back in the app or through system navigation. Use in-app for the app navigation stack and system when the platform back behavior is required.'; + 'Navigate back in the app or through system navigation. Use in-app for the app navigation stack and system when the platform back behavior is required. On a single-screen iPhone, an unreadable app tree can use a top-leading tap with before/after visual verification. A capture failure reports an unknown outcome.'; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 68a609bf9a..10653d3e10 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -118,6 +118,7 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: visible in-app back UI on Apple targets, and the back keyevent on Android. `--in-app` names this default explicitly. +- On a single-screen iPhone, `back` can use a top-leading tap when the app tree is unreadable. It compares the screen before and after the tap. If the screen cannot be captured, the command reports an unknown outcome; inspect the screen before you retry. - `back --system` sends system back input: the Android back keyevent, the iOS back gesture, or the tvOS Siri Remote Menu button. macOS has no system back, so it reports unavailable. - `orientation portrait|portrait-upside-down|landscape-left|landscape-right` works on iOS and Android mobile targets only. On ADB-backed Android it turns auto-rotate off, and the lock persists after the session ends until something turns it back on. - `orientation auto` hands rotation back to the sensor on ADB-backed Android, local or Limrun (`accelerometer_rotation=1`, `user_rotation` untouched), and reports `orientation: "auto"`. It does not wait for the display to rotate, because there is no target rotation: the sensor decides, and an emulator usually settles on portrait. Every other backend refuses `auto` with `UNSUPPORTED_OPERATION`.