Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 61 additions & 5 deletions .github/workflows/xctest-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,25 @@ concurrency:

jobs:
full-suite:
name: iOS Runner Full XCTest Suite
name: iOS Runner Full XCTest Suite (${{ matrix.leg.name }})
runs-on: macos-26
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
leg:
# Both legs share the toolchain, the runner build cache, and the artifact steps above.
# The full suite runs on an iPhone, where the sole-screen rule is decided by the idiom
# alone. The iPad leg boots an iPad and runs only the sole-screen tests: the iPad half
# of the rule — the route opens only while the in-process full-screen evidence proves
# the app fills its screen — can execute nowhere else, and the evidence line that probe
# prints is what turns a revert to idiom-only routing red instead of silently green.
- name: iPhone full suite
device-name: iPhone 17 Pro
lane: nightly
- name: iPad sole-screen evidence
device-name: iPad Pro 11-inch (M4)
lane: nightly-ipad
env:
# Must match apple.yml. The runtime version rides in the derived-data cache key, and
# AGENT_DEVICE_XCUITEST_INCLUDE_UNIT_TESTS is BOTH an input to the cache's build-variant
Expand All @@ -38,7 +54,7 @@ jobs:
AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH: ${{ github.workspace }}/.tmp/ios-runner-derived
RESULT_BUNDLE_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/RunnerTests.xcresult
RESULT_SUMMARY_PATH: ${{ github.workspace }}/.tmp/xctest-nightly/test-results-summary.json
XCTEST_LANE: nightly
XCTEST_LANE: ${{ matrix.leg.lane }}
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
Expand Down Expand Up @@ -67,7 +83,7 @@ jobs:
uses: ./.github/actions/boot-ios-test-simulator
with:
runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
preferred-device-name: iPhone 17 Pro
preferred-device-name: ${{ matrix.leg.device-name }}

# Same command as apple.yml's targeted step minus every `-only-testing:` flag, so the
# xctestrun's own test plan decides what runs — with one exception.
Expand All @@ -84,6 +100,7 @@ jobs:
# A typo in that identifier silently re-arms the hang, so
# `pnpm check:xctest-selection` validates `-skip-testing:` exactly like `-only-testing:`.
- name: Run the full iOS runner XCTest suite
if: matrix.leg.lane == 'nightly'
run: |
set -euo pipefail
XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
Expand All @@ -102,6 +119,45 @@ jobs:
}
'

# The iPad leg's liveness check is its own grep below: it runs a named selection, not the
# whole bundle, so the count assertion the full-suite leg makes does not apply to it. The
# boot action falls back to any simulator when the preferred name is missing, so this step
# proves it booted the iPad that leg claims — the evidence line can only come from one.
- name: Run the sole-screen tests on iPad
if: matrix.leg.lane == 'nightly-ipad'
run: |
set -euo pipefail
UDID='${{ steps.ios-simulator.outputs.simulator-udid }}'
xcrun simctl list devices -j | node -e '
const fs = require("node:fs");
const udid = process.argv[1];
const payload = JSON.parse(fs.readFileSync(0, "utf8"));
const device = Object.values(payload.devices ?? {})
.flat()
.find((entry) => entry.udid === udid);
if (!device?.name?.startsWith("iPad")) {
console.error(`Booted ${device?.name ?? udid}; this leg needs an iPad simulator`);
process.exit(1);
}
' "$UDID"
XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
test -n "$XCTESTRUN_PATH"
xcodebuild test-without-building \
-xctestrun "$XCTESTRUN_PATH" \
-destination "platform=iOS Simulator,id=$UDID" \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testLongPressOnAnUnserializableTreeIsSynthesizedOnTheSoleScreen \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testGestureViewportOnAnUnserializableTreeIsTheSoleScreen \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSoleScreenDisplayIDIsNamedOnlyForAnAppThatFillsItsScreen \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackVerifiesSoleScreenWhenXCTestCannotReadFramesOrEnumerate \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testInAppBackAfterARejectedQueryTapsThroughTheRouteThatOpens \
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRankedInAppBackUsesSynthesisWithoutXCTestTap \
2>&1 | tee .tmp/xctest-nightly/runner.log
# The probe prints this line only when a non-phone idiom asks it. Its absence means the
# iPad never probed (a revert to idiom-only routing), not that a test failed.
grep -q 'AGENT_DEVICE_RUNNER_FULL_SCREEN_EVIDENCE result=fills-screen' \
.tmp/xctest-nightly/runner.log \
|| { echo 'The iPad sole-screen route never produced full-screen evidence.'; exit 1; }

# Best-effort and never the job's verdict on its own; the step below is what asserts.
# `--compact` first because a red night's summary is the large one, and the job summary
# has a 1 MiB cap — the plain form is the fallback if an Xcode version rejects the flag.
Expand All @@ -126,7 +182,7 @@ jobs:
# as red too. The failure list is capped so a badly red night cannot blow the 1 MiB
# job-summary limit and lose the headline with it.
- name: Report the run and assert the source-derived count
if: always()
if: always() && matrix.leg.lane == 'nightly'
run: |
set -euo pipefail
if [ ! -s "$RESULT_SUMMARY_PATH" ]; then
Expand All @@ -142,7 +198,7 @@ jobs:
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: xctest-nightly-results-${{ github.run_id }}-${{ github.run_attempt }}
name: xctest-nightly-${{ matrix.leg.lane }}-${{ github.run_id }}-${{ github.run_attempt }}
include-hidden-files: true
path: .tmp/xctest-nightly
if-no-files-found: warn
Original file line number Diff line number Diff line change
Expand Up @@ -672,11 +672,20 @@ - (void)viewDidLoad {
bar.items = @[item];
[self.view addSubview:bar];
} else {
// The control sits in the top bar band, whose height above the content differs by family
// (59 pt on an iPhone 17 Pro, 32 pt on an iPad Pro), so it anchors to the safe-area top
// instead of a hardcoded iPhone y.
UIButton *back = [UIButton buttonWithType:UIButtonTypeSystem];
back.frame = CGRectMake(12, 62, 88, 44);
back.translatesAutoresizingMaskIntoConstraints = NO;
[back setTitle:@"Go back" forState:UIControlStateNormal];
[back addTarget:self action:@selector(navigationBackPressed) forControlEvents:UIControlEventTouchUpInside];
[self.view addSubview:back];
[NSLayoutConstraint activateConstraints:@[
[back.leadingAnchor constraintEqualToAnchor:self.view.safeAreaLayoutGuide.leadingAnchor constant:12],
[back.topAnchor constraintEqualToAnchor:self.view.safeAreaLayoutGuide.topAnchor],
[back.widthAnchor constraintEqualToConstant:88],
[back.heightAnchor constraintEqualToConstant:44],
]];
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,12 @@ extension RunnerTests {
}

/// Samples the same sole-screen route that synthesized gestures use when the app tree refuses
/// its window. This route cannot name a screen on an iPad or a multi-screen iPhone.
func captureSoleAppScreen() -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
/// its window. On an iPad the route opens only while the app provably fills its sole screen, so
/// a windowed app keeps refusing instead of sampling the wrong frame.
func captureSoleAppScreen(app: XCUIApplication) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
var screen: AnyObject?
var displayID: UInt = 0
guard RunnerResolveSoleScreen(&screen, &displayID) == nil,
guard RunnerResolveSoleScreen(app, &screen, &displayID) == nil,
let captureScreen = screen as? XCUIScreen
else {
return .failure(.unresolvedScreen)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ static RunnerTapSynthesisStatus RunnerTrySynthesizeTap(
NSUInteger displayID = 0;
// A nil window is the caller's sole-screen route: the app tree could not name its window.
NSString *displayError = resolvedWindow == nil
? RunnerResolveSoleScreenDisplayID(&displayID)
? RunnerResolveSoleScreenDisplayID(application, &displayID)
: RunnerResolveWindowDisplayID(resolvedWindow, &displayID);
if (displayError != nil) return displayError;
NSInteger interfaceOrientation =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -179,18 +179,44 @@ extension RunnerTests {
return candidate.midY >= window.minY && candidate.midY <= maxY
}

static func topLeadingNavigationFallbackPoint(in frame: CGRect) -> CGPoint? {
/// The point the top-leading `back` fallback aims at: the leading slot of the bar that sits
/// directly under the top inset of the frame the tap lands in. A fraction of the frame height
/// cannot serve both families — an iPhone 17 Pro measures a 59-pt inset with its bar at y 62-116,
/// while an iPad Pro 11" measures 32 with its bar at y 32-86, where the iPhone's height-derived 96
/// lands below the bar. `22` is half of UIKit's standard 44-pt top bar. The bounds keep a runtime
/// that reports no inset inside a plausible bar, and a large inset from leaving it.
static func topLeadingNavigationFallbackPoint(
in frame: CGRect,
topSafeAreaInset: CGFloat
) -> CGPoint? {
guard SnapshotGeometry.isPositiveFinite(frame) else {
return nil
}
// Aim at the standard leading navigation slot, bounded for compact and tablet widths.
let xOffset = min(max(frame.width * 0.08, 28), 44)
// Sit below the status/dynamic-island region, inside the bar's leading control: on a 402x874
// iPhone the bar spans y 62-116 and its back control 62-106.
let yOffset = min(max(frame.height * 0.1, 56), 96)
let yOffset = min(max(topSafeAreaInset + 22, 56), 96)
return CGPoint(x: frame.minX + xOffset, y: frame.minY + yOffset)
}

/// The screen's top inset, read in-process from the runner's own window scene. The status-bar /
/// Dynamic Island band is a screen-level fact, so the runner's full-screen host window reports the
/// one the target app sees — and unlike any element frame it answers on a tree XCTest cannot
/// serialize. The largest top inset among the scene's windows wins: an alert or keyboard window
/// reports none, and the window that carries the band is the one to ask.
private func screenTopSafeAreaInset() -> CGFloat {
#if os(iOS)
let insets = UIApplication.shared.connectedScenes
.compactMap { $0 as? UIWindowScene }
.flatMap(\.windows)
.map(\.safeAreaInsets.top)
let inset = insets.max() ?? 0
NSLog("AGENT_DEVICE_RUNNER_BACK_FALLBACK_TOP_INSET read=%.1f windows=%zd", inset, insets.count)
return inset
#else
return 0
#endif
}

@MainActor
private func tapTopLeadingNavigationFallback(
app: XCUIApplication,
Expand All @@ -201,7 +227,10 @@ extension RunnerTests {
app: app,
policy: synthesizedGesturePolicy(.coordinateTap),
windowQueryRejected: windowQueryRejected
), let point = Self.topLeadingNavigationFallbackPoint(in: context.referenceFrame) else {
), let point = Self.topLeadingNavigationFallbackPoint(
in: context.referenceFrame,
topSafeAreaInset: screenTopSafeAreaInset()
) else {
return .unavailable
}
let before = captureNavigationFallbackVisualState(app: app, context: context)
Expand Down Expand Up @@ -238,7 +267,7 @@ extension RunnerTests {
return Self.navigationFallbackSample(
resolvingApp: {
context.resolvedWindow == nil
? self.captureSoleAppScreen()
? self.captureSoleAppScreen(app: app)
: self.captureResolvedAppScreen(app: app)
},
systemSurface: { self.captureResolvedAppScreen(app: self.springboard) },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -342,8 +342,8 @@ extension RunnerTests {
}

/// The viewport the portable gesture planner lays pointer paths in: the frame of the synthesized
/// context its record is routed through, so a tree with no window on an iPhone with one screen
/// plans on that screen, the same route a synthesized tap or scroll takes.
/// context its record is routed through, so a tree with no window whose app provably fills its
/// sole screen plans on that screen, the same route a synthesized tap or scroll takes.
@MainActor
func gestureViewportFrame(app: XCUIApplication) -> CGRect {
#if os(iOS)
Expand Down Expand Up @@ -517,14 +517,18 @@ extension RunnerTests {


#if os(iOS)
/// The reference frame of the sole-screen route, read without the app's accessibility tree: on an
/// iPhone with one screen a full-screen app's window is that screen. A deep app tree makes the
/// windows query fail with `kAXErrorIllegalArgument`, and without this route every synthesized
/// tap and scroll on such a screen failed even though synthesis itself needs no tree. Anything
/// that can show an app in part of a screen (iPad, more than one screen) keeps requiring a window.
/// The reference frame of the sole-screen route, read without the app's accessibility tree: when
/// the app provably fills the device's only screen, its window is that screen. A deep app tree
/// makes the windows query fail with `kAXErrorIllegalArgument`, and without this route every
/// synthesized tap and scroll on such a screen failed even though synthesis itself needs no tree.
/// `RunnerResolveSoleScreen` owns which devices prove that — an iPhone always, an iPad only while
/// the app fills the screen — so this reads the same answer the record's routing takes, and
/// anything else (a windowed iPad, a multi-screen foldable) keeps requiring a window.
@MainActor
func soleScreenReferenceFrame(app: XCUIApplication) -> CGRect? {
guard UIDevice.current.userInterfaceIdiom == .phone, XCUIScreen.screens.count == 1 else {
var resolvedScreen: AnyObject?
var resolvedDisplayID: UInt = 0
guard RunnerResolveSoleScreen(app, &resolvedScreen, &resolvedDisplayID) == nil else {
return nil
}
let screen = UIScreen.main
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#import <Foundation/Foundation.h>
#import <CoreGraphics/CoreGraphics.h>

NS_ASSUME_NONNULL_BEGIN

Expand Down Expand Up @@ -79,17 +80,57 @@ FOUNDATION_EXPORT NSString * _Nullable RunnerResolveWindowDisplayID(

// Reads the display ID of the device's only screen, for a gesture whose app window XCTest cannot
// resolve because the app's accessibility tree is too deep to serialize. On an iPhone with exactly
// one screen that screen hosts the full-screen app, so no window is needed to name it. Anywhere else
// this returns a message and leaves `displayID` untouched: an iPad can show the app in part of its
// screen, and only a window can say which panel lights a foldable's app.
FOUNDATION_EXPORT NSString * _Nullable RunnerResolveSoleScreenDisplayID(NSUInteger *displayID);
// one screen that screen hosts the full-screen app, so no window is needed to name it. On an iPad
// the same route opens only while the app provably fills that screen, decided in-process by the
// evidence seam below without the app's tree. Anywhere else this returns a message and leaves
// `displayID` untouched: a windowed iPad app sits in part of its screen, and only a window can say
// which panel lights a foldable's app.
FOUNDATION_EXPORT NSString * _Nullable RunnerResolveSoleScreenDisplayID(
id application,
NSUInteger *displayID
);

// Resolves the screen and display identity under the same sole-screen rule used by synthesis.
FOUNDATION_EXPORT NSString * _Nullable RunnerResolveSoleScreen(
id application,
id _Nullable *screen,
NSUInteger *displayID
);

// What the full-screen proof concluded. The decision is a function of these values, so a test
// supplies them directly instead of restating the device rule.
typedef NS_ENUM(NSUInteger, RunnerFullScreenEvidence) {
// The target app is the accessibility primary app and its frame is the screen's frame.
RunnerFullScreenEvidenceFillsScreen,
// The target app is not the accessibility primary app, so no screen belongs to it to fill.
RunnerFullScreenEvidenceNotPrimaryApp,
// The primary app's frame differs from the system app's frame: an iPad window that shows the app
// in part of the screen (Windowed Apps or Stage Manager).
RunnerFullScreenEvidenceWindowed,
// The runtime, the app, or the system app would not report a usable frame, which proves nothing.
RunnerFullScreenEvidenceUnknown,
};

// Decides the evidence from the values the AXRuntime probe reads. AXRuntime reports the primary
// app's window and the system app, which spans the screen, in one native space; this is the same
// comparison the Simulator snapshot bridge refuses a capture on (`app-window-not-full-screen`,
// ADR 0004). Neither read walks the app's tree, so the decision survives a tree XCTest cannot
// serialize. A pid that cannot name a process (<= 0) or an unreadable frame answers Unknown, never
// a guess.
FOUNDATION_EXPORT RunnerFullScreenEvidence RunnerFullScreenEvidenceFromForeground(
pid_t targetProcessID,
pid_t primaryProcessID,
CGRect appFrame,
CGRect screenFrame
);

// The sole-screen rule with one owner: an iPhone always routes through its only screen; any other
// idiom routes only while the evidence proves the app fills that screen.
FOUNDATION_EXPORT BOOL RunnerSoleScreenRouteOpensForEvidence(
BOOL isPhoneIdiom,
RunnerFullScreenEvidence evidence
);

FOUNDATION_EXPORT NSString * _Nullable RunnerRequireClass(
Class cls,
NSString *className,
Expand Down
Loading
Loading