diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index c8c622cfe6..adcab6215c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -155,7 +155,13 @@ extension RunnerTests { ) let textInput: XCUIElement? if !xCTestTextInputProbeSkipped { - textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y) + // The lookup compares candidate frames with this screen point (#3899). + textInput = coordinateTapTextInputAt( + app: activeApp, + x: x, + y: y, + contentTranslation: liveContentTranslation(app: activeApp) + ) } else { textInput = nil // Clear before dispatch so a throwing gesture cannot retain the preceding field's witness. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index ae1407d1d0..704b2bed10 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -352,7 +352,12 @@ extension RunnerTests { func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { let point = CGPoint(x: x, y: y) - let textInputCandidates = textInputCandidatesAt(app: app, point: point).map(\.element) + // The lookup compares candidate frames with this screen point (#3899). + let textInputCandidates = textInputCandidatesAt( + app: app, + point: point, + contentTranslation: liveContentTranslation(app: app) + ).map(\.element) for element in textInputCandidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { return text @@ -617,12 +622,14 @@ extension RunnerTests { #if !os(tvOS) private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { #if os(iOS) - // Coordinate taps, double taps, long presses, and drags anchor at the app origin, not the - // resolved window: on iOS the snapshot hands back app-space points, and the first qualifying - // window is not always the one under the finger — a SpringBoard `alert accept` can live in an - // alert window while the wallpaper or status-bar window qualifies first, so a window-relative - // anchor misses the button. Reference frames and the synthesized display ID still resolve - // through `resolveRunnerWindow`, so a foldable tap keeps its panel and stays on its display. + // A caller's point is a screen point: these gestures anchor at the app's origin, not the + // resolved window, so it is dispatched as given. The anchor sits at the origin of the space + // `app.frame` names, and for a windowed app the measured `app.frame` starts at the screen's + // origin (#3899). The first qualifying window is not always the one under the finger — a + // SpringBoard `alert accept` can live in an alert window while the wallpaper or status-bar + // window qualifies first, so a window-relative anchor misses the button. Reference frames and + // the synthesized display ID still resolve through `resolveRunnerWindow`, so a foldable tap + // keeps its panel and stays on its display (ADR 0025). let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) return origin.withOffset(CGVector(dx: x, dy: y)) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 5ac510f125..c198ea2534 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -149,6 +149,10 @@ extension RunnerTests { let element: XCUIElement? let refreshPoint: CGPoint? let prefersFocusedElement: Bool + /// The content vector of a placed window, read once for the command that built this target and + /// handed to every read it produces, which turns its snapshot frames into screen points (#3899). + /// Nil when the app fills the screen. + let contentTranslation: CGSize? let fromTapWitness: Bool /// The input the first resolved element was. Once bound, resolution refuses any other input. let boundIdentity: TextEntryInputIdentity? @@ -166,6 +170,7 @@ extension RunnerTests { element: XCUIElement?, refreshPoint: CGPoint?, prefersFocusedElement: Bool, + contentTranslation: CGSize? = nil, fromTapWitness: Bool = false, boundIdentity: TextEntryInputIdentity? = nil, inputAtRefreshPoint: TextInputAtPoint? = nil, @@ -174,23 +179,27 @@ extension RunnerTests { self.element = element self.refreshPoint = refreshPoint self.prefersFocusedElement = prefersFocusedElement + self.contentTranslation = contentTranslation self.fromTapWitness = fromTapWitness self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity self.inputAtRefreshPoint = inputAtRefreshPoint self.postFocusRead = postFocusRead } - /// The target moved to the input a resolution read: its element, a refresh point at its center, - /// and its identity while the target is unbound. + /// The target moved to the input a resolution read: its element, a refresh point at its frame's + /// center on the screen, and its identity while the target is unbound. A read with no snapshot has + /// no frame here, so its center cannot come from the live `element.frame` — the read a post-focus + /// handle cannot answer without recording a failure (#3060). func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget { guard let input else { return self } - let frame = input.snapshot?.frame ?? .zero + let frame = input.snapshot == nil ? .zero : input.frameOnScreen return TextEntryTarget( element: input.element, refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: prefersFocusedElement, + contentTranslation: contentTranslation, fromTapWitness: fromTapWitness, boundIdentity: boundIdentity ?? input.identity, inputAtRefreshPoint: inputAtRefreshPoint @@ -204,20 +213,27 @@ extension RunnerTests { /// Pro, iOS 27.0). Where a live read of an element that stopped answering its query, such as a /// removed input or a focused Flutter password field (#3060), records an XCTest failure that ends /// the runner session, the snapshot only throws. + /// + /// This type owns the probe-side space of #3899 (ADR 0004): a snapshot frame names a placed + /// window's content space while every point this read feeds is a screen point, so consumers take + /// `frameOnScreen` and compare raw frames only with each other. struct ResolvedTextEntryInput { let element: XCUIElement /// Nil when the element exists but could not be snapshotted. let snapshot: XCUIElementSnapshot? + /// The vector of the command that made this read (#3899); nil when the app fills the screen. + let contentTranslation: CGSize? - private init(element: XCUIElement, snapshot: XCUIElementSnapshot?) { + private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: CGSize?) { self.element = element self.snapshot = snapshot + self.contentTranslation = contentTranslation } /// Reads a handle that may name an input the app has removed; nil when it names none or the read /// raised. The snapshot of a missing element throws only after XCTest waits about two seconds for /// a match, so existence is read first. - init?(reading element: XCUIElement) { + init?(reading element: XCUIElement, contentTranslation: CGSize? = nil) { var exists = false var snapshot: XCUIElementSnapshot? if let exception = RunnerObjCExceptionCatcher.catchException({ @@ -228,15 +244,15 @@ extension RunnerTests { return nil } guard exists else { return nil } - self.init(element: element, snapshot: snapshot) + self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation) } /// Reads a handle its query has just listed, inside the caller's exception containment, so the /// snapshot is read first and existence only when the snapshot fails; nil when it names no element. - init?(readingListed element: XCUIElement) { + init?(readingListed element: XCUIElement, contentTranslation: CGSize? = nil) { let snapshot = try? element.snapshot() guard snapshot != nil || element.exists else { return nil } - self.init(element: element, snapshot: snapshot) + self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation) } var identity: TextEntryInputIdentity? { @@ -249,6 +265,19 @@ extension RunnerTests { } return element } + + /// The input's box in screen space: the snapshot frame plus `contentTranslation`, or the live + /// frame when there is no snapshot — that property already reports a placed window's content in + /// screen space (#3899). + var frameOnScreen: CGRect { + guard let snapshot else { + return element.frame + } + guard let contentTranslation else { + return snapshot.frame + } + return snapshot.frame.offsetBy(dx: contentTranslation.width, dy: contentTranslation.height) + } } /// A text entry target and its input as last read. `input()` hands the last read out again until @@ -260,7 +289,10 @@ extension RunnerTests { private var lastRead: ResolvedTextEntryInput? private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput? - init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?) { + init( + target: TextEntryTarget, + resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput? + ) { self.target = target.withResolvedInput(target.postFocusRead) lastRead = target.postFocusRead self.resolve = resolve @@ -494,7 +526,10 @@ extension RunnerTests { return resolveUnboundTextEntryInput(app: app, target: target) } func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? { - guard let input = ResolvedTextEntryInput(reading: candidate), input.identity == boundIdentity else { return nil } + guard let input = ResolvedTextEntryInput( + reading: candidate, + contentTranslation: target.contentTranslation + ), input.identity == boundIdentity else { return nil } return input } if target.prefersFocusedElement, let focused = focusedTextInput(app: app), let input = boundInput(focused) { @@ -503,12 +538,17 @@ extension RunnerTests { if let element = target.element, let input = boundInput(element) { return input } - if let input = target.inputAtRefreshPoint, let read = textInputIfStillResolves(input) { + if let input = target.inputAtRefreshPoint, + let read = textInputIfStillResolves(input, contentTranslation: target.contentTranslation) { return read } if target.inputAtRefreshPoint == nil, let refreshPoint = target.refreshPoint, - case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint), + case .matches(let candidates) = probeTextInputs( + app: app, + point: refreshPoint, + contentTranslation: target.contentTranslation + ), let match = candidates.first(where: { $0.identity == boundIdentity }) { return match } @@ -543,7 +583,11 @@ extension RunnerTests { guard let refreshPoint = target.refreshPoint else { return true } - switch probeTextInputs(app: app, point: refreshPoint) { + switch probeTextInputs( + app: app, + point: refreshPoint, + contentTranslation: target.contentTranslation + ) { case .absent: return true case .matches(let candidates): @@ -577,17 +621,25 @@ extension RunnerTests { /// Resolution for a target not bound to an input yet: the first element that exists. private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { - if target.prefersFocusedElement, let focused = focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) { + let translation = target.contentTranslation + if target.prefersFocusedElement, + let focused = focusedTextInput(app: app).flatMap({ ResolvedTextEntryInput(reading: $0, contentTranslation: translation) }) { return focused } - if let element = target.element, let input = ResolvedTextEntryInput(reading: element) { + if let element = target.element, + let input = ResolvedTextEntryInput(reading: element, contentTranslation: translation) { return input } if let refreshPoint = target.refreshPoint, - let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y) { + let refreshed = textInputAt( + app: app, + x: refreshPoint.x, + y: refreshPoint.y, + contentTranslation: translation + ) { return refreshed } - return focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) + return focusedTextInput(app: app).flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: translation) } } private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) { @@ -606,9 +658,12 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - guard let frame = input.snapshot?.frame, !frame.isEmpty else { + // A snapshotless read gets no point at all: `frameOnScreen`'s fallback is the live + // `element.frame` read this function exists to avoid (#3060). + guard input.snapshot?.frame.isEmpty == false else { return } + let frame = input.frameOnScreen _ = tapAt( app: app, x: frame.minX + max(2, frame.width - 4), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 34f289cccb..fb0c4592f8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -61,7 +61,8 @@ extension RunnerTests { func stabilizeTextInputBeforeTyping( app: XCUIApplication, target: XCUIElement?, - keyboardVisibleBeforeTap: Bool? = nil + keyboardVisibleBeforeTap: Bool? = nil, + contentTranslation: CGSize? = nil ) -> TextEntryStabilization { #if os(tvOS) return TextEntryStabilization(element: target, evidence: .focusEngine, polls: 0) @@ -79,7 +80,7 @@ extension RunnerTests { // presented, the keyboard query blocks and then finds none. An input that reports keyboard // focus is stronger evidence than the keyboard's arrival, which shows only that some input // took focus. - let read = target.flatMap(ResolvedTextEntryInput.init(reading:)) + let read = target.flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: contentTranslation) } if let read, let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) { return TextEntryStabilization(element: read.element, evidence: .inputReportsKeyboardFocus(read), polls: polls) } @@ -130,12 +131,20 @@ extension RunnerTests { return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true) } + // The one content-vector read for this command (#3899): every target built here carries it and + // every read those targets produce converts through it. + let contentTranslation = liveContentTranslation(app: app) let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - let input = textInputAt(app: app, x: x, y: y) + let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - if let frame = input?.attributes.frame, !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) + // The tap goes to the input's own center, and `tapAt` dispatches a screen point, so the point + // comes from `frameOnScreen` (#3899). + let inputCenter = input.map { $0.frameOnScreen }.flatMap { + $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) + } + if let inputCenter { + _ = tapAt(app: app, x: inputCenter.x, y: inputCenter.y) } else { _ = tapAt(app: app, x: x, y: y) } @@ -143,12 +152,17 @@ extension RunnerTests { // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. if keyboardVisibleBeforeTap, let target { - return focusedTextEntryTarget(target, requestedPoint: requestedPoint) + return focusedTextEntryTarget( + target, + requestedPoint: requestedPoint, + contentTranslation: contentTranslation + ) } let stabilized = stabilizeTextInputBeforeTyping( app: app, target: target, - keyboardVisibleBeforeTap: keyboardVisibleBeforeTap + keyboardVisibleBeforeTap: keyboardVisibleBeforeTap, + contentTranslation: contentTranslation ) NSLog( "%@%@ polls=%d", @@ -157,37 +171,66 @@ extension RunnerTests { stabilized.polls ) if case .inputReportsKeyboardFocus(let read) = stabilized.evidence { - return focusedTextEntryTarget(read.element, read: read, requestedPoint: requestedPoint) + return focusedTextEntryTarget( + read.element, + read: read, + requestedPoint: requestedPoint, + contentTranslation: contentTranslation + ) } let readyTarget = TextEntryTarget( element: stabilized.element ?? target, refreshPoint: requestedPoint, - prefersFocusedElement: false + prefersFocusedElement: false, + contentTranslation: contentTranslation ) let element = stabilized.focusConfirmed ? readyTarget.element : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element) - return focusedTextEntryTarget(element, requestedPoint: requestedPoint) + return focusedTextEntryTarget( + element, + requestedPoint: requestedPoint, + contentTranslation: contentTranslation + ) } /// The target the focus step hands to text entry, with the read its refresh point came from. - private func focusedTextEntryTarget(_ element: XCUIElement?, requestedPoint: CGPoint) -> TextEntryTarget { + private func focusedTextEntryTarget( + _ element: XCUIElement?, + requestedPoint: CGPoint, + contentTranslation: CGSize? + ) -> TextEntryTarget { guard let element else { - return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false) + return TextEntryTarget( + element: nil, + refreshPoint: requestedPoint, + prefersFocusedElement: false, + contentTranslation: contentTranslation + ) } - return focusedTextEntryTarget(element, read: ResolvedTextEntryInput(reading: element), requestedPoint: requestedPoint) + return focusedTextEntryTarget( + element, + read: ResolvedTextEntryInput(reading: element, contentTranslation: contentTranslation), + requestedPoint: requestedPoint, + contentTranslation: contentTranslation + ) } /// `read` is `element` read after the focus tap, or nil when the read found no element. private func focusedTextEntryTarget( _ element: XCUIElement, read: ResolvedTextEntryInput?, - requestedPoint: CGPoint + requestedPoint: CGPoint, + contentTranslation: CGSize? ) -> TextEntryTarget { return TextEntryTarget( element: element, - refreshPoint: textEntryRefreshPoint(frame: read?.snapshot?.frame, fallbackTo: requestedPoint), + refreshPoint: textEntryRefreshPoint( + read: read, + fallbackTo: requestedPoint + ), prefersFocusedElement: false, + contentTranslation: contentTranslation, postFocusRead: read?.snapshot == nil ? nil : read ) } @@ -200,14 +243,16 @@ extension RunnerTests { /// the target gets NO refresh point rather than the pre-focus one: this route's first resolve is /// not yet bound to an identity, and a field that went unqueryable on focus may have moved the /// layout so the old point names a different input, so fail closed with a typed reason instead of - /// typing into that other field. + /// typing into that other field. The frame is `read`'s snapshot frame plus its content vector, + /// never the live `element.frame` (#3899, #3060). private func textEntryRefreshPoint( - frame: CGRect?, + read: ResolvedTextEntryInput?, fallbackTo requestedPoint: CGPoint ) -> CGPoint? { - guard let frame else { + guard let read, read.snapshot != nil else { return nil } + let frame = read.frameOnScreen guard !frame.isEmpty else { return requestedPoint } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index f4c49a5a67..a8f19d0d3a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -85,22 +85,47 @@ extension RunnerTests { /// `ResolvedTextEntryInput` already reads elements this way. It is read again after the /// wait rather than reused, because focusing a field moves the layout: the repeat aims at where the /// field is then, which is the moment the read it replaced ran at. + /// + /// The re-read carries the live content vector, so the repeat taps a screen point (#3899). It is + /// read only on this branch: a tap that confirms focus dispatches no window lookup. func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { #if os(iOS) if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return } - guard let settled = safely("TEXT_ENTRY_READINESS_FRAME", { try? element.snapshot() }) else { - return + // Contained like the probe lookups: a transient window that records an XCTest issue costs the + // vector, not the command. + let settled: ResolvedTextEntryInput? + switch containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?, { _ in + ResolvedTextEntryInput(reading: element, contentTranslation: liveContentTranslation(app: app)) + }) { + case .success(let read): + settled = read + case .failure: + settled = nil } - let frame = settled.frame - if !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) - _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) + guard let settled else { + return } + retapAt(read: settled, app: app, element: element) #endif } + /// The readiness retap, split out so the unit lane can drive it and assert where the point lands. + /// A read with no snapshot taps nothing: its `frameOnScreen` is the live `element.frame`, the read + /// a handle that stopped answering its query cannot make without recording a failure (#3060). + func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { + guard read.snapshot != nil else { + return + } + let frame = read.frameOnScreen + guard !frame.isEmpty else { + return + } + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) + } + private func waitForTextEntryFocus(app: XCUIApplication, element: XCUIElement, timeout: TimeInterval) -> Bool { let deadline = Date().addingTimeInterval(timeout) while Date() < deadline { @@ -129,6 +154,7 @@ extension RunnerTests { /// fallback at all. func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool { #if os(iOS) + // No content vector: this corroborates raw frame against raw frame and dispatches no point. return ResolvedTextEntryInput(reading: element).map { keyboardFocusConfirmed(app: app, read: $0) } ?? false #else return false diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 642e9db8f6..40346eef1e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation final class TextInputProbeIssues { let thread = Thread.current @@ -17,18 +18,33 @@ enum TextInputProbeOutcome { } extension RunnerTests { - func textInputAt(app: XCUIApplication, x: Double, y: Double) -> ResolvedTextEntryInput? { - textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first + func textInputAt( + app: XCUIApplication, + x: Double, + y: Double, + contentTranslation: CGSize? + ) -> ResolvedTextEntryInput? { + let point = CGPoint(x: x, y: y) + return textInputCandidatesAt(app: app, point: point, contentTranslation: contentTranslation).first } - func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [ResolvedTextEntryInput] { + func textInputCandidatesAt( + app: XCUIApplication, + point: CGPoint, + contentTranslation: CGSize? + ) -> [ResolvedTextEntryInput] { safely("TEXT_INPUT_AT_POINT", []) { - queryTextInputs(app: app, point: point, shouldStop: { false }) + queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: { false }) } } - func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? { - switch probeTextInputs(app: app, point: CGPoint(x: x, y: y)) { + func coordinateTapTextInputAt( + app: XCUIApplication, + x: Double, + y: Double, + contentTranslation: CGSize? + ) -> XCUIElement? { + switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { case .matches(let inputs): return inputs.first?.element case .absent: @@ -53,11 +69,17 @@ extension RunnerTests { func coordinateTapTextInputIdentityAt( app: XCUIApplication, x: Double, - y: Double + y: Double, + contentTranslation: CGSize? ) -> CoordinateTextInputResolution { let probed = containingTextInputProbeIssues(fallback: CoordinateTextInputResolution.unavailable) { shouldStop -> CoordinateTextInputResolution in - guard let input = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first else { + guard let input = queryTextInputs( + app: app, + point: CGPoint(x: x, y: y), + contentTranslation: contentTranslation, + shouldStop: shouldStop + ).first else { return .absent } guard let identity = input.identity else { @@ -75,9 +97,13 @@ extension RunnerTests { /// `input`'s handle, read again, when it still resolves to the input it named when found, read /// inside the same issue containment. A read that cannot answer counts as no. - func textInputIfStillResolves(_ input: TextInputAtPoint) -> ResolvedTextEntryInput? { + func textInputIfStillResolves( + _ input: TextInputAtPoint, + contentTranslation: CGSize? + ) -> ResolvedTextEntryInput? { let probed = containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?) { _ in - ResolvedTextEntryInput(reading: input.element).flatMap { $0.identity == input.identity ? $0 : nil } + ResolvedTextEntryInput(reading: input.element, contentTranslation: contentTranslation) + .flatMap { $0.identity == input.identity ? $0 : nil } } guard case .success(let read) = probed else { return nil } return read @@ -93,9 +119,13 @@ extension RunnerTests { return read } - func probeTextInputs(app: XCUIApplication, point: CGPoint) -> TextInputProbeOutcome { + func probeTextInputs( + app: XCUIApplication, + point: CGPoint, + contentTranslation: CGSize? + ) -> TextInputProbeOutcome { switch containingTextInputProbeIssues(fallback: [], { shouldStop in - queryTextInputs(app: app, point: point, shouldStop: shouldStop) + queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) }) { case .success(let inputs): return inputs.isEmpty ? .absent : .matches(inputs) @@ -106,7 +136,7 @@ extension RunnerTests { /// Runs an optional probe whose XCTest issues are contained instead of recorded: an issue makes /// the probe unavailable rather than failing the command and invalidating the runner. - private func containingTextInputProbeIssues( + func containingTextInputProbeIssues( fallback: T, _ probe: (_ shouldStop: () -> Bool) -> T ) -> Result { @@ -150,6 +180,7 @@ extension RunnerTests { private func queryTextInputs( app: XCUIApplication, point: CGPoint, + contentTranslation: CGSize?, shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] @@ -166,9 +197,15 @@ extension RunnerTests { // cannot be snapshotted is read live, so its failure still reaches the caller's issue handling. for element in candidates { if shouldStop() { break } - guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue } - let attributes = candidate.attributes - if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) { + guard let candidate = ResolvedTextEntryInput( + readingListed: element, + contentTranslation: contentTranslation + ) else { continue } + if isCoordinateTextInputCandidate( + enabled: candidate.attributes.isEnabled, + frame: candidate.frameOnScreen, + point: point + ) { inputs.append(candidate) } } @@ -176,4 +213,29 @@ extension RunnerTests { guard !shouldStop() else { return [] } return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } } + + /// The vector that carries this session's snapshot-read app-content frames to their place on the + /// screen, or nil when the app fills the screen (#3899). Each command that dispatches points from + /// snapshot frames reads it once, at its own start, and hands it down; a read inside a typing loop + /// never calls this, because the two live box reads below cost about 90 ms each on a heavy React + /// Native screen. + /// + /// The signature and the vector are the capture's rule (`SnapshotContentTranslation`, ADR 0004) + /// read from the two live boxes that name it: the application's `frame` is a `(0,0)`-anchored copy + /// of the box the resolved window reports at its place on the screen. Any other pair of boxes + /// declares no vector, so full-screen, iPhone, and each foldable panel keep frames as read. + func liveContentTranslation(app: XCUIApplication) -> CGSize? { + // The window read is contained like the probe lookups it replaces: a transient window that + // records an XCTest issue costs the vector, not the runner session. + let read = containingTextInputProbeIssues(fallback: nil as CGSize?) { _ in + guard let resolved = resolveRunnerWindowElement(app: app) else { return nil } + return SnapshotContentTranslation.offset(reportedBox: app.frame, viewportBox: resolved.frame) + } + switch read { + case .success(let translation): + return translation + case .failure: + return nil + } + } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index acbb62ef54..8e49b4a3e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -27,12 +27,14 @@ extension RunnerTests { hasY: command.y != nil, xCTestChannelPenalized: xCTestChannelPenalized ), let x = command.x, let y = command.y { + // The one content-vector read for this command (#3899); the target built here carries it. + let contentTranslation = liveContentTranslation(app: activeApp) // Resolved before the tap: the focus it causes can move the field away from this point. // A replacement with no input at the point has nothing to bind to: its clear and its typed // text would reach the app-level first responder, so an empty point and a lookup that cannot // answer both refuse here, each under its own reason. let inputAtPoint: TextInputAtPoint - switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) { + switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { case .input(let input): inputAtPoint = input case .absent: @@ -54,6 +56,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: x, y: y), prefersFocusedElement: false, + contentTranslation: contentTranslation, inputAtRefreshPoint: inputAtPoint ) case .xctestFallback: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index a4a83c3df1..60979694f1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -6,7 +6,7 @@ extension RunnerTests { /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { - switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y) { + switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: nil) { case .input(let input): return input case .absent: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift index 8138eb41fb..1510dce82a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift @@ -662,6 +662,133 @@ extension RunnerTests { XCTAssertNotNil(keyQ, "`q` key must be presented under --depth \(options.depth ?? -1)") } + /// #3899: an iPad window that does not fill the screen publishes screen space. The measured shape is + /// iPad Pro 11-inch (M5, iOS 27.0) with Windowed Apps on and a Calendar window at (224,207,386x682): + /// the app's own window arrives as a (0,0)-anchored copy of that box and its subtree with it, so the + /// Add button reports (335,14,37,36) for a control drawn at (559,221). The one normalization pass + /// reads the declaration and publishes the subtree at its place on the screen, which is what lets the + /// fold keep every node instead of clipping each one to (w − x) × (h − y) and refusing the capture. + func testAWindowedSubtreeIsPublishedInScreenSpaceAndKeepsEveryNode() throws { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + func node( + _ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int + ) -> RawAXNode { + RawAXNode( + index: index, type: type, label: label, identifier: nil, value: nil, + rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height), + enabled: true, focused: nil, selected: nil, hittable: false, + depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil + ) + } + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let acquired = [ + node(0, "Application", "Calendar", reportedRoot, nil, 0), + node(1, "Window", "Window", reportedRoot, 0, 1), + node(2, "Other", "Toolbar", reportedRoot, 1, 2), + node(3, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3), + node(4, "Button", "Today", CGRect(x: 8, y: 583, width: 90, height: 38), 2, 3), + // A SpringBoard overlay under the app root reports the screen's own box: its subtree is already + // in screen space and must not move. + node(5, "Window", "Alert", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1), + node(6, "Button", "Discard Changes", CGRect(x: 300, y: 500, width: 240, height: 44), 5, 2), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: windowOnScreen, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ) + let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + let capture = try XCTUnwrap(try SnapshotPresentation.present( + SnapshotAcquisition( + hint: SnapshotPresentation.captureHint(for: options), + nodes: normalized, + truncated: false, + effectiveDepth: nil, + viewport: .reported(box: windowOnScreen) + ), + options: options + )) + + // One space: the root answers with the window's screen box, and the button with the position the + // screenshot draws. + XCTAssertEqual(capture.nodes.first?.rect, SnapshotRect(windowOnScreen)) + let add = try XCTUnwrap(capture.nodes.first { $0.label == "Add" }) + XCTAssertEqual(add.rect, SnapshotRect(x: 559, y: 221, width: 37, height: 36)) + XCTAssertEqual(add.hittable, true) + // Nothing is dropped for being "outside the clip": the whole acquired tree is still there. The one + // clipped node is the system overlay that really is larger than the app's window, which is the + // fold's normal answer to oversized content — not the mixed-space clip that used to publish + // (w − x) × (h − y) boxes and refuse this capture outright. + XCTAssertEqual(Set(capture.nodes.map(\.label)), Set(acquired.map(\.label))) + let clipNote = capture.qualityNotes.first { $0.code == .regularGeometryClipped } + XCTAssertEqual(clipNote?.droppedNodeCount, 0, "no node is dropped for the window's origin") + XCTAssertEqual(clipNote?.clippedNodeCount, 1, "only the overlay larger than the window is clipped") + XCTAssertEqual( + capture.nodes.first { $0.label == "Alert" }?.rect, + SnapshotRect(x: 224, y: 207, width: 386, height: 682), + "the overlay is clipped to the window's visible box, not to (w − x) × (h − y) of it" + ) + // The system window and its button stay where the platform reported them. + let discard = try XCTUnwrap(capture.nodes.first { $0.label == "Discard Changes" }) + XCTAssertEqual(discard.rect, SnapshotRect(x: 300, y: 500, width: 240, height: 44)) + } + + /// #3899 keeps a full-screen capture byte-identical: the same tree reported from the screen origin is + /// published exactly as reported, with no translation read from its zero-origin viewport. + func testAFullScreenCapturePublishesWhatThePlatformReported() throws { + let screen = CGRect(x: 0, y: 0, width: 834, height: 1210) + func node( + _ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int + ) -> RawAXNode { + RawAXNode( + index: index, type: type, label: label, identifier: nil, value: nil, + rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height), + enabled: true, focused: nil, selected: nil, hittable: false, + depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil + ) + } + let acquired = [ + node(0, "Application", "Calendar", screen, nil, 0), + node(1, "Window", "Window", screen, 0, 1), + node(2, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 1, 2), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: screen, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ) + XCTAssertEqual(normalized.map(\.rect), acquired.map(\.rect)) + XCTAssertEqual( + SnapshotContentTranslation.offset(reportedBox: screen, viewportBox: screen), + nil, + "a viewport at the screen origin carries no translation" + ) + } + + /// #3899 keeps ADR 0025 foldable panel taps: a panel reports its own box from its own display's + /// origin, so no translation is read and the panel's geometry stays exactly as measured. + func testAFoldablePanelReportsNoContentTranslation() { + let panel = CGRect(x: 0, y: 0, width: 951, height: 669) + XCTAssertNil( + SnapshotContentTranslation.offset(reportedBox: panel, viewportBox: panel), + "a panel box anchored at its display's origin names no translation" + ) + // A digitizer point on the panel is passed through, so the fold's display selection stays the only + // window-dependent choice the tap path makes. + XCTAssertEqual( + CoordinateSpaceRotation.native( + point: CGPoint(x: 476, y: 226), + in: panel, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ), + CGPoint(x: 476, y: 226) + ) + } + /// #2891: a capture whose viewport read failed publishes no `hittable` for a node whose answer is /// containment, while a root and a disabled node stay a declared `false`. Pinned on the /// encoded objects rather than on `capture.nodes`, because the promise is about the wire: a test diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index f70b57c907..9d27049ec2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -83,7 +83,7 @@ extension RunnerTests { } let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false) - let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y) } + let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y, contentTranslation: nil) } let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) } XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input") @@ -109,7 +109,12 @@ extension RunnerTests { "software keyboard is down: this simulator cannot show the keyboard a focus tap raises" ) - let stabilized = stabilizeTextInputBeforeTyping(app: app, target: field, keyboardVisibleBeforeTap: false) + let stabilized = stabilizeTextInputBeforeTyping( + app: app, + target: field, + keyboardVisibleBeforeTap: false, + contentTranslation: nil + ) guard case .inputReportsKeyboardFocus(let read) = stabilized.evidence else { return XCTFail("focus confirmed by \(stabilized.evidence?.name ?? "nothing"), not by the tapped input") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift new file mode 100644 index 0000000000..415a55a4b7 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift @@ -0,0 +1,57 @@ +import XCTest + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + /// The readiness retap dispatches `ResolvedTextEntryInput.frameOnScreen`'s center, so the snapshot + /// frame of a placed window passes through the read's content vector before it becomes a screen + /// point (#3899). Before that rule the retap tapped the bare `snapshot().frame` center, which on + /// a windowed iPad lands outside the window, and the fill after it failed with + /// TEXT_INPUT_NOT_FOCUSED — the issue's own failure shape. + /// + /// The fixture cannot be placed in a window, so the read is driven directly: the vector moves the + /// field's real snapshot frame exactly onto the press-counting witness button below it. A point + /// dispatched at `frameOnScreen`'s center presses that button; a point dispatched at the raw + /// snapshot center taps the field and moves no counter. The same read with no vector is the + /// closest negative: a full-screen app retaps the field itself and the counter stays put. + @MainActor + func testReadinessRetapDispatchesTheReadsScreenSpaceCenter() throws { + app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-tap-witness"] + app.launch() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) + let field = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + let witness = app.buttons["agent-device-tap-witness"] + XCTAssertTrue(witness.waitForFixtureExistence(timeout: appExistenceTimeout)) + let presses = app.staticTexts["agent-device-tap-witness-presses"] + XCTAssertTrue(presses.waitForFixtureExistence(timeout: appExistenceTimeout)) + let fieldFrame = field.frame + let witnessFrame = witness.frame + XCTAssertFalse(fieldFrame.isEmpty) + XCTAssertFalse(witnessFrame.isEmpty) + + // Closest negative: a read with no vector — a full-screen app — retaps the field's own center, + // which is not the button's point. + let untranslated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: nil)) + retapAt(read: untranslated, app: app, element: field) + XCTAssertEqual(presses.label, "presses=0", "a vectorless read dispatched a point off the field") + + // The read a probe makes on a placed window: the snapshot reports the field's box and the read + // carries the window's vector. Here the vector is chosen so the translated box is the button. + let vector = CGSize( + width: witnessFrame.midX - fieldFrame.midX, + height: witnessFrame.midY - fieldFrame.midY + ) + let translated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: vector)) + XCTAssertEqual(translated.frameOnScreen.midX, witnessFrame.midX, accuracy: 0.5) + XCTAssertEqual(translated.frameOnScreen.midY, witnessFrame.midY, accuracy: 0.5) + + retapAt(read: translated, app: app, element: field) + XCTAssertEqual( + presses.label, + "presses=1", + "the retap dispatched the raw snapshot center, not the read's screen-space center (#3899)" + ) + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index c9b306758a..6c7b54b49a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -15,7 +15,7 @@ extension RunnerTests { let completed = expectation(description: "optional probe completed inside runner wait") DispatchQueue.main.async { self.textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Optional probe issue during runner wait") - _ = self.probeTextInputs(app: self.app, point: point) + _ = self.probeTextInputs(app: self.app, point: point, contentTranslation: nil) completed.fulfill() } guard XCTWaiter.wait(for: [completed], timeout: 5) == .completed else { @@ -174,8 +174,8 @@ extension RunnerTests { } XCTExpectFailure("Required read and later issue belong to their caller", options: options) { textInputProbeIssueForTesting = expected - _ = textInputAt(app: app, x: point.x, y: point.y) - _ = probeTextInputs(app: app, point: point) + _ = textInputAt(app: app, x: point.x, y: point.y, contentTranslation: nil) + _ = probeTextInputs(app: app, point: point, contentTranslation: nil) record(expected) } XCTAssertEqual(observed, 2) @@ -192,7 +192,7 @@ extension RunnerTests { XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Failed to get matching snapshot: kAXErrorIllegalArgument") - let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY)) + let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: nil) guard case .unavailable = outcome else { return XCTFail("A suppressed AX issue must discard the matching candidate") } diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 300b8c910e..90cd0d5488 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -4,6 +4,9 @@ import CoreGraphics // Rotation between an app's interface space and the device's native (portrait-up) space, shared by // synthesized dispatch (forward) and capture (inverse). Rationale and measurements: ADR 0004 // "the coordinate space of a captured subtree" and contracts/fixtures/window-coordinate-space.json. +// +// Rotation moves axes only and never an origin; a placed window's offset from the screen belongs +// to `SnapshotContentTranslation` (#3899). public enum RunnerInterfaceOrientation { public static let unknown = 0 @@ -14,18 +17,16 @@ public enum RunnerInterfaceOrientation { } public enum CoordinateSpaceRotation { - // Dispatch rotates a point about the reference frame's top-left. The origin is localized away - // and never added back: an XCUIApplication session observes only its own scene surface, so its - // windows report (0,0)-anchored frames (measured on iPhone landscape both ways and on every - // system window in window-coordinate-space.json), and displacement between panels or panes is - // carried by the record's displayID, not by an origin. + // Dispatch turns a point's axes and never moves its origin: the digitizer's origin is the + // display's own, which is where a windowed app's screen-space points are already measured from + // (#3899). `frame` names the side lengths the turn swaps, nothing more. public static func native( point: CGPoint, in frame: CGRect, interfaceOrientation: Int ) -> CGPoint { - let localX = Double(point.x) - Double(frame.minX) - let localY = Double(point.y) - Double(frame.minY) + let localX = Double(point.x) + let localY = Double(point.y) let width = Double(frame.width) let height = Double(frame.height) switch interfaceOrientation { @@ -60,23 +61,23 @@ public enum CoordinateSpaceRotation { ) -> CGPoint { let width = Double(frame.width) let height = Double(frame.height) - let localX: Double - let localY: Double + let x: Double + let y: Double switch interfaceOrientation { case RunnerInterfaceOrientation.landscapeRight: - localX = Double(point.y) - localY = height - Double(point.x) + x = Double(point.y) + y = height - Double(point.x) case RunnerInterfaceOrientation.landscapeLeft: - localX = width - Double(point.y) - localY = Double(point.x) + x = width - Double(point.y) + y = Double(point.x) case RunnerInterfaceOrientation.portraitUpsideDown: - localX = width - Double(point.x) - localY = height - Double(point.y) + x = width - Double(point.x) + y = height - Double(point.y) default: - localX = Double(point.x) - localY = Double(point.y) + x = Double(point.x) + y = Double(point.y) } - return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY)) + return CGPoint(x: x, y: y) } public static func oriented( @@ -103,9 +104,48 @@ public enum CoordinateSpaceRotation { } } +/// The vector from a carrier's reported box to the screen space it is placed at (#3899). +/// +/// The signature is a `(0,0)`-anchored copy of the viewport box whose viewport starts elsewhere on +/// the screen; any other carrier box is not the app's own window, so its numbers say nothing about +/// where its subtree starts. Measured rows and the full rationale: ADR 0004 and the +/// `windowContentCases` table this replays against. +public enum SnapshotContentTranslation { + /// `nil` when the carrier's box declares no translation. A full-screen app, an iPhone, and each + /// foldable panel report origins at zero on both sides, so the rule reads none there and every + /// published rect stays as the platform reported it. + public static func offset(reportedBox: CGRect, viewportBox: CGRect) -> CGSize? { + guard SnapshotGeometry.isPositiveFinite(reportedBox), + SnapshotGeometry.isPositiveFinite(viewportBox) + else { + return nil + } + guard abs(reportedBox.minX) <= SnapshotGeometrySpace.quarterTurnTolerance, + abs(reportedBox.minY) <= SnapshotGeometrySpace.quarterTurnTolerance + else { + return nil + } + guard abs(viewportBox.minX) > SnapshotGeometrySpace.quarterTurnTolerance + || abs(viewportBox.minY) > SnapshotGeometrySpace.quarterTurnTolerance + else { + return nil + } + guard abs(reportedBox.width - viewportBox.width) <= SnapshotGeometrySpace.quarterTurnTolerance, + abs(reportedBox.height - viewportBox.height) <= SnapshotGeometrySpace.quarterTurnTolerance + else { + return nil + } + return CGSize(width: viewportBox.minX, height: viewportBox.minY) + } +} + public enum SnapshotGeometrySpace: Equatable { case appOrientation case deviceNative(appFrame: SnapshotViewport.Box, interfaceOrientation: Int) + /// The subtree was reported in a placed window's own content space and is published translated + /// into the viewport's screen space (`SnapshotContentTranslation`, #3899). The offset is the + /// vector from the window's reported box to the viewport box it sits at on screen. + case windowContent(offset: CGSize) public static let quarterTurnTolerance: Double = 1 @@ -119,10 +159,13 @@ public enum SnapshotGeometrySpace: Equatable { in: appFrame.rect, interfaceOrientation: interfaceOrientation ) + case .windowContent(let offset): + return reportedFrame.offsetBy(dx: offset.width, dy: offset.height) } } - /// Only a `.reported` viewport carries an orientation, so only it can anchor a rotation. + /// A `.reported` viewport is the screen-anchored fact both declarations read against; a `.derived` + /// viewport carries no screen anchor, so a subtree under it publishes what the platform reported. public static func space( reportedBySurfaceHost isSurfaceHost: Bool, reportedFrame: CGRect, @@ -130,13 +173,23 @@ public enum SnapshotGeometrySpace: Equatable { viewport: SnapshotViewport ) -> SnapshotGeometrySpace { guard isSurfaceHost else { return inherited } - guard case .reported(let appFrame, let interfaceOrientation) = viewport, - namesQuarterTurn(interfaceOrientation), - isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) - else { + guard case .reported(let appFrame, let interfaceOrientation) = viewport else { return .appOrientation } - return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + if namesQuarterTurn(interfaceOrientation), + isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) { + return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + } + if let offset = SnapshotContentTranslation.offset( + reportedBox: reportedFrame, + viewportBox: appFrame.rect + ) { + return .windowContent(offset: offset) + } + // A system surface under a placed window — the keyboard, a SpringBoard overlay — reports the + // screen's own box and is already in screen space; `windowContentCases` pins that inheriting + // the carrier's vector would move every key by the window origin. + return .appOrientation } public static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index c5df2bc22c..9b80b197d2 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -28,9 +28,22 @@ final class CoordinateSpaceTests: XCTestCase { let oriented: Frame } + struct WindowContentCase: Decodable { + let name: String + let reportedBox: Frame + let viewportBox: Frame + let offset: Offset? + } + + struct Offset: Decodable { + let width: Double + let height: Double + } + let constants: Constants let quarterTurnCases: [QuarterTurnCase] let rotationCases: [RotationCase] + let windowContentCases: [WindowContentCase] } /// Identity in portrait/unknown, 90° per landscape, 180° upside-down. @@ -63,8 +76,8 @@ final class CoordinateSpaceTests: XCTestCase { in: offsetLandscape, interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft ), - CGPoint(x: 268, y: 1040), - "non-zero frame origin is localized before rotation" + CGPoint(x: 288, y: 1030), + "a frame offset from the screen never moves the turned point (#3899)" ) } @@ -303,16 +316,34 @@ final class CoordinateSpaceTests: XCTestCase { ) } - /// Golden parity table (#2612): every case in contracts/fixtures/window-coordinate-space.json - /// must agree with the vitest twin - /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add - /// cases there, never fork the rule. Detection is replayed under both landscape turns, because + /// Golden parity table (#2612, #3899): every case in contracts/fixtures/window-coordinate-space.json + /// must agree here, and the quarter-turn rows with the vitest twin + /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). The + /// window-content rows replay here alone: the host channel's guest refuses a windowed tree + /// before its decoder can read one, so this capture is the rule's only production owner. Add + /// cases to the table, never fork the rule. Detection is replayed under both landscape turns, because /// which window declares the native space cannot depend on which way the app is turned; the /// rotation rows carry their own orientation. func testWindowCoordinateSpaceMatchesGoldenParityTable() throws { let fixture = try loadWindowCoordinateSpaceFixture() XCTAssertFalse(fixture.quarterTurnCases.isEmpty, "parity table must not be empty") XCTAssertFalse(fixture.rotationCases.isEmpty, "parity table must not be empty") + XCTAssertFalse(fixture.windowContentCases.isEmpty, "parity table must not be empty") + XCTAssertEqual( + Set(fixture.quarterTurnCases.map(\.name)).count, + fixture.quarterTurnCases.count, + "parity table case names must be unique" + ) + XCTAssertEqual( + Set(fixture.rotationCases.map(\.name)).count, + fixture.rotationCases.count, + "parity table case names must be unique" + ) + XCTAssertEqual( + Set(fixture.windowContentCases.map(\.name)).count, + fixture.windowContentCases.count, + "parity table case names must be unique" + ) XCTAssertEqual( fixture.constants.quarterTurnTolerance, SnapshotGeometrySpace.quarterTurnTolerance, @@ -349,6 +380,17 @@ final class CoordinateSpaceTests: XCTestCase { testCase.name ) } + for testCase in fixture.windowContentCases { + let expected = testCase.offset.map { CGSize(width: $0.width, height: $0.height) } + XCTAssertEqual( + SnapshotContentTranslation.offset( + reportedBox: testCase.reportedBox.cgRect, + viewportBox: testCase.viewportBox.cgRect + ), + expected, + testCase.name + ) + } } private func deviceNative(_ appFrame: CGRect, _ interfaceOrientation: Int) -> SnapshotGeometrySpace { @@ -486,6 +528,114 @@ final class CoordinateSpaceTests: XCTestCase { ) } + /// The measured windowed iPad Calendar window (#3899) driven through the pass: the carrier reports + /// a `(0,0)`-anchored copy of the viewport box, declares its subtree window-relative, and every + /// rect under it comes back at its place on the screen — the root at the window's screen box, the + /// Add button at (559,221) where the screenshot draws it, and a full-screen-capture tree untouched. + func testNormalizedTranslatesAWindowedSubtreeIntoScreenSpace() { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let reportedAddButton = CGRect(x: 335, y: 14, width: 37, height: 36) + let acquired = [ + coordinateNode(0, "Application", reportedRoot, nil, 0, label: "Calendar"), + coordinateNode(1, "Window", reportedRoot, 0, 1), + coordinateNode(2, "Other", reportedRoot, 1, 2), + coordinateNode(3, "Button", reportedAddButton, 2, 3, label: "Add"), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported(box: windowOnScreen, interfaceOrientation: RunnerInterfaceOrientation.portrait) + ) + XCTAssertEqual(normalized[0].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[1].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[2].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[3].rect.cgRect, CGRect(x: 559, y: 221, width: 37, height: 36)) + XCTAssertEqual(normalized[3].hittable, true) + + // The same window moved: the subtree follows the new origin. + let moved = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: CGRect(x: 120, y: 90, width: 386, height: 682), + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ) + XCTAssertEqual(moved[3].rect.cgRect, CGRect(x: 455, y: 104, width: 37, height: 36)) + } + + /// What must not move: a full-screen capture, and any subtree under a carrier that is not the app's + /// own placed window. Those trees keep exactly the rect the platform reported (#3899 keeps + /// full-screen and ADR 0025 panel behavior identical). + func testNormalizedTranslatesNothingForFullScreenAndForeignCarriers() { + // Full-screen iPad: reported boxes already start at the viewport. + let fullScreen = CGRect(x: 0, y: 0, width: 834, height: 1210) + let acquired = [ + coordinateNode(0, "Application", fullScreen, nil, 0), + coordinateNode(1, "Window", fullScreen, 0, 1), + coordinateNode(2, "Button", CGRect(x: 335, y: 14, width: 37, height: 36), 1, 2), + ] + XCTAssertEqual( + SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: fullScreen, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ).map(\.rect), + acquired.map(\.rect) + ) + + // A windowed app, measured shape: the app's own window declares the translation and carries the + // subtree; the keyboard window and the SpringBoard overlay report the screen's own box, so their + // subtrees stay where the platform put them. + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let mixed = [ + coordinateNode(0, "Application", reportedRoot, nil, 0), + coordinateNode(1, "Window", reportedRoot, 0, 1, label: "appWindow"), + coordinateNode(2, "Other", reportedRoot, 1, 2), + coordinateNode(3, "Button", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3, label: "Add"), + coordinateNode(4, "Window", CGRect(x: 0, y: 925, width: 834, height: 282), 0, 1, label: "keyboard"), + coordinateNode(5, "Key", CGRect(x: 300, y: 1100, width: 60, height: 60), 4, 2), + coordinateNode(6, "Window", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1, label: "springboard"), + coordinateNode(7, "Alert", CGRect(x: 300, y: 500, width: 240, height: 160), 6, 2), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: mixed, + viewport: .reported(box: windowOnScreen, interfaceOrientation: RunnerInterfaceOrientation.portrait) + ) + XCTAssertEqual(normalized[1].rect.cgRect, windowOnScreen, "appWindow") + XCTAssertEqual(normalized[2].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[3].rect.cgRect, CGRect(x: 559, y: 221, width: 37, height: 36)) + XCTAssertEqual(normalized[4].rect, mixed[4].rect, "keyboard") + XCTAssertEqual(normalized[5].rect, mixed[5].rect) + XCTAssertEqual(normalized[6].rect, mixed[6].rect, "springboard") + XCTAssertEqual(normalized[7].rect, mixed[7].rect) + } + + /// The dispatch direction for a windowed app: the caller's point is the point touched, so + /// `native(point:)` turns a screen-space point's axes without moving it toward the window origin + /// (#3899). Under portrait — the only orientation measured for a windowed app — it is the identity. + /// The landscape rows for a placed window stay the documented unmeasured gap (ADR 0004): the turn + /// here is asserted about the window's side lengths, while the digitizer turns about the screen's, + /// and no windowed landscape capture has been measured to say which basis is right. + func testNativePointForAWindowedAppKeepsTheCallersScreenPoint() { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + for orientation in [ + RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.unknown + ] { + XCTAssertEqual( + CoordinateSpaceRotation.native( + point: CGPoint(x: 578, y: 239), + in: windowOnScreen, + interfaceOrientation: orientation + ), + CGPoint(x: 578, y: 239), + "interfaceOrientation \(orientation)" + ) + } + } + func testNormalizedOfAnEmptyArrayIsEmpty() { XCTAssertEqual( SnapshotGeometrySpace.normalized( diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index ca92a01a71..dbd6bfd551 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -1,5 +1,5 @@ { - "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side.", + "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation) is the other space a captured subtree arrives in, and the one rule in this file that reads an origin. An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own screen box while the subtree under it arrives anchored at its own top-left: measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen (224,207,386x682), the application root reports (0,0,386,682) and its Add button (335,14,37,36) for a control drawn at screen (559,221). Published as reported, that tree mixes two spaces, clips its own content, and moves every tap by the window origin (#3899). SnapshotContentTranslation.offset(reportedBox:viewportBox:) reads the carrier box against the viewport box the platform reported, and names the viewport origin as the offset only when the carrier is the app's own window: a (0,0)-anchored box whose side lengths match the viewport's within the same tolerance, while the viewport starts somewhere else on the screen. A carrier that is not the app's own window names no space, so its numbers are left alone: the keyboard window and a SpringBoard overlay under a windowed app report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no offset there and every published rect stays exactly as it was measured before. `windowContentCases` are reportedBox + viewportBox to offset rows, replayed against that Swift function alone; a null offset means the rule declares no translation. The host decoder (packages/platform-apple/src/snapshot-source/window-coordinate-space.ts) replays only the quarter-turn rows: its channel's guest refuses a windowed tree (app-window-not-full-screen, #3899) before that decoder can read one, so the translation has exactly one producer-side owner. Because a translation is the one thing a rotation must not also do, `rotationCases` pin the turn as axes-only: `native` and `oriented` never move a box's origin, and a row whose app frame is offset from the screen comes back at the same place as its own zero-origin twin.", "constants": { "quarterTurnTolerance": 1 }, @@ -126,11 +126,11 @@ "oriented": { "x": 75, "y": 198, "width": 0, "height": 0 } }, { - "name": "landscape right localizes a non-zero app frame origin before the turn and restores it after", + "name": "a non-zero app frame origin does not travel through the turn: the same native box comes back at the same place (#3899)", "interfaceOrientation": 3, "app": { "x": 12, "y": 24, "width": 874, "height": 402 }, "native": { "x": 2, "y": 75, "width": 202, "height": 724 }, - "oriented": { "x": 87, "y": 222, "width": 724, "height": 202 } + "oriented": { "x": 75, "y": 198, "width": 724, "height": 202 } }, { "name": "landscape left turns the same plane the other way, which is where its own native column is", @@ -167,5 +167,91 @@ "native": { "x": 226, "y": 475, "width": 0, "height": 0 }, "oriented": { "x": 476, "y": 226, "width": 0, "height": 0 } } + ], + "windowContentCases": [ + { + "name": "RULE 3 (content translation): a Calendar window placed at (224,207) reports its own screen box and its subtree from its own top-left", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": { "width": 224, "height": 207 } + }, + { + "name": "the same window moved elsewhere on the screen carries its subtree by the new origin", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 120, "y": 90, "width": 386, "height": 682 }, + "offset": { "width": 120, "height": 90 } + }, + { + "name": "the app's own window box reports the screen box it is placed at and translates nothing", + "reportedBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a full-screen iPad window fills the screen from (0,0), so its subtree starts where the viewport starts", + "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "viewportBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "offset": null + }, + { + "name": "a full-screen iPhone window is the same no-op on the smaller screen", + "reportedBox": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "viewportBox": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "offset": null + }, + { + "name": "a Duo panel reports its own box from the origin of its own display, so no translation is read (ADR 0025)", + "reportedBox": { "x": 0, "y": 0, "width": 951, "height": 669 }, + "viewportBox": { "x": 0, "y": 0, "width": 951, "height": 669 }, + "offset": null + }, + { + "name": "the keyboard window under a windowed app reports the screen's box and stays in screen space", + "reportedBox": { "x": 0, "y": 925, "width": 834, "height": 282 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a SpringBoard overlay under a windowed app reports the whole screen and translates nothing", + "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a half-tolerance side-length miss is float representation, not a different box", + "reportedBox": { "x": 0, "y": 0, "width": 385.5, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": { "width": 224, "height": 207 } + }, + { + "name": "a carrier two points narrower than the viewport is some other box and declares no space", + "reportedBox": { "x": 0, "y": 0, "width": 384, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a carrier anchored inside the screen names some content box, not the app's window", + "reportedBox": { "x": 40, "y": 12, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a viewport the capture derived from its own root carries no screen anchor to translate toward", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a zero-width carrier carries no geometry to read a space from", + "reportedBox": { "x": 0, "y": 0, "width": 0, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a non-finite viewport declares nothing", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "infinite": true }, + "offset": null + } ] } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 0d7d1c78c3..67d1a05ad8 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -626,6 +626,51 @@ The rule detects un-normalized arrival, not a standing fact about iOS. The producers that the earlier sections of this ADR describe do normalize. A band taller than it is wide is what the output of a producer that did not normalize looks like. +## Amendment: a window that does not fill the screen (issue #3899) + +An iPad window that does not fill the screen splits a capture in two. +The measurement used iPad Pro 11-inch (M5, iOS 27.0) with Windowed Apps on and a Calendar window at screen `(224,207,386x682)`: + +- The app's root and its subtree report `(0,0)`-anchored frames. The window's `Add` button reads `(335,14,37,36)` and draws at screen `(559,221)`. +- The runner's live `XCUIApplication.windows` read reports the window's screen box `(224,207,386,682)`. +- A system window under the app root — the keyboard at `(0,925,834,282)`, a SpringBoard overlay at `(0,0,834,1210)` — reports the screen's own box. + +The viewport is built from the live read, so the capture published a screen-space viewport around window-relative rects. +That mix produced three failures from one cause: the fold clipped each node to `(w - x) x (h - y)` and the regular capture died with `regular-node-outside-cumulative-clip`; a published rect named no position in the screenshot; and a coordinate tap landed at the point minus the window origin. +The same drift appears in any app the user has placed, so Stage Manager behaves the same way. + +Decision. A window that does not fill the screen publishes screen space in points. +One capture publishes one space, as the #2612 amendment decided, and the space a placed window names is the screen's — because that is the space its viewport, its screenshot, and its digitizer already use. +The conversion runs where the platform's frame is still the platform's frame: inside the same `SnapshotGeometrySpace.normalized` pass that turns a quarter-turned subtree. +No per-command patch reads a window origin. + +A window declares its subtree's space from its own box, and this is the one rule in the table that reads an origin. +The carrier's box must be a `(0,0)`-anchored copy of the viewport box the platform reported, and the viewport must start somewhere else on the screen. +Then the subtree was reported in the carrier's content space, and the viewport's origin is the one vector that carries it to the screen. +Any other carrier box names no space, so the rule leaves its numbers alone: the keyboard window and the SpringBoard overlay above report the screen's own box and are already in screen space. +A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no vector there and every rect is unchanged. +`SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in the capture's language alone: the host bridge refuses a windowed tree (`app-window-not-full-screen`) before its decoder can read one, so the translation has one producer-side owner and the host's parity replay covers only the rules that decoder reads. + +Rotation and translation stay separate rules. +`CoordinateSpaceRotation` turns a box's axes and never moves its origin: the digitizer's origin is the display's own, which is where a placed window's screen-space points are already measured from. +Before this amendment, that function localized a non-zero frame origin away and never added it back. +Every configuration measured before this one has its app frame at the screen origin, so the localization was dead code there and the dispatch bug here. +Translation is the one rule that reads an origin, so it is the only place the fix can live. + +Capture is half of the promise; dispatch is the other half. +The measured dispatch spaces for a placed window are both screen-anchored: the synthesized path lands a point at its screen position, and XCTest's own `XCUICoordinate` on the app anchors at the screen's origin too. +`interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025). +The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (659,321) for a window at (324,307). +The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. +So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion. +The probe-side owner is `ResolvedTextEntryInput.frameOnScreen`. +The vector is read once per command and stored on the target and each read the target produces, so the probe matching, the focus tap, the refresh point, the caret tap, and the readiness retap all dispatch through that one property. +Where a probe read has no snapshot and falls back to the live element, its frame is already screen space and is not translated. +`app.frame` unions transformed subtrees, so content pushed off a placed window (a closed drawer at negative x) can stop the two live boxes matching and read no vector; the capture still converts from the snapshot root box, so a probe may then read window-local frames where the capture published screen space. A mid-command window move has the same effect on a vector read at command start. +Both are reads of a moving target, not a second space: a re-opened command reads the current vector. +The host bridge keeps refusing a windowed capture (`app-window-not-full-screen`) rather than converting one, so the TypeScript twin of the rule runs today only in the parity replay. +Portrait is the only orientation measured for a placed window; the quarter-turn table's windowed row pins the arithmetic (a frame origin never travels through the turn), not a device measurement, and landscape under Windowed Apps stays unverified until measured. + ## Amendment: modal containment in the presentation cut The bridge reads one window and reports every container attached to it. @@ -723,10 +768,9 @@ If the frames differ, the guest refuses the capture (`app-window-not-full-screen This refusal provides evidence about the screen arrangement, not the app, so it has capture scope. If the runtime cannot report either frame, the guest keeps the capture. -The XCTest fallback is not correct for a windowed app either. -The runner clips window-relative element frames against the screen-relative app frame. -That is a runner gap. -It is not a reason to publish a bridge tree that is known to be wrong. +The XCTest fallback is the correct capture for a windowed app, after the amendment above. +Before it, the runner clipped window-relative element frames against the screen-relative app frame, and that gap is what made this refusal protect a real failure. +The refusal stays: this channel reports window-relative frames and no channel here converts them. Rejected alternatives: diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index ca1a92920c..afd4dadacd 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -407,7 +407,7 @@ test.for([ ['foreground-owner-changed', 'while another surface owned the foreground'], [ 'app-window-not-full-screen', - 'while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen', + 'while the app window did not fill the screen; the runner republishes a windowed capture in screen space', ], ] as const)('a %s refusal sends only that capture to the runner', async ([code, consequence]) => { const source = sourceReturning({ stage: 'failed', failure: { kind: 'unsupported', code } }); diff --git a/packages/platform-apple/src/snapshot-route.ts b/packages/platform-apple/src/snapshot-route.ts index 5fcee4fca7..c977be2baf 100644 --- a/packages/platform-apple/src/snapshot-route.ts +++ b/packages/platform-apple/src/snapshot-route.ts @@ -512,7 +512,7 @@ function captureScopedFallbackWarning(code: string, consequence: CaptureScopedCo case 'foreground-ownership': return `${thisCapture}, while another surface owned the foreground.`; case 'window-not-full-screen': - return `${thisCapture}, while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen.`; + return `${thisCapture}, while the app window did not fill the screen; the runner republishes a windowed capture in screen space.`; case 'coordinate-space-unresolved': return `${thisCapture}, which reports captured geometry in the app's own orientation space.`; } diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index 8abe052783..a95d4908c8 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -38,10 +38,18 @@ type RotationCase = { oriented: FixtureFrame; }; +type WindowContentCase = { + name: string; + reportedBox: FixtureFrame; + viewportBox: FixtureFrame; + offset: { width: number; height: number } | null; +}; + type Fixture = { constants: { quarterTurnTolerance: number }; quarterTurnCases: QuarterTurnCase[]; rotationCases: RotationCase[]; + windowContentCases: WindowContentCase[]; }; const TABLE_PATH = path.resolve( @@ -82,8 +90,11 @@ test('the quarter-turn window rule agrees with every golden parity table case', const table = loadTable(); assert.ok(table.quarterTurnCases.length > 0, 'parity table must not be empty'); assert.ok(table.rotationCases.length > 0, 'parity table must not be empty'); + assert.ok(table.windowContentCases.length > 0, 'parity table must not be empty'); assertUniqueNames( - [...table.quarterTurnCases, ...table.rotationCases].map((fixture) => fixture.name), + [...table.quarterTurnCases, ...table.rotationCases, ...table.windowContentCases].map( + (fixture) => fixture.name, + ), 'parity table', ); for (const fixture of table.quarterTurnCases) { @@ -95,6 +106,11 @@ test('the quarter-turn window rule agrees with every golden parity table case', } }); +// `windowContentCases` is replayed in the Swift capture alone (CoordinateSpaceTests): this decoder +// never receives a windowed tree — the bridge guest refuses it first (#3899) — so a second replay +// here would pin arithmetic no production reader runs. The table still loads here, and its rows are +// read for the name-uniqueness guard below. + test('the quarter-turn tolerance belongs to the table, not this file', () => { assert.equal(loadTable().constants.quarterTurnTolerance, WINDOW_QUARTER_TURN_TOLERANCE); }); @@ -104,8 +120,12 @@ const LANDSCAPE_RIGHT = 3; const LANDSCAPE_LEFT = 4; /** - * The dispatch direction — an app-space point in the native space a synthesized event is performed in - * — as the runner's own `CoordinateSpaceRotation.native(point:in:interfaceOrientation:)` computes it. + * The dispatch direction — the point a caller passes, in the native space a synthesized event is + * performed in — as the runner's own `CoordinateSpaceRotation.native(point:in:interfaceOrientation:)` + * computes it. + * + * The turn moves a point's axes and never its origin (#3899): the digitizer's origin is the display's + * own, which is where a placed window's screen-space points are already measured from. * * The capture's inverse is NOT implemented here: it lives in Swift alone, and a TypeScript copy of it * would prove nothing about the code that runs. What this proves is that the table cannot be edited @@ -115,17 +135,15 @@ const LANDSCAPE_LEFT = 4; * forward map it ships with. */ function nativePoint(point: Point, app: Rect, interfaceOrientation: number): Point { - const localX = point.x - app.x; - const localY = point.y - app.y; switch (interfaceOrientation) { case LANDSCAPE_RIGHT: - return { x: app.height - localY, y: localX }; + return { x: app.height - point.y, y: point.x }; case LANDSCAPE_LEFT: - return { x: localY, y: app.width - localX }; + return { x: point.y, y: app.width - point.x }; case PORTRAIT_UPSIDE_DOWN: - return { x: app.width - localX, y: app.height - localY }; + return { x: app.width - point.x, y: app.height - point.y }; default: - return { x: localX, y: localY }; + return { x: point.x, y: point.y }; } } diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index 13c58fa59e..ee97982866 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -15,17 +15,31 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * two spaces coincide and nothing is turned. Rules that read the reported numbers refused app content * the keyboard was nowhere near and let a tap through into a key (#2612). * - * The box is the only evidence, so the rule reads side lengths and nothing else. Origins are not - * compared: a quarter turn says which axis a box runs along, not where it sits, and a window may - * legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it - * is left alone rather than guessed at — rotating geometry that did not need it is the worse failure. + * A window that does not fill the screen — iPad Windowed Apps or Stage Manager — declares the other + * space. Measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window whose screen box is + * `(224,207,386,682)`, the window itself reports that screen box while the subtree under it arrives + * anchored at its own top-left: the application root reads `(0,0,386,682)` and its `Add` button + * `(335,14,37,36)` — the button's screen position `(559,221)` minus the window origin. Published as + * reported, that tree mixes two spaces, clips its own content, and moves every tap by the window + * origin (#3899). That translation belongs to the capture that performs it — the runner's Swift + * `SnapshotContentTranslation` — not to this decoder, because the guest refuses a windowed tree + * before this file can read one. * - * The same rule is enforced in Swift for the capture that produces these trees, and one table proves - * they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and against - * `SnapshotGeometrySpace` in - * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also - * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only - * today); change either rule only through that table. + * A rule that reads a box reads what the box can say. The quarter turn is a statement about which + * axis a box runs along, so it compares side lengths and nothing else. A content translation is a + * statement about where a subtree starts, so it reads all four numbers of the carrier's box against + * the viewport box the platform reported: the carrier must be the app's own window — a `(0,0)`-anchored + * copy of the viewport's side lengths — and the viewport must start somewhere else on the screen. + * A full-screen window, an iPhone, and each foldable panel report origins at zero on both sides, so + * the rule reads no translation there and every published rect stays as the platform reported it. + * + * RULE 1 is enforced in both languages, because both channels read it. RULE 3 (the content + * translation) and RULE 2 (the rotation back, `CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`) + * live with the runner's capture alone. One table proves the shared rules are one rule: + * `contracts/fixtures/window-coordinate-space.json` is replayed here and against + * `SnapshotGeometrySpace`, `SnapshotContentTranslation`, and `CoordinateSpaceRotation` in + * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`; + * change any rule only through that table. * * Why this is a detector and not a repair: the bridge guest cannot read the app's interface * orientation. Measured in the #2659 spike (verdict on the issue, write-up in the diff of #2667):