From a7370b2fe95ec2f7886c94919a825a56d9e36553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 21:51:37 +0200 Subject: [PATCH 1/9] fix(ios): publish a windowed iPad app's capture in screen space (#3899) An iPad window that does not fill the screen reports its own screen box while its subtree arrives anchored at its own top-left. The capture published a screen-space viewport around window-relative rects: the fold clipped content to (w - x) x (h - y), a rect named no screenshot position, and a synthesized tap landed at the point minus the window origin. SnapshotContentTranslation reads the one vector from a carrier's (0,0)-anchored copy of the viewport box to the viewport itself, and SnapshotGeometrySpace.normalized translates the subtree it declares in the same pass that turns a quarter-turned subtree. CoordinateSpaceRotation now turns axes only; the digitizer origin stays where the display put it. Replayed from contracts/fixtures/window-coordinate-space.json in both languages. Refs #3899 --- ...unnerTests+SnapshotPresentationTests.swift | 127 ++++++++++++++ .../SnapshotCoordinateSpace.swift | 112 +++++++++--- .../CoordinateSpaceTests.swift | 162 +++++++++++++++++- .../fixtures/window-coordinate-space.json | 92 +++++++++- .../adr/0004-ios-snapshot-backend-strategy.md | 37 ++++ .../platform-apple/src/snapshot-route.test.ts | 2 +- .../window-coordinate-space.test.ts | 48 +++++- .../window-coordinate-space.ts | 69 +++++++- 8 files changed, 600 insertions(+), 49 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift index 8138eb41fb..1510dce82a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationTests.swift @@ -662,6 +662,133 @@ extension RunnerTests { XCTAssertNotNil(keyQ, "`q` key must be presented under --depth \(options.depth ?? -1)") } + /// #3899: an iPad window that does not fill the screen publishes screen space. The measured shape is + /// iPad Pro 11-inch (M5, iOS 27.0) with Windowed Apps on and a Calendar window at (224,207,386x682): + /// the app's own window arrives as a (0,0)-anchored copy of that box and its subtree with it, so the + /// Add button reports (335,14,37,36) for a control drawn at (559,221). The one normalization pass + /// reads the declaration and publishes the subtree at its place on the screen, which is what lets the + /// fold keep every node instead of clipping each one to (w − x) × (h − y) and refusing the capture. + func testAWindowedSubtreeIsPublishedInScreenSpaceAndKeepsEveryNode() throws { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + func node( + _ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int + ) -> RawAXNode { + RawAXNode( + index: index, type: type, label: label, identifier: nil, value: nil, + rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height), + enabled: true, focused: nil, selected: nil, hittable: false, + depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil + ) + } + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let acquired = [ + node(0, "Application", "Calendar", reportedRoot, nil, 0), + node(1, "Window", "Window", reportedRoot, 0, 1), + node(2, "Other", "Toolbar", reportedRoot, 1, 2), + node(3, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3), + node(4, "Button", "Today", CGRect(x: 8, y: 583, width: 90, height: 38), 2, 3), + // A SpringBoard overlay under the app root reports the screen's own box: its subtree is already + // in screen space and must not move. + node(5, "Window", "Alert", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1), + node(6, "Button", "Discard Changes", CGRect(x: 300, y: 500, width: 240, height: 44), 5, 2), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: windowOnScreen, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ) + let options = PresentationOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false) + let capture = try XCTUnwrap(try SnapshotPresentation.present( + SnapshotAcquisition( + hint: SnapshotPresentation.captureHint(for: options), + nodes: normalized, + truncated: false, + effectiveDepth: nil, + viewport: .reported(box: windowOnScreen) + ), + options: options + )) + + // One space: the root answers with the window's screen box, and the button with the position the + // screenshot draws. + XCTAssertEqual(capture.nodes.first?.rect, SnapshotRect(windowOnScreen)) + let add = try XCTUnwrap(capture.nodes.first { $0.label == "Add" }) + XCTAssertEqual(add.rect, SnapshotRect(x: 559, y: 221, width: 37, height: 36)) + XCTAssertEqual(add.hittable, true) + // Nothing is dropped for being "outside the clip": the whole acquired tree is still there. The one + // clipped node is the system overlay that really is larger than the app's window, which is the + // fold's normal answer to oversized content — not the mixed-space clip that used to publish + // (w − x) × (h − y) boxes and refuse this capture outright. + XCTAssertEqual(Set(capture.nodes.map(\.label)), Set(acquired.map(\.label))) + let clipNote = capture.qualityNotes.first { $0.code == .regularGeometryClipped } + XCTAssertEqual(clipNote?.droppedNodeCount, 0, "no node is dropped for the window's origin") + XCTAssertEqual(clipNote?.clippedNodeCount, 1, "only the overlay larger than the window is clipped") + XCTAssertEqual( + capture.nodes.first { $0.label == "Alert" }?.rect, + SnapshotRect(x: 224, y: 207, width: 386, height: 682), + "the overlay is clipped to the window's visible box, not to (w − x) × (h − y) of it" + ) + // The system window and its button stay where the platform reported them. + let discard = try XCTUnwrap(capture.nodes.first { $0.label == "Discard Changes" }) + XCTAssertEqual(discard.rect, SnapshotRect(x: 300, y: 500, width: 240, height: 44)) + } + + /// #3899 keeps a full-screen capture byte-identical: the same tree reported from the screen origin is + /// published exactly as reported, with no translation read from its zero-origin viewport. + func testAFullScreenCapturePublishesWhatThePlatformReported() throws { + let screen = CGRect(x: 0, y: 0, width: 834, height: 1210) + func node( + _ index: Int, _ type: String, _ label: String, _ rect: CGRect, _ parent: Int?, _ depth: Int + ) -> RawAXNode { + RawAXNode( + index: index, type: type, label: label, identifier: nil, value: nil, + rect: SnapshotRect(x: rect.minX, y: rect.minY, width: rect.width, height: rect.height), + enabled: true, focused: nil, selected: nil, hittable: false, + depth: depth, parentIndex: parent, hiddenContentAbove: nil, hiddenContentBelow: nil + ) + } + let acquired = [ + node(0, "Application", "Calendar", screen, nil, 0), + node(1, "Window", "Window", screen, 0, 1), + node(2, "Button", "Add", CGRect(x: 335, y: 14, width: 37, height: 36), 1, 2), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: screen, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ) + XCTAssertEqual(normalized.map(\.rect), acquired.map(\.rect)) + XCTAssertEqual( + SnapshotContentTranslation.offset(reportedBox: screen, viewportBox: screen), + nil, + "a viewport at the screen origin carries no translation" + ) + } + + /// #3899 keeps ADR 0025 foldable panel taps: a panel reports its own box from its own display's + /// origin, so no translation is read and the panel's geometry stays exactly as measured. + func testAFoldablePanelReportsNoContentTranslation() { + let panel = CGRect(x: 0, y: 0, width: 951, height: 669) + XCTAssertNil( + SnapshotContentTranslation.offset(reportedBox: panel, viewportBox: panel), + "a panel box anchored at its display's origin names no translation" + ) + // A digitizer point on the panel is passed through, so the fold's display selection stays the only + // window-dependent choice the tap path makes. + XCTAssertEqual( + CoordinateSpaceRotation.native( + point: CGPoint(x: 476, y: 226), + in: panel, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ), + CGPoint(x: 476, y: 226) + ) + } + /// #2891: a capture whose viewport read failed publishes no `hittable` for a node whose answer is /// containment, while a root and a disabled node stay a declared `false`. Pinned on the /// encoded objects rather than on `capture.nodes`, because the promise is about the wire: a test diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 300b8c910e..7718fbabc1 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -4,6 +4,10 @@ import CoreGraphics // Rotation between an app's interface space and the device's native (portrait-up) space, shared by // synthesized dispatch (forward) and capture (inverse). Rationale and measurements: ADR 0004 // "the coordinate space of a captured subtree" and contracts/fixtures/window-coordinate-space.json. +// +// Rotation moves axes only. It never moves an origin: a placed window's offset from the screen +// belongs to `SnapshotContentTranslation` (#3899), which is the one rule that turns a reported box +// into its screen-space box. Keeping the two apart is what lets a capture publish one space. public enum RunnerInterfaceOrientation { public static let unknown = 0 @@ -14,18 +18,18 @@ public enum RunnerInterfaceOrientation { } public enum CoordinateSpaceRotation { - // Dispatch rotates a point about the reference frame's top-left. The origin is localized away - // and never added back: an XCUIApplication session observes only its own scene surface, so its - // windows report (0,0)-anchored frames (measured on iPhone landscape both ways and on every - // system window in window-coordinate-space.json), and displacement between panels or panes is - // carried by the record's displayID, not by an origin. + // Dispatch turns a point's axes and never moves its origin: the turn says which axis a box runs + // along, not where that box sits (#3899). The digitizer's origin is the display's own, which is + // where a windowed app's screen-space points are already measured from, so a placed window's + // offset from the screen belongs to `SnapshotContentTranslation` — the one rule that reads an + // offset — and not to this function. The `frame` argument names the side lengths the turn swaps. public static func native( point: CGPoint, in frame: CGRect, interfaceOrientation: Int ) -> CGPoint { - let localX = Double(point.x) - Double(frame.minX) - let localY = Double(point.y) - Double(frame.minY) + let localX = Double(point.x) + let localY = Double(point.y) let width = Double(frame.width) let height = Double(frame.height) switch interfaceOrientation { @@ -60,23 +64,23 @@ public enum CoordinateSpaceRotation { ) -> CGPoint { let width = Double(frame.width) let height = Double(frame.height) - let localX: Double - let localY: Double + let x: Double + let y: Double switch interfaceOrientation { case RunnerInterfaceOrientation.landscapeRight: - localX = Double(point.y) - localY = height - Double(point.x) + x = Double(point.y) + y = height - Double(point.x) case RunnerInterfaceOrientation.landscapeLeft: - localX = width - Double(point.y) - localY = Double(point.x) + x = width - Double(point.y) + y = Double(point.x) case RunnerInterfaceOrientation.portraitUpsideDown: - localX = width - Double(point.x) - localY = height - Double(point.y) + x = width - Double(point.x) + y = height - Double(point.y) default: - localX = Double(point.x) - localY = Double(point.y) + x = Double(point.x) + y = Double(point.y) } - return CGPoint(x: localX + Double(frame.minX), y: localY + Double(frame.minY)) + return CGPoint(x: x, y: y) } public static func oriented( @@ -103,9 +107,57 @@ public enum CoordinateSpaceRotation { } } +/// Whether a carrier's own box declares that its subtree reports in the carrier's content space +/// rather than in the space the viewport names (#3899). +/// +/// An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own frame +/// at its place on the screen while its subtree arrives anchored at its own top-left: measured on +/// iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window whose `frame` is `(224,207,386,682)`, the +/// application root reports `(0,0,386,682)` and its Add button `(335,14,37,36)` at screen +/// `(559,221)`. A full-screen window, an iPhone, and each foldable panel report origins at zero on +/// both sides, so the rule reads zero translation there and every published rect is unchanged. +/// +/// The signature is the carrier's box as a `(0,0)`-anchored copy of the viewport box: the side +/// lengths match within the tolerance and the viewport — the app's screen-anchored box — starts +/// somewhere else on the screen. Any other carrier box names no space: it is not the app's own +/// window, so its numbers say nothing about where the app's subtree starts. +public enum SnapshotContentTranslation { + /// The vector from a carrier's reported box to the screen space it is placed at, or `nil` when the + /// carrier's box declares no translation. One box offset from one viewport box, decided from the + /// same tolerance the quarter-turn rule reads; replayed against + /// `contracts/fixtures/window-coordinate-space.json` in `windowContentCases`. + public static func offset(reportedBox: CGRect, viewportBox: CGRect) -> CGSize? { + guard SnapshotGeometry.isPositiveFinite(reportedBox), + SnapshotGeometry.isPositiveFinite(viewportBox) + else { + return nil + } + guard abs(reportedBox.minX) <= SnapshotGeometrySpace.quarterTurnTolerance, + abs(reportedBox.minY) <= SnapshotGeometrySpace.quarterTurnTolerance + else { + return nil + } + guard abs(viewportBox.minX) > SnapshotGeometrySpace.quarterTurnTolerance + || abs(viewportBox.minY) > SnapshotGeometrySpace.quarterTurnTolerance + else { + return nil + } + guard abs(reportedBox.width - viewportBox.width) <= SnapshotGeometrySpace.quarterTurnTolerance, + abs(reportedBox.height - viewportBox.height) <= SnapshotGeometrySpace.quarterTurnTolerance + else { + return nil + } + return CGSize(width: viewportBox.minX, height: viewportBox.minY) + } +} + public enum SnapshotGeometrySpace: Equatable { case appOrientation case deviceNative(appFrame: SnapshotViewport.Box, interfaceOrientation: Int) + /// The subtree was reported in a placed window's own content space and is published translated + /// into the viewport's screen space (`SnapshotContentTranslation`, #3899). The offset is the + /// vector from the window's reported box to the viewport box it sits at on screen. + case windowContent(offset: CGSize) public static let quarterTurnTolerance: Double = 1 @@ -119,10 +171,15 @@ public enum SnapshotGeometrySpace: Equatable { in: appFrame.rect, interfaceOrientation: interfaceOrientation ) + case .windowContent(let offset): + return reportedFrame.offsetBy(dx: offset.width, dy: offset.height) } } - /// Only a `.reported` viewport carries an orientation, so only it can anchor a rotation. + /// A carrier declares its own subtree's space from its own box. A `.reported` viewport is the + /// screen-anchored fact both declarations read against: a quarter turn needs its orientation, and + /// a window-content offset needs its origin. A `.derived` viewport carries no screen anchor, so a + /// subtree under it publishes what the platform reported rather than a guessed space. public static func space( reportedBySurfaceHost isSurfaceHost: Bool, reportedFrame: CGRect, @@ -130,13 +187,20 @@ public enum SnapshotGeometrySpace: Equatable { viewport: SnapshotViewport ) -> SnapshotGeometrySpace { guard isSurfaceHost else { return inherited } - guard case .reported(let appFrame, let interfaceOrientation) = viewport, - namesQuarterTurn(interfaceOrientation), - isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) - else { + guard case .reported(let appFrame, let interfaceOrientation) = viewport else { return .appOrientation } - return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + if namesQuarterTurn(interfaceOrientation), + isQuarterTurned(reportedFrame, relativeTo: appFrame.rect) { + return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation) + } + if let offset = SnapshotContentTranslation.offset( + reportedBox: reportedFrame, + viewportBox: appFrame.rect + ) { + return .windowContent(offset: offset) + } + return .appOrientation } public static func namesQuarterTurn(_ interfaceOrientation: Int) -> Bool { diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index c5df2bc22c..82a76fb74a 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -28,9 +28,22 @@ final class CoordinateSpaceTests: XCTestCase { let oriented: Frame } + struct WindowContentCase: Decodable { + let name: String + let reportedBox: Frame + let viewportBox: Frame + let offset: Offset? + } + + struct Offset: Decodable { + let width: Double + let height: Double + } + let constants: Constants let quarterTurnCases: [QuarterTurnCase] let rotationCases: [RotationCase] + let windowContentCases: [WindowContentCase] } /// Identity in portrait/unknown, 90° per landscape, 180° upside-down. @@ -63,9 +76,9 @@ final class CoordinateSpaceTests: XCTestCase { in: offsetLandscape, interfaceOrientation: RunnerInterfaceOrientation.landscapeLeft ), - CGPoint(x: 268, y: 1040), - "non-zero frame origin is localized before rotation" - ) + CGPoint(x: 288, y: 1030), + "a frame offset from the screen never moves the turned point (#3899)" + ); } func testNativeSynthesizedVectorRotatesByInterfaceOrientation() { @@ -303,7 +316,7 @@ final class CoordinateSpaceTests: XCTestCase { ) } - /// Golden parity table (#2612): every case in contracts/fixtures/window-coordinate-space.json + /// Golden parity table (#2612, #3899): every case in contracts/fixtures/window-coordinate-space.json /// must agree with the vitest twin /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add /// cases there, never fork the rule. Detection is replayed under both landscape turns, because @@ -313,6 +326,22 @@ final class CoordinateSpaceTests: XCTestCase { let fixture = try loadWindowCoordinateSpaceFixture() XCTAssertFalse(fixture.quarterTurnCases.isEmpty, "parity table must not be empty") XCTAssertFalse(fixture.rotationCases.isEmpty, "parity table must not be empty") + XCTAssertFalse(fixture.windowContentCases.isEmpty, "parity table must not be empty") + XCTAssertEqual( + Set(fixture.quarterTurnCases.map(\.name)).count, + fixture.quarterTurnCases.count, + "parity table case names must be unique" + ) + XCTAssertEqual( + Set(fixture.rotationCases.map(\.name)).count, + fixture.rotationCases.count, + "parity table case names must be unique" + ) + XCTAssertEqual( + Set(fixture.windowContentCases.map(\.name)).count, + fixture.windowContentCases.count, + "parity table case names must be unique" + ) XCTAssertEqual( fixture.constants.quarterTurnTolerance, SnapshotGeometrySpace.quarterTurnTolerance, @@ -349,6 +378,17 @@ final class CoordinateSpaceTests: XCTestCase { testCase.name ) } + for testCase in fixture.windowContentCases { + let expected = testCase.offset.map { CGSize(width: $0.width, height: $0.height) } + XCTAssertEqual( + SnapshotContentTranslation.offset( + reportedBox: testCase.reportedBox.cgRect, + viewportBox: testCase.viewportBox.cgRect + ), + expected, + testCase.name + ) + } } private func deviceNative(_ appFrame: CGRect, _ interfaceOrientation: Int) -> SnapshotGeometrySpace { @@ -486,6 +526,120 @@ final class CoordinateSpaceTests: XCTestCase { ) } + /// The measured windowed iPad Calendar window (#3899) driven through the pass: the carrier reports + /// a `(0,0)`-anchored copy of the viewport box, declares its subtree window-relative, and every + /// rect under it comes back at its place on the screen — the root at the window's screen box, the + /// Add button at (559,221) where the screenshot draws it, and a full-screen-capture tree untouched. + func testNormalizedTranslatesAWindowedSubtreeIntoScreenSpace() { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let reportedAddButton = CGRect(x: 335, y: 14, width: 37, height: 36) + let acquired = [ + coordinateNode(0, "Application", reportedRoot, nil, 0, label: "Calendar"), + coordinateNode(1, "Window", reportedRoot, 0, 1), + coordinateNode(2, "Other", reportedRoot, 1, 2), + coordinateNode(3, "Button", reportedAddButton, 2, 3, label: "Add"), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported(box: windowOnScreen, interfaceOrientation: RunnerInterfaceOrientation.portrait) + ) + XCTAssertEqual(normalized[0].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[1].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[2].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[3].rect.cgRect, CGRect(x: 559, y: 221, width: 37, height: 36)) + XCTAssertEqual(normalized[3].hittable, true) + + // The same window moved: the subtree follows the new origin. + let moved = SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: CGRect(x: 120, y: 90, width: 386, height: 682), + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ) + XCTAssertEqual(moved[3].rect.cgRect, CGRect(x: 455, y: 104, width: 37, height: 36)) + } + + /// What must not move: a full-screen capture, and any subtree under a carrier that is not the app's + /// own placed window. Those trees keep exactly the rect the platform reported (#3899 keeps + /// full-screen and ADR 0025 panel behavior identical). + func testNormalizedTranslatesNothingForFullScreenAndForeignCarriers() { + // Full-screen iPad: reported boxes already start at the viewport. + let fullScreen = CGRect(x: 0, y: 0, width: 834, height: 1210) + let acquired = [ + coordinateNode(0, "Application", fullScreen, nil, 0), + coordinateNode(1, "Window", fullScreen, 0, 1), + coordinateNode(2, "Button", CGRect(x: 335, y: 14, width: 37, height: 36), 1, 2), + ] + XCTAssertEqual( + SnapshotGeometrySpace.normalized( + nodes: acquired, + viewport: .reported( + box: fullScreen, + interfaceOrientation: RunnerInterfaceOrientation.portrait + ) + ).map(\.rect), + acquired.map(\.rect) + ); + + // A windowed app, measured shape: the app's own window declares the translation and carries the + // subtree; the keyboard window and the SpringBoard overlay report the screen's own box, so their + // subtrees stay where the platform put them. + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + let reportedRoot = CGRect(x: 0, y: 0, width: 386, height: 682) + let mixed = [ + coordinateNode(0, "Application", reportedRoot, nil, 0), + coordinateNode(1, "Window", reportedRoot, 0, 1, label: "appWindow"), + coordinateNode(2, "Other", reportedRoot, 1, 2), + coordinateNode(3, "Button", CGRect(x: 335, y: 14, width: 37, height: 36), 2, 3, label: "Add"), + coordinateNode(4, "Window", CGRect(x: 0, y: 925, width: 834, height: 282), 0, 1, label: "keyboard"), + coordinateNode(5, "Key", CGRect(x: 300, y: 1100, width: 60, height: 60), 4, 2), + coordinateNode(6, "Window", CGRect(x: 0, y: 0, width: 834, height: 1210), 0, 1, label: "springboard"), + coordinateNode(7, "Alert", CGRect(x: 300, y: 500, width: 240, height: 160), 6, 2), + ] + let normalized = SnapshotGeometrySpace.normalized( + nodes: mixed, + viewport: .reported(box: windowOnScreen, interfaceOrientation: RunnerInterfaceOrientation.portrait) + ) + XCTAssertEqual(normalized[1].rect.cgRect, windowOnScreen, "appWindow") + XCTAssertEqual(normalized[2].rect.cgRect, windowOnScreen) + XCTAssertEqual(normalized[3].rect.cgRect, CGRect(x: 559, y: 221, width: 37, height: 36)) + XCTAssertEqual(normalized[4].rect, mixed[4].rect, "keyboard") + XCTAssertEqual(normalized[5].rect, mixed[5].rect) + XCTAssertEqual(normalized[6].rect, mixed[6].rect, "springboard") + XCTAssertEqual(normalized[7].rect, mixed[7].rect) + } + + /// The dispatch direction for a windowed app: the caller's point is the point touched, so + /// `native(point:)` turns a screen-space point's axes without moving it toward the window origin + /// (#3899). Under portrait — the only orientation measured for a windowed app — it is the identity. + func testNativePointForAWindowedAppKeepsTheCallersScreenPoint() { + let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) + for orientation in [ + RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.unknown + ] { + XCTAssertEqual( + CoordinateSpaceRotation.native( + point: CGPoint(x: 578, y: 239), + in: windowOnScreen, + interfaceOrientation: orientation + ), + CGPoint(x: 578, y: 239), + "interfaceOrientation \(orientation)" + ) + } + // A turn moves axes only: the same point under a landscape window keeps its own origin story. + XCTAssertEqual( + CoordinateSpaceRotation.native( + point: CGPoint(x: 578, y: 239), + in: windowOnScreen, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ), + CGPoint(x: 682 - 239, y: 578) + ) + } + func testNormalizedOfAnEmptyArrayIsEmpty() { XCTAssertEqual( SnapshotGeometrySpace.normalized( diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index ca92a01a71..d42ef974ce 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -1,5 +1,5 @@ { - "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side.", + "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation) is the other space a captured subtree arrives in, and the one rule in this file that reads an origin. An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own screen box while the subtree under it arrives anchored at its own top-left: measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen (224,207,386x682), the application root reports (0,0,386,682) and its Add button (335,14,37,36) for a control drawn at screen (559,221). Published as reported, that tree mixes two spaces, clips its own content, and moves every tap by the window origin (#3899). SnapshotContentTranslation.offset(reportedBox:viewportBox:) reads the carrier box against the viewport box the platform reported, and names the viewport origin as the offset only when the carrier is the app's own window: a (0,0)-anchored box whose side lengths match the viewport's within the same tolerance, while the viewport starts somewhere else on the screen. A carrier that is not the app's own window names no space, so its numbers are left alone: the keyboard window and a SpringBoard overlay under a windowed app report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no offset there and every published rect stays exactly as it was measured before. `windowContentCases` are reportedBox + viewportBox to offset rows, replayed against that Swift function and against windowContentTranslation in packages/platform-apple/src/snapshot-source/window-coordinate-space.ts; a null offset means the rule declares no translation. Because a translation is the one thing a rotation must not also do, `rotationCases` pin the turn as axes-only: `native` and `oriented` never move a box's origin, and a row whose app frame is offset from the screen comes back at the same place as its own zero-origin twin.", "constants": { "quarterTurnTolerance": 1 }, @@ -126,11 +126,11 @@ "oriented": { "x": 75, "y": 198, "width": 0, "height": 0 } }, { - "name": "landscape right localizes a non-zero app frame origin before the turn and restores it after", + "name": "a non-zero app frame origin does not travel through the turn: the same native box comes back at the same place (#3899)", "interfaceOrientation": 3, "app": { "x": 12, "y": 24, "width": 874, "height": 402 }, "native": { "x": 2, "y": 75, "width": 202, "height": 724 }, - "oriented": { "x": 87, "y": 222, "width": 724, "height": 202 } + "oriented": { "x": 75, "y": 198, "width": 724, "height": 202 } }, { "name": "landscape left turns the same plane the other way, which is where its own native column is", @@ -167,5 +167,91 @@ "native": { "x": 226, "y": 475, "width": 0, "height": 0 }, "oriented": { "x": 476, "y": 226, "width": 0, "height": 0 } } + ], + "windowContentCases": [ + { + "name": "RULE 3 (content translation): a Calendar window placed at (224,207) reports its own screen box and its subtree from its own top-left", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": { "width": 224, "height": 207 } + }, + { + "name": "the same window moved elsewhere on the screen carries its subtree by the new origin", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 120, "y": 90, "width": 386, "height": 682 }, + "offset": { "width": 120, "height": 90 } + }, + { + "name": "the app's own window box reports the screen box it is placed at and translates nothing", + "reportedBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a full-screen iPad window fills the screen from (0,0), so its subtree starts where the viewport starts", + "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "viewportBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "offset": null + }, + { + "name": "a full-screen iPhone window is the same no-op on the smaller screen", + "reportedBox": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "viewportBox": { "x": 0, "y": 0, "width": 402, "height": 874 }, + "offset": null + }, + { + "name": "a Duo panel reports its own box from the origin of its own display, so no translation is read (ADR 0025)", + "reportedBox": { "x": 0, "y": 0, "width": 951, "height": 669 }, + "viewportBox": { "x": 0, "y": 0, "width": 951, "height": 669 }, + "offset": null + }, + { + "name": "the keyboard window under a windowed app reports the screen's box and stays in screen space", + "reportedBox": { "x": 0, "y": 925, "width": 834, "height": 282 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a SpringBoard overlay under a windowed app reports the whole screen and translates nothing", + "reportedBox": { "x": 0, "y": 0, "width": 834, "height": 1210 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a half-tolerance origin miss is float representation, not a different placement", + "reportedBox": { "x": 0, "y": 0, "width": 385.5, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": { "width": 224, "height": 207 } + }, + { + "name": "a carrier two points narrower than the viewport is some other box and declares no space", + "reportedBox": { "x": 0, "y": 0, "width": 384, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a carrier anchored inside the screen names some content box, not the app's window", + "reportedBox": { "x": 40, "y": 12, "width": 386, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a viewport the capture derived from its own root carries no screen anchor to translate toward", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a zero-width carrier carries no geometry to read a space from", + "reportedBox": { "x": 0, "y": 0, "width": 0, "height": 682 }, + "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, + "offset": null + }, + { + "name": "a non-finite viewport declares nothing", + "reportedBox": { "x": 0, "y": 0, "width": 386, "height": 682 }, + "viewportBox": { "infinite": true }, + "offset": null + } ] } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 0d7d1c78c3..d56a194091 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -626,6 +626,43 @@ The rule detects un-normalized arrival, not a standing fact about iOS. The producers that the earlier sections of this ADR describe do normalize. A band taller than it is wide is what the output of a producer that did not normalize looks like. +## Amendment: a window that does not fill the screen (issue #3899) + +An iPad window that does not fill the screen splits a capture in two. +The measurement used iPad Pro 11-inch (M5, iOS 27.0) with Windowed Apps on and a Calendar window at screen `(224,207,386x682)`: + +- The app's root and its subtree report `(0,0)`-anchored frames. The window's `Add` button reads `(335,14,37,36)` and draws at screen `(559,221)`. +- The runner's live `XCUIApplication.windows` read reports the window's screen box `(224,207,386,682)`. +- A system window under the app root — the keyboard at `(0,925,834,282)`, a SpringBoard overlay at `(0,0,834,1210)` — reports the screen's own box. + +The viewport is built from the live read, so the capture published a screen-space viewport around window-relative rects. +That mix produced three failures from one cause: the fold clipped each node to `(w - x) x (h - y)` and the regular capture died with `regular-node-outside-cumulative-clip`; a published rect named no position in the screenshot; and a coordinate tap landed at the point minus the window origin. +The same drift appears in any app the user has placed, so Stage Manager behaves the same way. + +Decision. A window that does not fill the screen publishes screen space in points. +One capture publishes one space, as the #2612 amendment decided, and the space a placed window names is the screen's — because that is the space its viewport, its screenshot, and its digitizer already use. +The conversion runs where the platform's frame is still the platform's frame: inside the same `SnapshotGeometrySpace.normalized` pass that turns a quarter-turned subtree. +No per-command patch reads a window origin. + +A window declares its subtree's space from its own box, and this is the one rule in the table that reads an origin. +The carrier's box must be a `(0,0)`-anchored copy of the viewport box the platform reported, and the viewport must start somewhere else on the screen. +Then the subtree was reported in the carrier's content space, and the viewport's origin is the one vector that carries it to the screen. +Any other carrier box names no space, so the rule leaves its numbers alone: the keyboard window and the SpringBoard overlay above report the screen's own box and are already in screen space. +A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no vector there and every rect is unchanged. +`SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in both languages. + +Rotation and translation stay separate rules, and the split is the point. +`CoordinateSpaceRotation` turns a box's axes and never moves its origin: the digitizer's origin is the display's own, which is where a placed window's screen-space points are already measured from. +Before this amendment, that function localized a non-zero frame origin away and never added it back. +Every configuration measured before this one has its app frame at the screen origin, so the localization was dead code there and the dispatch bug here. +Translation is the one rule that reads an origin, so it is the only place the fix can live. + +Capture is half of the promise; dispatch is the other half. +The measured dispatch spaces for a placed window are both screen-anchored: the synthesized path lands a point at its screen position, and XCTest's own `XCUICoordinate` on the app anchors at the screen's origin too. +`interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025). +The live `XCUIElement.frame` reads that the tap path already compares — the selector match, the input probe, the tree's sweep candidates — report screen space on a placed window, as the ref-tap path proved on the device. +Only the snapshot channel arrives window-relative, and only that channel converts. + ## Amendment: modal containment in the presentation cut The bridge reads one window and reports every container attached to it. diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index ca1a92920c..3872fca846 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -407,7 +407,7 @@ test.for([ ['foreground-owner-changed', 'while another surface owned the foreground'], [ 'app-window-not-full-screen', - 'while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen', + "while the app window did not fill the screen; XCTest publishes a windowed app's geometry in screen space (#3899)", ], ] as const)('a %s refusal sends only that capture to the runner', async ([code, consequence]) => { const source = sourceReturning({ stage: 'failed', failure: { kind: 'unsupported', code } }); diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index 8abe052783..e416406beb 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -6,6 +6,7 @@ import type { Point, Rect } from '@agent-device/kernel/snapshot'; import { isQuarterTurnedWindowFrame, WINDOW_QUARTER_TURN_TOLERANCE, + windowContentTranslation, } from './window-coordinate-space.ts'; // ADR 0011 Layer 2 golden parity table (#2612): the SAME JSON is asserted against the Swift twin @@ -38,10 +39,18 @@ type RotationCase = { oriented: FixtureFrame; }; +type WindowContentCase = { + name: string; + reportedBox: FixtureFrame; + viewportBox: FixtureFrame; + offset: { width: number; height: number } | null; +}; + type Fixture = { constants: { quarterTurnTolerance: number }; quarterTurnCases: QuarterTurnCase[]; rotationCases: RotationCase[]; + windowContentCases: WindowContentCase[]; }; const TABLE_PATH = path.resolve( @@ -82,8 +91,11 @@ test('the quarter-turn window rule agrees with every golden parity table case', const table = loadTable(); assert.ok(table.quarterTurnCases.length > 0, 'parity table must not be empty'); assert.ok(table.rotationCases.length > 0, 'parity table must not be empty'); + assert.ok(table.windowContentCases.length > 0, 'parity table must not be empty'); assertUniqueNames( - [...table.quarterTurnCases, ...table.rotationCases].map((fixture) => fixture.name), + [...table.quarterTurnCases, ...table.rotationCases, ...table.windowContentCases].map( + (fixture) => fixture.name, + ), 'parity table', ); for (const fixture of table.quarterTurnCases) { @@ -95,6 +107,22 @@ test('the quarter-turn window rule agrees with every golden parity table case', } }); +test('the window-content translation rule agrees with every golden parity table case', () => { + for (const fixture of loadTable().windowContentCases) { + const translation = windowContentTranslation( + toRect(fixture.reportedBox), + toRect(fixture.viewportBox), + ); + assert.deepEqual( + translation, + fixture.offset === null + ? null + : { offsetX: fixture.offset.width, offsetY: fixture.offset.height }, + fixture.name, + ); + } +}); + test('the quarter-turn tolerance belongs to the table, not this file', () => { assert.equal(loadTable().constants.quarterTurnTolerance, WINDOW_QUARTER_TURN_TOLERANCE); }); @@ -104,8 +132,12 @@ const LANDSCAPE_RIGHT = 3; const LANDSCAPE_LEFT = 4; /** - * The dispatch direction — an app-space point in the native space a synthesized event is performed in - * — as the runner's own `CoordinateSpaceRotation.native(point:in:interfaceOrientation:)` computes it. + * The dispatch direction — the point a caller passes, in the native space a synthesized event is + * performed in — as the runner's own `CoordinateSpaceRotation.native(point:in:interfaceOrientation:)` + * computes it. + * + * The turn moves a point's axes and never its origin (#3899): the digitizer's origin is the display's + * own, which is where a placed window's screen-space points are already measured from. * * The capture's inverse is NOT implemented here: it lives in Swift alone, and a TypeScript copy of it * would prove nothing about the code that runs. What this proves is that the table cannot be edited @@ -115,17 +147,15 @@ const LANDSCAPE_LEFT = 4; * forward map it ships with. */ function nativePoint(point: Point, app: Rect, interfaceOrientation: number): Point { - const localX = point.x - app.x; - const localY = point.y - app.y; switch (interfaceOrientation) { case LANDSCAPE_RIGHT: - return { x: app.height - localY, y: localX }; + return { x: app.height - point.y, y: point.x }; case LANDSCAPE_LEFT: - return { x: localY, y: app.width - localX }; + return { x: point.y, y: app.width - point.x }; case PORTRAIT_UPSIDE_DOWN: - return { x: app.width - localX, y: app.height - localY }; + return { x: app.width - point.x, y: app.height - point.y }; default: - return { x: localX, y: localY }; + return { x: point.x, y: point.y }; } } diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index 13c58fa59e..b08c20964f 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -15,17 +15,28 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * two spaces coincide and nothing is turned. Rules that read the reported numbers refused app content * the keyboard was nowhere near and let a tap through into a key (#2612). * - * The box is the only evidence, so the rule reads side lengths and nothing else. Origins are not - * compared: a quarter turn says which axis a box runs along, not where it sits, and a window may - * legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it - * is left alone rather than guessed at — rotating geometry that did not need it is the worse failure. + * A window that does not fill the screen — iPad Windowed Apps or Stage Manager — declares the other + * space. Measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window whose screen box is + * `(224,207,386,682)`, the window itself reports that screen box while the subtree under it arrives + * anchored at its own top-left: the application root reads `(0,0,386,682)` and its `Add` button + * `(335,14,37,36)` — the button's screen position `(559,221)` minus the window origin. Published as + * reported, that tree mixes two spaces, clips its own content, and moves every tap by the window + * origin (#3899). `windowContentTranslation` names the one vector that puts the subtree back. * - * The same rule is enforced in Swift for the capture that produces these trees, and one table proves - * they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and against - * `SnapshotGeometrySpace` in + * A rule that reads a box reads what the box can say. The quarter turn is a statement about which + * axis a box runs along, so it compares side lengths and nothing else. A content translation is a + * statement about where a subtree starts, so it reads all four numbers of the carrier's box against + * the viewport box the platform reported: the carrier must be the app's own window — a `(0,0)`-anchored + * copy of the viewport's side lengths — and the viewport must start somewhere else on the screen. + * A full-screen window, an iPhone, and each foldable panel report origins at zero on both sides, so + * the rule reads no translation there and every published rect stays as the platform reported it. + * + * The same rules are enforced in Swift for the capture that produces these trees, and one table + * proves they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and + * against `SnapshotGeometrySpace` and `SnapshotContentTranslation` in * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only - * today); change either rule only through that table. + * today); change any rule only through that table. * * Why this is a detector and not a repair: the bridge guest cannot read the app's interface * orientation. Measured in the #2659 spike (verdict on the issue, write-up in the diff of #2667): @@ -59,3 +70,45 @@ export function isQuarterTurnedWindowFrame(windowFrame: Rect, appFrame: Rect): b Math.abs(windowFrame.height - appFrame.width) <= WINDOW_QUARTER_TURN_TOLERANCE ); } + +/** The vector that carries a window-relative subtree into the screen space the viewport names. */ +export type WindowContentTranslation = { + offsetX: number; + offsetY: number; +}; + +/** + * The vector from a carrier's own box to the screen space it is placed at, or null when the carrier's + * box declares no translation. The carrier's subtree was reported in the carrier's content space, so + * adding the vector to each reported rect publishes the subtree in the viewport's screen space (#3899). + * + * Null on anything that is not the app's own placed window: a box that is not a `(0,0)`-anchored copy + * of the viewport's side lengths names no space, and a viewport that already starts at the screen + * origin needs no move. Replayed against `windowContentCases` in + * `contracts/fixtures/window-coordinate-space.json`, in the Swift twin and here. + */ +export function windowContentTranslation( + reportedBox: Rect, + viewportBox: Rect, +): WindowContentTranslation | null { + if (!isPositiveFiniteRect(reportedBox) || !isPositiveFiniteRect(viewportBox)) return null; + if ( + Math.abs(reportedBox.x) > WINDOW_QUARTER_TURN_TOLERANCE || + Math.abs(reportedBox.y) > WINDOW_QUARTER_TURN_TOLERANCE + ) { + return null; + } + if ( + Math.abs(viewportBox.x) <= WINDOW_QUARTER_TURN_TOLERANCE && + Math.abs(viewportBox.y) <= WINDOW_QUARTER_TURN_TOLERANCE + ) { + return null; + } + if ( + Math.abs(reportedBox.width - viewportBox.width) > WINDOW_QUARTER_TURN_TOLERANCE || + Math.abs(reportedBox.height - viewportBox.height) > WINDOW_QUARTER_TURN_TOLERANCE + ) { + return null; + } + return { offsetX: viewportBox.x, offsetY: viewportBox.y }; +} From 8b0427f76cb8c9cd24309b32e61affbebcbdc25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 21:55:21 +0200 Subject: [PATCH 2/9] revert: keep the snapshot-route warning owned by #3914 The refusal wording belongs to the snapshot-route failure policy, not to this fix. Refs #3899 --- packages/platform-apple/src/snapshot-route.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index 3872fca846..ca1a92920c 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -407,7 +407,7 @@ test.for([ ['foreground-owner-changed', 'while another surface owned the foreground'], [ 'app-window-not-full-screen', - "while the app window did not fill the screen; XCTest publishes a windowed app's geometry in screen space (#3899)", + 'while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen', ], ] as const)('a %s refusal sends only that capture to the runner', async ([code, consequence]) => { const source = sourceReturning({ stage: 'failed', failure: { kind: 'unsupported', code } }); From 6bb1ff8b8faaa722cca19b362ef1802cd6d18f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 22:17:06 +0200 Subject: [PATCH 3/9] fix(ios): read a placed window's snapshot frames at their screen place (#3899) XCUIElement.frame reports a placed window's content in screen space, but XCUIElementSnapshot.frame reports it in the window's own content space, so the text-input probe saw two spaces on an iPad Windowed Apps window and refused a fill whose point was on the screen. The probe, the focus tap, and the typing reader now read the same SnapshotContentTranslation vector the capture reads, from the two live boxes that name it. Refs #3899 --- .../RunnerTests+TextEntry.swift | 23 +++++++++++---- .../RunnerTests+TextEntryFocus.swift | 5 +++- .../RunnerTests+TextInputProbe.swift | 28 ++++++++++++++++++- .../RunnerTests+TextTyping.swift | 14 ++++++++-- .../adr/0004-ios-snapshot-backend-strategy.md | 5 ++-- 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 9f8abe79e5..508c711af6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -182,11 +182,18 @@ extension RunnerTests { /// The target moved to the input a resolution read: its element, a refresh point at its center, /// and its identity while the target is unbound. - func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget { + /// + /// `translation` is the live snapshot-frame vector for the reading session (#3899): the snapshot + /// frame of a placed window's input names the window's content space, while a refresh point is + /// compared against screen-space probes, so the center enters the target translated. + func withResolvedInput( + _ input: ResolvedTextEntryInput?, + translation: CGSize? = nil + ) -> TextEntryTarget { guard let input else { return self } - let frame = input.snapshot?.frame ?? .zero + let frame = snapshotFrameOnScreen(input.snapshot?.frame ?? .zero, translation: translation) return TextEntryTarget( element: input.element, refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY), @@ -259,9 +266,15 @@ extension RunnerTests { private(set) var target: TextEntryTarget private var lastRead: ResolvedTextEntryInput? private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput? + private let translation: CGSize? - init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?) { - self.target = target.withResolvedInput(target.postFocusRead) + init( + target: TextEntryTarget, + translation: CGSize? = nil, + resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput? + ) { + self.translation = translation + self.target = target.withResolvedInput(target.postFocusRead, translation: translation) lastRead = target.postFocusRead self.resolve = resolve } @@ -271,7 +284,7 @@ extension RunnerTests { return lastRead } let read = resolve(target) - target = target.withResolvedInput(read) + target = target.withResolvedInput(read, translation: translation) lastRead = read return read } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 34f289cccb..0b009bddaa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -134,7 +134,10 @@ extension RunnerTests { let input = textInputAt(app: app, x: x, y: y) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - if let frame = input?.attributes.frame, !frame.isEmpty { + // The input's frame arrives from a snapshot, which reports a placed window's content in the + // window's own space; the tap below is a screen point (#3899). + if let read = input, !read.attributes.frame.isEmpty { + let frame = snapshotFrameOnScreen(read.attributes.frame, translation: liveContentTranslation(app: app)) _ = tapAt(app: app, x: frame.midX, y: frame.midY) } else { _ = tapAt(app: app, x: x, y: y) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 642e9db8f6..4d348c4355 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -1,4 +1,5 @@ import XCTest +import AgentDeviceSnapshotPresentation final class TextInputProbeIssues { let thread = Thread.current @@ -153,6 +154,7 @@ extension RunnerTests { shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] + let translation = liveContentTranslation(app: app) for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] { if shouldStop() { break } let candidates = query.allElementsBoundByIndex @@ -168,7 +170,8 @@ extension RunnerTests { if shouldStop() { break } guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue } let attributes = candidate.attributes - if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: attributes.frame, point: point) { + let frame = snapshotFrameOnScreen(attributes.frame, translation: translation) + if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: frame, point: point) { inputs.append(candidate) } } @@ -176,4 +179,27 @@ extension RunnerTests { guard !shouldStop() else { return [] } return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } } + + /// The vector that carries this session's snapshot-read app-content frames to their place on the + /// screen, or nil when the app fills the screen (#3899). + /// + /// The live `XCUIElement.frame` property already reports a placed window's content in screen space, + /// but `XCUIElementSnapshot.frame` reports it in the window's own content space, so the same probe + /// sees two spaces on an iPad Windowed Apps window. The signature and the vector are the capture's + /// rule (`SnapshotContentTranslation`, ADR 0004) read from the two live boxes that name it: the + /// application's `frame` is a `(0,0)`-anchored copy of the box the resolved window reports at its + /// place on the screen. Any other pair of boxes declares no vector, so full-screen, iPhone, and + /// each foldable panel keep their frames exactly as read. + func liveContentTranslation(app: XCUIApplication) -> CGSize? { + safely("LIVE_CONTENT_TRANSLATION", nil) { + guard let resolved = resolveRunnerWindowElement(app: app) else { return nil } + return SnapshotContentTranslation.offset(reportedBox: app.frame, viewportBox: resolved.frame) + } + } +} + +/// `frame` as the screen names it, given `liveContentTranslation`'s vector. +func snapshotFrameOnScreen(_ frame: CGRect, translation: CGSize?) -> CGRect { + guard let translation else { return frame } + return frame.offsetBy(dx: translation.width, dy: translation.height) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index c0ad58ca1c..760e215661 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -16,7 +16,13 @@ extension RunnerTests { ) -> TextEntryResult { let totalStartedAt = Date() let resolve = { (target: TextEntryTarget) in self.resolveTextEntryInput(app: app, target: target) } - var reader = TextEntryReader(target: target, resolve: resolve) + // Reads inside the typing loop answer from snapshots, which report a placed window's content in + // the window's own space; the reader's refresh points are screen points (#3899). + var reader = TextEntryReader( + target: target, + translation: liveContentTranslation(app: app), + resolve: resolve + ) guard !text.isEmpty else { // Replacing a field WITH the empty string is the clear-field primitive (#2063): the typing // is vacuous but the clear is not. Returning here unconditionally left the old value in @@ -74,7 +80,11 @@ extension RunnerTests { case .completed(let result): return result case .fallback(let fallbackTarget): - reader = TextEntryReader(target: fallbackTarget, resolve: resolve) + reader = TextEntryReader( + target: fallbackTarget, + translation: liveContentTranslation(app: app), + resolve: resolve + ) } } let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType( diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index d56a194091..4e8b143f13 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -660,8 +660,9 @@ Translation is the one rule that reads an origin, so it is the only place the fi Capture is half of the promise; dispatch is the other half. The measured dispatch spaces for a placed window are both screen-anchored: the synthesized path lands a point at its screen position, and XCTest's own `XCUICoordinate` on the app anchors at the screen's origin too. `interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025). -The live `XCUIElement.frame` reads that the tap path already compares — the selector match, the input probe, the tree's sweep candidates — report screen space on a placed window, as the ref-tap path proved on the device. -Only the snapshot channel arrives window-relative, and only that channel converts. +The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (641,321) for a window at (324,307). +The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. +So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion. ## Amendment: modal containment in the presentation cut From 7fa359023e1132a19fcb69fe83e5e0df1be8edae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 22:51:08 +0200 Subject: [PATCH 4/9] fix(ios): keep caret repositioning and keyboard evidence decision stable on a placed window [ #3899 ] --- .../RunnerTests+TextEntry.swift | 5 ++++- .../RunnerTests+TextTyping.swift | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 508c711af6..d32579e051 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -537,9 +537,12 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - guard let frame = input.snapshot?.frame, !frame.isEmpty else { + // The snapshot frame names the window's own content space on a placed window, so it enters + // `tapAt` translated (#3899). + guard let read = input.snapshot?.frame, !read.isEmpty else { return } + let frame = snapshotFrameOnScreen(read, translation: liveContentTranslation(app: app)) _ = tapAt( app: app, x: frame.minX + max(2, frame.width - 4), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 760e215661..2328e0c966 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -87,11 +87,20 @@ extension RunnerTests { ) } } + // The keyboard evidence is decided once, at the command's start. Measured on iPad Pro 11-inch + // (M5, iOS 27.0) with Windowed Apps (#3899): `app.keyboards` answered `false` 50 ms after + // answering `true` for the same on-screen keyboard, while the app's own AX reads kept failing + // with `kAXErrorServerNotFound` under a synthesized post burst. A per-post re-query therefore + // refused a receiver mid-burst and reported `TEXT_INPUT_NOT_FOCUSED` for a focused field, one + // character per call. An app that loses the keyboard mid-command keeps the start-of-command + // answer: the extra posts land on the digitizer and do nothing, the outcome a tap into empty + // space already has. + let softwareKeyboardVisibleAtStart = isKeyboardVisible(app: app) let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType( repairMode: repairMode, text: text, fromTapWitness: reader.target.fromTapWitness, - softwareKeyboardVisible: isKeyboardVisible(app: app) + softwareKeyboardVisible: softwareKeyboardVisibleAtStart ) let initialResolveStartedAt = Date() let initialTarget = reader.input() @@ -110,7 +119,7 @@ extension RunnerTests { // Dispatching text through XCTest without a resolved target or evidence of a focused // responder records a test failure. That tears down the long-lived runner and turns a // single invalid request into a restart cascade, so fail before entering that channel. - guard initialTarget != nil || (reader.target.prefersFocusedElement && isKeyboardVisible(app: app)) else { + guard initialTarget != nil || (reader.target.prefersFocusedElement && softwareKeyboardVisibleAtStart) else { logTextEntryPhase( commandId: commandId, phase: "total", @@ -205,7 +214,8 @@ extension RunnerTests { textEntryRoute = "xctest-element" reader.dispatch { currentTarget.element.typeText(value) } return nil - } else if reader.target.boundIdentity == nil && reader.target.prefersFocusedElement && isKeyboardVisible(app: app) { + } else if reader.target.boundIdentity == nil && reader.target.prefersFocusedElement + && softwareKeyboardVisibleAtStart { #if os(iOS) // Two ways this post leaves the synthesized channel, and both hand the text to // application-wide typing: the command's own budget refused it, or XCTest's private synthesis From a8dbd01e273e87f6c9e204ba05ce78b3d9bb7476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 23 Oct 2026 00:08:37 +0200 Subject: [PATCH 5/9] fix(ios): translate only snapshot frames and pin the unmeasured landscape case [ #3899 ] --- .../RunnerTests+TextEntry.swift | 4 +- .../RunnerTests+TextEntryFocus.swift | 56 +++++++++++++++---- .../RunnerTests+TextInputProbe.swift | 10 +++- .../CoordinateSpaceTests.swift | 4 +- .../adr/0004-ios-snapshot-backend-strategy.md | 5 ++ 5 files changed, 62 insertions(+), 17 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index d32579e051..b928947aeb 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -188,7 +188,7 @@ extension RunnerTests { /// compared against screen-space probes, so the center enters the target translated. func withResolvedInput( _ input: ResolvedTextEntryInput?, - translation: CGSize? = nil + translation: CGSize? ) -> TextEntryTarget { guard let input else { return self @@ -270,7 +270,7 @@ extension RunnerTests { init( target: TextEntryTarget, - translation: CGSize? = nil, + translation: CGSize?, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput? ) { self.translation = translation diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 0b009bddaa..1ac9056733 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -135,9 +135,10 @@ extension RunnerTests { let target = input?.element let requestedPoint = CGPoint(x: x, y: y) // The input's frame arrives from a snapshot, which reports a placed window's content in the - // window's own space; the tap below is a screen point (#3899). - if let read = input, !read.attributes.frame.isEmpty { - let frame = snapshotFrameOnScreen(read.attributes.frame, translation: liveContentTranslation(app: app)) + // window's own space; the tap below is a screen point (#3899). An input read live (no + // snapshot) already names screen space, so only the snapshot frame is translated here. + if let snapshotFrame = input?.snapshot?.frame, !snapshotFrame.isEmpty { + let frame = snapshotFrameOnScreen(snapshotFrame, translation: liveContentTranslation(app: app)) _ = tapAt(app: app, x: frame.midX, y: frame.midY) } else { _ = tapAt(app: app, x: x, y: y) @@ -146,7 +147,11 @@ extension RunnerTests { // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. if keyboardVisibleBeforeTap, let target { - return focusedTextEntryTarget(target, requestedPoint: requestedPoint) + return focusedTextEntryTarget( + target, + requestedPoint: requestedPoint, + translation: liveContentTranslation(app: app) + ) } let stabilized = stabilizeTextInputBeforeTyping( app: app, @@ -160,7 +165,12 @@ extension RunnerTests { stabilized.polls ) if case .inputReportsKeyboardFocus(let read) = stabilized.evidence { - return focusedTextEntryTarget(read.element, read: read, requestedPoint: requestedPoint) + return focusedTextEntryTarget( + read.element, + read: read, + requestedPoint: requestedPoint, + translation: liveContentTranslation(app: app) + ) } let readyTarget = TextEntryTarget( element: stabilized.element ?? target, @@ -170,26 +180,44 @@ extension RunnerTests { let element = stabilized.focusConfirmed ? readyTarget.element : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? readyTarget.element) - return focusedTextEntryTarget(element, requestedPoint: requestedPoint) + return focusedTextEntryTarget( + element, + requestedPoint: requestedPoint, + translation: liveContentTranslation(app: app) + ) } /// The target the focus step hands to text entry, with the read its refresh point came from. - private func focusedTextEntryTarget(_ element: XCUIElement?, requestedPoint: CGPoint) -> TextEntryTarget { + private func focusedTextEntryTarget( + _ element: XCUIElement?, + requestedPoint: CGPoint, + translation: CGSize? + ) -> TextEntryTarget { guard let element else { return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false) } - return focusedTextEntryTarget(element, read: ResolvedTextEntryInput(reading: element), requestedPoint: requestedPoint) + return focusedTextEntryTarget( + element, + read: ResolvedTextEntryInput(reading: element), + requestedPoint: requestedPoint, + translation: translation + ) } /// `read` is `element` read after the focus tap, or nil when the read found no element. private func focusedTextEntryTarget( _ element: XCUIElement, read: ResolvedTextEntryInput?, - requestedPoint: CGPoint + requestedPoint: CGPoint, + translation: CGSize? ) -> TextEntryTarget { return TextEntryTarget( element: element, - refreshPoint: textEntryRefreshPoint(frame: read?.snapshot?.frame, fallbackTo: requestedPoint), + refreshPoint: textEntryRefreshPoint( + frame: read?.snapshot?.frame, + fallbackTo: requestedPoint, + translation: translation + ), prefersFocusedElement: false, postFocusRead: read?.snapshot == nil ? nil : read ) @@ -206,7 +234,8 @@ extension RunnerTests { /// typing into that other field. private func textEntryRefreshPoint( frame: CGRect?, - fallbackTo requestedPoint: CGPoint + fallbackTo requestedPoint: CGPoint, + translation: CGSize? ) -> CGPoint? { guard let frame else { return nil @@ -214,6 +243,9 @@ extension RunnerTests { guard !frame.isEmpty else { return requestedPoint } - return CGPoint(x: frame.midX, y: frame.midY) + // The read's frame names the window's content space on a placed window; a refresh point is + // compared against screen-space probes, so the center enters the target translated (#3899). + let onScreen = snapshotFrameOnScreen(frame, translation: translation) + return CGPoint(x: onScreen.midX, y: onScreen.midY) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 4d348c4355..9a449a51c9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -170,7 +170,15 @@ extension RunnerTests { if shouldStop() { break } guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue } let attributes = candidate.attributes - let frame = snapshotFrameOnScreen(attributes.frame, translation: translation) + // Only the snapshot names the window's content space; a candidate read live (no snapshot) + // already reports screen space, and translating it again would move the candidate off the + // point it is being matched against (#3899). + let frame: CGRect + if let snapshot = candidate.snapshot { + frame = snapshotFrameOnScreen(snapshot.frame, translation: translation) + } else { + frame = attributes.frame + } if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: frame, point: point) { inputs.append(candidate) } diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 82a76fb74a..1e97adfc83 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -78,7 +78,7 @@ final class CoordinateSpaceTests: XCTestCase { ), CGPoint(x: 288, y: 1030), "a frame offset from the screen never moves the turned point (#3899)" - ); + ) } func testNativeSynthesizedVectorRotatesByInterfaceOrientation() { @@ -581,7 +581,7 @@ final class CoordinateSpaceTests: XCTestCase { ) ).map(\.rect), acquired.map(\.rect) - ); + ) // A windowed app, measured shape: the app's own window declares the translation and carries the // subtree; the keyboard window and the SpringBoard overlay report the screen's own box, so their diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 4e8b143f13..ddfb0ea37d 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -663,6 +663,11 @@ The measured dispatch spaces for a placed window are both screen-anchored: the s The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (641,321) for a window at (324,307). The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion. +Where a probe read has no snapshot and falls back to the live element, its frame is already screen space and is not translated. +`app.frame` unions transformed subtrees, so content pushed off a placed window (a closed drawer at negative x) can stop the two live boxes matching and read no vector; the capture still converts from the snapshot root box, so a probe may then read window-local frames where the capture published screen space. A mid-command window move has the same effect on a vector read at command start. +Both are reads of a moving target, not a second space: a re-opened command reads the current vector. +The host bridge keeps refusing a windowed capture (`app-window-not-full-screen`) rather than converting one, so the TypeScript twin of the rule runs today only in the parity replay. +Portrait is the only orientation measured for a placed window; the quarter-turn table's windowed row pins the arithmetic (a frame origin never travels through the turn), not a device measurement, and landscape under Windowed Apps stays unverified until measured. ## Amendment: modal containment in the presentation cut From ec7aecbf92fb97635445913d9e64603e4473dc24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 23 Oct 2026 00:33:53 +0200 Subject: [PATCH 6/9] refactor(ios): keep the window-content translation with the capture that runs it [ #3899 ] --- .../CoordinateSpaceTests.swift | 8 ++- .../fixtures/window-coordinate-space.json | 2 +- .../adr/0004-ios-snapshot-backend-strategy.md | 2 +- .../window-coordinate-space.test.ts | 20 ++---- .../window-coordinate-space.ts | 66 ++++++------------- 5 files changed, 30 insertions(+), 68 deletions(-) diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index 1e97adfc83..ceb6f35cdb 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -317,9 +317,11 @@ final class CoordinateSpaceTests: XCTestCase { } /// Golden parity table (#2612, #3899): every case in contracts/fixtures/window-coordinate-space.json - /// must agree with the vitest twin - /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). Add - /// cases there, never fork the rule. Detection is replayed under both landscape turns, because + /// must agree here, and the quarter-turn rows with the vitest twin + /// (packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts). The + /// window-content rows replay here alone: the host channel's guest refuses a windowed tree + /// before its decoder can read one, so this capture is the rule's only production owner. Add + /// cases to the table, never fork the rule. Detection is replayed under both landscape turns, because /// which window declares the native space cannot depend on which way the app is turned; the /// rotation rows carry their own orientation. func testWindowCoordinateSpaceMatchesGoldenParityTable() throws { diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index d42ef974ce..1211632d64 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -1,5 +1,5 @@ { - "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation) is the other space a captured subtree arrives in, and the one rule in this file that reads an origin. An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own screen box while the subtree under it arrives anchored at its own top-left: measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen (224,207,386x682), the application root reports (0,0,386,682) and its Add button (335,14,37,36) for a control drawn at screen (559,221). Published as reported, that tree mixes two spaces, clips its own content, and moves every tap by the window origin (#3899). SnapshotContentTranslation.offset(reportedBox:viewportBox:) reads the carrier box against the viewport box the platform reported, and names the viewport origin as the offset only when the carrier is the app's own window: a (0,0)-anchored box whose side lengths match the viewport's within the same tolerance, while the viewport starts somewhere else on the screen. A carrier that is not the app's own window names no space, so its numbers are left alone: the keyboard window and a SpringBoard overlay under a windowed app report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no offset there and every published rect stays exactly as it was measured before. `windowContentCases` are reportedBox + viewportBox to offset rows, replayed against that Swift function and against windowContentTranslation in packages/platform-apple/src/snapshot-source/window-coordinate-space.ts; a null offset means the rule declares no translation. Because a translation is the one thing a rotation must not also do, `rotationCases` pin the turn as axes-only: `native` and `oriented` never move a box's origin, and a row whose app frame is offset from the screen comes back at the same place as its own zero-origin twin.", + "description": "Golden table for the coordinate space a captured window's own box declares (#2612), the twin of tap-keyboard-occlusion-policy.json on the geometry that guard reads. iOS hosts some system surfaces in the device's native (portrait-up) space even while the app is rotated, so their whole subtree arrives quarter-turned: measured on iPhone 17 Pro (iOS 26.2) with the system keyboard up over a landscape app frame (0,0,874,402), UIRemoteKeyboardWindow reports its own box as (0,0,402,874) — the app's box with the two side lengths swapped — while the app's own UIWindow, UITextEffectsWindow and a dev-menu overlay window all report (0,0,874,402); in portrait the same keyboard window reports (0,0,402,874) against a (0,0,402,874) app frame, i.e. nothing is turned. RULE 1 (detection) is enforced twice, in packages/contracts/src/window-coordinate-space.ts and in SnapshotGeometrySpace inside apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift, and decides from side lengths alone: a window is quarter-turned relative to an app frame when the app frame is not square (abs(app.width - app.height) > quarterTurnTolerance), both frames are finite, positive and non-empty, and the window's side lengths match the app's on the swapped axes (abs(window.width - app.height) <= quarterTurnTolerance && abs(window.height - app.width) <= quarterTurnTolerance). Origins are deliberately never compared: the turn is a statement about which axis a box runs along, and a window may legitimately be placed elsewhere. A square app frame cannot be told from its own quarter turn, so it is left alone rather than guessed at. The tolerance is 1 because measured captures match the swap to the point and this absorbs float representation only. RULE 2 (rotation) is the capture's way back for the runner's own UIInterfaceOrientation raw values — 1 portrait, 2 portraitUpsideDown, 3 landscapeRight, 4 landscapeLeft, and 0 unknown, where portrait and unknown publish what the platform reported rather than inventing a turn — implemented by CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:) in that same Swift file, whose rect helper turns opposite corners of the reported box and re-orders them so the result keeps a top-left origin. `rotationCases` are native -> oriented rows: the landscape ones measured on the device above, the 180 degree and identity ones derived from that table. Capture turns back with the exact inverse of what dispatch turns forward, so each row also has to come back to its own `native` box under the forward mapping; that is why one measured-looking landscape-left pairing whose native and oriented boxes are identical is absent — the table computes no such row. Infinity has no JSON spelling, so the unusable frame a platform hands back when it could not resolve a box is named {\"infinite\": true} and decodes to CGRect.infinite on the Swift side and to an infinite rect on the TypeScript side. RULE 3 (content translation) is the other space a captured subtree arrives in, and the one rule in this file that reads an origin. An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own screen box while the subtree under it arrives anchored at its own top-left: measured on iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window at screen (224,207,386x682), the application root reports (0,0,386,682) and its Add button (335,14,37,36) for a control drawn at screen (559,221). Published as reported, that tree mixes two spaces, clips its own content, and moves every tap by the window origin (#3899). SnapshotContentTranslation.offset(reportedBox:viewportBox:) reads the carrier box against the viewport box the platform reported, and names the viewport origin as the offset only when the carrier is the app's own window: a (0,0)-anchored box whose side lengths match the viewport's within the same tolerance, while the viewport starts somewhere else on the screen. A carrier that is not the app's own window names no space, so its numbers are left alone: the keyboard window and a SpringBoard overlay under a windowed app report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no offset there and every published rect stays exactly as it was measured before. `windowContentCases` are reportedBox + viewportBox to offset rows, replayed against that Swift function alone; a null offset means the rule declares no translation. The host decoder (packages/platform-apple/src/snapshot-source/window-coordinate-space.ts) replays only the quarter-turn rows: its channel's guest refuses a windowed tree (app-window-not-full-screen, #3899) before that decoder can read one, so the translation has exactly one producer-side owner. Because a translation is the one thing a rotation must not also do, `rotationCases` pin the turn as axes-only: `native` and `oriented` never move a box's origin, and a row whose app frame is offset from the screen comes back at the same place as its own zero-origin twin.", "constants": { "quarterTurnTolerance": 1 }, diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index ddfb0ea37d..11f0799d22 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -649,7 +649,7 @@ The carrier's box must be a `(0,0)`-anchored copy of the viewport box the platfo Then the subtree was reported in the carrier's content space, and the viewport's origin is the one vector that carries it to the screen. Any other carrier box names no space, so the rule leaves its numbers alone: the keyboard window and the SpringBoard overlay above report the screen's own box and are already in screen space. A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no vector there and every rect is unchanged. -`SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in both languages. +`SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in the capture's language alone: the host bridge refuses a windowed tree (`app-window-not-full-screen`) before its decoder can read one, so the translation has one producer-side owner and the host's parity replay covers only the rules that decoder reads. Rotation and translation stay separate rules, and the split is the point. `CoordinateSpaceRotation` turns a box's axes and never moves its origin: the digitizer's origin is the display's own, which is where a placed window's screen-space points are already measured from. diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts index e416406beb..a95d4908c8 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.test.ts @@ -6,7 +6,6 @@ import type { Point, Rect } from '@agent-device/kernel/snapshot'; import { isQuarterTurnedWindowFrame, WINDOW_QUARTER_TURN_TOLERANCE, - windowContentTranslation, } from './window-coordinate-space.ts'; // ADR 0011 Layer 2 golden parity table (#2612): the SAME JSON is asserted against the Swift twin @@ -107,21 +106,10 @@ test('the quarter-turn window rule agrees with every golden parity table case', } }); -test('the window-content translation rule agrees with every golden parity table case', () => { - for (const fixture of loadTable().windowContentCases) { - const translation = windowContentTranslation( - toRect(fixture.reportedBox), - toRect(fixture.viewportBox), - ); - assert.deepEqual( - translation, - fixture.offset === null - ? null - : { offsetX: fixture.offset.width, offsetY: fixture.offset.height }, - fixture.name, - ); - } -}); +// `windowContentCases` is replayed in the Swift capture alone (CoordinateSpaceTests): this decoder +// never receives a windowed tree — the bridge guest refuses it first (#3899) — so a second replay +// here would pin arithmetic no production reader runs. The table still loads here, and its rows are +// read for the name-uniqueness guard below. test('the quarter-turn tolerance belongs to the table, not this file', () => { assert.equal(loadTable().constants.quarterTurnTolerance, WINDOW_QUARTER_TURN_TOLERANCE); diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index b08c20964f..4ce6887031 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -21,7 +21,9 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * anchored at its own top-left: the application root reads `(0,0,386,682)` and its `Add` button * `(335,14,37,36)` — the button's screen position `(559,221)` minus the window origin. Published as * reported, that tree mixes two spaces, clips its own content, and moves every tap by the window - * origin (#3899). `windowContentTranslation` names the one vector that puts the subtree back. + * origin (#3899). That translation belongs to the capture that performs it — the runner's Swift + * `SnapshotContentTranslation` — not to this decoder, because the guest refuses a windowed tree + * before this file can read one; see the note at the foot of this file. * * A rule that reads a box reads what the box can say. The quarter turn is a statement about which * axis a box runs along, so it compares side lengths and nothing else. A content translation is a @@ -31,12 +33,13 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * A full-screen window, an iPhone, and each foldable panel report origins at zero on both sides, so * the rule reads no translation there and every published rect stays as the platform reported it. * - * The same rules are enforced in Swift for the capture that produces these trees, and one table - * proves they are one rule: `contracts/fixtures/window-coordinate-space.json` is replayed here and - * against `SnapshotGeometrySpace` and `SnapshotContentTranslation` in - * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`. The table also - * pins the way back (`CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`, Swift-only - * today); change any rule only through that table. + * RULE 1 is enforced in both languages, because both channels read it. RULE 3 (the content + * translation) and RULE 2 (the rotation back, `CoordinateSpaceRotation.oriented(rect:in:interfaceOrientation:)`) + * live with the runner's capture alone. One table proves the shared rules are one rule: + * `contracts/fixtures/window-coordinate-space.json` is replayed here and against + * `SnapshotGeometrySpace`, `SnapshotContentTranslation`, and `CoordinateSpaceRotation` in + * `apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift`; + * change any rule only through that table. * * Why this is a detector and not a repair: the bridge guest cannot read the app's interface * orientation. Measured in the #2659 spike (verdict on the issue, write-up in the diff of #2667): @@ -71,44 +74,13 @@ export function isQuarterTurnedWindowFrame(windowFrame: Rect, appFrame: Rect): b ); } -/** The vector that carries a window-relative subtree into the screen space the viewport names. */ -export type WindowContentTranslation = { - offsetX: number; - offsetY: number; -}; - -/** - * The vector from a carrier's own box to the screen space it is placed at, or null when the carrier's - * box declares no translation. The carrier's subtree was reported in the carrier's content space, so - * adding the vector to each reported rect publishes the subtree in the viewport's screen space (#3899). - * - * Null on anything that is not the app's own placed window: a box that is not a `(0,0)`-anchored copy - * of the viewport's side lengths names no space, and a viewport that already starts at the screen - * origin needs no move. Replayed against `windowContentCases` in - * `contracts/fixtures/window-coordinate-space.json`, in the Swift twin and here. +/* + * A window that does not fill the screen (#3899) is translated by the capture that owns it, and this + * file is not that capture. The bridge guest refuses a windowed tree (`app-window-not-full-screen`) + * before this decoder ever sees one, so no host tree arrives in the window's content space and there + * is nothing here to translate. The rule lives with the producer that converts — + * `SnapshotContentTranslation` beside the Swift capture — and `windowContentCases` in + * `contracts/fixtures/window-coordinate-space.json` is replayed there alone. The quarter-turn + * detection above stays in both languages because this decoder does read it: it is what refuses a + * turned tree the reader cannot resolve. */ -export function windowContentTranslation( - reportedBox: Rect, - viewportBox: Rect, -): WindowContentTranslation | null { - if (!isPositiveFiniteRect(reportedBox) || !isPositiveFiniteRect(viewportBox)) return null; - if ( - Math.abs(reportedBox.x) > WINDOW_QUARTER_TURN_TOLERANCE || - Math.abs(reportedBox.y) > WINDOW_QUARTER_TURN_TOLERANCE - ) { - return null; - } - if ( - Math.abs(viewportBox.x) <= WINDOW_QUARTER_TURN_TOLERANCE && - Math.abs(viewportBox.y) <= WINDOW_QUARTER_TURN_TOLERANCE - ) { - return null; - } - if ( - Math.abs(reportedBox.width - viewportBox.width) > WINDOW_QUARTER_TURN_TOLERANCE || - Math.abs(reportedBox.height - viewportBox.height) > WINDOW_QUARTER_TURN_TOLERANCE - ) { - return null; - } - return { offsetX: viewportBox.x, offsetY: viewportBox.y }; -} From 009b65c8d0aa7abbab8fdf0baa4b8a04f9809c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 24 Oct 2026 10:31:09 +0200 Subject: [PATCH 7/9] refactor(ios): make ResolvedTextEntryInput own the probe-side screen frame (#3899) The readiness retap re-read the input as a bare XCUIElementSnapshot and tapped its raw center through tapAt, which takes screen points, so on a windowed iPad the retap landed outside the window. The live window lookup also ran whole-app queries four to six times per fill on every iOS device. Every snapshot frame that becomes a dispatched point now passes through the one content translation, held on ResolvedTextEntryInput.frameOnScreen. The vector is read once per command and stored on TextEntryTarget.contentTranslation; the probe matching, focus tap, refresh point, caret tap and readiness retap all read that one property. The threaded translation: parameters and the per-read lookups are gone, and the once-per-command keyboard read in the typing command is reverted to the base behavior with the retap fixed. --- .../RunnerTests+CommandExecution.swift | 9 +- .../RunnerTests+Interaction.swift | 23 +++- .../RunnerTests+TextEntry.swift | 123 +++++++++++++----- .../RunnerTests+TextEntryFocus.swift | 82 +++++++----- .../RunnerTests+TextEntryReadiness.swift | 44 ++++++- .../RunnerTests+TextInputProbe.swift | 103 ++++++++++----- .../RunnerTests+TextTyping.swift | 30 +---- .../RunnerTests+TypeExecution.swift | 6 +- ...RunnerTests+CoordinateTextEntryTests.swift | 2 +- .../RunnerTests+TextEntryLatencyTests.swift | 9 +- .../RunnerTests+TextEntryReadinessTests.swift | 57 ++++++++ .../RunnerTests+TextInputProbeTests.swift | 8 +- .../SnapshotCoordinateSpace.swift | 5 + .../CoordinateSpaceTests.swift | 12 +- .../fixtures/window-coordinate-space.json | 2 +- .../adr/0004-ios-snapshot-backend-strategy.md | 4 +- .../platform-apple/src/snapshot-route.test.ts | 2 +- packages/platform-apple/src/snapshot-route.ts | 2 +- .../window-coordinate-space.ts | 10 +- 19 files changed, 364 insertions(+), 169 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index c8c622cfe6..ea6a445abf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -155,7 +155,14 @@ extension RunnerTests { ) let textInput: XCUIElement? if !xCTestTextInputProbeSkipped { - textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y) + // The lookup compares the point, which is a screen point, with candidate frames; on a + // placed window the snapshot frames need the content vector (#3899). + textInput = coordinateTapTextInputAt( + app: activeApp, + x: x, + y: y, + contentTranslation: liveContentTranslation(app: activeApp) + ) } else { textInput = nil // Clear before dispatch so a throwing gesture cannot retain the preceding field's witness. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index ae1407d1d0..4b134a059d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -352,7 +352,13 @@ extension RunnerTests { func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { let point = CGPoint(x: x, y: y) - let textInputCandidates = textInputCandidatesAt(app: app, point: point).map(\.element) + // The point is a screen point; candidate snapshot frames of a placed window need the content + // vector to be compared with it (#3899). + let textInputCandidates = textInputCandidatesAt( + app: app, + point: point, + contentTranslation: liveContentTranslation(app: app) + ).map(\.element) for element in textInputCandidates where prefersExpandedTextRead(element) { if let text = readableText(for: element) { return text @@ -617,12 +623,15 @@ extension RunnerTests { #if !os(tvOS) private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { #if os(iOS) - // Coordinate taps, double taps, long presses, and drags anchor at the app origin, not the - // resolved window: on iOS the snapshot hands back app-space points, and the first qualifying - // window is not always the one under the finger — a SpringBoard `alert accept` can live in an - // alert window while the wallpaper or status-bar window qualifies first, so a window-relative - // anchor misses the button. Reference frames and the synthesized display ID still resolve - // through `resolveRunnerWindow`, so a foldable tap keeps its panel and stays on its display. + // A caller's point is a screen point: coordinate taps, double taps, long presses, and drags + // anchor at the application's origin, not the resolved window, so the point is dispatched as + // given. That holds because `app.coordinate(withNormalizedOffset: .zero)` sits at the origin of + // the space `app.frame` names, and for a windowed app the measured `app.frame` starts at the + // screen's origin. The first qualifying window is not always the one under the finger — a + // SpringBoard `alert accept` can live in an alert window while the wallpaper or status-bar + // window qualifies first, so a window-relative anchor misses the button. Reference frames and + // the synthesized display ID still resolve through `resolveRunnerWindow`, so a foldable tap + // keeps its panel and stays on its display (#3899, ADR 0025). let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) return origin.withOffset(CGVector(dx: x, dy: y)) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 079bb14a36..097213aac1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -149,6 +149,11 @@ extension RunnerTests { let element: XCUIElement? let refreshPoint: CGPoint? let prefersFocusedElement: Bool + /// The live snapshot-frame vector for the reading session, read once when this target was built + /// (#3899). Every snapshot frame the target's reads produce is compared with screen points, so + /// every such read carries this vector (`ResolvedTextEntryInput.frameOnScreen`). Nil on a device + /// whose app fills the screen, where a snapshot frame is already a screen frame. + let contentTranslation: CGSize? let fromTapWitness: Bool /// The input the first resolved element was. Once bound, resolution refuses any other input. let boundIdentity: TextEntryInputIdentity? @@ -166,6 +171,7 @@ extension RunnerTests { element: XCUIElement?, refreshPoint: CGPoint?, prefersFocusedElement: Bool, + contentTranslation: CGSize? = nil, fromTapWitness: Bool = false, boundIdentity: TextEntryInputIdentity? = nil, inputAtRefreshPoint: TextInputAtPoint? = nil, @@ -174,30 +180,34 @@ extension RunnerTests { self.element = element self.refreshPoint = refreshPoint self.prefersFocusedElement = prefersFocusedElement + self.contentTranslation = contentTranslation self.fromTapWitness = fromTapWitness self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity self.inputAtRefreshPoint = inputAtRefreshPoint self.postFocusRead = postFocusRead } - /// The target moved to the input a resolution read: its element, a refresh point at its center, - /// and its identity while the target is unbound. - /// - /// `translation` is the live snapshot-frame vector for the reading session (#3899): the snapshot - /// frame of a placed window's input names the window's content space, while a refresh point is - /// compared against screen-space probes, so the center enters the target translated. - func withResolvedInput( - _ input: ResolvedTextEntryInput?, - translation: CGSize? - ) -> TextEntryTarget { + /// The target moved to the input a resolution read: its element and identity, and — when the + /// read carries a snapshot — a refresh point at that frame's center on the screen (`frameOnScreen` + /// plus this target's content vector, #3899). A read with no snapshot keeps the old refresh + /// point: its center would come from the live `element.frame`, the read a post-focus handle + /// cannot answer without recording a failure (#3060). + func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget { guard let input else { return self } - let frame = snapshotFrameOnScreen(input.snapshot?.frame ?? .zero, translation: translation) + let nextRefreshPoint: CGPoint? + if let snapshot = input.snapshot { + let frame = input.frameOnScreen + nextRefreshPoint = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY) + } else { + nextRefreshPoint = refreshPoint + } return TextEntryTarget( element: input.element, - refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY), + refreshPoint: nextRefreshPoint, prefersFocusedElement: prefersFocusedElement, + contentTranslation: contentTranslation, fromTapWitness: fromTapWitness, boundIdentity: boundIdentity ?? input.identity, inputAtRefreshPoint: inputAtRefreshPoint @@ -211,20 +221,31 @@ extension RunnerTests { /// Pro, iOS 27.0). Where a live read of an element that stopped answering its query, such as a /// removed input or a focused Flutter password field (#3060), records an XCTest failure that ends /// the runner session, the snapshot only throws. + /// + /// This type owns the probe-side space of #3899 (ADR 0004). `XCUIElementSnapshot.frame` reports a + /// placed window's content in the window's own space, while every point this read feeds — a refresh + /// point, a focus tap, a probe match, a readiness retap — is a screen point. So the read carries + /// the content vector it was taken under, and `frameOnScreen` is the only frame consumers use: a + /// snapshot frame plus the vector, or the live frame when there is no snapshot. Raw frames + /// (`snapshot.frame`) are compared only with each other. struct ResolvedTextEntryInput { let element: XCUIElement /// Nil when the element exists but could not be snapshotted. let snapshot: XCUIElementSnapshot? + /// The live snapshot-frame vector read once for the command that made this read (#3899). Nil + /// when the app fills the screen, where a snapshot frame is already a screen frame. + let contentTranslation: CGSize? - private init(element: XCUIElement, snapshot: XCUIElementSnapshot?) { + private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: CGSize?) { self.element = element self.snapshot = snapshot + self.contentTranslation = contentTranslation } /// Reads a handle that may name an input the app has removed; nil when it names none or the read /// raised. The snapshot of a missing element throws only after XCTest waits about two seconds for /// a match, so existence is read first. - init?(reading element: XCUIElement) { + init?(reading element: XCUIElement, contentTranslation: CGSize? = nil) { var exists = false var snapshot: XCUIElementSnapshot? if let exception = RunnerObjCExceptionCatcher.catchException({ @@ -235,15 +256,15 @@ extension RunnerTests { return nil } guard exists else { return nil } - self.init(element: element, snapshot: snapshot) + self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation) } /// Reads a handle its query has just listed, inside the caller's exception containment, so the /// snapshot is read first and existence only when the snapshot fails; nil when it names no element. - init?(readingListed element: XCUIElement) { + init?(readingListed element: XCUIElement, contentTranslation: CGSize? = nil) { let snapshot = try? element.snapshot() guard snapshot != nil || element.exists else { return nil } - self.init(element: element, snapshot: snapshot) + self.init(element: element, snapshot: snapshot, contentTranslation: contentTranslation) } var identity: TextEntryInputIdentity? { @@ -256,25 +277,37 @@ extension RunnerTests { } return element } + + /// The input's box in screen space: the snapshot frame carried by `contentTranslation` to its + /// place on the screen, or the live frame when there is no snapshot — the live + /// `XCUIElement.frame` already reports a placed window's content in screen space (#3899). + var frameOnScreen: CGRect { + guard let snapshot else { + return element.frame + } + guard let contentTranslation else { + return snapshot.frame + } + return snapshot.frame.offsetBy(dx: contentTranslation.width, dy: contentTranslation.height) + } } /// A text entry target and its input as last read. `input()` hands the last read out again until /// `dispatch` runs, and resolves the target again after that, so a bound target refuses an input /// that took its place. Each resolution moves the target to the input it read. Every post and every /// pause between reads goes through `dispatch`; the pause a typing plan takes itself follows a post. + /// The reader hands the target's content vector to every read it resolves, so one vector rules the + /// command (#3899). struct TextEntryReader { private(set) var target: TextEntryTarget private var lastRead: ResolvedTextEntryInput? private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput? - private let translation: CGSize? init( target: TextEntryTarget, - translation: CGSize?, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput? ) { - self.translation = translation - self.target = target.withResolvedInput(target.postFocusRead, translation: translation) + self.target = target.withResolvedInput(target.postFocusRead) lastRead = target.postFocusRead self.resolve = resolve } @@ -284,7 +317,7 @@ extension RunnerTests { return lastRead } let read = resolve(target) - target = target.withResolvedInput(read, translation: translation) + target = target.withResolvedInput(read) lastRead = read return read } @@ -507,7 +540,10 @@ extension RunnerTests { return resolveUnboundTextEntryInput(app: app, target: target) } func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? { - guard let input = ResolvedTextEntryInput(reading: candidate), input.identity == boundIdentity else { return nil } + guard let input = ResolvedTextEntryInput( + reading: candidate, + contentTranslation: target.contentTranslation + ), input.identity == boundIdentity else { return nil } return input } if target.prefersFocusedElement, let focused = focusedTextInput(app: app), let input = boundInput(focused) { @@ -516,12 +552,17 @@ extension RunnerTests { if let element = target.element, let input = boundInput(element) { return input } - if let input = target.inputAtRefreshPoint, let read = textInputIfStillResolves(input) { + if let input = target.inputAtRefreshPoint, + let read = textInputIfStillResolves(input, contentTranslation: target.contentTranslation) { return read } if target.inputAtRefreshPoint == nil, let refreshPoint = target.refreshPoint, - case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint), + case .matches(let candidates) = probeTextInputs( + app: app, + point: refreshPoint, + contentTranslation: target.contentTranslation + ), let match = candidates.first(where: { $0.identity == boundIdentity }) { return match } @@ -556,7 +597,11 @@ extension RunnerTests { guard let refreshPoint = target.refreshPoint else { return true } - switch probeTextInputs(app: app, point: refreshPoint) { + switch probeTextInputs( + app: app, + point: refreshPoint, + contentTranslation: target.contentTranslation + ) { case .absent: return true case .matches(let candidates): @@ -590,17 +635,25 @@ extension RunnerTests { /// Resolution for a target not bound to an input yet: the first element that exists. private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? { - if target.prefersFocusedElement, let focused = focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) { + let translation = target.contentTranslation + if target.prefersFocusedElement, + let focused = focusedTextInput(app: app).flatMap({ ResolvedTextEntryInput(reading: $0, contentTranslation: translation) }) { return focused } - if let element = target.element, let input = ResolvedTextEntryInput(reading: element) { + if let element = target.element, + let input = ResolvedTextEntryInput(reading: element, contentTranslation: translation) { return input } if let refreshPoint = target.refreshPoint, - let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y) { + let refreshed = textInputAt( + app: app, + x: refreshPoint.x, + y: refreshPoint.y, + contentTranslation: translation + ) { return refreshed } - return focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) + return focusedTextInput(app: app).flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: translation) } } private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) { @@ -619,12 +672,14 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - // The snapshot frame names the window's own content space on a placed window, so it enters - // `tapAt` translated (#3899). - guard let read = input.snapshot?.frame, !read.isEmpty else { + // The read's snapshot frame names the window's own content space on a placed window, so the + // point enters `tapAt` through `frameOnScreen`, which carries the read's content vector (#3899). + // A read with no snapshot gets no point at all: `frameOnScreen`'s fallback is the live + // `element.frame` read this function exists to avoid. + guard input.snapshot?.frame.isEmpty == false else { return } - let frame = snapshotFrameOnScreen(read, translation: liveContentTranslation(app: app)) + let frame = input.frameOnScreen _ = tapAt( app: app, x: frame.minX + max(2, frame.width - 4), diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 1ac9056733..5d4140d917 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -61,7 +61,8 @@ extension RunnerTests { func stabilizeTextInputBeforeTyping( app: XCUIApplication, target: XCUIElement?, - keyboardVisibleBeforeTap: Bool? = nil + keyboardVisibleBeforeTap: Bool? = nil, + contentTranslation: CGSize? = nil ) -> TextEntryStabilization { #if os(tvOS) return TextEntryStabilization(element: target, evidence: .focusEngine, polls: 0) @@ -78,8 +79,9 @@ extension RunnerTests { // The tapped input's own read comes before the keyboard. While a software keyboard is // presented, the keyboard query blocks and then finds none. An input that reports keyboard // focus is stronger evidence than the keyboard's arrival, which shows only that some input - // took focus. - let read = target.flatMap(ResolvedTextEntryInput.init(reading:)) + // took focus. The read carries the focus step's content vector (#3899): the hand-on read + // becomes the entry's first read, and its frames must enter screen space. + let read = target.flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: contentTranslation) } if let read, let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) { return TextEntryStabilization(element: read.element, evidence: .inputReportsKeyboardFocus(read), polls: polls) } @@ -130,16 +132,23 @@ extension RunnerTests { return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true) } + // The one content-vector read for this command (#3899): every target this focus step builds + // carries it, every read those targets produce converts through it, and the retap and reader + // downstream reuse it. On a device whose app fills the screen it is nil and every frame stays + // exactly as read. + let contentTranslation = liveContentTranslation(app: app) let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) - let input = textInputAt(app: app, x: x, y: y) + let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - // The input's frame arrives from a snapshot, which reports a placed window's content in the - // window's own space; the tap below is a screen point (#3899). An input read live (no - // snapshot) already names screen space, so only the snapshot frame is translated here. - if let snapshotFrame = input?.snapshot?.frame, !snapshotFrame.isEmpty { - let frame = snapshotFrameOnScreen(snapshotFrame, translation: liveContentTranslation(app: app)) - _ = tapAt(app: app, x: frame.midX, y: frame.midY) + // The focus tap goes to the input's own center, and the input's frame arrives from a snapshot, + // which reports a placed window's content in the window's own space while `tapAt` dispatches a + // screen point (#3899). An input read live (no snapshot) already names screen space. + let inputCenter = input.map { $0.frameOnScreen }.flatMap { + $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) + } + if let inputCenter { + _ = tapAt(app: app, x: inputCenter.x, y: inputCenter.y) } else { _ = tapAt(app: app, x: x, y: y) } @@ -150,13 +159,14 @@ extension RunnerTests { return focusedTextEntryTarget( target, requestedPoint: requestedPoint, - translation: liveContentTranslation(app: app) + contentTranslation: contentTranslation ) } let stabilized = stabilizeTextInputBeforeTyping( app: app, target: target, - keyboardVisibleBeforeTap: keyboardVisibleBeforeTap + keyboardVisibleBeforeTap: keyboardVisibleBeforeTap, + contentTranslation: contentTranslation ) NSLog( "%@%@ polls=%d", @@ -169,7 +179,7 @@ extension RunnerTests { read.element, read: read, requestedPoint: requestedPoint, - translation: liveContentTranslation(app: app) + contentTranslation: contentTranslation ) } let readyTarget = TextEntryTarget( @@ -183,7 +193,7 @@ extension RunnerTests { return focusedTextEntryTarget( element, requestedPoint: requestedPoint, - translation: liveContentTranslation(app: app) + contentTranslation: contentTranslation ) } @@ -191,16 +201,23 @@ extension RunnerTests { private func focusedTextEntryTarget( _ element: XCUIElement?, requestedPoint: CGPoint, - translation: CGSize? + contentTranslation: CGSize? ) -> TextEntryTarget { guard let element else { - return TextEntryTarget(element: nil, refreshPoint: requestedPoint, prefersFocusedElement: false) + // No element: the target keeps its refresh point, and the reads a later resolve makes through + // that point carry this command's vector (#3899). + return TextEntryTarget( + element: nil, + refreshPoint: requestedPoint, + prefersFocusedElement: false, + contentTranslation: contentTranslation + ) } return focusedTextEntryTarget( element, - read: ResolvedTextEntryInput(reading: element), + read: ResolvedTextEntryInput(reading: element, contentTranslation: contentTranslation), requestedPoint: requestedPoint, - translation: translation + contentTranslation: contentTranslation ) } @@ -209,16 +226,16 @@ extension RunnerTests { _ element: XCUIElement, read: ResolvedTextEntryInput?, requestedPoint: CGPoint, - translation: CGSize? + contentTranslation: CGSize? ) -> TextEntryTarget { return TextEntryTarget( element: element, refreshPoint: textEntryRefreshPoint( - frame: read?.snapshot?.frame, - fallbackTo: requestedPoint, - translation: translation + read: read, + fallbackTo: requestedPoint ), prefersFocusedElement: false, + contentTranslation: contentTranslation, postFocusRead: read?.snapshot == nil ? nil : read ) } @@ -231,21 +248,22 @@ extension RunnerTests { /// the target gets NO refresh point rather than the pre-focus one: this route's first resolve is /// not yet bound to an identity, and a field that went unqueryable on focus may have moved the /// layout so the old point names a different input, so fail closed with a typed reason instead of - /// typing into that other field. + /// typing into that other field. The frame is the read's snapshot frame plus its content vector + /// (#3899) — never the live `element.frame` fallback `frameOnScreen` uses for a snapshotless read. private func textEntryRefreshPoint( - frame: CGRect?, - fallbackTo requestedPoint: CGPoint, - translation: CGSize? + read: ResolvedTextEntryInput?, + fallbackTo requestedPoint: CGPoint ) -> CGPoint? { - guard let frame else { + guard let snapshot = read?.snapshot else { return nil } - guard !frame.isEmpty else { + guard !snapshot.frame.isEmpty else { return requestedPoint } - // The read's frame names the window's content space on a placed window; a refresh point is - // compared against screen-space probes, so the center enters the target translated (#3899). - let onScreen = snapshotFrameOnScreen(frame, translation: translation) - return CGPoint(x: onScreen.midX, y: onScreen.midY) + let frame = snapshot.frame.offsetBy( + dx: read?.contentTranslation?.width ?? 0, + dy: read?.contentTranslation?.height ?? 0 + ) + return CGPoint(x: frame.midX, y: frame.midY) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index f4c49a5a67..52bd98268d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -85,22 +85,52 @@ extension RunnerTests { /// `ResolvedTextEntryInput` already reads elements this way. It is read again after the /// wait rather than reused, because focusing a field moves the layout: the repeat aims at where the /// field is then, which is the moment the read it replaced ran at. + /// + /// The read enters `retapAt` as a `ResolvedTextEntryInput`, so the repeat taps + /// `frameOnScreen` — the snapshot frame plus the content vector of a placed window (#3899). The base + /// read carries no vector; the re-read carries the live one, read only on this branch, so a tap that + /// confirms focus dispatches no window lookup. func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { #if os(iOS) if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return } - guard let settled = safely("TEXT_ENTRY_READINESS_FRAME", { try? element.snapshot() }) else { - return + // The re-read enters as a `ResolvedTextEntryInput` carrying the content vector of a placed + // window (#3899), read only on this branch, so a tap that confirms focus dispatches no window + // lookup. The window read is contained like the probe lookups: a transient window that records + // an XCTest issue costs the vector, not the command. + let settled: ResolvedTextEntryInput? + switch containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?, { _ in + ResolvedTextEntryInput(reading: element, contentTranslation: liveContentTranslation(app: app)) + }) { + case .success(let read): + settled = read + case .failure: + settled = nil } - let frame = settled.frame - if !frame.isEmpty { - _ = tapAt(app: app, x: frame.midX, y: frame.midY) - _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) + guard let settled else { + return } + retapAt(read: settled, app: app, element: element) #endif } + /// The readiness retap, split out so the unit lane can drive it and assert where the point lands. + /// `tapAt` takes a screen point, so the point is `frameOnScreen`'s center. A read with no snapshot + /// taps nothing: `frameOnScreen`'s fallback is the live `element.frame` read — the one that records + /// an XCTest failure on a handle that stopped answering its query (#3060). + func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { + guard read.snapshot != nil else { + return + } + let frame = read.frameOnScreen + guard !frame.isEmpty else { + return + } + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) + } + private func waitForTextEntryFocus(app: XCUIApplication, element: XCUIElement, timeout: TimeInterval) -> Bool { let deadline = Date().addingTimeInterval(timeout) while Date() < deadline { @@ -129,6 +159,8 @@ extension RunnerTests { /// fallback at all. func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool { #if os(iOS) + // The read needs no content vector: this corroborates identity and position raw-frame against + // raw-frame, and never turns a frame into a dispatched point (#3899's rule). return ResolvedTextEntryInput(reading: element).map { keyboardFocusConfirmed(app: app, read: $0) } ?? false #else return false diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 9a449a51c9..39f0e8b6d6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -18,18 +18,32 @@ enum TextInputProbeOutcome { } extension RunnerTests { - func textInputAt(app: XCUIApplication, x: Double, y: Double) -> ResolvedTextEntryInput? { - textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first + func textInputAt( + app: XCUIApplication, + x: Double, + y: Double, + contentTranslation: CGSize? + ) -> ResolvedTextEntryInput? { + textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first } - func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [ResolvedTextEntryInput] { + func textInputCandidatesAt( + app: XCUIApplication, + point: CGPoint, + contentTranslation: CGSize? + ) -> [ResolvedTextEntryInput] { safely("TEXT_INPUT_AT_POINT", []) { - queryTextInputs(app: app, point: point, shouldStop: { false }) + queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: { false }) } } - func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? { - switch probeTextInputs(app: app, point: CGPoint(x: x, y: y)) { + func coordinateTapTextInputAt( + app: XCUIApplication, + x: Double, + y: Double, + contentTranslation: CGSize? + ) -> XCUIElement? { + switch probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { case .matches(let inputs): return inputs.first?.element case .absent: @@ -54,11 +68,17 @@ extension RunnerTests { func coordinateTapTextInputIdentityAt( app: XCUIApplication, x: Double, - y: Double + y: Double, + contentTranslation: CGSize? ) -> CoordinateTextInputResolution { let probed = containingTextInputProbeIssues(fallback: CoordinateTextInputResolution.unavailable) { shouldStop -> CoordinateTextInputResolution in - guard let input = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first else { + guard let input = queryTextInputs( + app: app, + point: CGPoint(x: x, y: y), + contentTranslation: contentTranslation, + shouldStop: shouldStop + ).first else { return .absent } guard let identity = input.identity else { @@ -76,9 +96,13 @@ extension RunnerTests { /// `input`'s handle, read again, when it still resolves to the input it named when found, read /// inside the same issue containment. A read that cannot answer counts as no. - func textInputIfStillResolves(_ input: TextInputAtPoint) -> ResolvedTextEntryInput? { + func textInputIfStillResolves( + _ input: TextInputAtPoint, + contentTranslation: CGSize? + ) -> ResolvedTextEntryInput? { let probed = containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?) { _ in - ResolvedTextEntryInput(reading: input.element).flatMap { $0.identity == input.identity ? $0 : nil } + ResolvedTextEntryInput(reading: input.element, contentTranslation: contentTranslation) + .flatMap { $0.identity == input.identity ? $0 : nil } } guard case .success(let read) = probed else { return nil } return read @@ -94,9 +118,13 @@ extension RunnerTests { return read } - func probeTextInputs(app: XCUIApplication, point: CGPoint) -> TextInputProbeOutcome { + func probeTextInputs( + app: XCUIApplication, + point: CGPoint, + contentTranslation: CGSize? + ) -> TextInputProbeOutcome { switch containingTextInputProbeIssues(fallback: [], { shouldStop in - queryTextInputs(app: app, point: point, shouldStop: shouldStop) + queryTextInputs(app: app, point: point, contentTranslation: contentTranslation, shouldStop: shouldStop) }) { case .success(let inputs): return inputs.isEmpty ? .absent : .matches(inputs) @@ -106,8 +134,9 @@ extension RunnerTests { } /// Runs an optional probe whose XCTest issues are contained instead of recorded: an issue makes - /// the probe unavailable rather than failing the command and invalidating the runner. - private func containingTextInputProbeIssues( + /// the probe unavailable rather than failing the command and invalidating the runner. Visible to + /// the readiness retap, whose window re-read needs the same containment (#3899). + func containingTextInputProbeIssues( fallback: T, _ probe: (_ shouldStop: () -> Bool) -> T ) -> Result { @@ -151,10 +180,10 @@ extension RunnerTests { private func queryTextInputs( app: XCUIApplication, point: CGPoint, + contentTranslation: CGSize?, shouldStop: () -> Bool ) -> [ResolvedTextEntryInput] { var inputs: [ResolvedTextEntryInput] = [] - let translation = liveContentTranslation(app: app) for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] { if shouldStop() { break } let candidates = query.allElementsBoundByIndex @@ -168,18 +197,15 @@ extension RunnerTests { // cannot be snapshotted is read live, so its failure still reaches the caller's issue handling. for element in candidates { if shouldStop() { break } - guard let candidate = ResolvedTextEntryInput(readingListed: element) else { continue } - let attributes = candidate.attributes - // Only the snapshot names the window's content space; a candidate read live (no snapshot) - // already reports screen space, and translating it again would move the candidate off the - // point it is being matched against (#3899). - let frame: CGRect - if let snapshot = candidate.snapshot { - frame = snapshotFrameOnScreen(snapshot.frame, translation: translation) - } else { - frame = attributes.frame - } - if isCoordinateTextInputCandidate(enabled: attributes.isEnabled, frame: frame, point: point) { + guard let candidate = ResolvedTextEntryInput( + readingListed: element, + contentTranslation: contentTranslation + ) else { continue } + if isCoordinateTextInputCandidate( + enabled: candidate.attributes.isEnabled, + frame: candidate.frameOnScreen, + point: point + ) { inputs.append(candidate) } } @@ -189,7 +215,12 @@ extension RunnerTests { } /// The vector that carries this session's snapshot-read app-content frames to their place on the - /// screen, or nil when the app fills the screen (#3899). + /// screen, or nil when the app fills the screen (#3899). Read once at the start of each command + /// that dispatches points derived from snapshot frames — the focus step, the coordinate fill route, + /// the readiness retap, a coordinate text read — and carried on the types the command builds from + /// it (`TextEntryTarget.contentTranslation`, `ResolvedTextEntryInput.contentTranslation`). No read + /// inside a loop re-reads it: the whole-app queries below cost about 90 ms each on heavy React + /// Native screens, and a per-read re-query would undo the fill-read work of #3881, #3906 and #3909. /// /// The live `XCUIElement.frame` property already reports a placed window's content in screen space, /// but `XCUIElementSnapshot.frame` reports it in the window's own content space, so the same probe @@ -199,15 +230,17 @@ extension RunnerTests { /// place on the screen. Any other pair of boxes declares no vector, so full-screen, iPhone, and /// each foldable panel keep their frames exactly as read. func liveContentTranslation(app: XCUIApplication) -> CGSize? { - safely("LIVE_CONTENT_TRANSLATION", nil) { + // The window read is contained like the probe lookups it replaces: a transient window that + // records an XCTest issue costs the vector, not the runner session. + let read = containingTextInputProbeIssues(fallback: nil as CGSize?) { _ in guard let resolved = resolveRunnerWindowElement(app: app) else { return nil } return SnapshotContentTranslation.offset(reportedBox: app.frame, viewportBox: resolved.frame) } + switch read { + case .success(let translation): + return translation + case .failure: + return nil + } } } - -/// `frame` as the screen names it, given `liveContentTranslation`'s vector. -func snapshotFrameOnScreen(_ frame: CGRect, translation: CGSize?) -> CGRect { - guard let translation else { return frame } - return frame.offsetBy(dx: translation.width, dy: translation.height) -} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift index 75aaffeece..142f9c4415 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift @@ -16,13 +16,7 @@ extension RunnerTests { ) -> TextEntryResult { let totalStartedAt = Date() let resolve = { (target: TextEntryTarget) in self.resolveTextEntryInput(app: app, target: target) } - // Reads inside the typing loop answer from snapshots, which report a placed window's content in - // the window's own space; the reader's refresh points are screen points (#3899). - var reader = TextEntryReader( - target: target, - translation: liveContentTranslation(app: app), - resolve: resolve - ) + var reader = TextEntryReader(target: target, resolve: resolve) guard !text.isEmpty else { // Replacing a field WITH the empty string is the clear-field primitive (#2063): the typing // is vacuous but the clear is not. Returning here unconditionally left the old value in @@ -80,27 +74,14 @@ extension RunnerTests { case .completed(let result): return result case .fallback(let fallbackTarget): - reader = TextEntryReader( - target: fallbackTarget, - translation: liveContentTranslation(app: app), - resolve: resolve - ) + reader = TextEntryReader(target: fallbackTarget, resolve: resolve) } } - // The keyboard evidence is decided once, at the command's start. Measured on iPad Pro 11-inch - // (M5, iOS 27.0) with Windowed Apps (#3899): `app.keyboards` answered `false` 50 ms after - // answering `true` for the same on-screen keyboard, while the app's own AX reads kept failing - // with `kAXErrorServerNotFound` under a synthesized post burst. A per-post re-query therefore - // refused a receiver mid-burst and reported `TEXT_INPUT_NOT_FOCUSED` for a focused field, one - // character per call. An app that loses the keyboard mid-command keeps the start-of-command - // answer: the extra posts land on the digitizer and do nothing, the outcome a tap into empty - // space already has. - let softwareKeyboardVisibleAtStart = isKeyboardVisible(app: app) let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType( repairMode: repairMode, text: text, fromTapWitness: reader.target.fromTapWitness, - softwareKeyboardVisible: softwareKeyboardVisibleAtStart + softwareKeyboardVisible: isKeyboardVisible(app: app) ) let initialResolveStartedAt = Date() let initialTarget = reader.input() @@ -119,7 +100,7 @@ extension RunnerTests { // Dispatching text through XCTest without a resolved target or evidence of a focused // responder records a test failure. That tears down the long-lived runner and turns a // single invalid request into a restart cascade, so fail before entering that channel. - guard initialTarget != nil || (reader.target.prefersFocusedElement && softwareKeyboardVisibleAtStart) else { + guard initialTarget != nil || (reader.target.prefersFocusedElement && isKeyboardVisible(app: app)) else { logTextEntryPhase( commandId: commandId, phase: "total", @@ -214,8 +195,7 @@ extension RunnerTests { textEntryRoute = "xctest-element" reader.dispatch { currentTarget.element.typeText(value) } return nil - } else if reader.target.boundIdentity == nil && reader.target.prefersFocusedElement - && softwareKeyboardVisibleAtStart { + } else if reader.target.boundIdentity == nil && reader.target.prefersFocusedElement && isKeyboardVisible(app: app) { #if os(iOS) // Two ways this post leaves the synthesized channel, and both hand the text to // application-wide typing: the command's own budget refused it, or XCTest's private synthesis diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index acbb62ef54..bea8d91e60 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -27,12 +27,15 @@ extension RunnerTests { hasY: command.y != nil, xCTestChannelPenalized: xCTestChannelPenalized ), let x = command.x, let y = command.y { + // The one content-vector read for this command (#3899): the target built here carries it, and + // every read that target produces converts through it, as the focus route's targets do. + let contentTranslation = liveContentTranslation(app: activeApp) // Resolved before the tap: the focus it causes can move the field away from this point. // A replacement with no input at the point has nothing to bind to: its clear and its typed // text would reach the app-level first responder, so an empty point and a lookup that cannot // answer both refuse here, each under its own reason. let inputAtPoint: TextInputAtPoint - switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y) { + switch coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { case .input(let input): inputAtPoint = input case .absent: @@ -54,6 +57,7 @@ extension RunnerTests { element: nil, refreshPoint: CGPoint(x: x, y: y), prefersFocusedElement: false, + contentTranslation: contentTranslation, inputAtRefreshPoint: inputAtPoint ) case .xctestFallback: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index a4a83c3df1..60979694f1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -6,7 +6,7 @@ extension RunnerTests { /// cannot answer. A fixture that does not expose its own field is a broken lane, not a refusal. @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { - switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y) { + switch coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: nil) { case .input(let input): return input case .absent: diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift index f70b57c907..9d27049ec2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryLatencyTests.swift @@ -83,7 +83,7 @@ extension RunnerTests { } let target = TextEntryTarget(element: nil, refreshPoint: point, prefersFocusedElement: false) - let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y) } + let (lookup, lookupReads) = try countingReads { textInputAt(app: app, x: point.x, y: point.y, contentTranslation: nil) } let (input, resolutionReads) = try countingReads { resolveTextEntryInput(app: app, target: target) } XCTAssertEqual(lookup?.snapshot?.identifier, "agent-device-hardware-keyboard-input") @@ -109,7 +109,12 @@ extension RunnerTests { "software keyboard is down: this simulator cannot show the keyboard a focus tap raises" ) - let stabilized = stabilizeTextInputBeforeTyping(app: app, target: field, keyboardVisibleBeforeTap: false) + let stabilized = stabilizeTextInputBeforeTyping( + app: app, + target: field, + keyboardVisibleBeforeTap: false, + contentTranslation: nil + ) guard case .inputReportsKeyboardFocus(let read) = stabilized.evidence else { return XCTFail("focus confirmed by \(stabilized.evidence?.name ?? "nothing"), not by the tapped input") diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift new file mode 100644 index 0000000000..415a55a4b7 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryReadinessTests.swift @@ -0,0 +1,57 @@ +import XCTest + +extension RunnerTests { +#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS) + /// The readiness retap dispatches `ResolvedTextEntryInput.frameOnScreen`'s center, so the snapshot + /// frame of a placed window passes through the read's content vector before it becomes a screen + /// point (#3899). Before that rule the retap tapped the bare `snapshot().frame` center, which on + /// a windowed iPad lands outside the window, and the fill after it failed with + /// TEXT_INPUT_NOT_FOCUSED — the issue's own failure shape. + /// + /// The fixture cannot be placed in a window, so the read is driven directly: the vector moves the + /// field's real snapshot frame exactly onto the press-counting witness button below it. A point + /// dispatched at `frameOnScreen`'s center presses that button; a point dispatched at the raw + /// snapshot center taps the field and moves no counter. The same read with no vector is the + /// closest negative: a full-screen app retaps the field itself and the counter stays put. + @MainActor + func testReadinessRetapDispatchesTheReadsScreenSpaceCenter() throws { + app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-tap-witness"] + app.launch() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) + let field = app.textFields["agent-device-hardware-keyboard-input"] + XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) + let witness = app.buttons["agent-device-tap-witness"] + XCTAssertTrue(witness.waitForFixtureExistence(timeout: appExistenceTimeout)) + let presses = app.staticTexts["agent-device-tap-witness-presses"] + XCTAssertTrue(presses.waitForFixtureExistence(timeout: appExistenceTimeout)) + let fieldFrame = field.frame + let witnessFrame = witness.frame + XCTAssertFalse(fieldFrame.isEmpty) + XCTAssertFalse(witnessFrame.isEmpty) + + // Closest negative: a read with no vector — a full-screen app — retaps the field's own center, + // which is not the button's point. + let untranslated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: nil)) + retapAt(read: untranslated, app: app, element: field) + XCTAssertEqual(presses.label, "presses=0", "a vectorless read dispatched a point off the field") + + // The read a probe makes on a placed window: the snapshot reports the field's box and the read + // carries the window's vector. Here the vector is chosen so the translated box is the button. + let vector = CGSize( + width: witnessFrame.midX - fieldFrame.midX, + height: witnessFrame.midY - fieldFrame.midY + ) + let translated = try XCTUnwrap(ResolvedTextEntryInput(reading: field, contentTranslation: vector)) + XCTAssertEqual(translated.frameOnScreen.midX, witnessFrame.midX, accuracy: 0.5) + XCTAssertEqual(translated.frameOnScreen.midY, witnessFrame.midY, accuracy: 0.5) + + retapAt(read: translated, app: app, element: field) + XCTAssertEqual( + presses.label, + "presses=1", + "the retap dispatched the raw snapshot center, not the read's screen-space center (#3899)" + ) + } +#endif +} diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift index c9b306758a..6c7b54b49a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift @@ -15,7 +15,7 @@ extension RunnerTests { let completed = expectation(description: "optional probe completed inside runner wait") DispatchQueue.main.async { self.textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Optional probe issue during runner wait") - _ = self.probeTextInputs(app: self.app, point: point) + _ = self.probeTextInputs(app: self.app, point: point, contentTranslation: nil) completed.fulfill() } guard XCTWaiter.wait(for: [completed], timeout: 5) == .completed else { @@ -174,8 +174,8 @@ extension RunnerTests { } XCTExpectFailure("Required read and later issue belong to their caller", options: options) { textInputProbeIssueForTesting = expected - _ = textInputAt(app: app, x: point.x, y: point.y) - _ = probeTextInputs(app: app, point: point) + _ = textInputAt(app: app, x: point.x, y: point.y, contentTranslation: nil) + _ = probeTextInputs(app: app, point: point, contentTranslation: nil) record(expected) } XCTAssertEqual(observed, 2) @@ -192,7 +192,7 @@ extension RunnerTests { XCTAssertTrue(field.waitForFixtureExistence(timeout: appExistenceTimeout)) let frame = field.frame textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Failed to get matching snapshot: kAXErrorIllegalArgument") - let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY)) + let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY), contentTranslation: nil) guard case .unavailable = outcome else { return XCTFail("A suppressed AX issue must discard the matching candidate") } diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 7718fbabc1..95396fc3a4 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -200,6 +200,11 @@ public enum SnapshotGeometrySpace: Equatable { ) { return .windowContent(offset: offset) } + // A host that declares no space of its own publishes what the platform reported. A system + // surface under a placed window — the keyboard at (0,925,834,282), a SpringBoard overlay at + // (0,0,834,1210) — reports the screen's own box and is already in screen space; the two + // `windowContentCases` rows for those windows pin that inheriting the carrier's vector would + // move every key by the window origin (#3899's own failure shape). return .appOrientation } diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift index ceb6f35cdb..9b80b197d2 100644 --- a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/CoordinateSpaceTests.swift @@ -616,6 +616,9 @@ final class CoordinateSpaceTests: XCTestCase { /// The dispatch direction for a windowed app: the caller's point is the point touched, so /// `native(point:)` turns a screen-space point's axes without moving it toward the window origin /// (#3899). Under portrait — the only orientation measured for a windowed app — it is the identity. + /// The landscape rows for a placed window stay the documented unmeasured gap (ADR 0004): the turn + /// here is asserted about the window's side lengths, while the digitizer turns about the screen's, + /// and no windowed landscape capture has been measured to say which basis is right. func testNativePointForAWindowedAppKeepsTheCallersScreenPoint() { let windowOnScreen = CGRect(x: 224, y: 207, width: 386, height: 682) for orientation in [ @@ -631,15 +634,6 @@ final class CoordinateSpaceTests: XCTestCase { "interfaceOrientation \(orientation)" ) } - // A turn moves axes only: the same point under a landscape window keeps its own origin story. - XCTAssertEqual( - CoordinateSpaceRotation.native( - point: CGPoint(x: 578, y: 239), - in: windowOnScreen, - interfaceOrientation: RunnerInterfaceOrientation.landscapeRight - ), - CGPoint(x: 682 - 239, y: 578) - ) } func testNormalizedOfAnEmptyArrayIsEmpty() { diff --git a/contracts/fixtures/window-coordinate-space.json b/contracts/fixtures/window-coordinate-space.json index 1211632d64..dbd6bfd551 100644 --- a/contracts/fixtures/window-coordinate-space.json +++ b/contracts/fixtures/window-coordinate-space.json @@ -218,7 +218,7 @@ "offset": null }, { - "name": "a half-tolerance origin miss is float representation, not a different placement", + "name": "a half-tolerance side-length miss is float representation, not a different box", "reportedBox": { "x": 0, "y": 0, "width": 385.5, "height": 682 }, "viewportBox": { "x": 224, "y": 207, "width": 386, "height": 682 }, "offset": { "width": 224, "height": 207 } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index 11f0799d22..db819ae872 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -660,9 +660,11 @@ Translation is the one rule that reads an origin, so it is the only place the fi Capture is half of the promise; dispatch is the other half. The measured dispatch spaces for a placed window are both screen-anchored: the synthesized path lands a point at its screen position, and XCTest's own `XCUICoordinate` on the app anchors at the screen's origin too. `interactionCoordinate` therefore passes the caller's screen point through, and `resolveRunnerWindow` keeps naming the window a foldable's tap resolves its display on (ADR 0025). -The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (641,321) for a window at (324,307). +The live `XCUIElement.frame` property reports screen space on a placed window, as the selector-tap path proved on the device: it measured the `Add` button at (659,321) for a window at (324,307). The one live read that does not is `XCUIElementSnapshot.frame`, which reports the window's content space like the capture channel does. So the snapshot capture converts in `SnapshotGeometrySpace.normalized`, and the text-input probe converts its snapshot reads through the same `SnapshotContentTranslation` rule read from the two live boxes (`app.frame` against the resolved window's frame); every other live read needs no conversion. +The probe-side owner is `ResolvedTextEntryInput.frameOnScreen`. +The vector is read once per command and stored on the target and each read the target produces, so the probe matching, the focus tap, the refresh point, the caret tap, and the readiness retap all dispatch through that one property. Where a probe read has no snapshot and falls back to the live element, its frame is already screen space and is not translated. `app.frame` unions transformed subtrees, so content pushed off a placed window (a closed drawer at negative x) can stop the two live boxes matching and read no vector; the capture still converts from the snapshot root box, so a probe may then read window-local frames where the capture published screen space. A mid-command window move has the same effect on a vector read at command start. Both are reads of a moving target, not a second space: a re-opened command reads the current vector. diff --git a/packages/platform-apple/src/snapshot-route.test.ts b/packages/platform-apple/src/snapshot-route.test.ts index ca1a92920c..afd4dadacd 100644 --- a/packages/platform-apple/src/snapshot-route.test.ts +++ b/packages/platform-apple/src/snapshot-route.test.ts @@ -407,7 +407,7 @@ test.for([ ['foreground-owner-changed', 'while another surface owned the foreground'], [ 'app-window-not-full-screen', - 'while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen', + 'while the app window did not fill the screen; the runner republishes a windowed capture in screen space', ], ] as const)('a %s refusal sends only that capture to the runner', async ([code, consequence]) => { const source = sourceReturning({ stage: 'failed', failure: { kind: 'unsupported', code } }); diff --git a/packages/platform-apple/src/snapshot-route.ts b/packages/platform-apple/src/snapshot-route.ts index 5fcee4fca7..c977be2baf 100644 --- a/packages/platform-apple/src/snapshot-route.ts +++ b/packages/platform-apple/src/snapshot-route.ts @@ -512,7 +512,7 @@ function captureScopedFallbackWarning(code: string, consequence: CaptureScopedCo case 'foreground-ownership': return `${thisCapture}, while another surface owned the foreground.`; case 'window-not-full-screen': - return `${thisCapture}, while the app window did not fill the screen; XCTest geometry for a windowed app can also be offset from the screen.`; + return `${thisCapture}, while the app window did not fill the screen; the runner republishes a windowed capture in screen space.`; case 'coordinate-space-unresolved': return `${thisCapture}, which reports captured geometry in the app's own orientation space.`; } diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index 4ce6887031..b181eef498 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -75,12 +75,6 @@ export function isQuarterTurnedWindowFrame(windowFrame: Rect, appFrame: Rect): b } /* - * A window that does not fill the screen (#3899) is translated by the capture that owns it, and this - * file is not that capture. The bridge guest refuses a windowed tree (`app-window-not-full-screen`) - * before this decoder ever sees one, so no host tree arrives in the window's content space and there - * is nothing here to translate. The rule lives with the producer that converts — - * `SnapshotContentTranslation` beside the Swift capture — and `windowContentCases` in - * `contracts/fixtures/window-coordinate-space.json` is replayed there alone. The quarter-turn - * detection above stays in both languages because this decoder does read it: it is what refuses a - * turned tree the reader cannot resolve. + * The #3899 window-content translation belongs to the Swift capture's `SnapshotContentTranslation`, + * replayed from `windowContentCases` there alone. */ From b4ec02750f2e5908322cbc852254f8e3d1e488ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 24 Oct 2026 10:34:58 +0200 Subject: [PATCH 8/9] docs(ios): name the probe-side owner and the screen-space dispatch basis (#3899) Amends ADR 0004 to name ResolvedTextEntryInput as the probe-side owner of the content vector, cites the measured Add-button rect at (659,321), records that the runner republishes a windowed capture in screen space where the bridge-refusal warning said it was offset, and names screen space plus the app.frame origin assumption in the interactionCoordinate comment. --- .../RunnerTests+TextEntryFocus.swift | 3 ++- docs/adr/0004-ios-snapshot-backend-strategy.md | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 5d4140d917..cd1ef382b2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -185,7 +185,8 @@ extension RunnerTests { let readyTarget = TextEntryTarget( element: stabilized.element ?? target, refreshPoint: requestedPoint, - prefersFocusedElement: false + prefersFocusedElement: false, + contentTranslation: contentTranslation ) let element = stabilized.focusConfirmed ? readyTarget.element diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index db819ae872..a1dfccf286 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -768,10 +768,9 @@ If the frames differ, the guest refuses the capture (`app-window-not-full-screen This refusal provides evidence about the screen arrangement, not the app, so it has capture scope. If the runtime cannot report either frame, the guest keeps the capture. -The XCTest fallback is not correct for a windowed app either. -The runner clips window-relative element frames against the screen-relative app frame. -That is a runner gap. -It is not a reason to publish a bridge tree that is known to be wrong. +The XCTest fallback is the correct capture for a windowed app, after the amendment above. +Before it, the runner clipped window-relative element frames against the screen-relative app frame, and that gap is what made this refusal protect a real failure. +The refusal stays: this channel reports window-relative frames and no channel here converts them. Rejected alternatives: From 70f6faa55235d69fa33f504c901013168e3dcff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 24 Oct 2026 10:54:43 +0200 Subject: [PATCH 9/9] refactor(ios): cut the screen-space comments and the duplicated frame offset (#3899) The vector on the target replaces the offset arithmetic in textEntryRefreshPoint, which read the snapshot frame and added the vector itself. One owner now: ResolvedTextEntryInput.frameOnScreen. The comments that restated the capture's rule at every call site are cut to the constraint the code cannot show. SnapshotCoordinateSpace.swift ships in the npm package without comment stripping, so its prose lives in ADR 0004 and the golden table instead; the same rule text is removed from the TypeScript decoder's foot note. --- .../RunnerTests+CommandExecution.swift | 3 +- .../RunnerTests+Interaction.swift | 14 +++-- .../RunnerTests+TextEntry.swift | 52 +++++++------------ .../RunnerTests+TextEntryFocus.swift | 29 ++++------- .../RunnerTests+TextEntryReadiness.swift | 20 +++---- .../RunnerTests+TextInputProbe.swift | 27 ++++------ .../RunnerTests+TypeExecution.swift | 3 +- .../SnapshotCoordinateSpace.swift | 52 +++++++------------ .../adr/0004-ios-snapshot-backend-strategy.md | 2 +- .../window-coordinate-space.ts | 7 +-- 10 files changed, 74 insertions(+), 135 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index ea6a445abf..adcab6215c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -155,8 +155,7 @@ extension RunnerTests { ) let textInput: XCUIElement? if !xCTestTextInputProbeSkipped { - // The lookup compares the point, which is a screen point, with candidate frames; on a - // placed window the snapshot frames need the content vector (#3899). + // The lookup compares candidate frames with this screen point (#3899). textInput = coordinateTapTextInputAt( app: activeApp, x: x, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 4b134a059d..704b2bed10 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -352,8 +352,7 @@ extension RunnerTests { func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { let point = CGPoint(x: x, y: y) - // The point is a screen point; candidate snapshot frames of a placed window need the content - // vector to be compared with it (#3899). + // The lookup compares candidate frames with this screen point (#3899). let textInputCandidates = textInputCandidatesAt( app: app, point: point, @@ -623,15 +622,14 @@ extension RunnerTests { #if !os(tvOS) private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate { #if os(iOS) - // A caller's point is a screen point: coordinate taps, double taps, long presses, and drags - // anchor at the application's origin, not the resolved window, so the point is dispatched as - // given. That holds because `app.coordinate(withNormalizedOffset: .zero)` sits at the origin of - // the space `app.frame` names, and for a windowed app the measured `app.frame` starts at the - // screen's origin. The first qualifying window is not always the one under the finger — a + // A caller's point is a screen point: these gestures anchor at the app's origin, not the + // resolved window, so it is dispatched as given. The anchor sits at the origin of the space + // `app.frame` names, and for a windowed app the measured `app.frame` starts at the screen's + // origin (#3899). The first qualifying window is not always the one under the finger — a // SpringBoard `alert accept` can live in an alert window while the wallpaper or status-bar // window qualifies first, so a window-relative anchor misses the button. Reference frames and // the synthesized display ID still resolve through `resolveRunnerWindow`, so a foldable tap - // keeps its panel and stays on its display (#3899, ADR 0025). + // keeps its panel and stays on its display (ADR 0025). let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) return origin.withOffset(CGVector(dx: x, dy: y)) #else diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift index 097213aac1..c198ea2534 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift @@ -149,10 +149,9 @@ extension RunnerTests { let element: XCUIElement? let refreshPoint: CGPoint? let prefersFocusedElement: Bool - /// The live snapshot-frame vector for the reading session, read once when this target was built - /// (#3899). Every snapshot frame the target's reads produce is compared with screen points, so - /// every such read carries this vector (`ResolvedTextEntryInput.frameOnScreen`). Nil on a device - /// whose app fills the screen, where a snapshot frame is already a screen frame. + /// The content vector of a placed window, read once for the command that built this target and + /// handed to every read it produces, which turns its snapshot frames into screen points (#3899). + /// Nil when the app fills the screen. let contentTranslation: CGSize? let fromTapWitness: Bool /// The input the first resolved element was. Once bound, resolution refuses any other input. @@ -187,25 +186,18 @@ extension RunnerTests { self.postFocusRead = postFocusRead } - /// The target moved to the input a resolution read: its element and identity, and — when the - /// read carries a snapshot — a refresh point at that frame's center on the screen (`frameOnScreen` - /// plus this target's content vector, #3899). A read with no snapshot keeps the old refresh - /// point: its center would come from the live `element.frame`, the read a post-focus handle - /// cannot answer without recording a failure (#3060). + /// The target moved to the input a resolution read: its element, a refresh point at its frame's + /// center on the screen, and its identity while the target is unbound. A read with no snapshot has + /// no frame here, so its center cannot come from the live `element.frame` — the read a post-focus + /// handle cannot answer without recording a failure (#3060). func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget { guard let input else { return self } - let nextRefreshPoint: CGPoint? - if let snapshot = input.snapshot { - let frame = input.frameOnScreen - nextRefreshPoint = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY) - } else { - nextRefreshPoint = refreshPoint - } + let frame = input.snapshot == nil ? .zero : input.frameOnScreen return TextEntryTarget( element: input.element, - refreshPoint: nextRefreshPoint, + refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY), prefersFocusedElement: prefersFocusedElement, contentTranslation: contentTranslation, fromTapWitness: fromTapWitness, @@ -222,18 +214,14 @@ extension RunnerTests { /// removed input or a focused Flutter password field (#3060), records an XCTest failure that ends /// the runner session, the snapshot only throws. /// - /// This type owns the probe-side space of #3899 (ADR 0004). `XCUIElementSnapshot.frame` reports a - /// placed window's content in the window's own space, while every point this read feeds — a refresh - /// point, a focus tap, a probe match, a readiness retap — is a screen point. So the read carries - /// the content vector it was taken under, and `frameOnScreen` is the only frame consumers use: a - /// snapshot frame plus the vector, or the live frame when there is no snapshot. Raw frames - /// (`snapshot.frame`) are compared only with each other. + /// This type owns the probe-side space of #3899 (ADR 0004): a snapshot frame names a placed + /// window's content space while every point this read feeds is a screen point, so consumers take + /// `frameOnScreen` and compare raw frames only with each other. struct ResolvedTextEntryInput { let element: XCUIElement /// Nil when the element exists but could not be snapshotted. let snapshot: XCUIElementSnapshot? - /// The live snapshot-frame vector read once for the command that made this read (#3899). Nil - /// when the app fills the screen, where a snapshot frame is already a screen frame. + /// The vector of the command that made this read (#3899); nil when the app fills the screen. let contentTranslation: CGSize? private init(element: XCUIElement, snapshot: XCUIElementSnapshot?, contentTranslation: CGSize?) { @@ -278,9 +266,9 @@ extension RunnerTests { return element } - /// The input's box in screen space: the snapshot frame carried by `contentTranslation` to its - /// place on the screen, or the live frame when there is no snapshot — the live - /// `XCUIElement.frame` already reports a placed window's content in screen space (#3899). + /// The input's box in screen space: the snapshot frame plus `contentTranslation`, or the live + /// frame when there is no snapshot — that property already reports a placed window's content in + /// screen space (#3899). var frameOnScreen: CGRect { guard let snapshot else { return element.frame @@ -296,8 +284,6 @@ extension RunnerTests { /// `dispatch` runs, and resolves the target again after that, so a bound target refuses an input /// that took its place. Each resolution moves the target to the input it read. Every post and every /// pause between reads goes through `dispatch`; the pause a typing plan takes itself follows a post. - /// The reader hands the target's content vector to every read it resolves, so one vector rules the - /// command (#3899). struct TextEntryReader { private(set) var target: TextEntryTarget private var lastRead: ResolvedTextEntryInput? @@ -672,10 +658,8 @@ extension RunnerTests { // zero-size one (how a departed handle reads), there is no point to dispatch — an edge-tap // from a frame the field moved after records a failure or navigates away, the shape // `clearTextInput`'s comment refuses — so degrade to the caller's point-free delete burst. - // The read's snapshot frame names the window's own content space on a placed window, so the - // point enters `tapAt` through `frameOnScreen`, which carries the read's content vector (#3899). - // A read with no snapshot gets no point at all: `frameOnScreen`'s fallback is the live - // `element.frame` read this function exists to avoid. + // A snapshotless read gets no point at all: `frameOnScreen`'s fallback is the live + // `element.frame` read this function exists to avoid (#3060). guard input.snapshot?.frame.isEmpty == false else { return } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index cd1ef382b2..fb0c4592f8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -79,8 +79,7 @@ extension RunnerTests { // The tapped input's own read comes before the keyboard. While a software keyboard is // presented, the keyboard query blocks and then finds none. An input that reports keyboard // focus is stronger evidence than the keyboard's arrival, which shows only that some input - // took focus. The read carries the focus step's content vector (#3899): the hand-on read - // becomes the entry's first read, and its frames must enter screen space. + // took focus. let read = target.flatMap { ResolvedTextEntryInput(reading: $0, contentTranslation: contentTranslation) } if let read, let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) { return TextEntryStabilization(element: read.element, evidence: .inputReportsKeyboardFocus(read), polls: polls) @@ -132,18 +131,15 @@ extension RunnerTests { return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true) } - // The one content-vector read for this command (#3899): every target this focus step builds - // carries it, every read those targets produce converts through it, and the retap and reader - // downstream reuse it. On a device whose app fills the screen it is nil and every frame stays - // exactly as read. + // The one content-vector read for this command (#3899): every target built here carries it and + // every read those targets produce converts through it. let contentTranslation = liveContentTranslation(app: app) let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) let input = textInputAt(app: app, x: x, y: y, contentTranslation: contentTranslation) let target = input?.element let requestedPoint = CGPoint(x: x, y: y) - // The focus tap goes to the input's own center, and the input's frame arrives from a snapshot, - // which reports a placed window's content in the window's own space while `tapAt` dispatches a - // screen point (#3899). An input read live (no snapshot) already names screen space. + // The tap goes to the input's own center, and `tapAt` dispatches a screen point, so the point + // comes from `frameOnScreen` (#3899). let inputCenter = input.map { $0.frameOnScreen }.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } @@ -205,8 +201,6 @@ extension RunnerTests { contentTranslation: CGSize? ) -> TextEntryTarget { guard let element else { - // No element: the target keeps its refresh point, and the reads a later resolve makes through - // that point carry this command's vector (#3899). return TextEntryTarget( element: nil, refreshPoint: requestedPoint, @@ -249,22 +243,19 @@ extension RunnerTests { /// the target gets NO refresh point rather than the pre-focus one: this route's first resolve is /// not yet bound to an identity, and a field that went unqueryable on focus may have moved the /// layout so the old point names a different input, so fail closed with a typed reason instead of - /// typing into that other field. The frame is the read's snapshot frame plus its content vector - /// (#3899) — never the live `element.frame` fallback `frameOnScreen` uses for a snapshotless read. + /// typing into that other field. The frame is `read`'s snapshot frame plus its content vector, + /// never the live `element.frame` (#3899, #3060). private func textEntryRefreshPoint( read: ResolvedTextEntryInput?, fallbackTo requestedPoint: CGPoint ) -> CGPoint? { - guard let snapshot = read?.snapshot else { + guard let read, read.snapshot != nil else { return nil } - guard !snapshot.frame.isEmpty else { + let frame = read.frameOnScreen + guard !frame.isEmpty else { return requestedPoint } - let frame = snapshot.frame.offsetBy( - dx: read?.contentTranslation?.width ?? 0, - dy: read?.contentTranslation?.height ?? 0 - ) return CGPoint(x: frame.midX, y: frame.midY) } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift index 52bd98268d..a8f19d0d3a 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift @@ -86,19 +86,15 @@ extension RunnerTests { /// wait rather than reused, because focusing a field moves the layout: the repeat aims at where the /// field is then, which is the moment the read it replaced ran at. /// - /// The read enters `retapAt` as a `ResolvedTextEntryInput`, so the repeat taps - /// `frameOnScreen` — the snapshot frame plus the content vector of a placed window (#3899). The base - /// read carries no vector; the re-read carries the live one, read only on this branch, so a tap that - /// confirms focus dispatches no window lookup. + /// The re-read carries the live content vector, so the repeat taps a screen point (#3899). It is + /// read only on this branch: a tap that confirms focus dispatches no window lookup. func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { #if os(iOS) if waitForTextEntryFocus(app: app, element: element, timeout: TextEntryTiming.readinessTimeout) { return } - // The re-read enters as a `ResolvedTextEntryInput` carrying the content vector of a placed - // window (#3899), read only on this branch, so a tap that confirms focus dispatches no window - // lookup. The window read is contained like the probe lookups: a transient window that records - // an XCTest issue costs the vector, not the command. + // Contained like the probe lookups: a transient window that records an XCTest issue costs the + // vector, not the command. let settled: ResolvedTextEntryInput? switch containingTextInputProbeIssues(fallback: nil as ResolvedTextEntryInput?, { _ in ResolvedTextEntryInput(reading: element, contentTranslation: liveContentTranslation(app: app)) @@ -116,9 +112,8 @@ extension RunnerTests { } /// The readiness retap, split out so the unit lane can drive it and assert where the point lands. - /// `tapAt` takes a screen point, so the point is `frameOnScreen`'s center. A read with no snapshot - /// taps nothing: `frameOnScreen`'s fallback is the live `element.frame` read — the one that records - /// an XCTest failure on a handle that stopped answering its query (#3060). + /// A read with no snapshot taps nothing: its `frameOnScreen` is the live `element.frame`, the read + /// a handle that stopped answering its query cannot make without recording a failure (#3060). func retapAt(read: ResolvedTextEntryInput, app: XCUIApplication, element: XCUIElement) { guard read.snapshot != nil else { return @@ -159,8 +154,7 @@ extension RunnerTests { /// fallback at all. func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool { #if os(iOS) - // The read needs no content vector: this corroborates identity and position raw-frame against - // raw-frame, and never turns a frame into a dispatched point (#3899's rule). + // No content vector: this corroborates raw frame against raw frame and dispatches no point. return ResolvedTextEntryInput(reading: element).map { keyboardFocusConfirmed(app: app, read: $0) } ?? false #else return false diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 39f0e8b6d6..40346eef1e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -24,7 +24,8 @@ extension RunnerTests { y: Double, contentTranslation: CGSize? ) -> ResolvedTextEntryInput? { - textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first + let point = CGPoint(x: x, y: y) + return textInputCandidatesAt(app: app, point: point, contentTranslation: contentTranslation).first } func textInputCandidatesAt( @@ -134,8 +135,7 @@ extension RunnerTests { } /// Runs an optional probe whose XCTest issues are contained instead of recorded: an issue makes - /// the probe unavailable rather than failing the command and invalidating the runner. Visible to - /// the readiness retap, whose window re-read needs the same containment (#3899). + /// the probe unavailable rather than failing the command and invalidating the runner. func containingTextInputProbeIssues( fallback: T, _ probe: (_ shouldStop: () -> Bool) -> T @@ -215,20 +215,15 @@ extension RunnerTests { } /// The vector that carries this session's snapshot-read app-content frames to their place on the - /// screen, or nil when the app fills the screen (#3899). Read once at the start of each command - /// that dispatches points derived from snapshot frames — the focus step, the coordinate fill route, - /// the readiness retap, a coordinate text read — and carried on the types the command builds from - /// it (`TextEntryTarget.contentTranslation`, `ResolvedTextEntryInput.contentTranslation`). No read - /// inside a loop re-reads it: the whole-app queries below cost about 90 ms each on heavy React - /// Native screens, and a per-read re-query would undo the fill-read work of #3881, #3906 and #3909. + /// screen, or nil when the app fills the screen (#3899). Each command that dispatches points from + /// snapshot frames reads it once, at its own start, and hands it down; a read inside a typing loop + /// never calls this, because the two live box reads below cost about 90 ms each on a heavy React + /// Native screen. /// - /// The live `XCUIElement.frame` property already reports a placed window's content in screen space, - /// but `XCUIElementSnapshot.frame` reports it in the window's own content space, so the same probe - /// sees two spaces on an iPad Windowed Apps window. The signature and the vector are the capture's - /// rule (`SnapshotContentTranslation`, ADR 0004) read from the two live boxes that name it: the - /// application's `frame` is a `(0,0)`-anchored copy of the box the resolved window reports at its - /// place on the screen. Any other pair of boxes declares no vector, so full-screen, iPhone, and - /// each foldable panel keep their frames exactly as read. + /// The signature and the vector are the capture's rule (`SnapshotContentTranslation`, ADR 0004) + /// read from the two live boxes that name it: the application's `frame` is a `(0,0)`-anchored copy + /// of the box the resolved window reports at its place on the screen. Any other pair of boxes + /// declares no vector, so full-screen, iPhone, and each foldable panel keep frames as read. func liveContentTranslation(app: XCUIApplication) -> CGSize? { // The window read is contained like the probe lookups it replaces: a transient window that // records an XCTest issue costs the vector, not the runner session. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index bea8d91e60..8e49b4a3e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -27,8 +27,7 @@ extension RunnerTests { hasY: command.y != nil, xCTestChannelPenalized: xCTestChannelPenalized ), let x = command.x, let y = command.y { - // The one content-vector read for this command (#3899): the target built here carries it, and - // every read that target produces converts through it, as the focus route's targets do. + // The one content-vector read for this command (#3899); the target built here carries it. let contentTranslation = liveContentTranslation(app: activeApp) // Resolved before the tap: the focus it causes can move the field away from this point. // A replacement with no input at the point has nothing to bind to: its clear and its typed diff --git a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift index 95396fc3a4..90cd0d5488 100644 --- a/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +++ b/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift @@ -5,9 +5,8 @@ import CoreGraphics // synthesized dispatch (forward) and capture (inverse). Rationale and measurements: ADR 0004 // "the coordinate space of a captured subtree" and contracts/fixtures/window-coordinate-space.json. // -// Rotation moves axes only. It never moves an origin: a placed window's offset from the screen -// belongs to `SnapshotContentTranslation` (#3899), which is the one rule that turns a reported box -// into its screen-space box. Keeping the two apart is what lets a capture publish one space. +// Rotation moves axes only and never an origin; a placed window's offset from the screen belongs +// to `SnapshotContentTranslation` (#3899). public enum RunnerInterfaceOrientation { public static let unknown = 0 @@ -18,11 +17,9 @@ public enum RunnerInterfaceOrientation { } public enum CoordinateSpaceRotation { - // Dispatch turns a point's axes and never moves its origin: the turn says which axis a box runs - // along, not where that box sits (#3899). The digitizer's origin is the display's own, which is - // where a windowed app's screen-space points are already measured from, so a placed window's - // offset from the screen belongs to `SnapshotContentTranslation` — the one rule that reads an - // offset — and not to this function. The `frame` argument names the side lengths the turn swaps. + // Dispatch turns a point's axes and never moves its origin: the digitizer's origin is the + // display's own, which is where a windowed app's screen-space points are already measured from + // (#3899). `frame` names the side lengths the turn swaps, nothing more. public static func native( point: CGPoint, in frame: CGRect, @@ -107,25 +104,16 @@ public enum CoordinateSpaceRotation { } } -/// Whether a carrier's own box declares that its subtree reports in the carrier's content space -/// rather than in the space the viewport names (#3899). +/// The vector from a carrier's reported box to the screen space it is placed at (#3899). /// -/// An iPad window that does not fill the screen (Windowed Apps, Stage Manager) reports its own frame -/// at its place on the screen while its subtree arrives anchored at its own top-left: measured on -/// iPad Pro 11-inch (M5, iOS 27.0) with a Calendar window whose `frame` is `(224,207,386,682)`, the -/// application root reports `(0,0,386,682)` and its Add button `(335,14,37,36)` at screen -/// `(559,221)`. A full-screen window, an iPhone, and each foldable panel report origins at zero on -/// both sides, so the rule reads zero translation there and every published rect is unchanged. -/// -/// The signature is the carrier's box as a `(0,0)`-anchored copy of the viewport box: the side -/// lengths match within the tolerance and the viewport — the app's screen-anchored box — starts -/// somewhere else on the screen. Any other carrier box names no space: it is not the app's own -/// window, so its numbers say nothing about where the app's subtree starts. +/// The signature is a `(0,0)`-anchored copy of the viewport box whose viewport starts elsewhere on +/// the screen; any other carrier box is not the app's own window, so its numbers say nothing about +/// where its subtree starts. Measured rows and the full rationale: ADR 0004 and the +/// `windowContentCases` table this replays against. public enum SnapshotContentTranslation { - /// The vector from a carrier's reported box to the screen space it is placed at, or `nil` when the - /// carrier's box declares no translation. One box offset from one viewport box, decided from the - /// same tolerance the quarter-turn rule reads; replayed against - /// `contracts/fixtures/window-coordinate-space.json` in `windowContentCases`. + /// `nil` when the carrier's box declares no translation. A full-screen app, an iPhone, and each + /// foldable panel report origins at zero on both sides, so the rule reads none there and every + /// published rect stays as the platform reported it. public static func offset(reportedBox: CGRect, viewportBox: CGRect) -> CGSize? { guard SnapshotGeometry.isPositiveFinite(reportedBox), SnapshotGeometry.isPositiveFinite(viewportBox) @@ -176,10 +164,8 @@ public enum SnapshotGeometrySpace: Equatable { } } - /// A carrier declares its own subtree's space from its own box. A `.reported` viewport is the - /// screen-anchored fact both declarations read against: a quarter turn needs its orientation, and - /// a window-content offset needs its origin. A `.derived` viewport carries no screen anchor, so a - /// subtree under it publishes what the platform reported rather than a guessed space. + /// A `.reported` viewport is the screen-anchored fact both declarations read against; a `.derived` + /// viewport carries no screen anchor, so a subtree under it publishes what the platform reported. public static func space( reportedBySurfaceHost isSurfaceHost: Bool, reportedFrame: CGRect, @@ -200,11 +186,9 @@ public enum SnapshotGeometrySpace: Equatable { ) { return .windowContent(offset: offset) } - // A host that declares no space of its own publishes what the platform reported. A system - // surface under a placed window — the keyboard at (0,925,834,282), a SpringBoard overlay at - // (0,0,834,1210) — reports the screen's own box and is already in screen space; the two - // `windowContentCases` rows for those windows pin that inheriting the carrier's vector would - // move every key by the window origin (#3899's own failure shape). + // A system surface under a placed window — the keyboard, a SpringBoard overlay — reports the + // screen's own box and is already in screen space; `windowContentCases` pins that inheriting + // the carrier's vector would move every key by the window origin. return .appOrientation } diff --git a/docs/adr/0004-ios-snapshot-backend-strategy.md b/docs/adr/0004-ios-snapshot-backend-strategy.md index a1dfccf286..67d1a05ad8 100644 --- a/docs/adr/0004-ios-snapshot-backend-strategy.md +++ b/docs/adr/0004-ios-snapshot-backend-strategy.md @@ -651,7 +651,7 @@ Any other carrier box names no space, so the rule leaves its numbers alone: the A full-screen window, an iPhone, and each foldable panel of ADR 0025 report origins at zero on both sides, so the rule reads no vector there and every rect is unchanged. `SnapshotContentTranslation.offset(reportedBox:viewportBox:)` is that rule, replayed from `contracts/fixtures/window-coordinate-space.json` in the capture's language alone: the host bridge refuses a windowed tree (`app-window-not-full-screen`) before its decoder can read one, so the translation has one producer-side owner and the host's parity replay covers only the rules that decoder reads. -Rotation and translation stay separate rules, and the split is the point. +Rotation and translation stay separate rules. `CoordinateSpaceRotation` turns a box's axes and never moves its origin: the digitizer's origin is the display's own, which is where a placed window's screen-space points are already measured from. Before this amendment, that function localized a non-zero frame origin away and never added it back. Every configuration measured before this one has its app frame at the screen origin, so the localization was dead code there and the dispatch bug here. diff --git a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts index b181eef498..ee97982866 100644 --- a/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts +++ b/packages/platform-apple/src/snapshot-source/window-coordinate-space.ts @@ -23,7 +23,7 @@ import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; * reported, that tree mixes two spaces, clips its own content, and moves every tap by the window * origin (#3899). That translation belongs to the capture that performs it — the runner's Swift * `SnapshotContentTranslation` — not to this decoder, because the guest refuses a windowed tree - * before this file can read one; see the note at the foot of this file. + * before this file can read one. * * A rule that reads a box reads what the box can say. The quarter turn is a statement about which * axis a box runs along, so it compares side lengths and nothing else. A content translation is a @@ -73,8 +73,3 @@ export function isQuarterTurnedWindowFrame(windowFrame: Rect, appFrame: Rect): b Math.abs(windowFrame.height - appFrame.width) <= WINDOW_QUARTER_TURN_TOLERANCE ); } - -/* - * The #3899 window-content translation belongs to the Swift capture's `SnapshotContentTranslation`, - * replayed from `windowContentCases` there alone. - */