From 19d2823ddd7ac7278df72282f73998a451495f02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 16:59:22 +0200 Subject: [PATCH 01/12] feat(snapshot): carry provider keyboard observations through acquisition --- .../tap-keyboard-occlusion-policy.json | 244 +++++++++++++++++- .../src/ios-snapshot-acquisition.ts | 8 +- .../ios-snapshot-runtime-viewport.test.ts | 18 ++ .../src/snapshot/ios-snapshot-runtime.ts | 1 + packages/contracts/src/ios-snapshot.ts | 8 +- .../src/tap-keyboard-occlusion.test.ts | 7 +- .../contracts/src/tap-keyboard-occlusion.ts | 45 +++- packages/kernel/src/record.test.ts | 13 + packages/kernel/src/record.ts | 17 +- packages/kernel/src/snapshot.ts | 30 +-- .../src/ios-snapshot-adapter.test.ts | 22 ++ .../src/ios-snapshot-adapter.ts | 8 +- .../src/webdriver-client.ts | 6 +- .../src/webdriver-interactor.test.ts | 54 ++++ .../src/webdriver-interactor.ts | 22 +- .../src/webdriver-ios-snapshot.test.ts | 24 ++ .../src/webdriver-ios-snapshot.ts | 7 + .../client-snapshot-keyboard-fact.test.ts | 10 + src/commands/schema/cli-help-workflows.ts | 2 +- .../limrun-ios-snapshot-owner.test.ts | 1 + website/docs/docs/snapshots.md | 14 + 21 files changed, 511 insertions(+), 50 deletions(-) diff --git a/contracts/fixtures/tap-keyboard-occlusion-policy.json b/contracts/fixtures/tap-keyboard-occlusion-policy.json index 73e6456528..45b7ac7292 100644 --- a/contracts/fixtures/tap-keyboard-occlusion-policy.json +++ b/contracts/fixtures/tap-keyboard-occlusion-policy.json @@ -1,5 +1,5 @@ { - "description": "Golden table for keyboard occlusion on interaction tap points (#2589), consumed by packages/contracts/src/tap-keyboard-occlusion.ts. A visible software keyboard is its own system surface: it is never a covering sibling of app content, so the same-window occlusion classifier cannot see it, and it leaves the app window rect intact, so the viewport rule passes a covered tab bar. The band runs from the topmost keyboard node down to the bottom of the viewport, across the keyboard's own columns; the tap point is the point the interaction dispatches: the rect center the platform aims at for a ref handed over by name, or the touch point a parent with interactive descendants resolves instead. `nodes` are captured snapshot nodes as they appear in the projection the interaction resolved against — note that regular and interactive projections prune the keyboard's own container, leaving Key nodes beside app chrome. Three rules decide whether reported geometry may be measured at all. Docking: the band is derived only where the keyboard's own reported geometry reaches the viewport's bottom edge within 80 units — measured on iPhone 17 Pro (26.2) the reported key plane stops 58 pt short of an 874 pt portrait viewport and lands exactly on a 402 pt landscape one, and on iPad Pro 11-inch (M4) 3 pt short of a 1210 pt viewport — so an iPad floating keyboard, an app-drawn keypad, or an IME that stops above its own navigation bar stays unmeasured and fails open. The budget is counted in the same units as the rects it guards, which is what a measured Android IME asks for rather than what a dp table would give: with three-button navigation on Pixel 7 (Android 16, 1080x2400 at 420 dpi), Gboard's reported region runs to the physical display bottom at 2400 while the 126 px navigation bar occupies 2274..2400 inside it, so no Android keyboard measured so far needs slack for the bar. Width: geometry that comes back taller than it is wide did not arrive in the app's orientation space, i.e. it came from a capture that normalized no space — the producers that can name the app's interface orientation either turn it back (the runner's tree tiers) or refuse the capture (the Simulator AX bridge) — and is refused measurement for the same reason. Tiling: the keys themselves must form one unbroken run of columns, because a split iPad keyboard docks both clusters at the bottom edge while its surface container still spans the whole screen, and the app content between the clusters is visible and tappable. Reported seams are zero on every measured tree — iPhone 17 Pro's reported row adjoins across 395 pt, iPad Pro 11-inch's across 743.5 pt, and Gboard's across 1070 px — so the allowance covers rounding rather than layout, and a gapped band is refused measurement like the two above. That rule is deliberately blunt in both directions, and the last two cases name the trade: a sparse layout such as the iOS emoji panel reports corner keys over a wide container, does not tile, and reads `undetermined` — the guard abstains there rather than inventing which gaps are app content; and a projection that reports the container with no keys at all has nothing left to check, so the band is the container's own claim. Every captured iOS and Android keyboard measured so far reports its keys, so the second shape is the harness's, not a captured one. `target` names either a resolved node index or a bare coordinate point; both paths read one decision and choose their own consequence. A case that reaches a band declares its whole shape — the frame the refusal and the disclosure both quote, and the control rects that excuse a bare coordinate, which is where plane exclusion shows up.", + "description": "Golden table for keyboard occlusion on interaction tap points (#2589), consumed by packages/contracts/src/tap-keyboard-occlusion.ts. A visible software keyboard is its own system surface: it is never a covering sibling of app content, so the same-window occlusion classifier cannot see it, and it leaves the app window rect intact, so the viewport rule passes a covered tab bar. The band runs from the topmost keyboard node down to the bottom of the viewport, across the keyboard's own columns; the tap point is the point the interaction dispatches: the rect center the platform aims at for a ref handed over by name, or the touch point a parent with interactive descendants resolves instead. `nodes` are captured snapshot nodes as they appear in the projection the interaction resolved against \u2014 note that regular and interactive projections prune the keyboard's own container, leaving Key nodes beside app chrome. Three rules decide whether reported geometry may be measured at all. Docking: the band is derived only where the keyboard's own reported geometry reaches the viewport's bottom edge within 80 units \u2014 measured on iPhone 17 Pro (26.2) the reported key plane stops 58 pt short of an 874 pt portrait viewport and lands exactly on a 402 pt landscape one, and on iPad Pro 11-inch (M4) 3 pt short of a 1210 pt viewport \u2014 so an iPad floating keyboard, an app-drawn keypad, or an IME that stops above its own navigation bar stays unmeasured and fails open. The budget is counted in the same units as the rects it guards, which is what a measured Android IME asks for rather than what a dp table would give: with three-button navigation on Pixel 7 (Android 16, 1080x2400 at 420 dpi), Gboard's reported region runs to the physical display bottom at 2400 while the 126 px navigation bar occupies 2274..2400 inside it, so no Android keyboard measured so far needs slack for the bar. Width: geometry that comes back taller than it is wide did not arrive in the app's orientation space, i.e. it came from a capture that normalized no space \u2014 the producers that can name the app's interface orientation either turn it back (the runner's tree tiers) or refuse the capture (the Simulator AX bridge) \u2014 and is refused measurement for the same reason. Tiling: the keys themselves must form one unbroken run of columns, because a split iPad keyboard docks both clusters at the bottom edge while its surface container still spans the whole screen, and the app content between the clusters is visible and tappable. Reported seams are zero on every measured tree \u2014 iPhone 17 Pro's reported row adjoins across 395 pt, iPad Pro 11-inch's across 743.5 pt, and Gboard's across 1070 px \u2014 so the allowance covers rounding rather than layout, and a gapped band is refused measurement like the two above. That rule is deliberately blunt in both directions, and the last two cases name the trade: a sparse layout such as the iOS emoji panel reports corner keys over a wide container, does not tile, and reads `undetermined` \u2014 the guard abstains there rather than inventing which gaps are app content; and a projection that reports the container with no keys at all has nothing left to check, so the band is the container's own claim. Every captured iOS and Android keyboard measured so far reports its keys, so the second shape is the harness's, not a captured one. `target` names either a resolved node index or a bare coordinate point; both paths read one decision and choose their own consequence. A case that reaches a band declares its whole shape \u2014 the frame the refusal and the disclosure both quote, and the control rects that excuse a bare coordinate, which is where plane exclusion shows up. Provider facts distinguish measured visible frames, derived docked-tree bands, visibility-only probes, explicit absence, and unknown. Missing keyboard nodes are undetermined, never absence. A carried derived band was established before projection and stays usable after projection removes keyboard nodes; it obeys the same docking/orientation/tiling rules at acquisition. A measured floating frame needs no bottom extension.", "constants": { "occlusionReason": "tap_keyboard_occludes_target" }, @@ -1687,7 +1687,7 @@ "index": 1 }, "expected": { - "kind": "no-keyboard" + "kind": "undetermined" } }, { @@ -3992,6 +3992,246 @@ } ] } + }, + { + "name": "a carried docked-tree band survives projection that removes all keyboard nodes", + "viewport": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + }, + "nodes": [ + { + "index": 0, + "type": "Application", + "rect": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + } + }, + { + "index": 1, + "type": "Button", + "label": "Form", + "parentIndex": 0, + "rect": { + "x": 162, + "y": 795, + "width": 74, + "height": 54 + } + } + ], + "target": { + "index": 1 + }, + "keyboard": { + "kind": "derived", + "derivation": "docked-tree", + "frame": { + "x": 0, + "y": 583, + "width": 402, + "height": 291 + } + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 0, + "y": 583, + "width": 402, + "height": 291 + }, + "controlRects": [] + } + }, + { + "name": "a positive visibility probe without geometry cannot decide occlusion", + "viewport": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + }, + "nodes": [ + { + "index": 0, + "type": "Application", + "rect": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + } + }, + { + "index": 1, + "type": "Button", + "label": "Form", + "parentIndex": 0, + "rect": { + "x": 162, + "y": 795, + "width": 74, + "height": 54 + } + } + ], + "target": { + "index": 1 + }, + "keyboard": { + "kind": "visibility-only", + "visible": true + }, + "expected": { + "kind": "undetermined" + } + }, + { + "name": "a negative visibility probe establishes absence", + "viewport": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + }, + "nodes": [ + { + "index": 0, + "type": "Application", + "rect": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + } + }, + { + "index": 1, + "type": "Button", + "label": "Form", + "parentIndex": 0, + "rect": { + "x": 162, + "y": 795, + "width": 74, + "height": 54 + } + } + ], + "target": { + "index": 1 + }, + "keyboard": { + "kind": "visibility-only", + "visible": false + }, + "expected": { + "kind": "no-keyboard" + } + }, + { + "name": "an unknown provider observation with no keyboard nodes never establishes absence", + "viewport": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + }, + "nodes": [ + { + "index": 0, + "type": "Application", + "rect": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + } + }, + { + "index": 1, + "type": "Button", + "label": "Form", + "parentIndex": 0, + "rect": { + "x": 162, + "y": 795, + "width": 74, + "height": 54 + } + } + ], + "target": { + "index": 1 + }, + "keyboard": { + "kind": "unmeasurable", + "reason": "keyboard-visibility-probe-failed" + }, + "expected": { + "kind": "undetermined" + } + }, + { + "name": "a directly measured floating keyboard frame can refuse the target in its rectangle", + "viewport": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + }, + "nodes": [ + { + "index": 0, + "type": "Application", + "rect": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + } + }, + { + "index": 1, + "type": "Button", + "label": "Form", + "parentIndex": 0, + "rect": { + "x": 162, + "y": 795, + "width": 74, + "height": 54 + } + } + ], + "target": { + "index": 1 + }, + "keyboard": { + "kind": "visible", + "frame": { + "x": 100, + "y": 800, + "width": 220, + "height": 65 + } + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 100, + "y": 800, + "width": 220, + "height": 65 + }, + "controlRects": [] + } } ] } diff --git a/packages/capture-kit/src/ios-snapshot-acquisition.ts b/packages/capture-kit/src/ios-snapshot-acquisition.ts index 106c5a0b4f..f9474bf63e 100644 --- a/packages/capture-kit/src/ios-snapshot-acquisition.ts +++ b/packages/capture-kit/src/ios-snapshot-acquisition.ts @@ -10,7 +10,11 @@ import type { } from '@agent-device/contracts/ios-snapshot'; import { normalizeType } from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; -import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; +import type { + RawSnapshotNode, + Rect, + SnapshotKeyboardBandFact, +} from '@agent-device/kernel/snapshot'; /** * The facts a *provider* acquisition leaves unproven, keyed only by the producers whose residue @@ -96,6 +100,7 @@ export function createIosSnapshotAcquisition( producer: IosProviderAcquisitionProducer; nodes: readonly RawSnapshotNode[]; viewport: IosViewportEvidence; + keyboard?: SnapshotKeyboardBandFact; lineage: IosSnapshotLineage; }>, ): SnapshotRuntimeAcquiredResult { @@ -107,6 +112,7 @@ export function createIosSnapshotAcquisition( intent: 'full', nodes: input.nodes, viewport: input.viewport, + ...(input.keyboard ? { keyboard: input.keyboard } : {}), lineage: input.lineage, residue: deriveIosSnapshotAcquisitionResidue(producer, input.viewport), }, diff --git a/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts index 5c5e68f1a6..1a7de6205f 100644 --- a/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts @@ -82,3 +82,21 @@ test('an empty iOS acquisition still publishes the viewport it read (#3182)', () assert.deepEqual(presented.nodes, []); assert.deepEqual(presented.viewport, { width: 390, height: 844 }); }); + +test('provider keyboard evidence survives projection even when the tree has no keyboard nodes', () => { + const keyboard = { + kind: 'derived' as const, + derivation: 'docked-tree' as const, + frame: { x: 0, y: 580, width: 390, height: 264 }, + }; + const acquired = createIosSnapshotAcquisition({ + producer: 'appium-source', + nodes: NODES, + viewport: { kind: 'reported', rect: SCREEN }, + lineage: {}, + keyboard, + }); + for (const options of [{ depth: 0 }, { interactiveOnly: true }, { raw: true }]) { + assert.deepEqual(presentIosSnapshotAcquisition(acquired, options).keyboard, keyboard); + } +}); diff --git a/packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts b/packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts index 89677873f9..04d1c8a02c 100644 --- a/packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts +++ b/packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts @@ -49,6 +49,7 @@ export function presentIosSnapshotAcquisition( backend: 'xctest', producer: acquired.acquisition.producer, nodes: [...presentation.payload.nodes], + ...(acquired.acquisition.keyboard ? { keyboard: acquired.acquisition.keyboard } : {}), comparisonIdentity: presentation.comparisonIdentity, ...(acquired.acquisition.truncated === undefined ? {} diff --git a/packages/contracts/src/ios-snapshot.ts b/packages/contracts/src/ios-snapshot.ts index 07bdb2b68f..a608372385 100644 --- a/packages/contracts/src/ios-snapshot.ts +++ b/packages/contracts/src/ios-snapshot.ts @@ -1,4 +1,9 @@ -import type { RawSnapshotNode, Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { + RawSnapshotNode, + Rect, + SnapshotNode, + SnapshotKeyboardBandFact, +} from '@agent-device/kernel/snapshot'; export type IosSnapshotProducer = | 'apple-runner' @@ -154,6 +159,7 @@ type IosSnapshotAcquisitionForIntent = Read nodes: readonly RawSnapshotNode[]; truncated?: boolean; viewport: IosViewportEvidence; + keyboard?: SnapshotKeyboardBandFact; lineage: IosSnapshotLineage; residue: readonly IosAcquisitionResidue[]; }>; diff --git a/packages/contracts/src/tap-keyboard-occlusion.test.ts b/packages/contracts/src/tap-keyboard-occlusion.test.ts index 350bef09a8..9d2d26312e 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.test.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.test.ts @@ -30,6 +30,7 @@ type FixtureCase = { viewport: Rect | null; nodes: FixtureNode[]; target: { index: number } | { point: { x: number; y: number } }; + keyboard?: SnapshotKeyboardBandFact; expected: { kind: string; frame?: Rect; controlRects?: readonly Rect[] }; }; @@ -74,6 +75,7 @@ test('keyboard tap occlusion agrees with every golden parity table case', () => viewport: fixture.viewport, point, node, + keyboard: fixture.keyboard, }); assert.equal(occlusion.kind, fixture.expected.kind, fixture.name); const surface = @@ -167,7 +169,7 @@ test('a measured band is the band, even in a tree that reports no keyboard at al // The tree rule's answer here is `no-keyboard`. A refusal off a tree with no key nodes can only // have come from the fact, which is the assertion that the rule was not consulted at all. const fixture = tableCase('no keyboard in the tree means nothing to refuse'); - assert.equal(fixture.expected.kind, 'no-keyboard'); + assert.equal(fixture.expected.kind, 'undetermined'); const band: SnapshotKeyboardBandFact = { kind: 'visible', frame: { x: 0, y: 583, width: 402, height: 291 }, @@ -218,8 +220,7 @@ test('an unmeasurable fact leaves every golden-table decision exactly as the tre // either side of the refusal, or the table would stop describing this guard. for (const fixture of loadTable().cases) { const withFact = resolveWithFact(fixture, { - kind: 'unmeasurable', - reason: 'keyboard-frame-query-timeout', + ...(fixture.keyboard ?? { kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' }), }); assert.equal(withFact.kind, fixture.expected.kind, `${fixture.name}: kind`); } diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index 92a1731800..3b0637618a 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -79,7 +79,7 @@ export type KeyboardSurface = { }; export type KeyboardTapOcclusion = - /** No keyboard in the tree: nothing to refuse. */ + /** An explicit observation established absence. */ | { kind: 'no-keyboard' } /** * Keyboard nodes exist but the band cannot be measured — no resolvable viewport, no usable keyboard @@ -312,6 +312,28 @@ function resolveVisibleKeyboardSurface( }; } +/** Observes provider keyboard evidence before depth, scope, or interactive projection. */ +export function observeProviderKeyboard(params: { + nodes: readonly RawSnapshotNode[]; + viewport: Rect | null; + visibility?: boolean | 'unsupported' | 'failed'; +}): SnapshotKeyboardBandFact { + if (params.visibility === false) return { kind: 'visibility-only', visible: false }; + const surface = resolveVisibleKeyboardSurface(params.nodes, params.viewport); + if (surface) return { kind: 'derived', derivation: 'docked-tree', frame: surface.frame }; + if (params.visibility === true) return { kind: 'visibility-only', visible: true }; + return { + kind: 'unmeasurable', + reason: params.nodes.some(isKeyboardAnchorNode) + ? 'keyboard-tree-geometry-unusable' + : params.visibility === 'failed' + ? 'keyboard-visibility-probe-failed' + : params.visibility === 'unsupported' + ? 'keyboard-visibility-probe-unsupported' + : 'keyboard-not-observed', + }; +} + /** * Whether this tap point belongs to the visible keyboard rather than to app content behind it. * @@ -320,12 +342,12 @@ function resolveVisibleKeyboardSurface( * still above the key plane keeps tapping. Callers choose the consequence: an acting element path * refuses, while a coordinate path that never captured this tree discloses instead of refusing. * - * A producer that measured the band publishes it in `keyboard`, and its frame IS the band: no rule + * A measured or acquisition-derived frame in `keyboard` survives projection. Measured frames need no rule * about docking, column runs, or which way the tree was turned is consulted, because the producer * that measured does not need them to answer (#2660). The keyboard's own controls still come from the * tree — which node the keyboard rather than the app is responsible for is a different question from - * where the band is, and only the tree can answer it. Without a fact, or with one that measured - * nothing, every rule below applies exactly as it did. + * where the band is, and only the tree can answer it. Unknown or visibility-only positive observations + * may derive geometry from the current tree; missing nodes leave the decision undetermined. */ export function resolveKeyboardTapOcclusion(params: { nodes: readonly RawSnapshotNode[]; @@ -336,17 +358,18 @@ export function resolveKeyboardTapOcclusion(params: { /** The band this capture's producer measured, when it measured one (#2660). */ keyboard?: SnapshotKeyboardBandFact; }): KeyboardTapOcclusion { - // A producer that looked for the keyboard and found none settles the question the tree rule would - // otherwise be guessing about, stale key nodes in the captured tree included. - if (params.keyboard?.kind === 'absent') return { kind: 'no-keyboard' }; + if ( + params.keyboard?.kind === 'absent' || + (params.keyboard?.kind === 'visibility-only' && !params.keyboard.visible) + ) { + return { kind: 'no-keyboard' }; + } const surface = - params.keyboard?.kind === 'visible' + params.keyboard?.kind === 'visible' || params.keyboard?.kind === 'derived' ? { frame: params.keyboard.frame, controlRects: collectKeyboardControlRects(params.nodes) } : resolveVisibleKeyboardSurface(params.nodes, params.viewport); if (!surface) { - return params.nodes.some(isKeyboardAnchorNode) - ? { kind: 'undetermined' } - : { kind: 'no-keyboard' }; + return { kind: 'undetermined' }; } if (!params.node) return classifyKeyboardPoint(params.point, surface); // A resolved element is excused only by belonging to the keyboard, never by sitting under a key: diff --git a/packages/kernel/src/record.test.ts b/packages/kernel/src/record.test.ts index 7ad8fcaf70..0a9e55f7a3 100644 --- a/packages/kernel/src/record.test.ts +++ b/packages/kernel/src/record.test.ts @@ -74,3 +74,16 @@ test('a band is a plane, so a zero-size frame never reads as visible', () => { }); } }); + +test('keyboard provenance validation rejects malformed derived bands and visibility probes', () => { + for (const [fact, reason] of [ + [{ kind: 'derived', frame: FRAME }, 'unrecognized-derivation'], + [ + { kind: 'derived', derivation: 'docked-tree', frame: { ...FRAME, width: 0 } }, + 'invalid-derived-frame', + ], + [{ kind: 'visibility-only', visible: 'false' }, 'invalid-visibility-probe'], + ] as const) { + assert.deepEqual(readSnapshotKeyboardBandFact(fact), { kind: 'unmeasurable', reason }); + } +}); diff --git a/packages/kernel/src/record.ts b/packages/kernel/src/record.ts index 88f0309c81..192a40d87e 100644 --- a/packages/kernel/src/record.ts +++ b/packages/kernel/src/record.ts @@ -105,7 +105,15 @@ export function readSnapshotKeyboardBandFact(value: unknown): SnapshotKeyboardBa ? { kind: 'unmeasurable', reason: value.reason } : { kind: 'unmeasurable', reason: 'unreported-reason' }; } - if (value.kind === 'visible') { + if (value.kind === 'visibility-only') { + return typeof value.visible === 'boolean' + ? { kind: 'visibility-only', visible: value.visible } + : { kind: 'unmeasurable', reason: 'invalid-visibility-probe' }; + } + if (value.kind === 'derived' && value.derivation !== 'docked-tree') { + return { kind: 'unmeasurable', reason: 'unrecognized-derivation' }; + } + if (value.kind === 'visible' || value.kind === 'derived') { const frame = parseRect(value.frame); // Same rule as `isPositiveFiniteRect` in kernel/rect, inlined: this module is a leaf that // many facades evaluate, and a rect import here would land in every one of their closures. @@ -116,9 +124,10 @@ export function readSnapshotKeyboardBandFact(value: unknown): SnapshotKeyboardBa [frame.x, frame.y, frame.width, frame.height].every(Number.isFinite) && frame.width > 0 && frame.height > 0; - return plottable - ? { kind: 'visible', frame } - : { kind: 'unmeasurable', reason: 'invalid-visible-frame' }; + if (!plottable) return { kind: 'unmeasurable', reason: `invalid-${value.kind}-frame` }; + return value.kind === 'derived' + ? { kind: 'derived', derivation: 'docked-tree', frame } + : { kind: 'visible', frame }; } return { kind: 'unmeasurable', reason: 'unrecognized-kind' }; } diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index aac8cb83aa..8828db4017 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -351,33 +351,17 @@ export type HiddenContentHint = { }; /** - * What a capture's producer can say about the software keyboard on screen, measured while the tree - * was captured rather than rebuilt from it afterwards. - * - * A keyboard is its own system surface, so it never reaches the tree as a covering sibling of app - * content, and a consumer that wants to refuse a tap behind it has to learn where it is from - * somewhere (#2589). A producer that can measure the band directly — the Apple runner, from its - * `app.keyboards` query, and the Android helper, from the input method window in the window list it - * captures — publishes one fact per capture and says nothing else about it. A consumer therefore gets three - * answers and no fourth: a band in the same space as every node rect, a proven absence, or a - * producer that could not look. - * - * A producer that publishes nothing has declared nothing, so absence from a result means the same - * thing as `unmeasurable` — which is why the field stays optional on every carrier, including the - * three client-side paths that rebuild a state from a bare backend result (#2199). Those consumers - * then derive the band from the tree they hold: the rule that stays for the producers that publish - * no fact (#2660). + * Keyboard evidence observed during acquisition, in the capture's node coordinate space. + * `visible` is a measured frame; `derived` is a frame established by the shared docked-tree + * policy before projection; `visibility-only` is a boolean probe with no geometry; + * `unmeasurable` is unknown. `absent` requires an explicit absence observation. + * Missing keyboard nodes or an omitted fact never prove absence. */ export type SnapshotKeyboardBandFact = - /** The band the keyboard occupies, in the same orientation space as this capture's node rects. */ | { kind: 'visible'; frame: Rect } - /** The producer looked for the keyboard and found none. */ + | { kind: 'derived'; derivation: 'docked-tree'; frame: Rect } + | { kind: 'visibility-only'; visible: boolean } | { kind: 'absent' } - /** - * The producer cannot measure the band on this path, with a stable reason code. Typed rather than - * inferred from absence so a log says which path failed to measure without the consumer having to - * guess which producer it was talking to. - */ | { kind: 'unmeasurable'; reason: string }; /** diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts index 5b455e31ba..b333e714f4 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts @@ -98,3 +98,25 @@ test('the Limrun acquired result keeps provider provenance and SDK screen dimens height: LIMRUN_SNAPSHOT_SCREEN.height, }); }); + +test('Limrun carries docked tree evidence and never infers absence from missing nodes', async () => { + const tree = limrunSnapshotTree(); + expect( + (await captureLimrunIosSnapshot(createLimrunSnapshotSession(tree))).acquisition.keyboard, + ).toEqual({ kind: 'unmeasurable', reason: 'keyboard-not-observed' }); + tree.children!.push({ + elementType: 'Keyboard', + frame: { x: 0, y: 140, width: 320, height: 100 }, + }); + expect( + (await captureLimrunIosSnapshot(createLimrunSnapshotSession(tree))).acquisition.keyboard, + ).toEqual({ + kind: 'derived', + derivation: 'docked-tree', + frame: { x: 0, y: 140, width: 320, height: 100 }, + }); + tree.children!.at(-1)!.frame = { x: 100, y: 60, width: 140, height: 80 }; + expect( + (await captureLimrunIosSnapshot(createLimrunSnapshotSession(tree))).acquisition.keyboard, + ).toEqual({ kind: 'unmeasurable', reason: 'keyboard-tree-geometry-unusable' }); +}); diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.ts b/packages/provider-limrun/src/ios-snapshot-adapter.ts index f74c7425f7..04f3aefc91 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.ts @@ -4,6 +4,7 @@ import { } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import type { SnapshotRuntimeAcquiredResult } from '@agent-device/contracts/interactor-types'; import type { LimrunIosSession } from './ios.ts'; +import { observeProviderKeyboard } from '@agent-device/contracts/tap-keyboard-occlusion'; import { flattenIosTree, type IosTreeNode } from './snapshot.ts'; export async function captureLimrunIosSnapshot( @@ -11,9 +12,14 @@ export async function captureLimrunIosSnapshot( ): Promise { const tree = (await session.client.elementTree()) as IosTreeNode[]; const viewport = readLimrunViewport(tree, session.client.deviceInfo); + const nodes = flattenIosTree(tree); return createIosSnapshotAcquisition({ producer: 'limrun-ios-tree', - nodes: flattenIosTree(tree), + nodes, + keyboard: observeProviderKeyboard({ + nodes, + viewport: viewport.kind === 'missing' ? null : viewport.rect, + }), viewport, lineage: { targetId: session.instanceId }, }); diff --git a/packages/provider-webdriver/src/webdriver-client.ts b/packages/provider-webdriver/src/webdriver-client.ts index 06276c4cf9..f5380e68f6 100644 --- a/packages/provider-webdriver/src/webdriver-client.ts +++ b/packages/provider-webdriver/src/webdriver-client.ts @@ -173,12 +173,16 @@ export class WebDriverClient { * driver does not implement — or one that answers with a non-boolean — is * reported as unsupported; everything else throws. */ - async isKeyboardShown(timeoutMs?: number): Promise { + async isKeyboardShown( + timeoutMs?: number, + signal?: AbortSignal, + ): Promise { let value: unknown; try { value = await this.sessionRequest('GET', '/appium/device/is_keyboard_shown', undefined, { retryAttempts: 0, ...(timeoutMs === undefined ? {} : { timeoutMs }), + ...(signal === undefined ? {} : { signal }), }); } catch (error) { if (isWebDriverRouteUnsupported(error)) return 'unsupported'; diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index 6b86e0f670..df23d6a326 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -703,3 +703,57 @@ test('a failing echo probe after enter or dismiss leaves the visibility unknown }); assert.deepEqual(dismissed.transcript, ['probe', 'hide', 'probe']); }); + +test('snapshot carries a bounded visibility probe on both Appium platforms', async () => { + for (const backend of ['android', 'xctest'] as const) { + for (const visible of [true, false, 'unsupported', 'failed'] as const) { + const controller = new AbortController(); + const probe = vi.fn(async () => { + if (visible === 'failed') throw new AppError('COMMAND_FAILED', 'grid failure'); + return visible; + }); + const interactor = createWebDriverInteractor({ + client: { + source: async () => + backend === 'android' + ? ANDROID_ONBOARDING_SOURCE + : '', + isKeyboardShown: probe, + } as unknown as WebDriverClient, + backend, + capabilities: createCloudWebDriverCapabilities({ + provider: 'test', + platform: backend === 'android' ? 'android' : 'ios', + }), + }); + const result = await interactor.snapshot({ signal: controller.signal }); + const keyboard = 'stage' in result ? result.acquisition.keyboard : result.keyboard; + assert.deepEqual( + keyboard, + typeof visible === 'boolean' + ? { kind: 'visibility-only', visible } + : { kind: 'unmeasurable', reason: `keyboard-visibility-probe-${visible}` }, + ); + assert.deepEqual(probe.mock.calls, [[500, controller.signal]]); + } + } +}); + +test('snapshot cancellation during an optional keyboard probe is not swallowed', async () => { + const controller = new AbortController(); + const interactor = createWebDriverInteractor({ + client: { + source: async () => ANDROID_ONBOARDING_SOURCE, + isKeyboardShown: async () => { + controller.abort(); + throw controller.signal.reason; + }, + } as unknown as WebDriverClient, + backend: 'android', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }), + }); + await assert.rejects( + interactor.snapshot({ signal: controller.signal }), + (error: unknown) => error === controller.signal.reason, + ); +}); diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index 35f3d99c72..ad1dbb8d7b 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -1,3 +1,4 @@ +import { observeProviderKeyboard } from '@agent-device/contracts/tap-keyboard-occlusion'; import type { DeviceRotation } from '@agent-device/contracts/device'; import type { BackMode } from '@agent-device/contracts/back-mode'; import type { @@ -324,18 +325,35 @@ class WebDriverInteractor implements Interactor { */ private async captureSource(signal?: AbortSignal) { const source = await this.readSource(signal); + signal?.throwIfAborted(); + const visibility = await this.readSnapshotKeyboardVisibility(signal); if (this.backend === 'xctest') { const { acquireWebDriverIosSnapshot } = await import('./webdriver-ios-snapshot.ts'); - return acquireWebDriverIosSnapshot(source, this.targetId); + return acquireWebDriverIosSnapshot(source, this.targetId, visibility); } const { parseWebDriverSourceFacts } = await import('./webdriver-source.ts'); + const nodes = parseWebDriverSourceFacts(source, 'android').nodes; return { backend: 'android' as const, producer: 'appium-source' as const, - nodes: parseWebDriverSourceFacts(source, 'android').nodes, + nodes, + keyboard: observeProviderKeyboard({ nodes, viewport: null, visibility }), }; } + private async readSnapshotKeyboardVisibility( + signal?: AbortSignal, + ): Promise { + try { + const visibility = await this.client.isKeyboardShown(500, signal); + signal?.throwIfAborted(); + return visibility; + } catch { + signal?.throwIfAborted(); + return 'failed'; + } + } + private async readSource(signal?: AbortSignal): Promise { try { return await this.client.source(signal === undefined ? {} : { signal }); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index 6169bceb28..08d37f2bb5 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -61,3 +61,27 @@ test('Appium iOS adapter carries missing viewport evidence without presenting or { kind: 'missing-viewport', reason: 'not-provided' }, ]); }); + +test('Appium carries visibility-only and derived keyboard facts before projection', () => { + assert.deepEqual(acquireWebDriverIosSnapshot(SOURCE, undefined, true).acquisition.keyboard, { + kind: 'visibility-only', + visible: true, + }); + assert.deepEqual(acquireWebDriverIosSnapshot(SOURCE, undefined, false).acquisition.keyboard, { + kind: 'visibility-only', + visible: false, + }); + assert.deepEqual(acquireWebDriverIosSnapshot(SOURCE).acquisition.keyboard, { + kind: 'unmeasurable', + reason: 'keyboard-not-observed', + }); + const withKeyboard = SOURCE.replace( + '', + '', + ); + assert.deepEqual(acquireWebDriverIosSnapshot(withKeyboard).acquisition.keyboard, { + kind: 'derived', + derivation: 'docked-tree', + frame: { x: 0, y: 580, width: 390, height: 264 }, + }); +}); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 432c5a6597..7a9549c218 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -3,6 +3,7 @@ import { resolveIosViewportEvidenceFromRoots, } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import type { SnapshotRuntimeAcquiredResult } from '@agent-device/contracts/interactor-types'; +import { observeProviderKeyboard } from '@agent-device/contracts/tap-keyboard-occlusion'; import { parseWebDriverSourceFacts } from './webdriver-source.ts'; const APPIUM_PRODUCER = 'appium-source' as const; @@ -11,6 +12,7 @@ const APPIUM_PRODUCER = 'appium-source' as const; export function acquireWebDriverIosSnapshot( source: string, targetId?: string, + visibility?: boolean | 'unsupported' | 'failed', ): SnapshotRuntimeAcquiredResult { const sourceFacts = parseWebDriverSourceFacts(source); const viewport = @@ -20,6 +22,11 @@ export function acquireWebDriverIosSnapshot( producer: APPIUM_PRODUCER, nodes: sourceFacts.nodes, viewport, + keyboard: observeProviderKeyboard({ + nodes: sourceFacts.nodes, + viewport: viewport.kind === 'missing' ? null : viewport.rect, + visibility, + }), lineage: targetId ? { targetId } : {}, }); } diff --git a/src/__tests__/client-snapshot-keyboard-fact.test.ts b/src/__tests__/client-snapshot-keyboard-fact.test.ts index dd8114d73b..cb665885eb 100644 --- a/src/__tests__/client-snapshot-keyboard-fact.test.ts +++ b/src/__tests__/client-snapshot-keyboard-fact.test.ts @@ -61,3 +61,13 @@ test('client capture.snapshot reports no keyboard field for a capture that carri assert.equal('keyboard' in result, false); }); + +test('client preserves provider keyboard provenance through wire normalization', async () => { + for (const keyboard of [ + { kind: 'visibility-only', visible: true }, + { kind: 'visibility-only', visible: false }, + { kind: 'derived', derivation: 'docked-tree', frame: BAND }, + ]) { + assert.deepEqual((await clientAnswering({ keyboard }).capture.snapshot()).keyboard, keyboard); + } +}); diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index b3de99d982..18dbd2fb32 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -52,7 +52,7 @@ Targets: Recovery: Network/typeahead result missing: wait text "Expected result" or wait . A target that should be gone/disappear: use wait absent ; wait exists ... is rejected in favor of the plain selector wait. - Keyboard visible over the next target: an element whose center sits behind the keyboard is refused with tap_keyboard_occludes_target, because the touch would activate a key instead of the target. End editing first -- the app's own Done/Cancel control, or keyboard enter when submission is wanted -- then retry. A target whose center stays above the keys still presses; a raw coordinate behind the keyboard taps anyway and reports the reason in warning. + Keyboard visible over the next target: when a measured or derived keyboard frame covers the tap point, the element is refused with tap_keyboard_occludes_target, because the touch would activate a key instead of the target. End editing first -- the app's own Done/Cancel control, or keyboard enter when submission is wanted -- then retry. Missing keyboard nodes never prove absence; a visibility-only probe cannot establish the covered area. A target whose center stays above the keys still presses; a raw coordinate behind the keyboard taps anyway and reports the reason in warning. Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect) so you know what to dismiss; the cover is diagnosis only -- never target its ref. Dismiss/close it or scroll the target clear, then retry. Sparse or recovered accessibility snapshot: use screenshot as visual truth, leave the bad screen if needed, then retry snapshot -i. Non-hittable success hint: verify with the settled diff or snapshot; retarget by a better ref/selector if the UI did not change. diff --git a/test/integration/provider-scenarios/limrun-ios-snapshot-owner.test.ts b/test/integration/provider-scenarios/limrun-ios-snapshot-owner.test.ts index c629c4690d..fb8e6fbfb4 100644 --- a/test/integration/provider-scenarios/limrun-ios-snapshot-owner.test.ts +++ b/test/integration/provider-scenarios/limrun-ios-snapshot-owner.test.ts @@ -127,6 +127,7 @@ test('Limrun iOS acquisition reaches one host presenter and the public snapshot true, ); assert.equal(result.truncated, undefined); + assert.deepEqual(result.keyboard, { kind: 'unmeasurable', reason: 'keyboard-not-observed' }); assert.deepEqual(result.warnings, [ 'iOS snapshot acquisition does not provide hittability evidence; regular snapshots omit unverified hittability while raw snapshots preserve supplied facts.', 'iOS snapshot acquisition does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index 4cddc48783..afd54f64eb 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -277,6 +277,20 @@ tree failed (`2` is an input method window); `windowCount` counts only the windo when every listed window was read. When it names an input method window, the capture reports the keyboard band as unmeasurable rather than absent. An older helper omits it. +The response-level `keyboard` field tells you what the capture observed: + +- `visible` carries a directly measured `frame` in the same coordinates as node bounds. +- `derived` carries a `frame` inferred from a docked keyboard tree, with `derivation: "docked-tree"`. +- `visibility-only` carries a boolean `visible` from a provider probe, without a frame. +- `absent` means the producer explicitly established absence. +- `unmeasurable` carries a `reason` when the producer cannot establish a frame or visibility. + +Appium iOS and Android snapshots include the visibility probe; Appium iOS and limrun iOS can also +report a derived docked band. Missing keyboard nodes never mean the keyboard is hidden. Element taps +use measured or derived frames to refuse a target behind the keyboard. A positive visibility probe +alone cannot establish occlusion; floating, split, or unnormalized tree geometry stays uncertain. +Run `snapshot -i` again after ending editing before retrying a refused target. + The Android keyboard band is the input method window's bounds, the box around the area it takes touches in. On Android 13 (API 33) and later the helper also checks that this area is one rectangle. A floating keyboard's is not (the panel plus the gesture strip, with app content between them), so the From 0de6c90387423624ab8ce38bf29dec4a1e20ee13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 17:14:01 +0200 Subject: [PATCH 02/12] fix(snapshot): preserve keyboard ownership through projection --- .../src/ios-snapshot-acquisition.ts | 6 +- .../ios-snapshot-runtime-viewport.test.ts | 92 +++++++++++++++++++ .../contracts/src/tap-keyboard-occlusion.ts | 26 ++++-- packages/kernel/src/snapshot.ts | 9 +- .../src/webdriver-interactor.test.ts | 2 +- .../src/webdriver-interactor.ts | 2 +- .../src/webdriver-ios-snapshot.test.ts | 11 +++ .../capture/runtime/snapshot-unchanged.ts | 1 + 8 files changed, 136 insertions(+), 13 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-acquisition.ts b/packages/capture-kit/src/ios-snapshot-acquisition.ts index f9474bf63e..2a331b3921 100644 --- a/packages/capture-kit/src/ios-snapshot-acquisition.ts +++ b/packages/capture-kit/src/ios-snapshot-acquisition.ts @@ -8,6 +8,7 @@ import type { IosSnapshotProducer, IosViewportEvidence, } from '@agent-device/contracts/ios-snapshot'; +import { annotateKeyboardOwnership } from '@agent-device/contracts/tap-keyboard-occlusion'; import { normalizeType } from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import type { @@ -110,7 +111,10 @@ export function createIosSnapshotAcquisition( acquisition: { producer: input.producer, intent: 'full', - nodes: input.nodes, + nodes: + input.keyboard?.kind === 'derived' || input.keyboard?.kind === 'visible' + ? annotateKeyboardOwnership(input.nodes) + : input.nodes, viewport: input.viewport, ...(input.keyboard ? { keyboard: input.keyboard } : {}), lineage: input.lineage, diff --git a/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts index 1a7de6205f..633dfa5309 100644 --- a/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts @@ -1,6 +1,11 @@ +import { buildSnapshotState } from '../../snapshot-state.ts'; import assert from 'node:assert/strict'; import { test } from 'vitest'; import { createIosSnapshotAcquisition } from '@agent-device/capture-kit/ios-snapshot-acquisition'; +import { + observeProviderKeyboard, + resolveKeyboardTapOcclusion, +} from '@agent-device/contracts/tap-keyboard-occlusion'; import { presentIosSnapshotAcquisition } from '../ios-snapshot-runtime.ts'; import type { IosViewportEvidence } from '@agent-device/contracts/ios-snapshot'; import { AppError } from '@agent-device/kernel/errors'; @@ -100,3 +105,90 @@ test('provider keyboard evidence survives projection even when the tree has no k assert.deepEqual(presentIosSnapshotAcquisition(acquired, options).keyboard, keyboard); } }); + +test('a scoped keyboard control retains its acquired ownership after reindexing', () => { + const nodes: RawSnapshotNode[] = [ + ...NODES, + { index: 2, parentIndex: 0, type: 'Keyboard', rect: { x: 0, y: 580, width: 390, height: 264 } }, + { + index: 3, + parentIndex: 2, + type: 'Button', + label: 'Done', + rect: { x: 200, y: 600, width: 100, height: 40 }, + }, + ]; + const keyboard = observeProviderKeyboard({ nodes, viewport: SCREEN }); + const acquired = createIosSnapshotAcquisition({ + producer: 'appium-source', + nodes, + viewport: { kind: 'reported', rect: SCREEN }, + lineage: {}, + keyboard, + }); + for (const raw of [false, true]) { + const result = presentIosSnapshotAcquisition(acquired, { scope: 'Done', raw }); + const node = result.nodes!.find((n) => n.label === 'Done')!; + assert.equal( + resolveKeyboardTapOcclusion({ + nodes: result.nodes!, + viewport: SCREEN, + keyboard: result.keyboard, + node, + point: { x: 250, y: 620 }, + }).kind, + 'clear', + ); + } +}); + +test('depth filtering and state normalization never promote app content into keyboard ownership', () => { + const nodes: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'Application', rect: SCREEN }, + { index: 1, depth: 1, parentIndex: 0, type: 'Other', rect: SCREEN }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'Nested', + rect: { x: 10, y: 10, width: 100, height: 40 }, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'Keyboard', + rect: { x: 0, y: 580, width: 390, height: 264 }, + }, + { + index: 4, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Form', + rect: { x: 200, y: 600, width: 100, height: 40 }, + }, + ]; + const acquired = createIosSnapshotAcquisition({ + producer: 'appium-source', + nodes, + viewport: { kind: 'reported', rect: SCREEN }, + lineage: {}, + keyboard: observeProviderKeyboard({ nodes, viewport: SCREEN }), + }); + const result = presentIosSnapshotAcquisition(acquired, { raw: true, depth: 1 }); + const state = buildSnapshotState(result, { snapshotRaw: true }); + const node = state.nodes.find((n) => n.label === 'Form')!; + assert.equal(node.keyboardOwned, undefined); + assert.equal( + resolveKeyboardTapOcclusion({ + nodes: state.nodes, + viewport: SCREEN, + keyboard: state.keyboard, + node, + point: { x: 250, y: 620 }, + }).kind, + 'occluded', + ); +}); diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index 3b0637618a..9faee8203e 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -82,7 +82,7 @@ export type KeyboardTapOcclusion = /** An explicit observation established absence. */ | { kind: 'no-keyboard' } /** - * Keyboard nodes exist but the band cannot be measured — no resolvable viewport, no usable keyboard + * Keyboard visibility or its band cannot be established — no resolvable viewport, no usable keyboard * rect, or geometry that is not docked to the bottom edge the band would need. Fails open like every * other missing platform fact; the distinction from `no-keyboard` exists so a caller can disclose * that it could not check rather than claiming the target was clear. @@ -303,7 +303,7 @@ function resolveVisibleKeyboardSurface( reportedBottom: Math.max(...reportedRects.map((rect) => rect.y + rect.height)), viewport, }); - if (!frame) return null; + if (!frame || !isPositiveFiniteRect(frame)) return null; const planeIndices = collectKeyboardPlaneIndices(nodes, surfaceNodes); if (!keysFormOneColumnRun({ anchorNodes, anchorRects, planeIndices })) return null; return { @@ -312,6 +312,15 @@ function resolveVisibleKeyboardSurface( }; } +/** Preserves witnessed tree ownership on nodes before projection can remove their ancestors. */ +export function annotateKeyboardOwnership(nodes: readonly RawSnapshotNode[]): RawSnapshotNode[] { + const parents = new Map(nodes.map((node) => [node.index, node.parentIndex] as const)); + const surfaceIndices = new Set(nodes.filter(isKeyboardSurfaceNode).map((node) => node.index)); + return nodes.map((node) => + isKeyboardOwnedNode(node, parents, surfaceIndices) ? { ...node, keyboardOwned: true } : node, + ); +} + /** Observes provider keyboard evidence before depth, scope, or interactive projection. */ export function observeProviderKeyboard(params: { nodes: readonly RawSnapshotNode[]; @@ -342,9 +351,9 @@ export function observeProviderKeyboard(params: { * still above the key plane keeps tapping. Callers choose the consequence: an acting element path * refuses, while a coordinate path that never captured this tree discloses instead of refusing. * - * A measured or acquisition-derived frame in `keyboard` survives projection. Measured frames need no rule - * about docking, column runs, or which way the tree was turned is consulted, because the producer - * that measured does not need them to answer (#2660). The keyboard's own controls still come from the + * A measured or acquisition-derived frame in `keyboard` survives projection. Measured frames need no + * docking, column-run, or orientation inference; derived bands already passed those rules during + * acquisition. Keyboard ownership on surviving nodes retains its acquisition witness after projection. The keyboard's own controls still come from the * tree — which node the keyboard rather than the app is responsible for is a different question from * where the band is, and only the tree can answer it. Unknown or visibility-only positive observations * may derive geometry from the current tree; missing nodes leave the decision undetermined. @@ -378,7 +387,12 @@ export function resolveKeyboardTapOcclusion(params: { const surfaceIndices = new Set( params.nodes.filter(isKeyboardSurfaceNode).map((node) => node.index), ); - if (isKeyboardOwnedNode(params.node, parents, surfaceIndices)) return { kind: 'clear', surface }; + if ( + isKeyboardOwnedNode(params.node, parents, surfaceIndices) || + params.node.keyboardOwned === true + ) { + return { kind: 'clear', surface }; + } return containsPoint(surface.frame, params.point.x, params.point.y) ? { kind: 'occluded', surface } : { kind: 'clear', surface }; diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 8828db4017..d110732f6a 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -312,6 +312,8 @@ export type RawSnapshotNode = { appName?: string; windowTitle?: string; surface?: string; + /** Keyboard ownership witnessed at acquisition; survives projections that remove its ancestors. */ + keyboardOwned?: true; hiddenContentAbove?: boolean; hiddenContentBelow?: boolean; interactionBlocked?: 'covered'; @@ -713,10 +715,9 @@ export type SnapshotState = { */ iosSystemSurfaceBundleId?: string; /** - * iOS and Android: the keyboard band this capture's producer measured, when it measured one. The - * tap-path keyboard guard prefers this over the band it would otherwise derive from `nodes`, - * because a producer that can query the keyboard directly answers in the tree's own space and - * needs no geometry to be plausible (#2660). Absent means the guard measures the tree as before. + * iOS and Android: keyboard observations made during acquisition. Measured and derived frames + * take precedence over reconstruction from the presented tree. Visibility-only and unknown facts + * carry no band; missing nodes or an omitted fact never establish absence. */ keyboard?: SnapshotKeyboardBandFact; /** diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index df23d6a326..2c38753384 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -734,7 +734,7 @@ test('snapshot carries a bounded visibility probe on both Appium platforms', asy ? { kind: 'visibility-only', visible } : { kind: 'unmeasurable', reason: `keyboard-visibility-probe-${visible}` }, ); - assert.deepEqual(probe.mock.calls, [[500, controller.signal]]); + assert.deepEqual(probe.mock.calls, [[1_500, controller.signal]]); } } }); diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index ad1dbb8d7b..1ce75195ca 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -345,7 +345,7 @@ class WebDriverInteractor implements Interactor { signal?: AbortSignal, ): Promise { try { - const visibility = await this.client.isKeyboardShown(500, signal); + const visibility = await this.client.isKeyboardShown(TEXT_ENTRY_PROBE_TIMEOUT_MS, signal); signal?.throwIfAborted(); return visibility; } catch { diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index 08d37f2bb5..8893a65367 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -85,3 +85,14 @@ test('Appium carries visibility-only and derived keyboard facts before projectio frame: { x: 0, y: 580, width: 390, height: 264 }, }); }); + +test('an offscreen provider keyboard never publishes an empty derived frame', () => { + const source = SOURCE.replace( + '', + '', + ); + assert.deepEqual(acquireWebDriverIosSnapshot(source).acquisition.keyboard, { + kind: 'unmeasurable', + reason: 'keyboard-tree-geometry-unusable', + }); +}); diff --git a/src/commands/capture/runtime/snapshot-unchanged.ts b/src/commands/capture/runtime/snapshot-unchanged.ts index 677df844a6..6f654bcd73 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.ts @@ -124,6 +124,7 @@ const PRESENTATION_SCALAR_FIELDS = { appName: true, windowTitle: true, surface: true, + keyboardOwned: true, hiddenContentAbove: true, hiddenContentBelow: true, interactionBlocked: true, From 6e3687975d1b1c6e3222c8b36ab787a4c076dfc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 17:17:33 +0200 Subject: [PATCH 03/12] fix(interaction): recognize scoped keyboard control coordinates --- .../tap-keyboard-occlusion-policy.json | 155 ++++++++++++++++++ .../src/tap-keyboard-occlusion.test.ts | 1 + .../contracts/src/tap-keyboard-occlusion.ts | 9 +- website/docs/docs/snapshots.md | 3 +- 4 files changed, 165 insertions(+), 3 deletions(-) diff --git a/contracts/fixtures/tap-keyboard-occlusion-policy.json b/contracts/fixtures/tap-keyboard-occlusion-policy.json index 45b7ac7292..2b909ce021 100644 --- a/contracts/fixtures/tap-keyboard-occlusion-policy.json +++ b/contracts/fixtures/tap-keyboard-occlusion-policy.json @@ -4232,6 +4232,161 @@ }, "controlRects": [] } + }, + { + "name": "a scoped witnessed keyboard button retains only control coordinate exemptions", + "viewport": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + }, + "nodes": [ + { + "index": 0, + "type": "Button", + "label": "Done", + "keyboardOwned": true, + "rect": { + "x": 200, + "y": 600, + "width": 100, + "height": 40 + } + } + ], + "target": { + "point": { + "x": 250, + "y": 620 + } + }, + "keyboard": { + "kind": "derived", + "derivation": "docked-tree", + "frame": { + "x": 0, + "y": 583, + "width": 402, + "height": 291 + } + }, + "expected": { + "kind": "clear", + "frame": { + "x": 0, + "y": 583, + "width": 402, + "height": 291 + }, + "controlRects": [ + { + "x": 200, + "y": 600, + "width": 100, + "height": 40 + } + ] + } + }, + { + "name": "a scoped witnessed keyboard other retains only control coordinate exemptions", + "viewport": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + }, + "nodes": [ + { + "index": 0, + "type": "Other", + "label": "Container", + "keyboardOwned": true, + "rect": { + "x": 200, + "y": 600, + "width": 100, + "height": 40 + } + } + ], + "target": { + "point": { + "x": 250, + "y": 620 + } + }, + "keyboard": { + "kind": "derived", + "derivation": "docked-tree", + "frame": { + "x": 0, + "y": 583, + "width": 402, + "height": 291 + } + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 0, + "y": 583, + "width": 402, + "height": 291 + }, + "controlRects": [] + } + }, + { + "name": "an offscreen keyboard plane cannot produce a nonpositive band", + "viewport": { + "x": 0, + "y": 0, + "width": 390, + "height": 844 + }, + "nodes": [ + { + "index": 0, + "type": "Application", + "rect": { + "x": 0, + "y": 0, + "width": 390, + "height": 844 + } + }, + { + "index": 1, + "type": "Button", + "label": "Continue", + "parentIndex": 0, + "rect": { + "x": 24, + "y": 700, + "width": 160, + "height": 48 + } + }, + { + "index": 2, + "type": "Keyboard", + "parentIndex": 0, + "rect": { + "x": 0, + "y": 844, + "width": 390, + "height": 233 + } + } + ], + "target": { + "index": 1 + }, + "expected": { + "kind": "undetermined" + } } ] } diff --git a/packages/contracts/src/tap-keyboard-occlusion.test.ts b/packages/contracts/src/tap-keyboard-occlusion.test.ts index 9d2d26312e..fef25e5793 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.test.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.test.ts @@ -18,6 +18,7 @@ type FixtureNode = { index: number; type?: string; role?: string; + keyboardOwned?: true; label?: string; identifier?: string; bundleId?: string; diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index 9faee8203e..ce2ab5ebee 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -279,7 +279,12 @@ function keysFormOneColumnRun(params: { /** The keyboard's own controls as the tree reports them, without deciding where its band is. */ function collectKeyboardControlRects(nodes: readonly RawSnapshotNode[]): Rect[] { - const surfaceNodes = nodes.filter(isKeyboardSurfaceNode); + const surfaceNodes = nodes.filter( + (node) => + isKeyboardSurfaceNode(node) || + (node.keyboardOwned === true && + [node.type, node.role].some((value) => normalizeType(value ?? '') === 'button')), + ); const planeIndices = collectKeyboardPlaneIndices(nodes, surfaceNodes); return usableRects(surfaceNodes.filter((node) => !planeIndices.has(node.index))); } @@ -308,7 +313,7 @@ function resolveVisibleKeyboardSurface( if (!keysFormOneColumnRun({ anchorNodes, anchorRects, planeIndices })) return null; return { frame, - controlRects: usableRects(surfaceNodes.filter((node) => !planeIndices.has(node.index))), + controlRects: collectKeyboardControlRects(nodes), }; } diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index afd54f64eb..f7a311ddcb 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -288,7 +288,8 @@ The response-level `keyboard` field tells you what the capture observed: Appium iOS and Android snapshots include the visibility probe; Appium iOS and limrun iOS can also report a derived docked band. Missing keyboard nodes never mean the keyboard is hidden. Element taps use measured or derived frames to refuse a target behind the keyboard. A positive visibility probe -alone cannot establish occlusion; floating, split, or unnormalized tree geometry stays uncertain. +alone cannot establish occlusion. When tree geometry cannot establish a docked band, the keyboard +remains uncertain. Run `snapshot -i` again after ending editing before retrying a refused target. The Android keyboard band is the input method window's bounds, the box around the area it takes From 80252926136b4802b3c9714d90ef5958fdf74ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 18:02:08 +0200 Subject: [PATCH 04/12] refactor(snapshot): separate keyboard frame validation --- packages/kernel/src/record.ts | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/packages/kernel/src/record.ts b/packages/kernel/src/record.ts index 192a40d87e..d965ed8017 100644 --- a/packages/kernel/src/record.ts +++ b/packages/kernel/src/record.ts @@ -114,24 +114,31 @@ export function readSnapshotKeyboardBandFact(value: unknown): SnapshotKeyboardBa return { kind: 'unmeasurable', reason: 'unrecognized-derivation' }; } if (value.kind === 'visible' || value.kind === 'derived') { - const frame = parseRect(value.frame); - // Same rule as `isPositiveFiniteRect` in kernel/rect, inlined: this module is a leaf that - // many facades evaluate, and a rect import here would land in every one of their closures. - // The viewport re-read (#3182) is the other side of that seam and lives in kernel/rect as - // `readSnapshotViewportSize`, because a viewport IS a rect; readers take the rect module there. - const plottable = - frame !== undefined && - [frame.x, frame.y, frame.width, frame.height].every(Number.isFinite) && - frame.width > 0 && - frame.height > 0; - if (!plottable) return { kind: 'unmeasurable', reason: `invalid-${value.kind}-frame` }; - return value.kind === 'derived' - ? { kind: 'derived', derivation: 'docked-tree', frame } - : { kind: 'visible', frame }; + return readKeyboardFrameFact(value.frame, value.kind); } return { kind: 'unmeasurable', reason: 'unrecognized-kind' }; } +function readKeyboardFrameFact( + value: unknown, + kind: 'visible' | 'derived', +): SnapshotKeyboardBandFact { + const frame = parseRect(value); + // Same rule as `isPositiveFiniteRect` in kernel/rect, inlined: this module is a leaf that + // many facades evaluate, and a rect import here would land in every one of their closures. + // The viewport re-read (#3182) is the other side of that seam and lives in kernel/rect as + // `readSnapshotViewportSize`, because a viewport IS a rect; readers take the rect module there. + const plottable = + frame !== undefined && + [frame.x, frame.y, frame.width, frame.height].every(Number.isFinite) && + frame.width > 0 && + frame.height > 0; + if (!plottable) return { kind: 'unmeasurable', reason: `invalid-${kind}-frame` }; + return kind === 'derived' + ? { kind: 'derived', derivation: 'docked-tree', frame } + : { kind: 'visible', frame }; +} + export function parsePoint(value: unknown): Point | undefined { if (!isRecord(value)) return undefined; const x = readNumberField(value, 'x'); From 8745fd86dab5c0c1629dbc41379b6d607bf57073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 18:49:40 +0200 Subject: [PATCH 05/12] fix(snapshot): preserve acquisition geometry decisions through projection --- .../tap-keyboard-occlusion-policy.json | 153 +++++++----------- .../ios-snapshot-runtime-viewport.test.ts | 51 ++++++ .../contracts/src/tap-keyboard-occlusion.ts | 24 ++- .../src/webdriver-interactor.test.ts | 35 ++++ .../src/webdriver-interactor.ts | 17 +- .../cloud-webdriver-ios-text-entry.test.ts | 12 +- 6 files changed, 189 insertions(+), 103 deletions(-) diff --git a/contracts/fixtures/tap-keyboard-occlusion-policy.json b/contracts/fixtures/tap-keyboard-occlusion-policy.json index 2b909ce021..4e26313c8e 100644 --- a/contracts/fixtures/tap-keyboard-occlusion-policy.json +++ b/contracts/fixtures/tap-keyboard-occlusion-policy.json @@ -1,5 +1,5 @@ { - "description": "Golden table for keyboard occlusion on interaction tap points (#2589), consumed by packages/contracts/src/tap-keyboard-occlusion.ts. A visible software keyboard is its own system surface: it is never a covering sibling of app content, so the same-window occlusion classifier cannot see it, and it leaves the app window rect intact, so the viewport rule passes a covered tab bar. The band runs from the topmost keyboard node down to the bottom of the viewport, across the keyboard's own columns; the tap point is the point the interaction dispatches: the rect center the platform aims at for a ref handed over by name, or the touch point a parent with interactive descendants resolves instead. `nodes` are captured snapshot nodes as they appear in the projection the interaction resolved against \u2014 note that regular and interactive projections prune the keyboard's own container, leaving Key nodes beside app chrome. Three rules decide whether reported geometry may be measured at all. Docking: the band is derived only where the keyboard's own reported geometry reaches the viewport's bottom edge within 80 units \u2014 measured on iPhone 17 Pro (26.2) the reported key plane stops 58 pt short of an 874 pt portrait viewport and lands exactly on a 402 pt landscape one, and on iPad Pro 11-inch (M4) 3 pt short of a 1210 pt viewport \u2014 so an iPad floating keyboard, an app-drawn keypad, or an IME that stops above its own navigation bar stays unmeasured and fails open. The budget is counted in the same units as the rects it guards, which is what a measured Android IME asks for rather than what a dp table would give: with three-button navigation on Pixel 7 (Android 16, 1080x2400 at 420 dpi), Gboard's reported region runs to the physical display bottom at 2400 while the 126 px navigation bar occupies 2274..2400 inside it, so no Android keyboard measured so far needs slack for the bar. Width: geometry that comes back taller than it is wide did not arrive in the app's orientation space, i.e. it came from a capture that normalized no space \u2014 the producers that can name the app's interface orientation either turn it back (the runner's tree tiers) or refuse the capture (the Simulator AX bridge) \u2014 and is refused measurement for the same reason. Tiling: the keys themselves must form one unbroken run of columns, because a split iPad keyboard docks both clusters at the bottom edge while its surface container still spans the whole screen, and the app content between the clusters is visible and tappable. Reported seams are zero on every measured tree \u2014 iPhone 17 Pro's reported row adjoins across 395 pt, iPad Pro 11-inch's across 743.5 pt, and Gboard's across 1070 px \u2014 so the allowance covers rounding rather than layout, and a gapped band is refused measurement like the two above. That rule is deliberately blunt in both directions, and the last two cases name the trade: a sparse layout such as the iOS emoji panel reports corner keys over a wide container, does not tile, and reads `undetermined` \u2014 the guard abstains there rather than inventing which gaps are app content; and a projection that reports the container with no keys at all has nothing left to check, so the band is the container's own claim. Every captured iOS and Android keyboard measured so far reports its keys, so the second shape is the harness's, not a captured one. `target` names either a resolved node index or a bare coordinate point; both paths read one decision and choose their own consequence. A case that reaches a band declares its whole shape \u2014 the frame the refusal and the disclosure both quote, and the control rects that excuse a bare coordinate, which is where plane exclusion shows up. Provider facts distinguish measured visible frames, derived docked-tree bands, visibility-only probes, explicit absence, and unknown. Missing keyboard nodes are undetermined, never absence. A carried derived band was established before projection and stays usable after projection removes keyboard nodes; it obeys the same docking/orientation/tiling rules at acquisition. A measured floating frame needs no bottom extension.", + "description": "Golden table for keyboard occlusion on interaction tap points (#2589), consumed by packages/contracts/src/tap-keyboard-occlusion.ts. A visible software keyboard is its own system surface: it is never a covering sibling of app content, so the same-window occlusion classifier cannot see it, and it leaves the app window rect intact, so the viewport rule passes a covered tab bar. The band runs from the topmost keyboard node down to the bottom of the viewport, across the keyboard's own columns; the tap point is the point the interaction dispatches: the rect center the platform aims at for a ref handed over by name, or the touch point a parent with interactive descendants resolves instead. `nodes` are captured snapshot nodes as they appear in the projection the interaction resolved against — note that regular and interactive projections prune the keyboard's own container, leaving Key nodes beside app chrome. Three rules decide whether reported geometry may be measured at all. Docking: the band is derived only where the keyboard's own reported geometry reaches the viewport's bottom edge within 80 units — measured on iPhone 17 Pro (26.2) the reported key plane stops 58 pt short of an 874 pt portrait viewport and lands exactly on a 402 pt landscape one, and on iPad Pro 11-inch (M4) 3 pt short of a 1210 pt viewport — so an iPad floating keyboard, an app-drawn keypad, or an IME that stops above its own navigation bar stays unmeasured and fails open. The budget is counted in the same units as the rects it guards, which is what a measured Android IME asks for rather than what a dp table would give: with three-button navigation on Pixel 7 (Android 16, 1080x2400 at 420 dpi), Gboard's reported region runs to the physical display bottom at 2400 while the 126 px navigation bar occupies 2274..2400 inside it, so no Android keyboard measured so far needs slack for the bar. Width: geometry that comes back taller than it is wide did not arrive in the app's orientation space, i.e. it came from a capture that normalized no space — the producers that can name the app's interface orientation either turn it back (the runner's tree tiers) or refuse the capture (the Simulator AX bridge) — and is refused measurement for the same reason. Tiling: the keys themselves must form one unbroken run of columns, because a split iPad keyboard docks both clusters at the bottom edge while its surface container still spans the whole screen, and the app content between the clusters is visible and tappable. Reported seams are zero on every measured tree — iPhone 17 Pro's reported row adjoins across 395 pt, iPad Pro 11-inch's across 743.5 pt, and Gboard's across 1070 px — so the allowance covers rounding rather than layout, and a gapped band is refused measurement like the two above. That rule is deliberately blunt in both directions, and the last two cases name the trade: a sparse layout such as the iOS emoji panel reports corner keys over a wide container, does not tile, and reads `undetermined` — the guard abstains there rather than inventing which gaps are app content; and a projection that reports the container with no keys at all has nothing left to check, so the band is the container's own claim. Every captured iOS and Android keyboard measured so far reports its keys, so the second shape is the harness's, not a captured one. `target` names either a resolved node index or a bare coordinate point; both paths read one decision and choose their own consequence. A case that reaches a band declares its whole shape — the frame the refusal and the disclosure both quote, and the control rects that excuse a bare coordinate, which is where plane exclusion shows up. Provider facts distinguish measured visible frames, derived docked-tree bands, visibility-only probes, explicit absence, and unknown. Missing keyboard nodes are undetermined, never absence. A carried derived band was established before projection and stays usable after projection removes keyboard nodes; it obeys the same docking/orientation/tiling rules at acquisition. A measured floating frame needs no bottom extension.", "constants": { "occlusionReason": "tap_keyboard_occludes_target" }, @@ -3995,28 +3995,12 @@ }, { "name": "a carried docked-tree band survives projection that removes all keyboard nodes", - "viewport": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - }, + "viewport": null, "nodes": [ { "index": 0, - "type": "Application", - "rect": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - } - }, - { - "index": 1, "type": "Button", "label": "Form", - "parentIndex": 0, "rect": { "x": 162, "y": 795, @@ -4026,7 +4010,7 @@ } ], "target": { - "index": 1 + "index": 0 }, "keyboard": { "kind": "derived", @@ -4051,38 +4035,32 @@ }, { "name": "a positive visibility probe without geometry cannot decide occlusion", - "viewport": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - }, + "viewport": null, "nodes": [ { "index": 0, - "type": "Application", - "rect": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - } - }, - { - "index": 1, "type": "Button", "label": "Form", - "parentIndex": 0, "rect": { "x": 162, "y": 795, "width": 74, "height": 54 } + }, + { + "index": 1, + "type": "Keyboard", + "rect": { + "x": 0, + "y": 583, + "width": 402, + "height": 291 + } } ], "target": { - "index": 1 + "index": 0 }, "keyboard": { "kind": "visibility-only", @@ -4094,28 +4072,12 @@ }, { "name": "a negative visibility probe establishes absence", - "viewport": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - }, + "viewport": null, "nodes": [ { "index": 0, - "type": "Application", - "rect": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - } - }, - { - "index": 1, "type": "Button", "label": "Form", - "parentIndex": 0, "rect": { "x": 162, "y": 795, @@ -4125,7 +4087,7 @@ } ], "target": { - "index": 1 + "index": 0 }, "keyboard": { "kind": "visibility-only", @@ -4146,19 +4108,8 @@ "nodes": [ { "index": 0, - "type": "Application", - "rect": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - } - }, - { - "index": 1, "type": "Button", "label": "Form", - "parentIndex": 0, "rect": { "x": 162, "y": 795, @@ -4168,7 +4119,7 @@ } ], "target": { - "index": 1 + "index": 0 }, "keyboard": { "kind": "unmeasurable", @@ -4180,28 +4131,12 @@ }, { "name": "a directly measured floating keyboard frame can refuse the target in its rectangle", - "viewport": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - }, + "viewport": null, "nodes": [ { "index": 0, - "type": "Application", - "rect": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - } - }, - { - "index": 1, "type": "Button", "label": "Form", - "parentIndex": 0, "rect": { "x": 162, "y": 795, @@ -4211,7 +4146,7 @@ } ], "target": { - "index": 1 + "index": 0 }, "keyboard": { "kind": "visible", @@ -4235,12 +4170,7 @@ }, { "name": "a scoped witnessed keyboard button retains only control coordinate exemptions", - "viewport": { - "x": 0, - "y": 0, - "width": 402, - "height": 874 - }, + "viewport": null, "nodes": [ { "index": 0, @@ -4387,6 +4317,47 @@ "expected": { "kind": "undetermined" } + }, + { + "name": "a tree band rejected at acquisition stays unknown after projection removes its gaps", + "viewport": { + "x": 0, + "y": 0, + "width": 402, + "height": 874 + }, + "nodes": [ + { + "index": 0, + "type": "Keyboard", + "rect": { + "x": 0, + "y": 583, + "width": 402, + "height": 291 + } + }, + { + "index": 1, + "type": "Button", + "rect": { + "x": 200, + "y": 800, + "width": 100, + "height": 40 + } + } + ], + "target": { + "index": 1 + }, + "keyboard": { + "kind": "unmeasurable", + "reason": "keyboard-tree-geometry-unusable" + }, + "expected": { + "kind": "undetermined" + } } ] } diff --git a/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts index 633dfa5309..b36ba008c5 100644 --- a/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts @@ -192,3 +192,54 @@ test('depth filtering and state normalization never promote app content into key 'occluded', ); }); + +test('depth projection cannot promote a rejected split keyboard into a trusted band', () => { + const viewport = SCREEN; + const nodes: RawSnapshotNode[] = [ + { index: 0, depth: 0, type: 'Application', rect: viewport }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Between clusters', + rect: { x: 150, y: 650, width: 40, height: 20 }, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'Keyboard', + rect: { x: 0, y: 580, width: 390, height: 264 }, + }, + ...[0, 310].map((x, offset) => ({ + index: 3 + offset, + depth: 2, + parentIndex: 2, + type: 'Key', + rect: { x, y: 600, width: 80, height: 40 }, + })), + ]; + for (const visibility of [undefined, true] as const) { + const acquired = createIosSnapshotAcquisition({ + producer: 'appium-source', + nodes, + viewport: { kind: 'reported', rect: viewport }, + lineage: {}, + keyboard: observeProviderKeyboard({ nodes, viewport, visibility }), + }); + const projected = presentIosSnapshotAcquisition(acquired, { raw: true, depth: 1 }); + const state = buildSnapshotState(projected, { snapshotRaw: true }); + const node = state.nodes.find((n) => n.label === 'Between clusters')!; + assert.equal( + resolveKeyboardTapOcclusion({ + nodes: state.nodes, + viewport, + keyboard: state.keyboard, + node, + point: { x: 170, y: 660 }, + }).kind, + 'undetermined', + ); + } +}); diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index ce2ab5ebee..f0d82ff584 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -35,6 +35,8 @@ import { normalizeType } from './snapshot-text.ts'; /** The one reason a tap refuses because the visible keyboard owns its tap point. */ export const TAP_KEYBOARD_OCCLUDES_TARGET_REASON = 'tap_keyboard_occludes_target'; +const KEYBOARD_TREE_GEOMETRY_UNUSABLE_REASON = 'keyboard-tree-geometry-unusable'; + /** * The hint every owner publishes beside the reason, naming the recovery that actually works. * `keyboard dismiss` taps the keyboard's own dismiss control and refuses when it exposes none @@ -339,7 +341,7 @@ export function observeProviderKeyboard(params: { return { kind: 'unmeasurable', reason: params.nodes.some(isKeyboardAnchorNode) - ? 'keyboard-tree-geometry-unusable' + ? KEYBOARD_TREE_GEOMETRY_UNUSABLE_REASON : params.visibility === 'failed' ? 'keyboard-visibility-probe-failed' : params.visibility === 'unsupported' @@ -348,6 +350,16 @@ export function observeProviderKeyboard(params: { }; } +function keyboardFactDisallowsTreeDerivation( + keyboard: SnapshotKeyboardBandFact | undefined, +): boolean { + return ( + keyboard?.kind === 'visibility-only' || + (keyboard?.kind === 'unmeasurable' && + keyboard.reason === KEYBOARD_TREE_GEOMETRY_UNUSABLE_REASON) + ); +} + /** * Whether this tap point belongs to the visible keyboard rather than to app content behind it. * @@ -358,10 +370,13 @@ export function observeProviderKeyboard(params: { * * A measured or acquisition-derived frame in `keyboard` survives projection. Measured frames need no * docking, column-run, or orientation inference; derived bands already passed those rules during - * acquisition. Keyboard ownership on surviving nodes retains its acquisition witness after projection. The keyboard's own controls still come from the + * acquisition. Keyboard ownership on surviving nodes retains its acquisition witness after projection. + * The keyboard's own controls still come from the * tree — which node the keyboard rather than the app is responsible for is a different question from - * where the band is, and only the tree can answer it. Unknown or visibility-only positive observations - * may derive geometry from the current tree; missing nodes leave the decision undetermined. + * where the band is, and only the tree can answer it. Visibility-only observations carry no band, and + * rejected acquisition geometry stays rejected + * after projection. Other failed direct measurements may still use the tree; missing nodes remain + * undetermined. */ export function resolveKeyboardTapOcclusion(params: { nodes: readonly RawSnapshotNode[]; @@ -378,6 +393,7 @@ export function resolveKeyboardTapOcclusion(params: { ) { return { kind: 'no-keyboard' }; } + if (keyboardFactDisallowsTreeDerivation(params.keyboard)) return { kind: 'undetermined' }; const surface = params.keyboard?.kind === 'visible' || params.keyboard?.kind === 'derived' ? { frame: params.keyboard.frame, controlRects: collectKeyboardControlRects(params.nodes) } diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index 2c38753384..1acfea8868 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test, vi } from 'vitest'; +import { resolveKeyboardTapOcclusion } from '@agent-device/contracts/tap-keyboard-occlusion'; import { buildGesturePlan } from '@agent-device/contracts/gesture-plan'; import { AppError } from '@agent-device/kernel/errors'; import { createCloudWebDriverCapabilities } from './capabilities.ts'; @@ -757,3 +758,37 @@ test('snapshot cancellation during an optional keyboard probe is not swallowed', (error: unknown) => error === controller.signal.reason, ); }); + +test('Android acquisition derives a docked IME before projection removes its tree', async () => { + const interactor = createWebDriverInteractor({ + client: { + source: async () => + '', + isKeyboardShown: async () => true, + } as unknown as WebDriverClient, + backend: 'android', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }), + }); + const snapshot = await interactor.snapshot(); + assert.ok(!('stage' in snapshot)); + assert.deepEqual(snapshot.keyboard, { + kind: 'derived', + derivation: 'docked-tree', + frame: { x: 0, y: 1500, width: 1080, height: 900 }, + }); + const form = snapshot.nodes.find((node) => node.label === 'Form')!; + assert.equal( + resolveKeyboardTapOcclusion({ + nodes: [form], + node: form, + viewport: null, + point: { x: 200, y: 1820 }, + keyboard: snapshot.keyboard, + }).kind, + 'occluded', + ); + assert.equal( + snapshot.nodes.find((node) => node.identifier?.includes('input_view'))?.keyboardOwned, + true, + ); +}); diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index 1ce75195ca..9b44688cd7 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -1,4 +1,8 @@ -import { observeProviderKeyboard } from '@agent-device/contracts/tap-keyboard-occlusion'; +import { resolveViewportRect } from '@agent-device/contracts/snapshot'; +import { + annotateKeyboardOwnership, + observeProviderKeyboard, +} from '@agent-device/contracts/tap-keyboard-occlusion'; import type { DeviceRotation } from '@agent-device/contracts/device'; import type { BackMode } from '@agent-device/contracts/back-mode'; import type { @@ -18,6 +22,7 @@ import { } from '@agent-device/contracts/scroll-gesture'; import type { TvRemoteButton } from '@agent-device/contracts/tv-remote'; import type { SettingOptions } from '@agent-device/contracts/settings'; +import { isPositiveFiniteRect, pickLargestRect } from '@agent-device/kernel/rect'; import { AppError } from '@agent-device/kernel/errors'; import { capabilitySupported, @@ -332,12 +337,18 @@ class WebDriverInteractor implements Interactor { return acquireWebDriverIosSnapshot(source, this.targetId, visibility); } const { parseWebDriverSourceFacts } = await import('./webdriver-source.ts'); - const nodes = parseWebDriverSourceFacts(source, 'android').nodes; + const nodes = annotateKeyboardOwnership(parseWebDriverSourceFacts(source, 'android').nodes); + const appRect = pickLargestRect( + nodes.flatMap((node) => + !node.keyboardOwned && isPositiveFiniteRect(node.rect) ? [node.rect] : [], + ), + ); + const viewport = appRect ? resolveViewportRect(nodes, appRect) : null; return { backend: 'android' as const, producer: 'appium-source' as const, nodes, - keyboard: observeProviderKeyboard({ nodes, viewport: null, visibility }), + keyboard: observeProviderKeyboard({ nodes, viewport, visibility }), }; } diff --git a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts index 249bc050c7..4523711672 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts @@ -154,9 +154,8 @@ test('cloud iOS fill lands text in the field it tapped', async () => { assert.equal(data.textEntryReadiness, 'focused-element'); // The keys reached the focused field, not the void. assert.equal(server.fieldValues.email, 'user@example.com'); - // Both pre-tap readings, the tap, then polling until focus lands. The keys - // come last — that ordering IS the fix. assert.deepEqual(textEntryTranscript(server), [ + 'keyboard', 'keyboard', 'active', 'tap', @@ -172,9 +171,12 @@ test('cloud iOS fill lands text in the field it tapped', async () => { { ...leaseFlags(lease.leaseId), snapshotInteractiveOnly: true }, { meta: leaseMeta(lease.leaseId) }, ); - const nodes = assertRpcOk<{ nodes?: Array<{ identifier?: string; value?: string }> }>( - snapshot, - ).nodes; + const snapshotData = assertRpcOk<{ + nodes?: Array<{ identifier?: string; value?: string }>; + keyboard?: { kind: string; visible?: boolean }; + }>(snapshot); + assert.deepEqual(snapshotData.keyboard, { kind: 'visibility-only', visible: true }); + const nodes = snapshotData.nodes; assert.equal(nodes?.find((node) => node.identifier === 'email')?.value, 'user@example.com'); }); }, 15_000); From 07ad69bb8319f9ec44a14f3c8ae80638e9c8552a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 19:04:37 +0200 Subject: [PATCH 06/12] fix(snapshot): retain lazy provider acquisition imports --- packages/capture-kit/src/ios-snapshot-acquisition.ts | 6 +----- .../__tests__/ios-snapshot-runtime-viewport.test.ts | 3 ++- packages/provider-limrun/src/ios-snapshot-adapter.ts | 7 +++++-- .../provider-webdriver/src/webdriver-interactor.test.ts | 1 + packages/provider-webdriver/src/webdriver-interactor.ts | 8 +++----- .../provider-webdriver/src/webdriver-ios-snapshot.test.ts | 6 ++++-- packages/provider-webdriver/src/webdriver-ios-snapshot.ts | 7 +++++-- 7 files changed, 21 insertions(+), 17 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-acquisition.ts b/packages/capture-kit/src/ios-snapshot-acquisition.ts index 2a331b3921..f9474bf63e 100644 --- a/packages/capture-kit/src/ios-snapshot-acquisition.ts +++ b/packages/capture-kit/src/ios-snapshot-acquisition.ts @@ -8,7 +8,6 @@ import type { IosSnapshotProducer, IosViewportEvidence, } from '@agent-device/contracts/ios-snapshot'; -import { annotateKeyboardOwnership } from '@agent-device/contracts/tap-keyboard-occlusion'; import { normalizeType } from '@agent-device/contracts/snapshot'; import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import type { @@ -111,10 +110,7 @@ export function createIosSnapshotAcquisition( acquisition: { producer: input.producer, intent: 'full', - nodes: - input.keyboard?.kind === 'derived' || input.keyboard?.kind === 'visible' - ? annotateKeyboardOwnership(input.nodes) - : input.nodes, + nodes: input.nodes, viewport: input.viewport, ...(input.keyboard ? { keyboard: input.keyboard } : {}), lineage: input.lineage, diff --git a/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts index b36ba008c5..932231fca3 100644 --- a/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts +++ b/packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts @@ -3,6 +3,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { createIosSnapshotAcquisition } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import { + annotateKeyboardOwnership, observeProviderKeyboard, resolveKeyboardTapOcclusion, } from '@agent-device/contracts/tap-keyboard-occlusion'; @@ -121,7 +122,7 @@ test('a scoped keyboard control retains its acquired ownership after reindexing' const keyboard = observeProviderKeyboard({ nodes, viewport: SCREEN }); const acquired = createIosSnapshotAcquisition({ producer: 'appium-source', - nodes, + nodes: annotateKeyboardOwnership(nodes), viewport: { kind: 'reported', rect: SCREEN }, lineage: {}, keyboard, diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.ts b/packages/provider-limrun/src/ios-snapshot-adapter.ts index 04f3aefc91..aec4560e85 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.ts @@ -4,7 +4,10 @@ import { } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import type { SnapshotRuntimeAcquiredResult } from '@agent-device/contracts/interactor-types'; import type { LimrunIosSession } from './ios.ts'; -import { observeProviderKeyboard } from '@agent-device/contracts/tap-keyboard-occlusion'; +import { + annotateKeyboardOwnership, + observeProviderKeyboard, +} from '@agent-device/contracts/tap-keyboard-occlusion'; import { flattenIosTree, type IosTreeNode } from './snapshot.ts'; export async function captureLimrunIosSnapshot( @@ -12,7 +15,7 @@ export async function captureLimrunIosSnapshot( ): Promise { const tree = (await session.client.elementTree()) as IosTreeNode[]; const viewport = readLimrunViewport(tree, session.client.deviceInfo); - const nodes = flattenIosTree(tree); + const nodes = annotateKeyboardOwnership(flattenIosTree(tree)); return createIosSnapshotAcquisition({ producer: 'limrun-ios-tree', nodes, diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index 1acfea8868..3dd89348b8 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -776,6 +776,7 @@ test('Android acquisition derives a docked IME before projection removes its tre derivation: 'docked-tree', frame: { x: 0, y: 1500, width: 1080, height: 900 }, }); + assert.ok(snapshot.nodes); const form = snapshot.nodes.find((node) => node.label === 'Form')!; assert.equal( resolveKeyboardTapOcclusion({ diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index 9b44688cd7..cc6da5f97d 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -1,8 +1,3 @@ -import { resolveViewportRect } from '@agent-device/contracts/snapshot'; -import { - annotateKeyboardOwnership, - observeProviderKeyboard, -} from '@agent-device/contracts/tap-keyboard-occlusion'; import type { DeviceRotation } from '@agent-device/contracts/device'; import type { BackMode } from '@agent-device/contracts/back-mode'; import type { @@ -337,6 +332,9 @@ class WebDriverInteractor implements Interactor { return acquireWebDriverIosSnapshot(source, this.targetId, visibility); } const { parseWebDriverSourceFacts } = await import('./webdriver-source.ts'); + const { resolveViewportRect } = await import('@agent-device/contracts/snapshot'); + const { annotateKeyboardOwnership, observeProviderKeyboard } = + await import('@agent-device/contracts/tap-keyboard-occlusion'); const nodes = annotateKeyboardOwnership(parseWebDriverSourceFacts(source, 'android').nodes); const appRect = pickLargestRect( nodes.flatMap((node) => diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index 8893a65367..bb0142faa0 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -77,9 +77,11 @@ test('Appium carries visibility-only and derived keyboard facts before projectio }); const withKeyboard = SOURCE.replace( '', - '', + '', ); - assert.deepEqual(acquireWebDriverIosSnapshot(withKeyboard).acquisition.keyboard, { + const acquired = acquireWebDriverIosSnapshot(withKeyboard).acquisition; + assert.equal(acquired.nodes.find((node) => node.label === 'Done')?.keyboardOwned, true); + assert.deepEqual(acquired.keyboard, { kind: 'derived', derivation: 'docked-tree', frame: { x: 0, y: 580, width: 390, height: 264 }, diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 7a9549c218..3d0f7f0e7d 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -3,7 +3,10 @@ import { resolveIosViewportEvidenceFromRoots, } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import type { SnapshotRuntimeAcquiredResult } from '@agent-device/contracts/interactor-types'; -import { observeProviderKeyboard } from '@agent-device/contracts/tap-keyboard-occlusion'; +import { + annotateKeyboardOwnership, + observeProviderKeyboard, +} from '@agent-device/contracts/tap-keyboard-occlusion'; import { parseWebDriverSourceFacts } from './webdriver-source.ts'; const APPIUM_PRODUCER = 'appium-source' as const; @@ -20,7 +23,7 @@ export function acquireWebDriverIosSnapshot( ({ kind: 'missing', reason: 'not-provided' } as const); return createIosSnapshotAcquisition({ producer: APPIUM_PRODUCER, - nodes: sourceFacts.nodes, + nodes: annotateKeyboardOwnership(sourceFacts.nodes), viewport, keyboard: observeProviderKeyboard({ nodes: sourceFacts.nodes, From 0b5e645372957901aac7aa1c8820ef8d21085e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 13 Oct 2026 19:27:06 +0200 Subject: [PATCH 07/12] fix(snapshot): preserve keyboard evidence and ownership boundaries --- .../tap-keyboard-occlusion-policy.json | 245 +++++++++++++++++- packages/contracts/src/interactor-types.ts | 9 +- .../src/tap-keyboard-occlusion.test.ts | 21 +- .../contracts/src/tap-keyboard-occlusion.ts | 71 +++-- packages/kernel/src/record.ts | 9 +- .../src/webdriver-interactor.test.ts | 10 +- .../src/webdriver-interactor.ts | 10 +- .../src/webdriver-ios-snapshot.test.ts | 29 +++ .../runtime/snapshot-unchanged.test.ts | 4 + src/commands/schema/cli-help-workflows.ts | 2 +- website/docs/docs/snapshots.md | 4 +- 11 files changed, 354 insertions(+), 60 deletions(-) diff --git a/contracts/fixtures/tap-keyboard-occlusion-policy.json b/contracts/fixtures/tap-keyboard-occlusion-policy.json index 4e26313c8e..36d3d45025 100644 --- a/contracts/fixtures/tap-keyboard-occlusion-policy.json +++ b/contracts/fixtures/tap-keyboard-occlusion-policy.json @@ -1,5 +1,5 @@ { - "description": "Golden table for keyboard occlusion on interaction tap points (#2589), consumed by packages/contracts/src/tap-keyboard-occlusion.ts. A visible software keyboard is its own system surface: it is never a covering sibling of app content, so the same-window occlusion classifier cannot see it, and it leaves the app window rect intact, so the viewport rule passes a covered tab bar. The band runs from the topmost keyboard node down to the bottom of the viewport, across the keyboard's own columns; the tap point is the point the interaction dispatches: the rect center the platform aims at for a ref handed over by name, or the touch point a parent with interactive descendants resolves instead. `nodes` are captured snapshot nodes as they appear in the projection the interaction resolved against — note that regular and interactive projections prune the keyboard's own container, leaving Key nodes beside app chrome. Three rules decide whether reported geometry may be measured at all. Docking: the band is derived only where the keyboard's own reported geometry reaches the viewport's bottom edge within 80 units — measured on iPhone 17 Pro (26.2) the reported key plane stops 58 pt short of an 874 pt portrait viewport and lands exactly on a 402 pt landscape one, and on iPad Pro 11-inch (M4) 3 pt short of a 1210 pt viewport — so an iPad floating keyboard, an app-drawn keypad, or an IME that stops above its own navigation bar stays unmeasured and fails open. The budget is counted in the same units as the rects it guards, which is what a measured Android IME asks for rather than what a dp table would give: with three-button navigation on Pixel 7 (Android 16, 1080x2400 at 420 dpi), Gboard's reported region runs to the physical display bottom at 2400 while the 126 px navigation bar occupies 2274..2400 inside it, so no Android keyboard measured so far needs slack for the bar. Width: geometry that comes back taller than it is wide did not arrive in the app's orientation space, i.e. it came from a capture that normalized no space — the producers that can name the app's interface orientation either turn it back (the runner's tree tiers) or refuse the capture (the Simulator AX bridge) — and is refused measurement for the same reason. Tiling: the keys themselves must form one unbroken run of columns, because a split iPad keyboard docks both clusters at the bottom edge while its surface container still spans the whole screen, and the app content between the clusters is visible and tappable. Reported seams are zero on every measured tree — iPhone 17 Pro's reported row adjoins across 395 pt, iPad Pro 11-inch's across 743.5 pt, and Gboard's across 1070 px — so the allowance covers rounding rather than layout, and a gapped band is refused measurement like the two above. That rule is deliberately blunt in both directions, and the last two cases name the trade: a sparse layout such as the iOS emoji panel reports corner keys over a wide container, does not tile, and reads `undetermined` — the guard abstains there rather than inventing which gaps are app content; and a projection that reports the container with no keys at all has nothing left to check, so the band is the container's own claim. Every captured iOS and Android keyboard measured so far reports its keys, so the second shape is the harness's, not a captured one. `target` names either a resolved node index or a bare coordinate point; both paths read one decision and choose their own consequence. A case that reaches a band declares its whole shape — the frame the refusal and the disclosure both quote, and the control rects that excuse a bare coordinate, which is where plane exclusion shows up. Provider facts distinguish measured visible frames, derived docked-tree bands, visibility-only probes, explicit absence, and unknown. Missing keyboard nodes are undetermined, never absence. A carried derived band was established before projection and stays usable after projection removes keyboard nodes; it obeys the same docking/orientation/tiling rules at acquisition. A measured floating frame needs no bottom extension.", + "description": "Golden table for keyboard occlusion on interaction tap points (#2589), consumed by packages/contracts/src/tap-keyboard-occlusion.ts. A visible software keyboard is its own system surface: it is never a covering sibling of app content, so the same-window occlusion classifier cannot see it, and it leaves the app window rect intact, so the viewport rule passes a covered tab bar. The band runs from the topmost keyboard node down to the bottom of the viewport, across the keyboard's own columns; the tap point is the point the interaction dispatches: the rect center the platform aims at for a ref handed over by name, or the touch point a parent with interactive descendants resolves instead. `nodes` are captured snapshot nodes as they appear in the projection the interaction resolved against \u2014 note that regular and interactive projections prune the keyboard's own container, leaving Key nodes beside app chrome. Three rules decide whether reported geometry may be measured at all. Docking: the band is derived only where the keyboard's own reported geometry reaches the viewport's bottom edge within 80 units \u2014 measured on iPhone 17 Pro (26.2) the reported key plane stops 58 pt short of an 874 pt portrait viewport and lands exactly on a 402 pt landscape one, and on iPad Pro 11-inch (M4) 3 pt short of a 1210 pt viewport \u2014 so an iPad floating keyboard, an app-drawn keypad, or an IME that stops above its own navigation bar stays unmeasured and fails open. The budget is counted in the same units as the rects it guards, which is what a measured Android IME asks for rather than what a dp table would give: with three-button navigation on Pixel 7 (Android 16, 1080x2400 at 420 dpi), Gboard's reported region runs to the physical display bottom at 2400 while the 126 px navigation bar occupies 2274..2400 inside it, so no Android keyboard measured so far needs slack for the bar. Width: geometry that comes back taller than it is wide did not arrive in the app's orientation space, i.e. it came from a capture that normalized no space \u2014 the producers that can name the app's interface orientation either turn it back (the runner's tree tiers) or refuse the capture (the Simulator AX bridge) \u2014 and is refused measurement for the same reason. Tiling: the keys themselves must form one unbroken run of columns, because a split iPad keyboard docks both clusters at the bottom edge while its surface container still spans the whole screen, and the app content between the clusters is visible and tappable. Reported seams are zero on every measured tree \u2014 iPhone 17 Pro's reported row adjoins across 395 pt, iPad Pro 11-inch's across 743.5 pt, and Gboard's across 1070 px \u2014 so the allowance covers rounding rather than layout, and a gapped band is refused measurement like the two above. That rule is deliberately blunt in both directions, and the last two cases name the trade: a sparse layout such as the iOS emoji panel reports corner keys over a wide container, does not tile, and reads `undetermined` \u2014 the guard abstains there rather than inventing which gaps are app content; and a projection that reports the container with no keys at all has nothing left to check, so the band is the container's own claim. Every captured iOS and Android keyboard measured so far reports its keys, so the second shape is the harness's, not a captured one. `target` names either a resolved node index or a bare coordinate point; both paths read one decision and choose their own consequence. A case that reaches a band declares its whole shape \u2014 the frame the refusal and the disclosure both quote, and the control rects that excuse a bare coordinate, which is where plane exclusion shows up. Provider facts distinguish measured visible frames, derived docked-tree bands, visibility-only probes, explicit absence, and unknown. Missing keyboard nodes are undetermined, never absence. A carried derived band was established before projection and stays usable after projection removes keyboard nodes; it obeys the same docking/orientation/tiling rules at acquisition. A measured floating frame needs no bottom extension.", "constants": { "occlusionReason": "tap_keyboard_occludes_target" }, @@ -4095,7 +4095,8 @@ }, "expected": { "kind": "no-keyboard" - } + }, + "observationVisibility": false }, { "name": "an unknown provider observation with no keyboard nodes never establishes absence", @@ -4358,6 +4359,246 @@ "expected": { "kind": "undetermined" } + }, + { + "name": "a docked provider tree outweighs a disagreeing negative visibility probe", + "viewport": { + "x": 0, + "y": 0, + "width": 390, + "height": 844 + }, + "nodes": [ + { + "index": 0, + "type": "Keyboard", + "rect": { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + } + }, + { + "index": 1, + "type": "Button", + "label": "Save draft", + "rect": { + "x": 200, + "y": 600, + "width": 100, + "height": 40 + } + } + ], + "target": { + "index": 1 + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + }, + "controlRects": [ + { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + } + ] + }, + "observationVisibility": false, + "keyboard": { + "kind": "derived", + "derivation": "docked-tree", + "frame": { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + } + } + }, + { + "name": "app content under _UIRemoteKeyboardPlaceholderView inside a Keyboard is refused", + "viewport": { + "x": 0, + "y": 0, + "width": 390, + "height": 844 + }, + "nodes": [ + { + "index": 0, + "type": "Keyboard", + "rect": { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + } + }, + { + "index": 1, + "type": "Button", + "label": "Save draft", + "rect": { + "x": 200, + "y": 600, + "width": 100, + "height": 40 + }, + "parentIndex": 2 + }, + { + "index": 2, + "type": "Other", + "role": "_UIRemoteKeyboardPlaceholderView", + "parentIndex": 0 + } + ], + "target": { + "index": 1 + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + }, + "controlRects": [ + { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + } + ] + } + }, + { + "name": "app content under TUISystemInputAssistantView inside a Keyboard is refused", + "viewport": { + "x": 0, + "y": 0, + "width": 390, + "height": 844 + }, + "nodes": [ + { + "index": 0, + "type": "Keyboard", + "rect": { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + } + }, + { + "index": 1, + "type": "Button", + "label": "Save draft", + "rect": { + "x": 200, + "y": 600, + "width": 100, + "height": 40 + }, + "parentIndex": 2 + }, + { + "index": 2, + "type": "Other", + "role": "TUISystemInputAssistantView", + "parentIndex": 0 + } + ], + "target": { + "index": 1 + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + }, + "controlRects": [ + { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + } + ] + } + }, + { + "name": "app content under TUISystemInputAssistantPageView inside a Keyboard is refused", + "viewport": { + "x": 0, + "y": 0, + "width": 390, + "height": 844 + }, + "nodes": [ + { + "index": 0, + "type": "Keyboard", + "rect": { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + } + }, + { + "index": 1, + "type": "Button", + "label": "Save draft", + "rect": { + "x": 200, + "y": 600, + "width": 100, + "height": 40 + }, + "parentIndex": 2 + }, + { + "index": 2, + "type": "Other", + "role": "TUISystemInputAssistantPageView", + "parentIndex": 0 + } + ], + "target": { + "index": 1 + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + }, + "controlRects": [ + { + "x": 0, + "y": 580, + "width": 390, + "height": 264 + } + ] + } } ] } diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index 73b727c97b..e9cefe95c8 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -266,9 +266,12 @@ export type SnapshotResult = Omit & */ systemSurface?: IosSystemSurfaceProvenance; /** - * The keyboard band the producer measured while capturing, when it can measure one (#2660). A - * producer that publishes nothing measured nothing, so the tap-path guard keeps deriving the band - * from `nodes`; see {@link SnapshotKeyboardBandFact}. + * Keyboard evidence acquired with this tree: measured `visible`, policy-derived `derived`, + * probe-only `visibility-only`, proven `absent`, or unknown `unmeasurable` facts. Measured and + * derived frames survive projection and take precedence over tree reconstruction. Visibility-only + * facts carry no band; rejected acquisition geometry stays rejected. Other failed measurements or + * an omitted fact may use tree geometry, but missing nodes never prove absence. + * See {@link SnapshotKeyboardBandFact}. */ keyboard?: SnapshotKeyboardBandFact; /** diff --git a/packages/contracts/src/tap-keyboard-occlusion.test.ts b/packages/contracts/src/tap-keyboard-occlusion.test.ts index fef25e5793..a26ab39987 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.test.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.test.ts @@ -8,6 +8,7 @@ import type { SnapshotKeyboardBandFact, } from '@agent-device/kernel/snapshot'; import { + observeProviderKeyboard, resolveKeyboardTapOcclusion, TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS, TAP_KEYBOARD_OCCLUDES_TARGET_REASON, @@ -32,6 +33,7 @@ type FixtureCase = { nodes: FixtureNode[]; target: { index: number } | { point: { x: number; y: number } }; keyboard?: SnapshotKeyboardBandFact; + observationVisibility?: boolean; expected: { kind: string; frame?: Rect; controlRects?: readonly Rect[] }; }; @@ -71,6 +73,17 @@ test('keyboard tap occlusion agrees with every golden parity table case', () => 'point' in fixture.target ? fixture.target.point : resolveCenter(byIndex.get(fixture.target.index)); + if (fixture.observationVisibility !== undefined) { + assert.deepEqual( + observeProviderKeyboard({ + nodes, + viewport: fixture.viewport, + visibility: fixture.observationVisibility, + }), + fixture.keyboard, + `${fixture.name}: acquisition observation`, + ); + } const occlusion = resolveKeyboardTapOcclusion({ nodes, viewport: fixture.viewport, @@ -128,7 +141,7 @@ function tableCase(name: string): FixtureCase { function resolveWithFact( fixture: FixtureCase, - keyboard: SnapshotKeyboardBandFact, + keyboard: SnapshotKeyboardBandFact | undefined, ): ReturnType { const nodes = fixture.nodes as RawSnapshotNode[]; const byIndex = new Map(nodes.map((node) => [node.index, node])); @@ -221,8 +234,10 @@ test('an unmeasurable fact leaves every golden-table decision exactly as the tre // either side of the refusal, or the table would stop describing this guard. for (const fixture of loadTable().cases) { const withFact = resolveWithFact(fixture, { - ...(fixture.keyboard ?? { kind: 'unmeasurable', reason: 'keyboard-frame-query-timeout' }), + kind: 'unmeasurable', + reason: 'keyboard-frame-query-timeout', }); - assert.equal(withFact.kind, fixture.expected.kind, `${fixture.name}: kind`); + const treeOnly = resolveWithFact(fixture, undefined); + assert.deepEqual(withFact, treeOnly, fixture.name); } }); diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index f0d82ff584..32a3f724bd 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -5,7 +5,14 @@ import type { SnapshotKeyboardBandFact, } from '@agent-device/kernel/snapshot'; import { containsPoint, isPositiveFiniteRect } from '@agent-device/kernel/rect'; -import { IOS_KEYBOARD_SURFACE_ROLES, isAndroidInputMethodNode } from './android-input-ownership.ts'; +import { + IOS_KEYBOARD_SURFACE_ROLES, + IOS_KEYBOARD_SETTLE_CHROME_ROLES, + IOS_KEYBOARD_SUBTREE_CHROME_ROLES, + IOS_KEYBOARD_CHROME_MARKER_ROLES, + isAndroidInputMethodNode, + hasIosKeyboardRole, +} from './android-input-ownership.ts'; import { normalizeType } from './snapshot-text.ts'; /** @@ -116,20 +123,34 @@ function isKeyboardSurfaceNode(node: RawSnapshotNode): boolean { return isKeyboardAnchorNode(node) || IOS_KEYBOARD_SURFACE_ROLES.has(node.role ?? ''); } -/** - * Whether this node belongs to the keyboard rather than to the app: a key, the key plane, an IME - * node, or anything inside one of them. Ownership walks ancestors only, so a flattened projection - * that lifts keys beside app chrome cannot promote app content into the keyboard. - */ +/** Keyboard hosts that own descendants, until an app-fillable marker interrupts the ancestry. */ +function isKeyboardSubtreeHost(node: RawSnapshotNode): boolean { + return isKeyboardAnchorNode(node) || hasIosKeyboardRole(node, IOS_KEYBOARD_SUBTREE_CHROME_ROLES); +} + +/** Per-node ownership and system-only ancestry never cross an app-fillable assistant marker. */ function isKeyboardOwnedNode( node: RawSnapshotNode, - parents: ReadonlyMap, - surfaceIndices: ReadonlySet, + byIndex: ReadonlyMap, ): boolean { - return ( - surfaceIndices.has(node.index) || - collectAncestorSurfaceIndices(parents, surfaceIndices, node.index).length > 0 - ); + if ( + isKeyboardSurfaceNode(node) || + hasIosKeyboardRole(node, IOS_KEYBOARD_SETTLE_CHROME_ROLES) || + hasIosKeyboardRole(node, IOS_KEYBOARD_CHROME_MARKER_ROLES) || + isKeyboardSubtreeHost(node) + ) + return true; + const visited = new Set(); + let parentIndex = node.parentIndex; + while (typeof parentIndex === 'number' && !visited.has(parentIndex)) { + visited.add(parentIndex); + const parent = byIndex.get(parentIndex); + if (!parent) return false; + if (hasIosKeyboardRole(parent, IOS_KEYBOARD_CHROME_MARKER_ROLES)) return false; + if (isKeyboardSubtreeHost(parent)) return true; + parentIndex = parent.parentIndex; + } + return false; } /** The band needs an edge to run to, which an absent or zero-area viewport does not report. */ @@ -321,10 +342,9 @@ function resolveVisibleKeyboardSurface( /** Preserves witnessed tree ownership on nodes before projection can remove their ancestors. */ export function annotateKeyboardOwnership(nodes: readonly RawSnapshotNode[]): RawSnapshotNode[] { - const parents = new Map(nodes.map((node) => [node.index, node.parentIndex] as const)); - const surfaceIndices = new Set(nodes.filter(isKeyboardSurfaceNode).map((node) => node.index)); + const byIndex = new Map(nodes.map((node) => [node.index, node] as const)); return nodes.map((node) => - isKeyboardOwnedNode(node, parents, surfaceIndices) ? { ...node, keyboardOwned: true } : node, + isKeyboardOwnedNode(node, byIndex) ? { ...node, keyboardOwned: true } : node, ); } @@ -334,10 +354,11 @@ export function observeProviderKeyboard(params: { viewport: Rect | null; visibility?: boolean | 'unsupported' | 'failed'; }): SnapshotKeyboardBandFact { - if (params.visibility === false) return { kind: 'visibility-only', visible: false }; const surface = resolveVisibleKeyboardSurface(params.nodes, params.viewport); if (surface) return { kind: 'derived', derivation: 'docked-tree', frame: surface.frame }; - if (params.visibility === true) return { kind: 'visibility-only', visible: true }; + if (typeof params.visibility === 'boolean') { + return { kind: 'visibility-only', visible: params.visibility }; + } return { kind: 'unmeasurable', reason: params.nodes.some(isKeyboardAnchorNode) @@ -363,8 +384,8 @@ function keyboardFactDisallowsTreeDerivation( /** * Whether this tap point belongs to the visible keyboard rather than to app content behind it. * - * The point is the rect center an interaction would activate — the same point - * `isTapPointInsideViewport` guards — so an element only partly under the keyboard whose center is + * The point is the dispatched tap point the interaction resolved — the same point + * `isTapPointInsideViewport` guards — so an element only partly under the keyboard whose tap point is * still above the key plane keeps tapping. Callers choose the consequence: an acting element path * refuses, while a coordinate path that never captured this tree discloses instead of refusing. * @@ -384,7 +405,7 @@ export function resolveKeyboardTapOcclusion(params: { point: Point; /** The resolved element, when the caller named one; absent for a bare coordinate. */ node?: RawSnapshotNode | null; - /** The band this capture's producer measured, when it measured one (#2660). */ + /** This capture's measured, acquisition-derived, visibility-only, or unknown keyboard evidence. */ keyboard?: SnapshotKeyboardBandFact; }): KeyboardTapOcclusion { if ( @@ -404,14 +425,8 @@ export function resolveKeyboardTapOcclusion(params: { if (!params.node) return classifyKeyboardPoint(params.point, surface); // A resolved element is excused only by belonging to the keyboard, never by sitting under a key: // the caller named the element, and behind the keyboard a tap does not reach it. - const parents = new Map(params.nodes.map((node) => [node.index, node.parentIndex] as const)); - const surfaceIndices = new Set( - params.nodes.filter(isKeyboardSurfaceNode).map((node) => node.index), - ); - if ( - isKeyboardOwnedNode(params.node, parents, surfaceIndices) || - params.node.keyboardOwned === true - ) { + const byIndex = new Map(params.nodes.map((node) => [node.index, node] as const)); + if (isKeyboardOwnedNode(params.node, byIndex) || params.node.keyboardOwned === true) { return { kind: 'clear', surface }; } return containsPoint(surface.frame, params.point.x, params.point.y) diff --git a/packages/kernel/src/record.ts b/packages/kernel/src/record.ts index d965ed8017..f80807ad26 100644 --- a/packages/kernel/src/record.ts +++ b/packages/kernel/src/record.ts @@ -90,11 +90,10 @@ export function parseRect(value: unknown): Rect | undefined { * distinguished from one that measured and reported. * * A payload that cannot be placed is restated as `unmeasurable` naming where it failed, never dropped - * and never trusted: silence would tell the tap guard the producer never looked. The reasons resolve - * in a fixed order, least to most trusted: a value that is not an object is `malformed-fact`; an - * unrecognized `kind` is `unrecognized-kind`; an `unmeasurable` that never named its failure is - * `unreported-reason`; a `visible` whose frame is not a positive finite rect is - * `invalid-visible-frame`. + * and never trusted: silence would tell the tap guard the producer never looked. Validation reasons + * identify the rejected shape: `malformed-fact`, `unrecognized-kind`, `unreported-reason`, + * `invalid-visibility-probe`, `unrecognized-derivation`, `invalid-visible-frame`, or + * `invalid-derived-frame`. They describe validation failures rather than levels of trust. */ export function readSnapshotKeyboardBandFact(value: unknown): SnapshotKeyboardBandFact | undefined { if (value === undefined) return undefined; diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index 3dd89348b8..4e09cd6466 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -759,7 +759,7 @@ test('snapshot cancellation during an optional keyboard probe is not swallowed', ); }); -test('Android acquisition derives a docked IME before projection removes its tree', async () => { +test('Android acquisition cannot derive a docked IME from app content bounds', async () => { const interactor = createWebDriverInteractor({ client: { source: async () => @@ -771,11 +771,7 @@ test('Android acquisition derives a docked IME before projection removes its tre }); const snapshot = await interactor.snapshot(); assert.ok(!('stage' in snapshot)); - assert.deepEqual(snapshot.keyboard, { - kind: 'derived', - derivation: 'docked-tree', - frame: { x: 0, y: 1500, width: 1080, height: 900 }, - }); + assert.deepEqual(snapshot.keyboard, { kind: 'visibility-only', visible: true }); assert.ok(snapshot.nodes); const form = snapshot.nodes.find((node) => node.label === 'Form')!; assert.equal( @@ -786,7 +782,7 @@ test('Android acquisition derives a docked IME before projection removes its tre point: { x: 200, y: 1820 }, keyboard: snapshot.keyboard, }).kind, - 'occluded', + 'undetermined', ); assert.equal( snapshot.nodes.find((node) => node.identifier?.includes('input_view'))?.keyboardOwned, diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index cc6da5f97d..88daef926a 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -17,7 +17,6 @@ import { } from '@agent-device/contracts/scroll-gesture'; import type { TvRemoteButton } from '@agent-device/contracts/tv-remote'; import type { SettingOptions } from '@agent-device/contracts/settings'; -import { isPositiveFiniteRect, pickLargestRect } from '@agent-device/kernel/rect'; import { AppError } from '@agent-device/kernel/errors'; import { capabilitySupported, @@ -332,21 +331,14 @@ class WebDriverInteractor implements Interactor { return acquireWebDriverIosSnapshot(source, this.targetId, visibility); } const { parseWebDriverSourceFacts } = await import('./webdriver-source.ts'); - const { resolveViewportRect } = await import('@agent-device/contracts/snapshot'); const { annotateKeyboardOwnership, observeProviderKeyboard } = await import('@agent-device/contracts/tap-keyboard-occlusion'); const nodes = annotateKeyboardOwnership(parseWebDriverSourceFacts(source, 'android').nodes); - const appRect = pickLargestRect( - nodes.flatMap((node) => - !node.keyboardOwned && isPositiveFiniteRect(node.rect) ? [node.rect] : [], - ), - ); - const viewport = appRect ? resolveViewportRect(nodes, appRect) : null; return { backend: 'android' as const, producer: 'appium-source' as const, nodes, - keyboard: observeProviderKeyboard({ nodes, viewport, visibility }), + keyboard: observeProviderKeyboard({ nodes, viewport: null, visibility }), }; } diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index bb0142faa0..c43d6b9bf5 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; +import { resolveKeyboardTapOcclusion } from '@agent-device/contracts/tap-keyboard-occlusion'; import { acquireWebDriverIosSnapshot } from './webdriver-ios-snapshot.ts'; const SOURCE = ` @@ -98,3 +99,31 @@ test('an offscreen provider keyboard never publishes an empty derived frame', () reason: 'keyboard-tree-geometry-unusable', }); }); + +test.each([ + '_UIRemoteKeyboardPlaceholderView', + 'TUISystemInputAssistantView', + 'TUISystemInputAssistantPageView', +])('app controls under %s do not inherit keyboard ownership', (role) => { + const source = SOURCE.replace( + '', + ``, + ); + const acquired = acquireWebDriverIosSnapshot(source).acquisition; + const appControl = acquired.nodes.find((node) => node.label === 'Save draft')!; + for (const nodes of [acquired.nodes, [appControl]]) { + assert.equal( + resolveKeyboardTapOcclusion({ + nodes, + node: appControl, + viewport: null, + point: { x: 250, y: 620 }, + keyboard: acquired.keyboard, + }).kind, + 'occluded', + ); + } + assert.equal(appControl.keyboardOwned, undefined); + assert.equal(acquired.nodes.find((node) => node.role === role)?.keyboardOwned, true); + assert.equal(acquired.nodes.find((node) => node.label === 'Done')?.keyboardOwned, true); +}); diff --git a/src/commands/capture/runtime/snapshot-unchanged.test.ts b/src/commands/capture/runtime/snapshot-unchanged.test.ts index 3f46e3729d..5f247fbc4b 100644 --- a/src/commands/capture/runtime/snapshot-unchanged.test.ts +++ b/src/commands/capture/runtime/snapshot-unchanged.test.ts @@ -82,6 +82,7 @@ test.each>([ { appName: 'Example' }, { windowTitle: 'Compose' }, { surface: 'app' }, + { keyboardOwned: true }, { hiddenContentAbove: true }, { hiddenContentBelow: true }, { interactionBlocked: 'covered' }, @@ -90,6 +91,9 @@ test.each>([ const current = snapshot('Create', { nodes: [{ ...previous.nodes[0]!, ...change }] }); expect(buildUnchangedSnapshotMetadata({ previous, current, options: {} })).toBeUndefined(); + expect( + buildUnchangedSnapshotMetadata({ previous: current, current: previous, options: {} }), + ).toBeUndefined(); }); test.each>([ diff --git a/src/commands/schema/cli-help-workflows.ts b/src/commands/schema/cli-help-workflows.ts index 18dbd2fb32..e064da5d1f 100644 --- a/src/commands/schema/cli-help-workflows.ts +++ b/src/commands/schema/cli-help-workflows.ts @@ -52,7 +52,7 @@ Targets: Recovery: Network/typeahead result missing: wait text "Expected result" or wait . A target that should be gone/disappear: use wait absent ; wait exists ... is rejected in favor of the plain selector wait. - Keyboard visible over the next target: when a measured or derived keyboard frame covers the tap point, the element is refused with tap_keyboard_occludes_target, because the touch would activate a key instead of the target. End editing first -- the app's own Done/Cancel control, or keyboard enter when submission is wanted -- then retry. Missing keyboard nodes never prove absence; a visibility-only probe cannot establish the covered area. A target whose center stays above the keys still presses; a raw coordinate behind the keyboard taps anyway and reports the reason in warning. + Keyboard visible over the next target: when a measured or derived keyboard frame covers an app target's dispatched tap point, the element is refused with tap_keyboard_occludes_target, because the touch would activate a key instead of the target. End editing first -- the app's own Done/Cancel control, or keyboard enter when submission is wanted -- then retry. Missing keyboard nodes never prove absence; a visibility-only probe cannot establish the covered area. An app target whose dispatched tap point stays above the keys passes this keyboard check; a raw coordinate behind the keyboard taps anyway and reports the reason in warning. Target covered by other visible UI: refused with target_covered. When the occlusion scan identified the covering element, error.details.coveredBy and the hint name it (role, label, rect) so you know what to dismiss; the cover is diagnosis only -- never target its ref. Dismiss/close it or scroll the target clear, then retry. Sparse or recovered accessibility snapshot: use screenshot as visual truth, leave the bad screen if needed, then retry snapshot -i. Non-hittable success hint: verify with the settled diff or snapshot; retarget by a better ref/selector if the UI did not change. diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index f7a311ddcb..d00a37e055 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -288,8 +288,8 @@ The response-level `keyboard` field tells you what the capture observed: Appium iOS and Android snapshots include the visibility probe; Appium iOS and limrun iOS can also report a derived docked band. Missing keyboard nodes never mean the keyboard is hidden. Element taps use measured or derived frames to refuse a target behind the keyboard. A positive visibility probe -alone cannot establish occlusion. When tree geometry cannot establish a docked band, the keyboard -remains uncertain. +alone cannot establish occlusion. When tree geometry cannot establish a docked band, visibility-only evidence may establish presence +or absence but cannot locate the band. Run `snapshot -i` again after ending editing before retrying a refused target. The Android keyboard band is the input method window's bounds, the box around the area it takes From e5cf1ca7c2e0a9507b486a51639e345f43bfeb5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 00:03:53 +0200 Subject: [PATCH 08/12] fix(snapshot): keep keyboard probes additive to tree evidence --- CHANGELOG.md | 5 + .../tap-keyboard-occlusion-policy.json | 253 +++++++++++++++++- packages/capture-kit/package.json | 4 + .../src/provider-keyboard-acquisition.ts | 17 ++ packages/contracts/src/interactor-types.ts | 2 +- .../contracts/src/tap-keyboard-occlusion.ts | 48 ++-- packages/kernel/src/record.test.ts | 23 ++ packages/kernel/src/record.ts | 13 + packages/kernel/src/snapshot.ts | 4 +- .../src/ios-snapshot-adapter.test.ts | 2 +- .../src/ios-snapshot-adapter.ts | 10 +- .../src/webdriver-interactor.test.ts | 68 ++--- .../src/webdriver-interactor.ts | 15 +- .../src/webdriver-ios-snapshot.test.ts | 25 +- .../src/webdriver-ios-snapshot.ts | 8 +- website/docs/docs/snapshots.md | 22 +- 16 files changed, 432 insertions(+), 87 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 packages/capture-kit/src/provider-keyboard-acquisition.ts diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..73ea8d6d04 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## Unreleased + +- Carry provider keyboard facts through snapshots. Appium captures, including wait polls and fill pre-captures, add one sequential keyboard visibility probe with a 500 ms timeout. Probe results preserve tap-time docked-tree checks; unavailable probes leave visibility unknown. diff --git a/contracts/fixtures/tap-keyboard-occlusion-policy.json b/contracts/fixtures/tap-keyboard-occlusion-policy.json index 36d3d45025..96237852a2 100644 --- a/contracts/fixtures/tap-keyboard-occlusion-policy.json +++ b/contracts/fixtures/tap-keyboard-occlusion-policy.json @@ -4,6 +4,172 @@ "occlusionReason": "tap_keyboard_occludes_target" }, "cases": [ + { + "name": "Gboard docked tree + visibility-only:true -> occluded", + "viewport": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "nodes": [ + { + "index": 0, + "type": "android.widget.FrameLayout", + "bundleId": "com.example.app", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + } + }, + { + "index": 1, + "type": "android.widget.Button", + "label": "Submit", + "bundleId": "com.example.app", + "parentIndex": 0, + "rect": { + "x": 440, + "y": 1800, + "width": 200, + "height": 120 + } + }, + { + "index": 2, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.inputmethod.latin", + "parentIndex": 0, + "rect": { + "x": 0, + "y": 1517, + "width": 1080, + "height": 883 + } + }, + { + "index": 8, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.inputmethod.latin", + "parentIndex": 2, + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + } + } + ], + "target": { + "index": 1 + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 0, + "y": 1517, + "width": 1080, + "height": 883 + }, + "controlRects": [ + { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + } + ] + }, + "keyboard": { + "kind": "visibility-only", + "visible": true + } + }, + { + "name": "Gboard docked tree + visibility-only:false -> occluded", + "viewport": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "nodes": [ + { + "index": 0, + "type": "android.widget.FrameLayout", + "bundleId": "com.example.app", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + } + }, + { + "index": 1, + "type": "android.widget.Button", + "label": "Submit", + "bundleId": "com.example.app", + "parentIndex": 0, + "rect": { + "x": 440, + "y": 1800, + "width": 200, + "height": 120 + } + }, + { + "index": 2, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.inputmethod.latin", + "parentIndex": 0, + "rect": { + "x": 0, + "y": 1517, + "width": 1080, + "height": 883 + } + }, + { + "index": 8, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.inputmethod.latin", + "parentIndex": 2, + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + } + } + ], + "target": { + "index": 1 + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 0, + "y": 1517, + "width": 1080, + "height": 883 + }, + "controlRects": [ + { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + } + ] + }, + "keyboard": { + "kind": "visibility-only", + "visible": false + } + }, { "name": "iPhone tab-bar item whose center is under the key plane is refused", "viewport": { @@ -4353,8 +4519,8 @@ "index": 1 }, "keyboard": { - "kind": "unmeasurable", - "reason": "keyboard-tree-geometry-unusable" + "kind": "tree-rejected", + "derivation": "docked-tree" }, "expected": { "kind": "undetermined" @@ -4599,6 +4765,89 @@ } ] } + }, + { + "name": "an unknown producer reason cannot reject docked tree geometry", + "viewport": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + }, + "nodes": [ + { + "index": 0, + "type": "android.widget.FrameLayout", + "bundleId": "com.example.app", + "rect": { + "x": 0, + "y": 0, + "width": 1080, + "height": 2400 + } + }, + { + "index": 1, + "type": "android.widget.Button", + "label": "Submit", + "bundleId": "com.example.app", + "parentIndex": 0, + "rect": { + "x": 440, + "y": 1800, + "width": 200, + "height": 120 + } + }, + { + "index": 2, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.inputmethod.latin", + "parentIndex": 0, + "rect": { + "x": 0, + "y": 1517, + "width": 1080, + "height": 883 + } + }, + { + "index": 8, + "type": "android.widget.FrameLayout", + "bundleId": "com.google.android.inputmethod.latin", + "parentIndex": 2, + "rect": { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + } + } + ], + "target": { + "index": 1 + }, + "expected": { + "kind": "occluded", + "frame": { + "x": 0, + "y": 1517, + "width": 1080, + "height": 883 + }, + "controlRects": [ + { + "x": 0, + "y": 2274, + "width": 1080, + "height": 126 + } + ] + }, + "keyboard": { + "kind": "unmeasurable", + "reason": "keyboard-tree-geometry-unusable" + } } ] } diff --git a/packages/capture-kit/package.json b/packages/capture-kit/package.json index 8b24bef561..71cb7703a4 100644 --- a/packages/capture-kit/package.json +++ b/packages/capture-kit/package.json @@ -317,6 +317,10 @@ "./durable-capture/session-binding": { "types": "./src/durable-capture/session-binding.ts", "default": "./src/durable-capture/session-binding.ts" + }, + "./provider-keyboard-acquisition": { + "types": "./src/provider-keyboard-acquisition.ts", + "default": "./src/provider-keyboard-acquisition.ts" } }, "devDependencies": { diff --git a/packages/capture-kit/src/provider-keyboard-acquisition.ts b/packages/capture-kit/src/provider-keyboard-acquisition.ts new file mode 100644 index 0000000000..01ba211c80 --- /dev/null +++ b/packages/capture-kit/src/provider-keyboard-acquisition.ts @@ -0,0 +1,17 @@ +import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; +import { + annotateKeyboardOwnership, + observeProviderKeyboard, +} from '@agent-device/contracts/tap-keyboard-occlusion'; + +/** Preserves provider keyboard geometry and ownership before snapshot projection. */ +export function acquireProviderKeyboard(params: { + nodes: readonly RawSnapshotNode[]; + viewport: Rect | null; + visibility?: boolean | 'unsupported' | 'failed'; +}) { + return { + nodes: annotateKeyboardOwnership(params.nodes), + keyboard: observeProviderKeyboard(params), + }; +} diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index e9cefe95c8..9c3fa56533 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -267,7 +267,7 @@ export type SnapshotResult = Omit & systemSurface?: IosSystemSurfaceProvenance; /** * Keyboard evidence acquired with this tree: measured `visible`, policy-derived `derived`, - * probe-only `visibility-only`, proven `absent`, or unknown `unmeasurable` facts. Measured and + * additive `visibility-only`, rejected `tree-rejected`, proven `absent`, or unknown `unmeasurable` facts. Measured and * derived frames survive projection and take precedence over tree reconstruction. Visibility-only * facts carry no band; rejected acquisition geometry stays rejected. Other failed measurements or * an omitted fact may use tree geometry, but missing nodes never prove absence. diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index 32a3f724bd..3b4776ccf4 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -42,8 +42,6 @@ import { normalizeType } from './snapshot-text.ts'; /** The one reason a tap refuses because the visible keyboard owns its tap point. */ export const TAP_KEYBOARD_OCCLUDES_TARGET_REASON = 'tap_keyboard_occludes_target'; -const KEYBOARD_TREE_GEOMETRY_UNUSABLE_REASON = 'keyboard-tree-geometry-unusable'; - /** * The hint every owner publishes beside the reason, naming the recovery that actually works. * `keyboard dismiss` taps the keyboard's own dismiss control and refuses when it exposes none @@ -356,14 +354,20 @@ export function observeProviderKeyboard(params: { }): SnapshotKeyboardBandFact { const surface = resolveVisibleKeyboardSurface(params.nodes, params.viewport); if (surface) return { kind: 'derived', derivation: 'docked-tree', frame: surface.frame }; + if (isMeasurableViewport(params.viewport) && params.nodes.some(isKeyboardAnchorNode)) { + return { + kind: 'tree-rejected', + derivation: 'docked-tree', + ...(typeof params.visibility === 'boolean' ? { visible: params.visibility } : {}), + }; + } if (typeof params.visibility === 'boolean') { return { kind: 'visibility-only', visible: params.visibility }; } return { kind: 'unmeasurable', - reason: params.nodes.some(isKeyboardAnchorNode) - ? KEYBOARD_TREE_GEOMETRY_UNUSABLE_REASON - : params.visibility === 'failed' + reason: + params.visibility === 'failed' ? 'keyboard-visibility-probe-failed' : params.visibility === 'unsupported' ? 'keyboard-visibility-probe-unsupported' @@ -371,16 +375,6 @@ export function observeProviderKeyboard(params: { }; } -function keyboardFactDisallowsTreeDerivation( - keyboard: SnapshotKeyboardBandFact | undefined, -): boolean { - return ( - keyboard?.kind === 'visibility-only' || - (keyboard?.kind === 'unmeasurable' && - keyboard.reason === KEYBOARD_TREE_GEOMETRY_UNUSABLE_REASON) - ); -} - /** * Whether this tap point belongs to the visible keyboard rather than to app content behind it. * @@ -394,9 +388,11 @@ function keyboardFactDisallowsTreeDerivation( * acquisition. Keyboard ownership on surviving nodes retains its acquisition witness after projection. * The keyboard's own controls still come from the * tree — which node the keyboard rather than the app is responsible for is a different question from - * where the band is, and only the tree can answer it. Visibility-only observations carry no band, and - * rejected acquisition geometry stays rejected - * after projection. Other failed direct measurements may still use the tree; missing nodes remain + * where the band is, and only the tree can answer it. Visibility-only observations are additive and never prevent + * tree derivation. Only + * `tree-rejected` records a failed full-tree evaluation with a usable viewport and prevents + * reconstruction after projection. Other failed direct measurements may still use the tree; + * missing nodes remain * undetermined. */ export function resolveKeyboardTapOcclusion(params: { @@ -408,19 +404,21 @@ export function resolveKeyboardTapOcclusion(params: { /** This capture's measured, acquisition-derived, visibility-only, or unknown keyboard evidence. */ keyboard?: SnapshotKeyboardBandFact; }): KeyboardTapOcclusion { - if ( - params.keyboard?.kind === 'absent' || - (params.keyboard?.kind === 'visibility-only' && !params.keyboard.visible) - ) { - return { kind: 'no-keyboard' }; + if (params.keyboard?.kind === 'absent') return { kind: 'no-keyboard' }; + if (params.keyboard?.kind === 'tree-rejected') { + return { kind: params.keyboard.visible === false ? 'no-keyboard' : 'undetermined' }; } - if (keyboardFactDisallowsTreeDerivation(params.keyboard)) return { kind: 'undetermined' }; const surface = params.keyboard?.kind === 'visible' || params.keyboard?.kind === 'derived' ? { frame: params.keyboard.frame, controlRects: collectKeyboardControlRects(params.nodes) } : resolveVisibleKeyboardSurface(params.nodes, params.viewport); if (!surface) { - return { kind: 'undetermined' }; + return { + kind: + params.keyboard?.kind === 'visibility-only' && !params.keyboard.visible + ? 'no-keyboard' + : 'undetermined', + }; } if (!params.node) return classifyKeyboardPoint(params.point, surface); // A resolved element is excused only by belonging to the keyboard, never by sitting under a key: diff --git a/packages/kernel/src/record.test.ts b/packages/kernel/src/record.test.ts index 0a9e55f7a3..2558818f59 100644 --- a/packages/kernel/src/record.test.ts +++ b/packages/kernel/src/record.test.ts @@ -83,7 +83,30 @@ test('keyboard provenance validation rejects malformed derived bands and visibil 'invalid-derived-frame', ], [{ kind: 'visibility-only', visible: 'false' }, 'invalid-visibility-probe'], + [{ kind: 'tree-rejected' }, 'unrecognized-derivation'], + [ + { kind: 'tree-rejected', derivation: 'docked-tree', visible: 'true' }, + 'invalid-visibility-probe', + ], ] as const) { assert.deepEqual(readSnapshotKeyboardBandFact(fact), { kind: 'unmeasurable', reason }); } }); + +test('typed tree rejection preserves optional visibility without trusting a reason string', () => { + for (const visible of [undefined, true, false]) { + const fact = { + kind: 'tree-rejected', + derivation: 'docked-tree', + ...(visible === undefined ? {} : { visible }), + }; + assert.deepEqual(readSnapshotKeyboardBandFact(fact), fact); + } + assert.deepEqual( + readSnapshotKeyboardBandFact({ + kind: 'unmeasurable', + reason: 'keyboard-tree-geometry-unusable', + }), + { kind: 'unmeasurable', reason: 'keyboard-tree-geometry-unusable' }, + ); +}); diff --git a/packages/kernel/src/record.ts b/packages/kernel/src/record.ts index f80807ad26..9fb961a576 100644 --- a/packages/kernel/src/record.ts +++ b/packages/kernel/src/record.ts @@ -104,6 +104,19 @@ export function readSnapshotKeyboardBandFact(value: unknown): SnapshotKeyboardBa ? { kind: 'unmeasurable', reason: value.reason } : { kind: 'unmeasurable', reason: 'unreported-reason' }; } + if (value.kind === 'tree-rejected') { + if (value.derivation !== 'docked-tree') { + return { kind: 'unmeasurable', reason: 'unrecognized-derivation' }; + } + if (value.visible !== undefined && typeof value.visible !== 'boolean') { + return { kind: 'unmeasurable', reason: 'invalid-visibility-probe' }; + } + return { + kind: 'tree-rejected', + derivation: 'docked-tree', + ...(typeof value.visible === 'boolean' ? { visible: value.visible } : {}), + }; + } if (value.kind === 'visibility-only') { return typeof value.visible === 'boolean' ? { kind: 'visibility-only', visible: value.visible } diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index d110732f6a..8dc94a470d 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -356,13 +356,15 @@ export type HiddenContentHint = { * Keyboard evidence observed during acquisition, in the capture's node coordinate space. * `visible` is a measured frame; `derived` is a frame established by the shared docked-tree * policy before projection; `visibility-only` is a boolean probe with no geometry; - * `unmeasurable` is unknown. `absent` requires an explicit absence observation. + * `tree-rejected` records rejected geometry evaluated against a usable viewport, with optional + * probe visibility; `unmeasurable` is unknown. `absent` requires an explicit absence observation. * Missing keyboard nodes or an omitted fact never prove absence. */ export type SnapshotKeyboardBandFact = | { kind: 'visible'; frame: Rect } | { kind: 'derived'; derivation: 'docked-tree'; frame: Rect } | { kind: 'visibility-only'; visible: boolean } + | { kind: 'tree-rejected'; derivation: 'docked-tree'; visible?: boolean } | { kind: 'absent' } | { kind: 'unmeasurable'; reason: string }; diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts index b333e714f4..478b397836 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.test.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.test.ts @@ -118,5 +118,5 @@ test('Limrun carries docked tree evidence and never infers absence from missing tree.children!.at(-1)!.frame = { x: 100, y: 60, width: 140, height: 80 }; expect( (await captureLimrunIosSnapshot(createLimrunSnapshotSession(tree))).acquisition.keyboard, - ).toEqual({ kind: 'unmeasurable', reason: 'keyboard-tree-geometry-unusable' }); + ).toEqual({ kind: 'tree-rejected', derivation: 'docked-tree' }); }); diff --git a/packages/provider-limrun/src/ios-snapshot-adapter.ts b/packages/provider-limrun/src/ios-snapshot-adapter.ts index aec4560e85..1df3211f51 100644 --- a/packages/provider-limrun/src/ios-snapshot-adapter.ts +++ b/packages/provider-limrun/src/ios-snapshot-adapter.ts @@ -4,10 +4,7 @@ import { } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import type { SnapshotRuntimeAcquiredResult } from '@agent-device/contracts/interactor-types'; import type { LimrunIosSession } from './ios.ts'; -import { - annotateKeyboardOwnership, - observeProviderKeyboard, -} from '@agent-device/contracts/tap-keyboard-occlusion'; +import { acquireProviderKeyboard } from '@agent-device/capture-kit/provider-keyboard-acquisition'; import { flattenIosTree, type IosTreeNode } from './snapshot.ts'; export async function captureLimrunIosSnapshot( @@ -15,11 +12,10 @@ export async function captureLimrunIosSnapshot( ): Promise { const tree = (await session.client.elementTree()) as IosTreeNode[]; const viewport = readLimrunViewport(tree, session.client.deviceInfo); - const nodes = annotateKeyboardOwnership(flattenIosTree(tree)); + const nodes = flattenIosTree(tree); return createIosSnapshotAcquisition({ producer: 'limrun-ios-tree', - nodes, - keyboard: observeProviderKeyboard({ + ...acquireProviderKeyboard({ nodes, viewport: viewport.kind === 'missing' ? null : viewport.rect, }), diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index 4e09cd6466..9c190f6b99 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test, vi } from 'vitest'; +import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; import { resolveKeyboardTapOcclusion } from '@agent-device/contracts/tap-keyboard-occlusion'; import { buildGesturePlan } from '@agent-device/contracts/gesture-plan'; import { AppError } from '@agent-device/kernel/errors'; @@ -735,7 +736,7 @@ test('snapshot carries a bounded visibility probe on both Appium platforms', asy ? { kind: 'visibility-only', visible } : { kind: 'unmeasurable', reason: `keyboard-visibility-probe-${visible}` }, ); - assert.deepEqual(probe.mock.calls, [[1_500, controller.signal]]); + assert.deepEqual(probe.mock.calls, [[500, controller.signal]]); } } }); @@ -759,33 +760,38 @@ test('snapshot cancellation during an optional keyboard probe is not swallowed', ); }); -test('Android acquisition cannot derive a docked IME from app content bounds', async () => { - const interactor = createWebDriverInteractor({ - client: { - source: async () => - '', - isKeyboardShown: async () => true, - } as unknown as WebDriverClient, - backend: 'android', - capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }), - }); - const snapshot = await interactor.snapshot(); - assert.ok(!('stage' in snapshot)); - assert.deepEqual(snapshot.keyboard, { kind: 'visibility-only', visible: true }); - assert.ok(snapshot.nodes); - const form = snapshot.nodes.find((node) => node.label === 'Form')!; - assert.equal( - resolveKeyboardTapOcclusion({ - nodes: [form], - node: form, - viewport: null, - point: { x: 200, y: 1820 }, - keyboard: snapshot.keyboard, - }).kind, - 'undetermined', - ); - assert.equal( - snapshot.nodes.find((node) => node.identifier?.includes('input_view'))?.keyboardOwned, - true, - ); -}); +test.each([true, false, 'failed'] as const)( + 'Android probe %s preserves tap-time docked IME derivation with a tree-resolved viewport', + async (visibility) => { + const interactor = createWebDriverInteractor({ + client: { + source: async () => + '', + isKeyboardShown: async () => { + if (visibility === 'failed') throw new Error('probe failed'); + return visibility; + }, + } as unknown as WebDriverClient, + backend: 'android', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }), + }); + const snapshot = await interactor.snapshot(); + assert.ok(!('stage' in snapshot)); + assert.ok(snapshot.nodes); + const form = snapshot.nodes.find((node) => node.label === 'Form')!; + assert.equal( + resolveKeyboardTapOcclusion({ + nodes: snapshot.nodes, + node: form, + viewport: createSnapshotVisibility(snapshot.nodes).resolveViewport(form.rect!), + point: { x: 200, y: 1820 }, + keyboard: snapshot.keyboard, + }).kind, + 'occluded', + ); + assert.equal( + snapshot.nodes.find((node) => node.identifier?.includes('input_view'))?.keyboardOwned, + true, + ); + }, +); diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index 88daef926a..ff6156c418 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -46,6 +46,7 @@ const TEXT_ENTRY_READINESS_POLL_MS = 100; * WebDriver client's much longer default request timeout. */ const TEXT_ENTRY_PROBE_TIMEOUT_MS = 1_500; +const SNAPSHOT_KEYBOARD_PROBE_TIMEOUT_MS = 500; /** The Android `KEYCODE_ENTER` the IME's return key sends. */ const ANDROID_KEYCODE_ENTER = 66; @@ -331,14 +332,13 @@ class WebDriverInteractor implements Interactor { return acquireWebDriverIosSnapshot(source, this.targetId, visibility); } const { parseWebDriverSourceFacts } = await import('./webdriver-source.ts'); - const { annotateKeyboardOwnership, observeProviderKeyboard } = - await import('@agent-device/contracts/tap-keyboard-occlusion'); - const nodes = annotateKeyboardOwnership(parseWebDriverSourceFacts(source, 'android').nodes); + const { acquireProviderKeyboard } = + await import('@agent-device/capture-kit/provider-keyboard-acquisition'); + const nodes = parseWebDriverSourceFacts(source, 'android').nodes; return { backend: 'android' as const, producer: 'appium-source' as const, - nodes, - keyboard: observeProviderKeyboard({ nodes, viewport: null, visibility }), + ...acquireProviderKeyboard({ nodes, viewport: null, visibility }), }; } @@ -346,7 +346,10 @@ class WebDriverInteractor implements Interactor { signal?: AbortSignal, ): Promise { try { - const visibility = await this.client.isKeyboardShown(TEXT_ENTRY_PROBE_TIMEOUT_MS, signal); + const visibility = await this.client.isKeyboardShown( + SNAPSHOT_KEYBOARD_PROBE_TIMEOUT_MS, + signal, + ); signal?.throwIfAborted(); return visibility; } catch { diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index c43d6b9bf5..0703d03716 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -95,8 +95,8 @@ test('an offscreen provider keyboard never publishes an empty derived frame', () '', ); assert.deepEqual(acquireWebDriverIosSnapshot(source).acquisition.keyboard, { - kind: 'unmeasurable', - reason: 'keyboard-tree-geometry-unusable', + kind: 'tree-rejected', + derivation: 'docked-tree', }); }); @@ -127,3 +127,24 @@ test.each([ assert.equal(acquired.nodes.find((node) => node.role === role)?.keyboardOwned, true); assert.equal(acquired.nodes.find((node) => node.label === 'Done')?.keyboardOwned, true); }); + +test.each([true, false, 'failed'] as const)( + 'Appium iOS probe %s without a viewport preserves later tree derivation', + (visibility) => { + const source = + ''; + const acquired = acquireWebDriverIosSnapshot(source, undefined, visibility).acquisition; + assert.equal(acquired.viewport.kind, 'missing'); + const node = acquired.nodes.find((node) => node.label === 'Continue')!; + assert.equal( + resolveKeyboardTapOcclusion({ + nodes: acquired.nodes, + node, + viewport: { x: 0, y: 0, width: 390, height: 844 }, + point: { x: 70, y: 720 }, + keyboard: acquired.keyboard, + }).kind, + 'occluded', + ); + }, +); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 3d0f7f0e7d..1a82cebb2c 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -3,10 +3,7 @@ import { resolveIosViewportEvidenceFromRoots, } from '@agent-device/capture-kit/ios-snapshot-acquisition'; import type { SnapshotRuntimeAcquiredResult } from '@agent-device/contracts/interactor-types'; -import { - annotateKeyboardOwnership, - observeProviderKeyboard, -} from '@agent-device/contracts/tap-keyboard-occlusion'; +import { acquireProviderKeyboard } from '@agent-device/capture-kit/provider-keyboard-acquisition'; import { parseWebDriverSourceFacts } from './webdriver-source.ts'; const APPIUM_PRODUCER = 'appium-source' as const; @@ -23,9 +20,8 @@ export function acquireWebDriverIosSnapshot( ({ kind: 'missing', reason: 'not-provided' } as const); return createIosSnapshotAcquisition({ producer: APPIUM_PRODUCER, - nodes: annotateKeyboardOwnership(sourceFacts.nodes), viewport, - keyboard: observeProviderKeyboard({ + ...acquireProviderKeyboard({ nodes: sourceFacts.nodes, viewport: viewport.kind === 'missing' ? null : viewport.rect, visibility, diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index d00a37e055..bdf50bd550 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -282,16 +282,28 @@ The response-level `keyboard` field tells you what the capture observed: - `visible` carries a directly measured `frame` in the same coordinates as node bounds. - `derived` carries a `frame` inferred from a docked keyboard tree, with `derivation: "docked-tree"`. - `visibility-only` carries a boolean `visible` from a provider probe, without a frame. +- `tree-rejected` means the full tree was checked with a usable viewport but could not establish a + docked band. Its optional `visible` carries probe visibility. Projecting that tree does not retry + the rejected geometry. - `absent` means the producer explicitly established absence. - `unmeasurable` carries a `reason` when the producer cannot establish a frame or visibility. -Appium iOS and Android snapshots include the visibility probe; Appium iOS and limrun iOS can also -report a derived docked band. Missing keyboard nodes never mean the keyboard is hidden. Element taps -use measured or derived frames to refuse a target behind the keyboard. A positive visibility probe -alone cannot establish occlusion. When tree geometry cannot establish a docked band, visibility-only evidence may establish presence -or absence but cannot locate the band. +Appium iOS and Android snapshots include a visibility probe. Each capture, including a `wait` poll +or the capture before `fill`, adds one sequential probe with a 500 ms timeout. A timeout leaves the +probe unknown; it does not discard the tree. Appium iOS and limrun iOS can also report a derived docked +band during capture. Android Appium can derive the band when a tap resolves the tree's viewport. + +Missing keyboard nodes never mean the keyboard is hidden. A positive probe establishes presence but +cannot locate the covered area. A negative probe establishes absence only when the tree also resolves +no docked band. Neither probe result overrides a resolvable docked tree. Element taps use measured or +derived frames to refuse an app target whose dispatched tap point is behind the keyboard. Run `snapshot -i` again after ending editing before retrying a refused target. +Provider snapshot nodes may include `keyboardOwned: true`. This means capture witnessed keyboard +ownership, so the keyboard's own controls remain tappable when projection removes their ancestors. +An omitted field does not prove app ownership. App-filled assistant hosts do not pass keyboard +ownership to their descendants. + The Android keyboard band is the input method window's bounds, the box around the area it takes touches in. On Android 13 (API 33) and later the helper also checks that this area is one rectangle. A floating keyboard's is not (the panel plus the gesture strip, with app content between them), so the From ff8f0a903197d0d0769cd93ff8206bed97c6c790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 00:22:59 +0200 Subject: [PATCH 09/12] fix(appium): preserve keyboard window package provenance --- .../src/webdriver-interactor.test.ts | 5 +++-- .../provider-webdriver/src/webdriver-source.test.ts | 11 +++++++++++ packages/provider-webdriver/src/webdriver-source.ts | 1 + website/docs/docs/snapshots.md | 1 + 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index 9c190f6b99..5576be8fb0 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -766,7 +766,7 @@ test.each([true, false, 'failed'] as const)( const interactor = createWebDriverInteractor({ client: { source: async () => - '', + '', isKeyboardShown: async () => { if (visibility === 'failed') throw new Error('probe failed'); return visibility; @@ -790,7 +790,8 @@ test.each([true, false, 'failed'] as const)( 'occluded', ); assert.equal( - snapshot.nodes.find((node) => node.identifier?.includes('input_view'))?.keyboardOwned, + snapshot.nodes.find((node) => node.bundleId === 'com.google.android.inputmethod.latin') + ?.keyboardOwned, true, ); }, diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index 7b22cead35..00a6bb4bc1 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -328,3 +328,14 @@ test('iOS WebDriver facts keep the tag as the type whatever a class attribute sa assert.equal(node!.type, 'XCUIElementTypeButton'); }); + +test('Android source preserves native package provenance on anonymous IME windows', () => { + const source = + ''; + const nodes = parseWebDriverSourceFacts(source, 'android').nodes; + assert.deepEqual( + nodes.map((node) => node.bundleId), + ['com.google.android.inputmethod.latin', 'com.google.android.inputmethod.latin'], + ); + assert.equal(nodes[0]?.identifier, undefined); +}); diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index 1e663c0c4f..6e6e607fac 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -173,6 +173,7 @@ function sourceNodeFromAttributes( ...labelFacts(attrs, platform), value: field === undefined ? nonEmptyWebDriverAttribute(attrs.value) : field.value, identifier: firstWebDriverAttribute(attrs, ['resource-id', 'id', 'accessibility-id', 'name']), + bundleId: platform === 'android' ? nonEmptyWebDriverAttribute(attrs.package) : undefined, rect, ...sourceStateFacts(attrs, rect, platform), ...(field === undefined ? {} : field.facts), diff --git a/website/docs/docs/snapshots.md b/website/docs/docs/snapshots.md index bdf50bd550..4b610db951 100644 --- a/website/docs/docs/snapshots.md +++ b/website/docs/docs/snapshots.md @@ -299,6 +299,7 @@ no docked band. Neither probe result overrides a resolvable docked tree. Element derived frames to refuse an app target whose dispatched tap point is behind the keyboard. Run `snapshot -i` again after ending editing before retrying a refused target. +Android Appium nodes carry the reported native package in `bundleId`. Provider snapshot nodes may include `keyboardOwned: true`. This means capture witnessed keyboard ownership, so the keyboard's own controls remain tappable when projection removes their ancestors. An omitted field does not prove app ownership. App-filled assistant hosts do not pass keyboard From 04f62494f47acb6106c660adb96006022e51cd62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 00:35:40 +0200 Subject: [PATCH 10/12] refactor(snapshot): isolate typed keyboard rejection validation --- packages/kernel/src/record.ts | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/kernel/src/record.ts b/packages/kernel/src/record.ts index 9fb961a576..0140ec9038 100644 --- a/packages/kernel/src/record.ts +++ b/packages/kernel/src/record.ts @@ -104,19 +104,7 @@ export function readSnapshotKeyboardBandFact(value: unknown): SnapshotKeyboardBa ? { kind: 'unmeasurable', reason: value.reason } : { kind: 'unmeasurable', reason: 'unreported-reason' }; } - if (value.kind === 'tree-rejected') { - if (value.derivation !== 'docked-tree') { - return { kind: 'unmeasurable', reason: 'unrecognized-derivation' }; - } - if (value.visible !== undefined && typeof value.visible !== 'boolean') { - return { kind: 'unmeasurable', reason: 'invalid-visibility-probe' }; - } - return { - kind: 'tree-rejected', - derivation: 'docked-tree', - ...(typeof value.visible === 'boolean' ? { visible: value.visible } : {}), - }; - } + if (value.kind === 'tree-rejected') return readKeyboardTreeRejection(value); if (value.kind === 'visibility-only') { return typeof value.visible === 'boolean' ? { kind: 'visibility-only', visible: value.visible } @@ -131,6 +119,20 @@ export function readSnapshotKeyboardBandFact(value: unknown): SnapshotKeyboardBa return { kind: 'unmeasurable', reason: 'unrecognized-kind' }; } +function readKeyboardTreeRejection(value: Record): SnapshotKeyboardBandFact { + if (value.derivation !== 'docked-tree') { + return { kind: 'unmeasurable', reason: 'unrecognized-derivation' }; + } + if (value.visible !== undefined && typeof value.visible !== 'boolean') { + return { kind: 'unmeasurable', reason: 'invalid-visibility-probe' }; + } + return { + kind: 'tree-rejected', + derivation: 'docked-tree', + ...(typeof value.visible === 'boolean' ? { visible: value.visible } : {}), + }; +} + function readKeyboardFrameFact( value: unknown, kind: 'visible' | 'derived', From e68b6814be81dd7580698367f7d4adda09a3aaa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 00:35:40 +0200 Subject: [PATCH 11/12] chore(gates): declare provider keyboard acquisition package entry --- scripts/layering/package-boundaries.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index b732afa650..b5d05dfc1e 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -468,6 +468,7 @@ test('the real tree parses, declares, and passes R11', () => { '@agent-device/capture-kit/png-size', '@agent-device/capture-kit/png-worker-client', '@agent-device/capture-kit/post-gesture-stability', + '@agent-device/capture-kit/provider-keyboard-acquisition', '@agent-device/capture-kit/quality-warnings', '@agent-device/capture-kit/react-native-overlay', '@agent-device/capture-kit/recording-artifact-fixtures', From a8e8f9ad73d8ef822835b334aad0050f34d829b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 14 Oct 2026 00:54:07 +0200 Subject: [PATCH 12/12] fix(snapshot): name unavailable viewport for observed keyboard --- .../contracts/src/tap-keyboard-occlusion.test.ts | 12 ++++++++++++ packages/contracts/src/tap-keyboard-occlusion.ts | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/contracts/src/tap-keyboard-occlusion.test.ts b/packages/contracts/src/tap-keyboard-occlusion.test.ts index a26ab39987..e98864f91b 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.test.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.test.ts @@ -241,3 +241,15 @@ test('an unmeasurable fact leaves every golden-table decision exactly as the tre assert.deepEqual(withFact, treeOnly, fixture.name); } }); + +test('keyboard anchors without an acquisition viewport name the missing geometry and preserve tap derivation', () => { + const fixture = tableCase('Gboard docked tree + visibility-only:true -> occluded'); + const fact = observeProviderKeyboard({ nodes: fixture.nodes, viewport: null }); + assert.deepEqual(fact, { kind: 'unmeasurable', reason: 'keyboard-viewport-unavailable' }); + assert.equal(resolveWithFact(fixture, fact).kind, 'occluded'); + const noKeyboard = tableCase('no keyboard in the tree means nothing to refuse'); + assert.deepEqual(observeProviderKeyboard({ nodes: noKeyboard.nodes, viewport: null }), { + kind: 'unmeasurable', + reason: 'keyboard-not-observed', + }); +}); diff --git a/packages/contracts/src/tap-keyboard-occlusion.ts b/packages/contracts/src/tap-keyboard-occlusion.ts index 3b4776ccf4..144144c911 100644 --- a/packages/contracts/src/tap-keyboard-occlusion.ts +++ b/packages/contracts/src/tap-keyboard-occlusion.ts @@ -366,8 +366,9 @@ export function observeProviderKeyboard(params: { } return { kind: 'unmeasurable', - reason: - params.visibility === 'failed' + reason: params.nodes.some(isKeyboardAnchorNode) + ? 'keyboard-viewport-unavailable' + : params.visibility === 'failed' ? 'keyboard-visibility-probe-failed' : params.visibility === 'unsupported' ? 'keyboard-visibility-probe-unsupported'