feat(snapshot): carry provider keyboard observations through acquisition

packages/capture-kit/src/ios-snapshot-acquisition.ts

@@ -10,7 +10,11 @@ import type {
 } from '@agent-device/contracts/ios-snapshot';
 import { normalizeType } from '@agent-device/contracts/snapshot';
 import { isPositiveFiniteRect } from '@agent-device/kernel/rect';
-import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot';
+import type {
+  RawSnapshotNode,
+  Rect,
+  SnapshotKeyboardBandFact,
+} from '@agent-device/kernel/snapshot';
 
 /**
  * The facts a *provider* acquisition leaves unproven, keyed only by the producers whose residue
@@ -96,6 +100,7 @@ export function createIosSnapshotAcquisition(
     producer: IosProviderAcquisitionProducer;
     nodes: readonly RawSnapshotNode[];
     viewport: IosViewportEvidence;
+    keyboard?: SnapshotKeyboardBandFact;
     lineage: IosSnapshotLineage;
   }>,
 ): SnapshotRuntimeAcquiredResult {
@@ -107,6 +112,7 @@ export function createIosSnapshotAcquisition(
       intent: 'full',
       nodes: input.nodes,
       viewport: input.viewport,
+      ...(input.keyboard ? { keyboard: input.keyboard } : {}),
       lineage: input.lineage,
       residue: deriveIosSnapshotAcquisitionResidue(producer, input.viewport),
     },

packages/capture-kit/src/snapshot/__tests__/ios-snapshot-runtime-viewport.test.ts

@@ -1,6 +1,12 @@
+import { buildSnapshotState } from '../../snapshot-state.ts';
 import assert from 'node:assert/strict';
 import { test } from 'vitest';
 import { createIosSnapshotAcquisition } from '@agent-device/capture-kit/ios-snapshot-acquisition';
+import {
+  annotateKeyboardOwnership,
+  observeProviderKeyboard,
+  resolveKeyboardTapOcclusion,
+} from '@agent-device/contracts/tap-keyboard-occlusion';
 import { presentIosSnapshotAcquisition } from '../ios-snapshot-runtime.ts';
 import type { IosViewportEvidence } from '@agent-device/contracts/ios-snapshot';
 import { AppError } from '@agent-device/kernel/errors';
@@ -82,3 +88,159 @@ test('an empty iOS acquisition still publishes the viewport it read (#3182)', ()
   assert.deepEqual(presented.nodes, []);
   assert.deepEqual(presented.viewport, { width: 390, height: 844 });
 });
+
+test('provider keyboard evidence survives projection even when the tree has no keyboard nodes', () => {
+  const keyboard = {
+    kind: 'derived' as const,
+    derivation: 'docked-tree' as const,
+    frame: { x: 0, y: 580, width: 390, height: 264 },
+  };
+  const acquired = createIosSnapshotAcquisition({
+    producer: 'appium-source',
+    nodes: NODES,
+    viewport: { kind: 'reported', rect: SCREEN },
+    lineage: {},
+    keyboard,
+  });
+  for (const options of [{ depth: 0 }, { interactiveOnly: true }, { raw: true }]) {
+    assert.deepEqual(presentIosSnapshotAcquisition(acquired, options).keyboard, keyboard);
+  }
+});
+
+test('a scoped keyboard control retains its acquired ownership after reindexing', () => {
+  const nodes: RawSnapshotNode[] = [
+    ...NODES,
+    { index: 2, parentIndex: 0, type: 'Keyboard', rect: { x: 0, y: 580, width: 390, height: 264 } },
+    {
+      index: 3,
+      parentIndex: 2,
+      type: 'Button',
+      label: 'Done',
+      rect: { x: 200, y: 600, width: 100, height: 40 },
+    },
+  ];
+  const keyboard = observeProviderKeyboard({ nodes, viewport: SCREEN });
+  const acquired = createIosSnapshotAcquisition({
+    producer: 'appium-source',
+    nodes: annotateKeyboardOwnership(nodes),
+    viewport: { kind: 'reported', rect: SCREEN },
+    lineage: {},
+    keyboard,
+  });
+  for (const raw of [false, true]) {
+    const result = presentIosSnapshotAcquisition(acquired, { scope: 'Done', raw });
+    const node = result.nodes!.find((n) => n.label === 'Done')!;
+    assert.equal(
+      resolveKeyboardTapOcclusion({
+        nodes: result.nodes!,
+        viewport: SCREEN,
+        keyboard: result.keyboard,
+        node,
+        point: { x: 250, y: 620 },
+      }).kind,
+      'clear',
+    );
+  }
+});
+
+test('depth filtering and state normalization never promote app content into keyboard ownership', () => {
+  const nodes: RawSnapshotNode[] = [
+    { index: 0, depth: 0, type: 'Application', rect: SCREEN },
+    { index: 1, depth: 1, parentIndex: 0, type: 'Other', rect: SCREEN },
+    {
+      index: 2,
+      depth: 2,
+      parentIndex: 1,
+      type: 'Button',
+      label: 'Nested',
+      rect: { x: 10, y: 10, width: 100, height: 40 },
+    },
+    {
+      index: 3,
+      depth: 1,
+      parentIndex: 0,
+      type: 'Keyboard',
+      rect: { x: 0, y: 580, width: 390, height: 264 },
+    },
+    {
+      index: 4,
+      depth: 1,
+      parentIndex: 0,
+      type: 'Button',
+      label: 'Form',
+      rect: { x: 200, y: 600, width: 100, height: 40 },
+    },
+  ];
+  const acquired = createIosSnapshotAcquisition({
+    producer: 'appium-source',
+    nodes,
+    viewport: { kind: 'reported', rect: SCREEN },
+    lineage: {},
+    keyboard: observeProviderKeyboard({ nodes, viewport: SCREEN }),
+  });
+  const result = presentIosSnapshotAcquisition(acquired, { raw: true, depth: 1 });
+  const state = buildSnapshotState(result, { snapshotRaw: true });
+  const node = state.nodes.find((n) => n.label === 'Form')!;
+  assert.equal(node.keyboardOwned, undefined);
+  assert.equal(
+    resolveKeyboardTapOcclusion({
+      nodes: state.nodes,
+      viewport: SCREEN,
+      keyboard: state.keyboard,
+      node,
+      point: { x: 250, y: 620 },
+    }).kind,
+    'occluded',
+  );
+});
+
+test('depth projection cannot promote a rejected split keyboard into a trusted band', () => {
+  const viewport = SCREEN;
+  const nodes: RawSnapshotNode[] = [
+    { index: 0, depth: 0, type: 'Application', rect: viewport },
+    {
+      index: 1,
+      depth: 1,
+      parentIndex: 0,
+      type: 'Button',
+      label: 'Between clusters',
+      rect: { x: 150, y: 650, width: 40, height: 20 },
+    },
+    {
+      index: 2,
+      depth: 1,
+      parentIndex: 0,
+      type: 'Keyboard',
+      rect: { x: 0, y: 580, width: 390, height: 264 },
+    },
+    ...[0, 310].map((x, offset) => ({
+      index: 3 + offset,
+      depth: 2,
+      parentIndex: 2,
+      type: 'Key',
+      rect: { x, y: 600, width: 80, height: 40 },
+    })),
+  ];
+  for (const visibility of [undefined, true] as const) {
+    const acquired = createIosSnapshotAcquisition({
+      producer: 'appium-source',
+      nodes,
+      viewport: { kind: 'reported', rect: viewport },
+      lineage: {},
+      keyboard: observeProviderKeyboard({ nodes, viewport, visibility }),
+    });
+    const projected = presentIosSnapshotAcquisition(acquired, { raw: true, depth: 1 });
+    const state = buildSnapshotState(projected, { snapshotRaw: true });
+    const node = state.nodes.find((n) => n.label === 'Between clusters')!;
+    assert.equal(
+      resolveKeyboardTapOcclusion({
+        nodes: state.nodes,
+        viewport,
+        keyboard: state.keyboard,
+        node,
+        point: { x: 170, y: 660 },
+      }).kind,
+      'undetermined',
+    );
+  }
+});

packages/capture-kit/src/snapshot/ios-snapshot-runtime.ts

@@ -49,6 +49,7 @@ export function presentIosSnapshotAcquisition(
       backend: 'xctest',
       producer: acquired.acquisition.producer,
       nodes: [...presentation.payload.nodes],
+      ...(acquired.acquisition.keyboard ? { keyboard: acquired.acquisition.keyboard } : {}),
       comparisonIdentity: presentation.comparisonIdentity,
       ...(acquired.acquisition.truncated === undefined
         ? {}

packages/contracts/src/interactor-types.ts

@@ -266,9 +266,12 @@ export type SnapshotResult = Omit<BackendSnapshotResult, 'backend' | 'nodes'> &
    */
   systemSurface?: IosSystemSurfaceProvenance;
   /**
-   * The keyboard band the producer measured while capturing, when it can measure one (#2660). A
-   * producer that publishes nothing measured nothing, so the tap-path guard keeps deriving the band
-   * from `nodes`; see {@link SnapshotKeyboardBandFact}.
+   * Keyboard evidence acquired with this tree: measured `visible`, policy-derived `derived`,
+   * probe-only `visibility-only`, proven `absent`, or unknown `unmeasurable` facts. Measured and
+   * derived frames survive projection and take precedence over tree reconstruction. Visibility-only
+   * facts carry no band; rejected acquisition geometry stays rejected. Other failed measurements or
+   * an omitted fact may use tree geometry, but missing nodes never prove absence.
+   * See {@link SnapshotKeyboardBandFact}.
    */
   keyboard?: SnapshotKeyboardBandFact;
   /**

packages/contracts/src/ios-snapshot.ts

@@ -1,4 +1,9 @@
-import type { RawSnapshotNode, Rect, SnapshotNode } from '@agent-device/kernel/snapshot';
+import type {
+  RawSnapshotNode,
+  Rect,
+  SnapshotNode,
+  SnapshotKeyboardBandFact,
+} from '@agent-device/kernel/snapshot';
 
 export type IosSnapshotProducer =
   | 'apple-runner'
@@ -154,6 +159,7 @@ type IosSnapshotAcquisitionForIntent<Intent extends IosAcquisitionIntent> = Read
   nodes: readonly RawSnapshotNode[];
   truncated?: boolean;
   viewport: IosViewportEvidence;
+  keyboard?: SnapshotKeyboardBandFact;
   lineage: IosSnapshotLineage;
   residue: readonly IosAcquisitionResidue[];
 }>;

packages/contracts/src/tap-keyboard-occlusion.test.ts

@@ -8,6 +8,7 @@ import type {
   SnapshotKeyboardBandFact,
 } from '@agent-device/kernel/snapshot';
 import {
+  observeProviderKeyboard,
   resolveKeyboardTapOcclusion,
   TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS,
   TAP_KEYBOARD_OCCLUDES_TARGET_REASON,
@@ -18,6 +19,7 @@ type FixtureNode = {
   index: number;
   type?: string;
   role?: string;
+  keyboardOwned?: true;
   label?: string;
   identifier?: string;
   bundleId?: string;
@@ -30,6 +32,8 @@ type FixtureCase = {
   viewport: Rect | null;
   nodes: FixtureNode[];
   target: { index: number } | { point: { x: number; y: number } };
+  keyboard?: SnapshotKeyboardBandFact;
+  observationVisibility?: boolean;
   expected: { kind: string; frame?: Rect; controlRects?: readonly Rect[] };
 };
 
@@ -69,11 +73,23 @@ test('keyboard tap occlusion agrees with every golden parity table case', () =>
       'point' in fixture.target
         ? fixture.target.point
         : resolveCenter(byIndex.get(fixture.target.index));
+    if (fixture.observationVisibility !== undefined) {
+      assert.deepEqual(
+        observeProviderKeyboard({
+          nodes,
+          viewport: fixture.viewport,
+          visibility: fixture.observationVisibility,
+        }),
+        fixture.keyboard,
+        `${fixture.name}: acquisition observation`,
+      );
+    }
     const occlusion = resolveKeyboardTapOcclusion({
       nodes,
       viewport: fixture.viewport,
       point,
       node,
+      keyboard: fixture.keyboard,
     });
     assert.equal(occlusion.kind, fixture.expected.kind, fixture.name);
     const surface =
@@ -125,7 +141,7 @@ function tableCase(name: string): FixtureCase {
 
 function resolveWithFact(
   fixture: FixtureCase,
-  keyboard: SnapshotKeyboardBandFact,
+  keyboard: SnapshotKeyboardBandFact | undefined,
 ): ReturnType<typeof resolveKeyboardTapOcclusion> {
   const nodes = fixture.nodes as RawSnapshotNode[];
   const byIndex = new Map(nodes.map((node) => [node.index, node]));
@@ -167,7 +183,7 @@ test('a measured band is the band, even in a tree that reports no keyboard at al
   // The tree rule's answer here is `no-keyboard`. A refusal off a tree with no key nodes can only
   // have come from the fact, which is the assertion that the rule was not consulted at all.
   const fixture = tableCase('no keyboard in the tree means nothing to refuse');
-  assert.equal(fixture.expected.kind, 'no-keyboard');
+  assert.equal(fixture.expected.kind, 'undetermined');
   const band: SnapshotKeyboardBandFact = {
     kind: 'visible',
     frame: { x: 0, y: 583, width: 402, height: 291 },
@@ -221,6 +237,7 @@ test('an unmeasurable fact leaves every golden-table decision exactly as the tre
       kind: 'unmeasurable',
       reason: 'keyboard-frame-query-timeout',
     });
-    assert.equal(withFact.kind, fixture.expected.kind, `${fixture.name}: kind`);
+    const treeOnly = resolveWithFact(fixture, undefined);
+    assert.deepEqual(withFact, treeOnly, fixture.name);
   }
 });