feat(snapshot)!: make a node's ref optional and mint it per producer

examples/sdk/client-session.ts

@@ -64,7 +64,8 @@ async function main(): Promise<void> {
 
     const snapshot = await client.capture.snapshot({ interactiveOnly: true });
     const target = snapshot.nodes.find((node) => node.role === 'button');
-    if (target) {
+    // A node carries a ref only when something can address it, and press needs that address.
+    if (target?.ref !== undefined) {
       await client.interactions.press({ ref: target.ref });
     }
   } catch (error) {

packages/capture-kit/src/screenshot-overlay.ts

@@ -76,7 +76,9 @@ export function buildScreenshotOverlayRefs(
   for (const node of snapshot.nodes) {
     if (!isOverlaySourceNode(snapshot, snapshotBounds, node)) continue;
     const target = resolveOverlayTarget(snapshot.nodes, node);
-    if (!target?.rect || !hasPositiveRect(target.rect)) continue;
+    // The overlay labels each box with the ref an agent would act on, so a target the capture
+    // issued no ref for gets no box: nothing can be addressed by it.
+    if (!target?.ref || !target.rect || !hasPositiveRect(target.rect)) continue;
     const label = resolveOverlayLabel(node, target, snapshot.nodes);
     const score = scoreOverlayCandidate(node, target, label);
     const overlaySourceRect = resolveOverlaySourceRect(snapshot, target, snapshot.nodes);

packages/capture-kit/src/snapshot-chrome.test.ts

@@ -230,7 +230,11 @@ test('the real interactive keyboard capture collapses to one [Keyboard] diff lin
       node.role === 'TUISystemInputAssistantView' ||
       node.role === 'TUIPredictionViewCell' ||
       node.role === 'UIKeyboardDockItemButton';
-    if (isKeyboardSurface) assert.equal(chromeRefs.has(node.ref), true, `node ${node.index}`);
+    if (isKeyboardSurface) {
+      // An iOS capture mints refs densely, so a keyboard surface always has one to be chrome under.
+      assert.ok(node.ref, `keyboard surface ${node.index} carries no ref`);
+      assert.equal(chromeRefs.has(node.ref), true, `node ${node.index}`);
+    }
   }
 
   // The stripped set collapses the ~50 keyboard-surface nodes; what survives is

packages/capture-kit/src/snapshot-chrome.ts

@@ -11,7 +11,7 @@ import {
   IOS_KEYBOARD_SURFACE_ROLES,
   isAndroidInputMethodNode,
 } from '@agent-device/contracts/android-input-ownership';
-import type { SnapshotNode } from '@agent-device/kernel/snapshot';
+import { snapshotRefBodies, type SnapshotNode } from '@agent-device/kernel/snapshot';
 import { normalizeType } from '@agent-device/contracts/snapshot';
 
 /**
@@ -143,9 +143,7 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome {
     chromeIndexes.add(index);
   }
   for (const index of markerIndexes) chromeIndexes.add(index);
-  const refs = new Set(
-    nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref),
-  );
+  const refs = new Set(snapshotRefBodies(nodes.filter((node) => chromeIndexes.has(node.index))));
   // The [Keyboard] container line itself survives the diff so "keyboard
   // appeared/left" stays one visible signal line; everything else collapses.
   const strippedIndexes = new Set(
@@ -307,11 +305,9 @@ function collectAndroidSettleChrome(
   );
   for (const index of systemChromeIndexes) strippedIndexes.add(index);
   const refs = new Set(
-    nodes
-      .filter(
-        (node) => node.ref && (imeIndexes.has(node.index) || systemChromeIndexes.has(node.index)),
-      )
-      .map((node) => node.ref),
+    snapshotRefBodies(
+      nodes.filter((node) => imeIndexes.has(node.index) || systemChromeIndexes.has(node.index)),
+    ),
   );
   if (strippedIndexes.size === 0 && refs.size === 0) return EMPTY_SETTLE_CHROME;
   return { strippedIndexes, refs };

packages/capture-kit/src/snapshot-state.ts

@@ -8,6 +8,7 @@ import {
   attachRefs,
   buildSnapshotPresentationKey,
   snapshotPresentationOptionsFromFlags,
+  snapshotRefMinting,
   type IosTargetActivation,
   type RawSnapshotNode,
   type SnapshotBackend,
@@ -79,6 +80,7 @@ export function buildSnapshotState(
     snapshotRaw
       ? scopedNodes
       : annotateCoveredSnapshotNodes(scopedNodes, overlayOptionsForBackend(data?.backend)),
+    snapshotRefMinting(data),
   );
   return {
     nodes,

packages/contracts/src/interaction.ts

@@ -283,10 +283,10 @@ export type SettleParams = {
  */
 export type FindReadResult =
   | { kind: 'found'; found: true; waitedMs?: number }
-  | { kind: 'text'; ref: string; text: string; node: SnapshotNode }
-  | { kind: 'attrs'; ref: string; node: SnapshotNode }
+  | { kind: 'text'; ref?: string; text: string; node: SnapshotNode }
+  | { kind: 'attrs'; ref?: string; node: SnapshotNode }
   /** #1625: the read-only inspection surface — every match, never a tap. */
-  | { kind: 'list'; matches: Array<{ ref: string; node: SnapshotNode }> };
+  | { kind: 'list'; matches: Array<{ ref?: string; node: SnapshotNode }> };
 
 export type SettleObservation = {
   settled: boolean;
@@ -513,8 +513,9 @@ export type FindCommandResponseData = {
   waitedMs?: number;
   text?: string;
   node?: SnapshotNode;
-  /** Every match of the read-only `list` action (#1625), each ref pinnable at `refsGeneration`. */
-  matches?: Array<{ ref: string; node: SnapshotNode }>;
+  /** Every match of the read-only `list` action (#1625), each ref pinnable at `refsGeneration`. A
+   * match on content the capture's producer issued no ref for carries none: readable, not pinnable. */
+  matches?: Array<{ ref?: string; node: SnapshotNode }>;
   locator?: string;
   query?: string;
   x?: number;

packages/kernel/src/snapshot-attach-refs.test.ts

@@ -1,5 +1,12 @@
 import { describe, expect, test } from 'vitest';
-import { attachRefs, type RawSnapshotNode } from './snapshot.ts';
+import {
+  attachRefs,
+  hasSnapshotRef,
+  snapshotRefBodies,
+  snapshotRefMinting,
+  type RawSnapshotNode,
+  type SnapshotProvenance,
+} from './snapshot.ts';
 
 const node = (index: number, extra: Partial<RawSnapshotNode> = {}): RawSnapshotNode => ({
   index,
@@ -23,6 +30,21 @@ describe('attachRefs', () => {
     expect(attached.map((n) => n.ref)).toEqual(['e1', 'e7', 'e3']);
   });
 
+  test('mints nothing for a node the backend left unaddressed', () => {
+    // A backend-issued tree is the web case: the backend resolves an action against refs of its own,
+    // so a ref minted here would either name a different element or nothing at all. The node stays
+    // in the tree — that is how its text reaches a reader — and simply carries no address.
+    const attached = attachRefs(
+      [node(0, { ref: 'e2' }), node(1), node(2, { ref: 'e4' })],
+      'backend-issued',
+    );
+
+    expect(attached.map((n) => n.ref)).toEqual(['e2', undefined, 'e4']);
+    // No `ref` key at all, so a JSON response says "nothing addresses this" instead of "undefined".
+    expect(Object.hasOwn(attached[1]!, 'ref')).toBe(false);
+    expect(attached.map((n) => n.index)).toEqual([0, 1, 2]);
+  });
+
   test('publishes the presenter role as kind, for Android and Apple vocabularies, on every node (#2656)', () => {
     const attached = attachRefs([
       node(0, { type: 'android.widget.Button' }),
@@ -32,3 +54,72 @@ describe('attachRefs', () => {
     expect(attached.map((n) => n.kind)).toEqual(['button', 'button', 'element']);
   });
 });
+
+describe('snapshotRefMinting', () => {
+  test('the producer that owns its ref namespace gets backend-issued minting', () => {
+    expect(snapshotRefMinting({ backend: 'web', producer: 'agent-browser' })).toBe(
+      'backend-issued',
+    );
+  });
+
+  // The mode keys on the producer, never on the platform channel: several producers can feed one
+  // channel (`xctest` alone has four) and only some of them mint refs of their own. A web adapter
+  // added later — a webdriver- or CDP-backed one — would arrive on the `web` channel minting
+  // nothing, and would need dense numbering to keep every node addressable as it does today.
+  test.each([
+    { backend: 'xctest', producer: 'apple-runner' },
+    { backend: 'xctest', producer: 'simulator-ax-bridge' },
+    { backend: 'xctest', producer: 'appium-source' },
+    { backend: 'xctest', producer: 'limrun-ios-tree' },
+    { backend: 'android', producer: 'android-uiautomator' },
+    { backend: 'android', producer: 'appium-source' },
+    { backend: 'harmonyos-arkui', producer: 'harmonyos-uitest' },
+    { backend: 'macos-helper', producer: 'macos-helper' },
+    { backend: 'linux-atspi', producer: 'linux-atspi' },
+  ] satisfies readonly SnapshotProvenance[])(
+    '$backend/$producer is minted positionally',
+    (provenance) => {
+      expect(snapshotRefMinting(provenance)).toBe('dense');
+    },
+  );
+
+  test('a capture with no provenance is minted positionally', () => {
+    expect(snapshotRefMinting(undefined)).toBe('dense');
+    expect(snapshotRefMinting({})).toBe('dense');
+    // Type-level half of the producer-keyed decision: a channel cannot be handed over on its own,
+    // so this function never has to guess which producer published a tree. `pnpm typecheck` fails
+    // the moment the parameter widens back to a partial pair.
+    // @ts-expect-error the minting mode needs the whole provenance pair, not the channel alone
+    snapshotRefMinting({ backend: 'web' });
+  });
+
+  test('a second producer on the web channel keeps dense numbering', () => {
+    // Forward-looking regression for the axis itself, not for a pair the table admits today: the
+    // provenance table has exactly one web producer, so a webdriver- or CDP-backed adapter is not
+    // expressible yet and has to be widened through here. When it lands, `agent-browser` is the
+    // entry that owns its refs and nothing else on the channel does — the case a channel check
+    // would get wrong by handing it `backend-issued` and stripping every ref in its tree.
+    const secondWebAdapter = {
+      backend: 'web',
+      producer: 'webdriver-source',
+    } as unknown as SnapshotProvenance;
+    expect(snapshotRefMinting(secondWebAdapter)).toBe('dense');
+  });
+});
+
+describe('snapshot addressability', () => {
+  const attached = attachRefs(
+    [node(0, { ref: 'e2' }), node(1), node(2, { ref: 'e4' })],
+    'backend-issued',
+  );
+
+  test('hasSnapshotRef narrows the ref for a caller that publishes it as an address', () => {
+    expect(attached.filter(hasSnapshotRef).map((n) => n.ref)).toEqual(['e2', 'e4']);
+    expect(attached.filter((n) => !hasSnapshotRef(n)).map((n) => n.index)).toEqual([1]);
+  });
+
+  test('snapshotRefBodies collects the refs a node set can be re-addressed by', () => {
+    expect(snapshotRefBodies(attached)).toEqual(['e2', 'e4']);
+    expect(snapshotRefBodies([])).toEqual([]);
+  });
+});

packages/kernel/src/snapshot.ts

@@ -386,15 +386,33 @@ export type SnapshotViewportSizeBrand = {
 };
 
 export type SnapshotNode = RawSnapshotNode & {
-  ref: string;
+  /**
+   * The ref an agent addresses this node by, or absent when nothing can address it.
+   *
+   * A backend that mints its own refs (web/agent-browser) owns that namespace and can only act on
+   * the refs it minted, so {@link attachRefs} mints nothing into it — a node the backend left
+   * unaddressed stays ref-less in the tree, which is how on-screen-but-not-actionable content
+   * (static text, a status region) reaches a reader without ever being offered as a tap target.
+   * Backends that mint no refs (Apple, Android, Linux) keep dense `e${index}` numbering, so every
+   * node of their trees carries a ref, as before.
+   *
+   * Every consumer that offers a node as an address must read this as an absence rather than print
+   * one: a ref-less node is selectable by text and by `find`/`wait`/`get`, and refused as a target.
+   *
+   * A ref-less node also carries no `rect`. Producers that mint refs measure geometry for the refs
+   * they issued (see `RawSnapshotNode.ref`), and producers that mint none get dense numbering, so
+   * every node a geometry-requiring path can reach — promotion, the touch point, the covered and
+   * keyboard refusals — has a ref. That is why those paths print `node.ref` without a guard.
+   */
+  ref?: string;
   /**
    * Normalized role, from the same table the text presenter reads (`formatRole` below). `type`
    * stays the raw platform class; `kind` is the presenter's own answer, published so a `--json`
    * consumer gets the presented role without re-deriving it from `type` (#2656). `attachRefs` is
-   * the one production construction path and always sets it; optional here (rather than required
-   * like `ref`) only so the many hand-built `SnapshotNode` fixtures across the daemon/selector test
-   * suites that predate #2656 and never route through `attachRefs` do not all need updating for a
-   * field their assertions never read.
+   * the one production construction path and always sets it; optional here only so the many
+   * hand-built `SnapshotNode` fixtures across the daemon/selector test suites that predate #2656 and
+   * never route through `attachRefs` do not all need updating for a field their assertions never
+   * read. An absent `ref` is a real answer about the node; an absent `kind` is a fixture shortcut.
    */
   kind?: string;
   /**
@@ -758,20 +776,90 @@ export type ScreenshotOverlayRef = {
 };
 
 /**
- * Assign a display ref to every node. A node that already carries a backend-minted
- * `ref` keeps it (see `RawSnapshotNode.ref`) — the web/agent-browser backend resolves
- * actions against its own refs, so re-minting a dense positional ref here would make
- * the snapshot show one ref while actions act on a different element. Backends that do
- * not mint refs get dense `e${index}` numbering, matching the historical behavior.
+ * How a capture's nodes get the ref an agent addresses them by.
+ *
+ * - `dense` — the capture's producer mints no refs, so this layer numbers the tree `e1…eN` and
+ *   every node becomes addressable by its position.
+ * - `backend-issued` — the producer resolves actions against refs of its own (`e${n}` minted by
+ *   the backend, in tree order and not dense), so a ref this layer invented would name nothing.
+ *   A node the producer did not issue a ref for stays ref-less: present to a reader, absent from
+ *   the addressable namespace.
  */
-export function attachRefs(nodes: RawSnapshotNode[]): SnapshotNode[] {
+export type SnapshotRefMinting = 'dense' | 'backend-issued';
+
+/**
+ * Assign a display ref to every node, in the mode the capture's producer needs (see
+ * `SnapshotRefMinting`).
+ *
+ * A node that already carries a backend-minted `ref` keeps it (see `RawSnapshotNode.ref`) — the
+ * web/agent-browser backend resolves actions against its own refs, so re-minting a dense positional
+ * ref here would make the snapshot show one ref while actions act on a different element.
+ *
+ * Under `dense` every node ends up addressable, as it always has. Under `backend-issued` only the
+ * nodes the producer issued a ref for do: the rest stay ref-less, readable and searchable by their
+ * text and tree position, and never an address — which is what `hasSnapshotRef` and
+ * `snapshotRefBodies` report, and what a caller publishing an address must check.
+ */
+export function attachRefs(
+  nodes: RawSnapshotNode[],
+  minting: SnapshotRefMinting = 'dense',
+): SnapshotNode[] {
   return nodes.map((node, idx) => ({
     ...node,
-    ref: node.ref ?? `e${idx + 1}`,
+    ...(node.ref === undefined && minting !== 'dense' ? {} : { ref: node.ref ?? `e${idx + 1}` }),
     kind: formatRole(node.type ?? 'Element'),
   }));
 }
 
+/**
+ * Whether a node can be addressed by ref, narrowing the ref for callers that publish it as an
+ * address (an overlay label, a settle tail entry, a ref pin). A ref-less node is readable and
+ * searchable, never addressable.
+ */
+export function hasSnapshotRef(node: SnapshotNode): node is SnapshotNode & { ref: string } {
+  return node.ref !== undefined;
+}
+
+/**
+ * The refs a set of ref-carrying records can be re-addressed by; a record with no ref contributes no
+ * evidence. Typed over the field rather than over `SnapshotNode` so every site holding an optional
+ * ref (a node, a competitor rect, a published match) answers the same question here instead of
+ * restating the filter.
+ */
+export function snapshotRefBodies(refCarriers: readonly { ref?: string }[]): string[] {
+  return refCarriers.flatMap((carrier) => (carrier.ref === undefined ? [] : [carrier.ref]));
+}
+
+/**
+ * The one producer that resolves actions against refs of its own, and so owns the ref namespace of
+ * every tree it publishes. A producer that mints nothing belongs nowhere in this list: it keeps
+ * dense positional numbering. Adding a second producer that mints its own refs means adding it here.
+ */
+const REF_NAMESPACE_OWNERS: readonly SnapshotProvenance['producer'][] = ['agent-browser'];
+
+/**
+ * The ref-minting mode of a capture, from the pair that produced it. This is the one place that
+ * knows which producers own their ref namespace, so the projection that publishes a tree and the
+ * projection that later re-attaches refs to it cannot disagree.
+ *
+ * It reads the producer and not the platform channel, per the rule the provenance table states:
+ * several producers can feed one channel with different guarantees, and a second web adapter
+ * (webdriver-backed, or anything else that lands on this channel) mints no refs of its own — it
+ * needs dense numbering, which a channel-keyed check would silently take away from it. The
+ * parameter is the whole pair or nothing, so a caller cannot present a channel and leave this to
+ * guess who acquired the tree.
+ */
+export function snapshotRefMinting(
+  provenance: SnapshotCaptureProvenance | undefined,
+): SnapshotRefMinting {
+  // agent-browser resolves click/fill/hover against refs it minted itself, in tree order and not
+  // densely (#2283). A ref minted beside it would either name a different element or nothing.
+  const producer = provenance?.producer;
+  return producer !== undefined && REF_NAMESPACE_OWNERS.includes(producer)
+    ? 'backend-issued'
+    : 'dense';
+}
+
 /**
  * Versioned-ref grammar (#1076): a ref argument may carry an optional
  * `~s<generation>` suffix pinning it to the session snapshot generation that

packages/replay-port/src/daemon-port/__tests__/target-classification.test.ts

@@ -44,7 +44,7 @@ test('classifyReplayTarget: real-capture fixture verifies by @ref when the tree
     requireRect: true,
     allowDisambiguation: true,
   });
-  assertVerified(result, { winnerRef: winner.ref, matchCount: 1 });
+  assertVerified(result, { winnerRef: winner.ref!, matchCount: 1 });
 });
 
 test('classifyReplayTarget: real-capture fixture — a relabeled node is identity-mismatch (path 3)', async () => {
@@ -547,7 +547,7 @@ test('#1269 e2e: a demoted shared-id row rebinds by role+label after the shared-
     requireRect: true,
     allowDisambiguation: true,
   });
-  assertVerified(result, { winnerRef: expected.ref, matchCount: 1 });
+  assertVerified(result, { winnerRef: expected.ref!, matchCount: 1 });
 
   // Contrast — the reorder is genuinely hostile to the id path: the shared id
   // matches all four rows (no unique bind → resolution refuses), while the
@@ -627,7 +627,7 @@ test('#1280 e2e: a retargeted press on a row container rebinds its labeled desce
     requireRect: true,
     allowDisambiguation: true,
   });
-  assertVerified(result, { winnerRef: expected.ref, matchCount: 1 });
+  assertVerified(result, { winnerRef: expected.ref!, matchCount: 1 });
 
   // Contrast: recording the CONTAINER itself (no retarget) leaves a
   // role-only identity — every row's wrapper shares it — that a fail-closed