diff --git a/examples/sdk/client-session.ts b/examples/sdk/client-session.ts index 539386ecd4..d1a3f9261a 100644 --- a/examples/sdk/client-session.ts +++ b/examples/sdk/client-session.ts @@ -64,7 +64,8 @@ async function main(): Promise { const snapshot = await client.capture.snapshot({ interactiveOnly: true }); const target = snapshot.nodes.find((node) => node.role === 'button'); - if (target) { + // A node carries a ref only when something can address it, and press needs that address. + if (target?.ref !== undefined) { await client.interactions.press({ ref: target.ref }); } } catch (error) { diff --git a/packages/capture-kit/src/screenshot-overlay.ts b/packages/capture-kit/src/screenshot-overlay.ts index be9236679f..3a3230b2b9 100644 --- a/packages/capture-kit/src/screenshot-overlay.ts +++ b/packages/capture-kit/src/screenshot-overlay.ts @@ -76,7 +76,9 @@ export function buildScreenshotOverlayRefs( for (const node of snapshot.nodes) { if (!isOverlaySourceNode(snapshot, snapshotBounds, node)) continue; const target = resolveOverlayTarget(snapshot.nodes, node); - if (!target?.rect || !hasPositiveRect(target.rect)) continue; + // The overlay labels each box with the ref an agent would act on, so a target the capture + // issued no ref for gets no box: nothing can be addressed by it. + if (!target?.ref || !target.rect || !hasPositiveRect(target.rect)) continue; const label = resolveOverlayLabel(node, target, snapshot.nodes); const score = scoreOverlayCandidate(node, target, label); const overlaySourceRect = resolveOverlaySourceRect(snapshot, target, snapshot.nodes); diff --git a/packages/capture-kit/src/snapshot-chrome.test.ts b/packages/capture-kit/src/snapshot-chrome.test.ts index 486efd6d7e..197f86adce 100644 --- a/packages/capture-kit/src/snapshot-chrome.test.ts +++ b/packages/capture-kit/src/snapshot-chrome.test.ts @@ -230,7 +230,11 @@ test('the real interactive keyboard capture collapses to one [Keyboard] diff lin node.role === 'TUISystemInputAssistantView' || node.role === 'TUIPredictionViewCell' || node.role === 'UIKeyboardDockItemButton'; - if (isKeyboardSurface) assert.equal(chromeRefs.has(node.ref), true, `node ${node.index}`); + if (isKeyboardSurface) { + // An iOS capture mints refs densely, so a keyboard surface always has one to be chrome under. + assert.ok(node.ref, `keyboard surface ${node.index} carries no ref`); + assert.equal(chromeRefs.has(node.ref), true, `node ${node.index}`); + } } // The stripped set collapses the ~50 keyboard-surface nodes; what survives is diff --git a/packages/capture-kit/src/snapshot-chrome.ts b/packages/capture-kit/src/snapshot-chrome.ts index 34ce0da898..10c897c520 100644 --- a/packages/capture-kit/src/snapshot-chrome.ts +++ b/packages/capture-kit/src/snapshot-chrome.ts @@ -11,7 +11,7 @@ import { IOS_KEYBOARD_SURFACE_ROLES, isAndroidInputMethodNode, } from '@agent-device/contracts/android-input-ownership'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { snapshotRefBodies, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { normalizeType } from '@agent-device/contracts/snapshot'; /** @@ -143,9 +143,7 @@ function collectKeyboardChrome(nodes: SnapshotNode[]): SettleChrome { chromeIndexes.add(index); } for (const index of markerIndexes) chromeIndexes.add(index); - const refs = new Set( - nodes.filter((node) => node.ref && chromeIndexes.has(node.index)).map((node) => node.ref), - ); + const refs = new Set(snapshotRefBodies(nodes.filter((node) => chromeIndexes.has(node.index)))); // The [Keyboard] container line itself survives the diff so "keyboard // appeared/left" stays one visible signal line; everything else collapses. const strippedIndexes = new Set( @@ -307,11 +305,9 @@ function collectAndroidSettleChrome( ); for (const index of systemChromeIndexes) strippedIndexes.add(index); const refs = new Set( - nodes - .filter( - (node) => node.ref && (imeIndexes.has(node.index) || systemChromeIndexes.has(node.index)), - ) - .map((node) => node.ref), + snapshotRefBodies( + nodes.filter((node) => imeIndexes.has(node.index) || systemChromeIndexes.has(node.index)), + ), ); if (strippedIndexes.size === 0 && refs.size === 0) return EMPTY_SETTLE_CHROME; return { strippedIndexes, refs }; diff --git a/packages/capture-kit/src/snapshot-state.ts b/packages/capture-kit/src/snapshot-state.ts index 7fe70d7eda..c1c361a3d8 100644 --- a/packages/capture-kit/src/snapshot-state.ts +++ b/packages/capture-kit/src/snapshot-state.ts @@ -8,6 +8,7 @@ import { attachRefs, buildSnapshotPresentationKey, snapshotPresentationOptionsFromFlags, + snapshotRefMinting, type IosTargetActivation, type RawSnapshotNode, type SnapshotBackend, @@ -79,6 +80,7 @@ export function buildSnapshotState( snapshotRaw ? scopedNodes : annotateCoveredSnapshotNodes(scopedNodes, overlayOptionsForBackend(data?.backend)), + snapshotRefMinting(data), ); return { nodes, diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 0308f90b4d..cde4faf5a0 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -283,10 +283,10 @@ export type SettleParams = { */ export type FindReadResult = | { kind: 'found'; found: true; waitedMs?: number } - | { kind: 'text'; ref: string; text: string; node: SnapshotNode } - | { kind: 'attrs'; ref: string; node: SnapshotNode } + | { kind: 'text'; ref?: string; text: string; node: SnapshotNode } + | { kind: 'attrs'; ref?: string; node: SnapshotNode } /** #1625: the read-only inspection surface — every match, never a tap. */ - | { kind: 'list'; matches: Array<{ ref: string; node: SnapshotNode }> }; + | { kind: 'list'; matches: Array<{ ref?: string; node: SnapshotNode }> }; export type SettleObservation = { settled: boolean; @@ -513,8 +513,9 @@ export type FindCommandResponseData = { waitedMs?: number; text?: string; node?: SnapshotNode; - /** Every match of the read-only `list` action (#1625), each ref pinnable at `refsGeneration`. */ - matches?: Array<{ ref: string; node: SnapshotNode }>; + /** Every match of the read-only `list` action (#1625), each ref pinnable at `refsGeneration`. A + * match on content the capture's producer issued no ref for carries none: readable, not pinnable. */ + matches?: Array<{ ref?: string; node: SnapshotNode }>; locator?: string; query?: string; x?: number; diff --git a/packages/kernel/src/snapshot-attach-refs.test.ts b/packages/kernel/src/snapshot-attach-refs.test.ts index dc125b55bb..ce02acf534 100644 --- a/packages/kernel/src/snapshot-attach-refs.test.ts +++ b/packages/kernel/src/snapshot-attach-refs.test.ts @@ -1,5 +1,12 @@ import { describe, expect, test } from 'vitest'; -import { attachRefs, type RawSnapshotNode } from './snapshot.ts'; +import { + attachRefs, + hasSnapshotRef, + snapshotRefBodies, + snapshotRefMinting, + type RawSnapshotNode, + type SnapshotProvenance, +} from './snapshot.ts'; const node = (index: number, extra: Partial = {}): RawSnapshotNode => ({ index, @@ -23,6 +30,21 @@ describe('attachRefs', () => { expect(attached.map((n) => n.ref)).toEqual(['e1', 'e7', 'e3']); }); + test('mints nothing for a node the backend left unaddressed', () => { + // A backend-issued tree is the web case: the backend resolves an action against refs of its own, + // so a ref minted here would either name a different element or nothing at all. The node stays + // in the tree — that is how its text reaches a reader — and simply carries no address. + const attached = attachRefs( + [node(0, { ref: 'e2' }), node(1), node(2, { ref: 'e4' })], + 'backend-issued', + ); + + expect(attached.map((n) => n.ref)).toEqual(['e2', undefined, 'e4']); + // No `ref` key at all, so a JSON response says "nothing addresses this" instead of "undefined". + expect(Object.hasOwn(attached[1]!, 'ref')).toBe(false); + expect(attached.map((n) => n.index)).toEqual([0, 1, 2]); + }); + test('publishes the presenter role as kind, for Android and Apple vocabularies, on every node (#2656)', () => { const attached = attachRefs([ node(0, { type: 'android.widget.Button' }), @@ -32,3 +54,72 @@ describe('attachRefs', () => { expect(attached.map((n) => n.kind)).toEqual(['button', 'button', 'element']); }); }); + +describe('snapshotRefMinting', () => { + test('the producer that owns its ref namespace gets backend-issued minting', () => { + expect(snapshotRefMinting({ backend: 'web', producer: 'agent-browser' })).toBe( + 'backend-issued', + ); + }); + + // The mode keys on the producer, never on the platform channel: several producers can feed one + // channel (`xctest` alone has four) and only some of them mint refs of their own. A web adapter + // added later — a webdriver- or CDP-backed one — would arrive on the `web` channel minting + // nothing, and would need dense numbering to keep every node addressable as it does today. + test.each([ + { backend: 'xctest', producer: 'apple-runner' }, + { backend: 'xctest', producer: 'simulator-ax-bridge' }, + { backend: 'xctest', producer: 'appium-source' }, + { backend: 'xctest', producer: 'limrun-ios-tree' }, + { backend: 'android', producer: 'android-uiautomator' }, + { backend: 'android', producer: 'appium-source' }, + { backend: 'harmonyos-arkui', producer: 'harmonyos-uitest' }, + { backend: 'macos-helper', producer: 'macos-helper' }, + { backend: 'linux-atspi', producer: 'linux-atspi' }, + ] satisfies readonly SnapshotProvenance[])( + '$backend/$producer is minted positionally', + (provenance) => { + expect(snapshotRefMinting(provenance)).toBe('dense'); + }, + ); + + test('a capture with no provenance is minted positionally', () => { + expect(snapshotRefMinting(undefined)).toBe('dense'); + expect(snapshotRefMinting({})).toBe('dense'); + // Type-level half of the producer-keyed decision: a channel cannot be handed over on its own, + // so this function never has to guess which producer published a tree. `pnpm typecheck` fails + // the moment the parameter widens back to a partial pair. + // @ts-expect-error the minting mode needs the whole provenance pair, not the channel alone + snapshotRefMinting({ backend: 'web' }); + }); + + test('a second producer on the web channel keeps dense numbering', () => { + // Forward-looking regression for the axis itself, not for a pair the table admits today: the + // provenance table has exactly one web producer, so a webdriver- or CDP-backed adapter is not + // expressible yet and has to be widened through here. When it lands, `agent-browser` is the + // entry that owns its refs and nothing else on the channel does — the case a channel check + // would get wrong by handing it `backend-issued` and stripping every ref in its tree. + const secondWebAdapter = { + backend: 'web', + producer: 'webdriver-source', + } as unknown as SnapshotProvenance; + expect(snapshotRefMinting(secondWebAdapter)).toBe('dense'); + }); +}); + +describe('snapshot addressability', () => { + const attached = attachRefs( + [node(0, { ref: 'e2' }), node(1), node(2, { ref: 'e4' })], + 'backend-issued', + ); + + test('hasSnapshotRef narrows the ref for a caller that publishes it as an address', () => { + expect(attached.filter(hasSnapshotRef).map((n) => n.ref)).toEqual(['e2', 'e4']); + expect(attached.filter((n) => !hasSnapshotRef(n)).map((n) => n.index)).toEqual([1]); + }); + + test('snapshotRefBodies collects the refs a node set can be re-addressed by', () => { + expect(snapshotRefBodies(attached)).toEqual(['e2', 'e4']); + expect(snapshotRefBodies([])).toEqual([]); + }); +}); diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 01117acd61..a483aa9cd2 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -386,15 +386,33 @@ export type SnapshotViewportSizeBrand = { }; export type SnapshotNode = RawSnapshotNode & { - ref: string; + /** + * The ref an agent addresses this node by, or absent when nothing can address it. + * + * A backend that mints its own refs (web/agent-browser) owns that namespace and can only act on + * the refs it minted, so {@link attachRefs} mints nothing into it — a node the backend left + * unaddressed stays ref-less in the tree, which is how on-screen-but-not-actionable content + * (static text, a status region) reaches a reader without ever being offered as a tap target. + * Backends that mint no refs (Apple, Android, Linux) keep dense `e${index}` numbering, so every + * node of their trees carries a ref, as before. + * + * Every consumer that offers a node as an address must read this as an absence rather than print + * one: a ref-less node is selectable by text and by `find`/`wait`/`get`, and refused as a target. + * + * A ref-less node also carries no `rect`. Producers that mint refs measure geometry for the refs + * they issued (see `RawSnapshotNode.ref`), and producers that mint none get dense numbering, so + * every node a geometry-requiring path can reach — promotion, the touch point, the covered and + * keyboard refusals — has a ref. That is why those paths print `node.ref` without a guard. + */ + ref?: string; /** * Normalized role, from the same table the text presenter reads (`formatRole` below). `type` * stays the raw platform class; `kind` is the presenter's own answer, published so a `--json` * consumer gets the presented role without re-deriving it from `type` (#2656). `attachRefs` is - * the one production construction path and always sets it; optional here (rather than required - * like `ref`) only so the many hand-built `SnapshotNode` fixtures across the daemon/selector test - * suites that predate #2656 and never route through `attachRefs` do not all need updating for a - * field their assertions never read. + * the one production construction path and always sets it; optional here only so the many + * hand-built `SnapshotNode` fixtures across the daemon/selector test suites that predate #2656 and + * never route through `attachRefs` do not all need updating for a field their assertions never + * read. An absent `ref` is a real answer about the node; an absent `kind` is a fixture shortcut. */ kind?: string; /** @@ -758,20 +776,90 @@ export type ScreenshotOverlayRef = { }; /** - * Assign a display ref to every node. A node that already carries a backend-minted - * `ref` keeps it (see `RawSnapshotNode.ref`) — the web/agent-browser backend resolves - * actions against its own refs, so re-minting a dense positional ref here would make - * the snapshot show one ref while actions act on a different element. Backends that do - * not mint refs get dense `e${index}` numbering, matching the historical behavior. + * How a capture's nodes get the ref an agent addresses them by. + * + * - `dense` — the capture's producer mints no refs, so this layer numbers the tree `e1…eN` and + * every node becomes addressable by its position. + * - `backend-issued` — the producer resolves actions against refs of its own (`e${n}` minted by + * the backend, in tree order and not dense), so a ref this layer invented would name nothing. + * A node the producer did not issue a ref for stays ref-less: present to a reader, absent from + * the addressable namespace. */ -export function attachRefs(nodes: RawSnapshotNode[]): SnapshotNode[] { +export type SnapshotRefMinting = 'dense' | 'backend-issued'; + +/** + * Assign a display ref to every node, in the mode the capture's producer needs (see + * `SnapshotRefMinting`). + * + * A node that already carries a backend-minted `ref` keeps it (see `RawSnapshotNode.ref`) — the + * web/agent-browser backend resolves actions against its own refs, so re-minting a dense positional + * ref here would make the snapshot show one ref while actions act on a different element. + * + * Under `dense` every node ends up addressable, as it always has. Under `backend-issued` only the + * nodes the producer issued a ref for do: the rest stay ref-less, readable and searchable by their + * text and tree position, and never an address — which is what `hasSnapshotRef` and + * `snapshotRefBodies` report, and what a caller publishing an address must check. + */ +export function attachRefs( + nodes: RawSnapshotNode[], + minting: SnapshotRefMinting = 'dense', +): SnapshotNode[] { return nodes.map((node, idx) => ({ ...node, - ref: node.ref ?? `e${idx + 1}`, + ...(node.ref === undefined && minting !== 'dense' ? {} : { ref: node.ref ?? `e${idx + 1}` }), kind: formatRole(node.type ?? 'Element'), })); } +/** + * Whether a node can be addressed by ref, narrowing the ref for callers that publish it as an + * address (an overlay label, a settle tail entry, a ref pin). A ref-less node is readable and + * searchable, never addressable. + */ +export function hasSnapshotRef(node: SnapshotNode): node is SnapshotNode & { ref: string } { + return node.ref !== undefined; +} + +/** + * The refs a set of ref-carrying records can be re-addressed by; a record with no ref contributes no + * evidence. Typed over the field rather than over `SnapshotNode` so every site holding an optional + * ref (a node, a competitor rect, a published match) answers the same question here instead of + * restating the filter. + */ +export function snapshotRefBodies(refCarriers: readonly { ref?: string }[]): string[] { + return refCarriers.flatMap((carrier) => (carrier.ref === undefined ? [] : [carrier.ref])); +} + +/** + * The one producer that resolves actions against refs of its own, and so owns the ref namespace of + * every tree it publishes. A producer that mints nothing belongs nowhere in this list: it keeps + * dense positional numbering. Adding a second producer that mints its own refs means adding it here. + */ +const REF_NAMESPACE_OWNERS: readonly SnapshotProvenance['producer'][] = ['agent-browser']; + +/** + * The ref-minting mode of a capture, from the pair that produced it. This is the one place that + * knows which producers own their ref namespace, so the projection that publishes a tree and the + * projection that later re-attaches refs to it cannot disagree. + * + * It reads the producer and not the platform channel, per the rule the provenance table states: + * several producers can feed one channel with different guarantees, and a second web adapter + * (webdriver-backed, or anything else that lands on this channel) mints no refs of its own — it + * needs dense numbering, which a channel-keyed check would silently take away from it. The + * parameter is the whole pair or nothing, so a caller cannot present a channel and leave this to + * guess who acquired the tree. + */ +export function snapshotRefMinting( + provenance: SnapshotCaptureProvenance | undefined, +): SnapshotRefMinting { + // agent-browser resolves click/fill/hover against refs it minted itself, in tree order and not + // densely (#2283). A ref minted beside it would either name a different element or nothing. + const producer = provenance?.producer; + return producer !== undefined && REF_NAMESPACE_OWNERS.includes(producer) + ? 'backend-issued' + : 'dense'; +} + /** * Versioned-ref grammar (#1076): a ref argument may carry an optional * `~s` suffix pinning it to the session snapshot generation that diff --git a/packages/replay-port/src/daemon-port/__tests__/target-classification.test.ts b/packages/replay-port/src/daemon-port/__tests__/target-classification.test.ts index 6dc7797658..e37262a63c 100644 --- a/packages/replay-port/src/daemon-port/__tests__/target-classification.test.ts +++ b/packages/replay-port/src/daemon-port/__tests__/target-classification.test.ts @@ -44,7 +44,7 @@ test('classifyReplayTarget: real-capture fixture verifies by @ref when the tree requireRect: true, allowDisambiguation: true, }); - assertVerified(result, { winnerRef: winner.ref, matchCount: 1 }); + assertVerified(result, { winnerRef: winner.ref!, matchCount: 1 }); }); test('classifyReplayTarget: real-capture fixture — a relabeled node is identity-mismatch (path 3)', async () => { @@ -547,7 +547,7 @@ test('#1269 e2e: a demoted shared-id row rebinds by role+label after the shared- requireRect: true, allowDisambiguation: true, }); - assertVerified(result, { winnerRef: expected.ref, matchCount: 1 }); + assertVerified(result, { winnerRef: expected.ref!, matchCount: 1 }); // Contrast — the reorder is genuinely hostile to the id path: the shared id // matches all four rows (no unique bind → resolution refuses), while the @@ -627,7 +627,7 @@ test('#1280 e2e: a retargeted press on a row container rebinds its labeled desce requireRect: true, allowDisambiguation: true, }); - assertVerified(result, { winnerRef: expected.ref, matchCount: 1 }); + assertVerified(result, { winnerRef: expected.ref!, matchCount: 1 }); // Contrast: recording the CONTAINER itself (no retarget) leaves a // role-only identity — every row's wrapper shares it — that a fail-closed diff --git a/packages/replay-port/src/daemon-port/target-classification.ts b/packages/replay-port/src/daemon-port/target-classification.ts index c24b5fc649..641afd0a7e 100644 --- a/packages/replay-port/src/daemon-port/target-classification.ts +++ b/packages/replay-port/src/daemon-port/target-classification.ts @@ -31,7 +31,13 @@ */ import type { Platform, PublicPlatform } from '@agent-device/kernel/device'; -import { findNodeByRef, normalizeRef, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { + findNodeByRef, + hasSnapshotRef, + normalizeRef, + snapshotRefBodies, + type SnapshotNode, +} from '@agent-device/kernel/snapshot'; import { findNodeByLabel } from '@agent-device/capture-kit/snapshot-node-lookup'; import { annotationLocalIdentity, @@ -123,10 +129,10 @@ export function classifyReplayTarget(params: { const classification = classifyTargetBindingMatch({ winnerRef: matching.winnerRef, - matchedRefs: matching.matchedNodes.map((node) => node.ref), - identitySetRefs: identitySet.map((node) => node.ref), - siblingMatchRefs: siblingMatches.map((node) => node.ref), - regionMemberRefs: regionMembers.map((node) => node.ref), + matchedRefs: snapshotRefBodies(matching.matchedNodes), + identitySetRefs: snapshotRefBodies(identitySet), + siblingMatchRefs: snapshotRefBodies(siblingMatches), + regionMemberRefs: snapshotRefBodies(regionMembers), viewportCandidateRef, }); @@ -195,9 +201,13 @@ function resolveRefTargetMatches( node !== null && (!requireRect || Boolean(node.rect)); const ref = normalizeRef(token); const byRef = ref ? findNodeByRef(nodes, ref) : null; - if (usable(byRef)) return { matchedNodes: [byRef], winnerRef: byRef.ref }; + if (usable(byRef) && byRef.ref !== undefined) { + return { matchedNodes: [byRef], winnerRef: byRef.ref }; + } + // A node the capture issued no ref for cannot be re-addressed, so the label fallback names no + // winner and the replay reports the divergence instead of acting on the wrong element. const byLabel = refLabel ? findNodeByLabel(nodes, refLabel) : null; - return usable(byLabel) + return usable(byLabel) && hasSnapshotRef(byLabel) ? { matchedNodes: [byLabel], winnerRef: byLabel.ref } : { matchedNodes: [], winnerRef: '' }; } @@ -214,7 +224,7 @@ function resolveSelectorTargetMatches( requireRect, allowDisambiguation, }); - if (resolution.kind === 'resolved') { + if (resolution.kind === 'resolved' && resolution.winner.ref !== undefined) { return { matchedNodes: [...resolution.matchedNodes], winnerRef: resolution.winner.ref }; } // A refusal is not always a changed screen. The rows that verify without diff --git a/packages/selectors/src/interaction-targeting.ts b/packages/selectors/src/interaction-targeting.ts index 356f40c9b9..d99e7e2dbe 100644 --- a/packages/selectors/src/interaction-targeting.ts +++ b/packages/selectors/src/interaction-targeting.ts @@ -337,9 +337,9 @@ function findPreferredActionableDescendant( if (!targetRect) return null; let current = node; - const visited = new Set(); - while (!visited.has(current.ref)) { - visited.add(current.ref); + const visited = new Set(); + while (!visited.has(current.index)) { + visited.add(current.index); const children = index ? (index.childrenByParentIndex.get(current.index) ?? []) : nodes.filter((candidate) => candidate.parentIndex === current.index); diff --git a/packages/selectors/src/interaction-touch-point.test.ts b/packages/selectors/src/interaction-touch-point.test.ts index 9160ed053f..5395518a67 100644 --- a/packages/selectors/src/interaction-touch-point.test.ts +++ b/packages/selectors/src/interaction-touch-point.test.ts @@ -175,6 +175,7 @@ test('fails closed when interactive descendants tile the parent', () => { assert.deepEqual(resolveInteractionTouchPoint(nodes, nodes[0]!), { kind: 'blocked', + competitorCount: 2, competitorRefs: ['e2', 'e3'], }); }); diff --git a/packages/selectors/src/interaction-touch-point.ts b/packages/selectors/src/interaction-touch-point.ts index 6c23d2ec12..d206ef325a 100644 --- a/packages/selectors/src/interaction-touch-point.ts +++ b/packages/selectors/src/interaction-touch-point.ts @@ -1,4 +1,9 @@ -import type { Point, Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { + snapshotRefBodies, + type Point, + type Rect, + type SnapshotNode, +} from '@agent-device/kernel/snapshot'; import { containsPoint } from '@agent-device/kernel/rect'; import { areRectsApproximatelyEqual, @@ -9,13 +14,17 @@ import { isSemanticTouchTarget } from './touch-semantics.ts'; export type InteractionTouchPointResolution = | { kind: 'resolved'; point: Point; strategy: 'center' | 'parent-owned' } - | { kind: 'blocked'; competitorRefs: string[] } + /** + * The descendants that own the blocked area: `competitorCount` is the total, and + * `competitorRefs` names them as far as the capture made that possible. + */ + | { kind: 'blocked'; competitorCount: number; competitorRefs: string[] } | { kind: 'invalid' }; const MIN_COMPETITOR_CLEARANCE = 12; const MAX_PARENT_EDGE_CLEARANCE = 12; -type InteractiveDescendantRect = { ref: string; index: number; rect: Rect }; +type InteractiveDescendantRect = { ref?: string; index: number; rect: Rect }; type RankedPoint = { point: Point; distanceSquared: number; clearance: number }; type PointClearance = { parentEdge: number; competitor: number }; @@ -43,7 +52,8 @@ export function resolveInteractionTouchPoint( if (!point) { return { kind: 'blocked', - competitorRefs: competitors.map((competitor) => competitor.ref), + competitorRefs: snapshotRefBodies(competitors), + competitorCount: competitors.length, }; } return { kind: 'resolved', point, strategy: 'parent-owned' }; diff --git a/packages/selectors/src/target-evidence.ts b/packages/selectors/src/target-evidence.ts index 401565ca7b..a1a35810b8 100644 --- a/packages/selectors/src/target-evidence.ts +++ b/packages/selectors/src/target-evidence.ts @@ -20,7 +20,7 @@ * identically by definition" (decision 3, "Record-time write"). */ -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { snapshotRefBodies, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { resolveRectCenter } from '@agent-device/kernel/rect-center'; import { findNearestScrollableContainer } from '@agent-device/capture-kit/ios-snapshot-tree'; import type { @@ -286,14 +286,16 @@ function runRecordTimeSelfCheck(params: { domain: DisambiguationDomain; }): TargetVerification { const { node, domain } = params; + // A node the capture's producer issued no ref for has no ref identity to compare: it can be + // found again by tree position but never re-addressed, so the self-check has nothing to verify. const winnerRef = node.ref; - const identitySetRefs = domain.identitySet.map((n) => n.ref); + if (!winnerRef) return 'unverifiable'; const classification = classifyTargetBindingMatch({ winnerRef, - matchedRefs: identitySetRefs, - identitySetRefs, - siblingMatchRefs: domain.siblingMatches.map((n) => n.ref), - regionMemberRefs: domain.regionMembers?.map((n) => n.ref), + matchedRefs: snapshotRefBodies(domain.identitySet), + identitySetRefs: snapshotRefBodies(domain.identitySet), + siblingMatchRefs: snapshotRefBodies(domain.siblingMatches), + regionMemberRefs: domain.regionMembers ? snapshotRefBodies(domain.regionMembers) : undefined, viewportCandidateRef: domain.orderedRegion[domain.viewportOrder]?.ref, }); return classification.outcome; diff --git a/src/__tests__/daemon-composition/replay-target/session-replay-target-guard.test.ts b/src/__tests__/daemon-composition/replay-target/session-replay-target-guard.test.ts index dae8cede39..fc451b6959 100644 --- a/src/__tests__/daemon-composition/replay-target/session-replay-target-guard.test.ts +++ b/src/__tests__/daemon-composition/replay-target/session-replay-target-guard.test.ts @@ -106,7 +106,7 @@ test('split resolver: verification verifies the covered deeper node while dispat // isolates A; the unfiltered disambiguation winner is also A (deepest) — // path 4 verified. Dispatch would resolve B: exactly the split the // post-resolution guard exists to catch. - assertVerified(result, { winnerRef: nodeA.ref, matchCount: 2 }); + assertVerified(result, { winnerRef: nodeA.ref!, matchCount: 2 }); }); test('split resolver: the post-resolution guard refuses pre-action when dispatch resolves a different element', async () => { @@ -275,7 +275,7 @@ test('same-identity duplicates: verification denotes the covered member A among }); // matchCount 2 (both match); path 6 sibling ordinal isolates A, which is // also the unfiltered disambiguation winner → verified on A. - assertVerified(result, { winnerRef: nodeA.ref, matchCount: 2 }); + assertVerified(result, { winnerRef: nodeA.ref!, matchCount: 2 }); }); test('same-identity duplicates: the structural guard refuses pre-action when dispatch resolves the OTHER duplicate (P1 regression)', async () => { diff --git a/src/commands/interaction/output.test.ts b/src/commands/interaction/output.test.ts index 9e46ea01f5..7198002931 100644 --- a/src/commands/interaction/output.test.ts +++ b/src/commands/interaction/output.test.ts @@ -71,6 +71,33 @@ describe('find CLI output', () => { ); }); + // The web backend's capture reports on-screen content it cannot address (static text, a status + // region). Such a match is still a match, so the row says it has no ref instead of printing an + // `@undefined` the daemon would reject. + test('list marks a match the capture issued no ref for', () => { + const output = formatFind({ + matches: [ + { ref: '@e5', node: { ref: 'e5', type: 'button', label: 'Add' } }, + { node: { type: 'StaticText', value: 'Item saved' } }, + ], + refsGeneration: 500014, + }); + + expect(output.text).toBe( + ['2 matches:', '= @e5~s500014 [button] "Add"', '= (no ref) [text] "Item saved"'].join('\n'), + ); + }); + + test('get text of a ref-less match prints its text and offers no ref', () => { + const output = formatFind({ + text: 'Item saved', + node: { type: 'StaticText', value: 'Item saved' }, + }); + + expect(output.text).toBe('Item saved'); + expect('ref' in (output.data as Record)).toBe(false); + }); + test('list without a generation still lists refs unpinned rather than hiding them', () => { const output = formatFind({ matches: [{ ref: '@e5', node: { ref: 'e5', type: 'Button', label: 'Add' } }], diff --git a/src/commands/interaction/output.ts b/src/commands/interaction/output.ts index 266bfc0711..eb5256411d 100644 --- a/src/commands/interaction/output.ts +++ b/src/commands/interaction/output.ts @@ -57,10 +57,14 @@ function formatFindMatchLines(matches: unknown[], refsGeneration: unknown): stri return [heading, ...lines].join('\n'); } +/** What a listed match prints where a paste-ready ref normally goes. */ +const FIND_NO_REF_ADDRESS = '(no ref)'; + function formatFindMatchLine(view: FindMatchView, refsGeneration: unknown): string { - const plain = view.ref ?? ''; - const body = plain.startsWith('@') ? plain.slice(1) : plain; - const ref = pinnedRefText(plain, refsGeneration) ?? `@${body}`; + const ref = + view.ref === undefined + ? FIND_NO_REF_ADDRESS + : (pinnedRefText(view.ref, refsGeneration) ?? view.ref); const role = formatRole(view.node?.type ?? 'Element'); const label = view.node ? displayLabel(view.node, role) : ''; return `= ${ref} [${role}]${label ? ` "${label}"` : ''}`; diff --git a/src/commands/interaction/runtime/keyboard-occlusion.ts b/src/commands/interaction/runtime/keyboard-occlusion.ts index 1491ff5892..7535dfce65 100644 --- a/src/commands/interaction/runtime/keyboard-occlusion.ts +++ b/src/commands/interaction/runtime/keyboard-occlusion.ts @@ -1,9 +1,9 @@ -import type { - Point, - Rect, - SnapshotKeyboardBandFact, - SnapshotNode, - SnapshotState, +import { + type Point, + type Rect, + type SnapshotKeyboardBandFact, + type SnapshotNode, + type SnapshotState, } from '@agent-device/kernel/snapshot'; import { AppError, discloseDispatch } from '@agent-device/kernel/errors'; import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; @@ -106,7 +106,8 @@ function keyboardOcclusionWarning(point: Point, surface: KeyboardSurface): strin function buildKeyboardOcclusionError(params: { label: string; action: InteractionAction; - ref: string; + /** The ref the target was named by; omitted with the detail when nothing addresses the node. */ + ref: string | undefined; targetRect: NonNullable; surface: KeyboardSurface; }): AppError { @@ -116,7 +117,7 @@ function buildKeyboardOcclusionError(params: { `${params.label} is behind the visible keyboard and cannot ${interactionVerb(params.action)} safely`, { ...TAP_KEYBOARD_OCCLUDES_TARGET_DETAILS, - ref: `@${params.ref}`, + ...(params.ref === undefined ? {} : { ref: `@${params.ref}` }), rect: params.targetRect, keyboardFrame: params.surface.frame, }, diff --git a/src/commands/interaction/runtime/resolution-touch-point.ts b/src/commands/interaction/runtime/resolution-touch-point.ts index 43a68e4532..410eef4f06 100644 --- a/src/commands/interaction/runtime/resolution-touch-point.ts +++ b/src/commands/interaction/runtime/resolution-touch-point.ts @@ -38,7 +38,7 @@ export function resolveNodeTouchPoint( reason: 'covered_by_interactive_descendants', ...failure.blockedTargetDetails, competitorRefs: resolution.competitorRefs.slice(0, 5).map((ref) => `@${ref}`), - competitorCount: resolution.competitorRefs.length, + competitorCount: resolution.competitorCount, hint: 'Tap the specific interactive child you intend, or use a more specific selector. Every safely tappable region of the parent belongs to one of its child controls.', }, ), diff --git a/src/commands/interaction/runtime/selector-read.ts b/src/commands/interaction/runtime/selector-read.ts index 03847d857c..8a4f6f27ee 100644 --- a/src/commands/interaction/runtime/selector-read.ts +++ b/src/commands/interaction/runtime/selector-read.ts @@ -143,10 +143,9 @@ export const findCommand: RuntimeCommand = async ( @@ -313,10 +312,20 @@ async function listFindMatches( : findBestMatchesByLocator(capture.snapshot.nodes, locator, options.query, {}).matches; return { kind: 'list', - matches: matched.map((node) => ({ ref: `@${node.ref}`, node })), + matches: matched.map((node) => ({ ...refAddress(node), node })), }; } +/** + * The address an agent acts on, when this node has one. Where the capture's producer mints its own + * refs (`backend-issued`, web) a node it left unaddressed carries none, and listing one would offer + * an agent something the next command refuses; where refs were minted densely by `attachRefs` + * (Apple, Android, Linux) every node has one. Read the field, never the mode. + */ +function refAddress(node: SnapshotNode): { ref?: string } { + return node.ref === undefined ? {} : { ref: `@${node.ref}` }; +} + async function findFirstLocatorMatch( runtime: AgentDeviceRuntime, options: FindReadCommandOptions, @@ -362,7 +371,7 @@ async function resolveSelectorNode( options: GetCommandOptions, sessionName: string, params: { selector: string; policy: GetPipelinePolicy; hooks?: SelectorPipelineHooks }, -): Promise<{ capture: CapturedSnapshot; node: SnapshotNode; selector: string; ref: string }> { +): Promise<{ capture: CapturedSnapshot; node: SnapshotNode; selector: string; ref?: string }> { const capture = await captureSelectorSnapshot( runtime, { ...options, session: sessionName }, @@ -389,6 +398,6 @@ async function resolveSelectorNode( capture, node: outcome.node, selector: outcome.selector, - ref: `@${outcome.node.ref}`, + ...refAddress(outcome.node), }; } diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 1dac8af721..f44e76226a 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -1,4 +1,4 @@ -import type { Point, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { hasSnapshotRef, type Point, type SnapshotNode } from '@agent-device/kernel/snapshot'; import type { AgentDeviceRuntime, CommandContext, @@ -373,14 +373,15 @@ export function buildSettleTailEntries( appBundleId?: string, ): Pick { const chromeRefs = collectSettleChromeRefs(settledNodes, appBundleId); - const candidates = settledNodes.filter( - (node) => - node.ref && - node.interactionBlocked !== 'covered' && - !excludeRefs.has(node.ref) && - !chromeRefs.has(node.ref) && - !STRUCTURAL_TAIL_ROLES.has(formatRole(node.type ?? 'Element')), - ); + const candidates = settledNodes + .filter( + (node) => + node.interactionBlocked !== 'covered' && + !STRUCTURAL_TAIL_ROLES.has(formatRole(node.type ?? 'Element')), + ) + // The tail is a list of next actionable targets, so only addressable nodes enter it. + .filter(hasSnapshotRef) + .filter((node) => !excludeRefs.has(node.ref) && !chromeRefs.has(node.ref)); if (candidates.length === 0) return {}; const tail: SettleTailEntry[] = candidates.slice(0, MAX_SETTLE_TAIL_ENTRIES).map((node) => { const role = formatRole(node.type ?? 'Element'); diff --git a/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts b/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts index 4c1896b0ab..594a50c060 100644 --- a/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts +++ b/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts @@ -1,6 +1,11 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; -import { attachRefs, type RawSnapshotNode, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { + attachRefs, + snapshotRefBodies, + type RawSnapshotNode, + type SnapshotNode, +} from '@agent-device/kernel/snapshot'; import { collectSettleChromeRefs, withoutSettleChrome, @@ -79,9 +84,9 @@ test('Android non-raw capture: status-bar leaves stay chrome after the walk drop // The whole systemui run drops together, including the anonymous Compose // plumbing nodes above/around the leaves that carry no identifier at all // (and the re-parented, now-anonymous battery leaf). - const systemUiRefs = nodes - .filter((node) => node.bundleId === 'com.android.systemui') - .map((node) => node.ref); + const systemUiRefs = snapshotRefBodies( + nodes.filter((node) => node.bundleId === 'com.android.systemui'), + ); assert.equal(systemUiRefs.length > 0, true); for (const ref of systemUiRefs) { assert.equal(chromeRefs.has(ref), true, 'expected every systemui-owned node to be chrome'); @@ -93,9 +98,9 @@ test('Android non-raw capture: status-bar leaves stay chrome after the walk drop assert.equal(chromeRefs.has(refForIdentifier(nodes, 'field-name')), false); assert.equal(chromeRefs.has(refForIdentifier(nodes, 'field-email')), false); assert.equal(chromeRefs.has(refForIdentifier(nodes, 'field-phone')), false); - const imeRefs = nodes - .filter((node) => node.bundleId === 'com.google.android.inputmethod.latin') - .map((node) => node.ref); + const imeRefs = snapshotRefBodies( + nodes.filter((node) => node.bundleId === 'com.google.android.inputmethod.latin'), + ); assert.equal(imeRefs.length > 0, true); for (const ref of imeRefs) { assert.equal(chromeRefs.has(ref), true, 'expected every IME-owned node to be chrome'); diff --git a/src/core/__tests__/snapshot-state.test.ts b/src/core/__tests__/snapshot-state.test.ts index 5daa6ae3b5..9830862235 100644 --- a/src/core/__tests__/snapshot-state.test.ts +++ b/src/core/__tests__/snapshot-state.test.ts @@ -674,3 +674,69 @@ test('buildSnapshotState leaves an unmeasured viewport absent (#3182)', () => { expect('viewport' in state).toBe(false); }); + +// The ref-minting mode reaches the stored tree through ONE argument here, and the whole web +// ref-collision fix (#2283) rests on it: an agent-browser tree that got dense numbering publishes +// refs its own backend cannot resolve. These two cases are that argument's only end-to-end pin — +// delete the `snapshotRefMinting(data)` argument and the first one goes red. +test('buildSnapshotState publishes only the refs an agent-browser capture issued (#2283)', () => { + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'document', + label: 'Checkout', + ref: 'e1', + rect: { x: 0, y: 0, width: 390, height: 720 }, + hittable: true, + }, + // On-screen text the backend reports without a ref: it cannot be addressed, so nothing + // here may hand it a positional one that collides with a real control. + { index: 1, depth: 1, parentIndex: 0, type: 'static text', value: 'Order summary' }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'button', + label: 'Submit order', + ref: 'e3', + rect: { x: 24, y: 120, width: 160, height: 44 }, + hittable: true, + }, + ], + backend: 'web', + producer: 'agent-browser', + }, + undefined, + ); + + expect('ref' in state.nodes[1]!).toBe(false); + // The producer's own numbering is non-dense (tree order, gaps where it minted nothing) and stays + // byte-identical: a renumbered tree would read as a working capture acting on another element. + expect(state.nodes.map((node) => node.ref)).toEqual(['e1', undefined, 'e3']); +}); + +test('buildSnapshotState numbers an Android capture densely so every node is addressable', () => { + const state = buildSnapshotState( + { + nodes: [ + { index: 0, depth: 0, type: 'android.widget.FrameLayout', label: 'Checkout' }, + { index: 1, depth: 1, parentIndex: 0, type: 'android.widget.TextView', label: 'Total' }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'android.widget.Button', + label: 'Submit order', + }, + ], + backend: 'android', + producer: 'android-uiautomator', + }, + undefined, + ); + + expect(state.nodes.map((node) => node.ref)).toEqual(['e1', 'e2', 'e3']); +}); diff --git a/src/daemon/__tests__/selector-recording-find-data.test.ts b/src/daemon/__tests__/selector-recording-find-data.test.ts new file mode 100644 index 0000000000..7c274a2a96 --- /dev/null +++ b/src/daemon/__tests__/selector-recording-find-data.test.ts @@ -0,0 +1,42 @@ +/** + * `toDaemonFindData` is where a find read's result becomes the response body, so it is the last + * place a ref-less match can be turned back into an address: the runtime hands over `ref?: string` + * and an unconditional `ref: match.ref` would publish a key whose value is `undefined`. + */ +import { expect, test } from 'vitest'; +import { toDaemonFindData } from '../selector-recording.ts'; + +test('a listed match with no ref publishes no ref key', () => { + const data = toDaemonFindData({ + kind: 'list', + matches: [ + { ref: '@e5', node: { index: 0, ref: 'e5', type: 'button', label: 'Add' } }, + { node: { index: 1, type: 'StaticText', value: 'Item saved' } }, + ], + }); + + const matches = data.matches as Array>; + expect(matches[0]).toMatchObject({ ref: '@e5' }); + expect('ref' in matches[1]!).toBe(false); +}); + +test('a ref-less text read publishes no ref key', () => { + const data = toDaemonFindData({ + kind: 'text', + text: 'Item saved', + node: { index: 0, type: 'StaticText', value: 'Item saved' }, + }); + + expect('ref' in data).toBe(false); + expect(data.text).toBe('Item saved'); +}); + +test('a read that resolved to a ref still publishes it', () => { + const data = toDaemonFindData({ + kind: 'attrs', + ref: '@e9', + node: { index: 0, ref: 'e9', type: 'button', label: 'Add' }, + }); + + expect(data.ref).toBe('@e9'); +}); diff --git a/src/daemon/selector-recording.ts b/src/daemon/selector-recording.ts index 38e044b079..3b06d57c04 100644 --- a/src/daemon/selector-recording.ts +++ b/src/daemon/selector-recording.ts @@ -49,18 +49,27 @@ export function toDaemonFindData(result: DaemonFindResult): Record ({ - ref: match.ref, + ...daemonRefAddress(match.ref), node: stripAndroidSystemChromeProvenanceFromNode(match.node), })), }; } return { - ref: result.ref, + ...daemonRefAddress(result.ref), ...(result.kind === 'text' ? { text: result.text } : {}), node: stripAndroidSystemChromeProvenanceFromNode(result.node), }; } +/** + * A match the producer issued no ref for carries no `ref` own-property. The wire form already hid + * that case, because `JSON.stringify` drops an `undefined` value; the response DATA object is what + * the CLI formatter and a Node consumer read, and that object is where the optional field has to hold. + */ +function daemonRefAddress(ref: string | undefined): { ref?: string } { + return ref === undefined ? {} : { ref }; +} + export function buildGetRecordResult( result: Record, property: 'text' | 'attrs', diff --git a/src/mcp/tool-ref-pins.ts b/src/mcp/tool-ref-pins.ts index cdf3b93525..d47e56c426 100644 --- a/src/mcp/tool-ref-pins.ts +++ b/src/mcp/tool-ref-pins.ts @@ -1,6 +1,6 @@ import type { SettleObservation } from '@agent-device/contracts/interaction'; import { readElementMatchCandidateRefs } from '@agent-device/kernel/errors'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import { snapshotRefBodies, type SnapshotNode } from '@agent-device/kernel/snapshot'; import { isCommandName, type CommandName } from '../commands/command-metadata.ts'; import type { CommandExecutionResult } from '../commands/command-surface.ts'; import { @@ -161,7 +161,7 @@ function mergeCommandResult( */ type SnapshotPinView = { refsGeneration?: number; - refs?: Array<{ ref: string }>; + refs?: Array<{ ref?: string }>; nodes?: SnapshotNode[]; }; @@ -175,13 +175,12 @@ function mergeSnapshotRefPins( refPinsByScope.delete(scopeKey); return; } - const bodies: string[] = []; - for (const { ref } of result.refs ?? []) { - bodies.push(ref); - } - for (const node of result.nodes ?? []) { - bodies.push(node.ref); - } + // Ref-less nodes (the web backend's on-screen-but-unaddressable content) mint no pin: there is + // no ref to pin, and an empty body would poison the scope's pin map. + const bodies = [ + ...snapshotRefBodies(result.refs ?? []), + ...snapshotRefBodies(result.nodes ?? []), + ]; mergeIntoScopedPins(refPinsByScope, scopeKey, bodies, refsGeneration); } diff --git a/test/integration/provider-scenarios/web-provider.test.ts b/test/integration/provider-scenarios/web-provider.test.ts index 4b84d87dd1..c9816a8094 100644 --- a/test/integration/provider-scenarios/web-provider.test.ts +++ b/test/integration/provider-scenarios/web-provider.test.ts @@ -25,6 +25,9 @@ test('web provider is scoped through the request router and dispatch path', asyn nodes: [ { index: 0, + // Refs the way agent-browser issues them: a web capture publishes its producer's refs + // and mints no replacement for a node it left unaddressed. + ref: 'e1', type: 'section', role: 'main', label: 'main', @@ -33,6 +36,7 @@ test('web provider is scoped through the request router and dispatch path', asyn }, { index: 1, + ref: 'e2', type: 'button', role: 'button', label: 'Launch', diff --git a/test/integration/provider-scenarios/web-world.ts b/test/integration/provider-scenarios/web-world.ts index f7e59acdee..35a0dbdaba 100644 --- a/test/integration/provider-scenarios/web-world.ts +++ b/test/integration/provider-scenarios/web-world.ts @@ -143,9 +143,13 @@ export async function createWebDesktopWorld(): Promise { } function webSnapshotNodes(state: WebPageState): RawSnapshotNode[] { + // Every element of the emulated page carries a ref, in the tree order the backend mints them in: + // a web capture publishes its producer's refs and mints no replacement, so a page that wants to be + // addressable has to issue them itself. return [ { index: 0, + ref: 'e1', role: 'document', label: state.openedTarget, rect: { x: 0, y: 0, width: 390, height: 720 }, @@ -156,6 +160,7 @@ function webSnapshotNodes(state: WebPageState): RawSnapshotNode[] { }, { index: 1, + ref: 'e2', role: 'static text', label: 'Ready', rect: { x: 24, y: 32, width: 160, height: 28 }, @@ -167,6 +172,7 @@ function webSnapshotNodes(state: WebPageState): RawSnapshotNode[] { }, { index: 2, + ref: 'e3', role: 'text field', label: 'Email', value: state.inputValue, @@ -179,6 +185,7 @@ function webSnapshotNodes(state: WebPageState): RawSnapshotNode[] { }, { index: 3, + ref: 'e4', role: 'button', label: 'Submit order', rect: BUTTON_RECT, @@ -190,6 +197,7 @@ function webSnapshotNodes(state: WebPageState): RawSnapshotNode[] { }, { index: 4, + ref: 'e5', role: 'static text', label: state.statusText, rect: { x: 24, y: 204, width: 180, height: 28 }, @@ -201,6 +209,7 @@ function webSnapshotNodes(state: WebPageState): RawSnapshotNode[] { }, { index: 5, + ref: 'e6', role: 'static text', label: state.scrolled ? 'Scrolled section' : 'Below the fold', rect: { x: 24, y: 620, width: 180, height: 28 },