From add06f4ac352e945f9aa06de970308576f863cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 12:55:44 +0200 Subject: [PATCH 1/2] refactor(capture): require complete runtime snapshots --- packages/contracts/src/backend.ts | 19 +- src/commands/capture/runtime/snapshot.test.ts | 389 ++++++++++-------- src/commands/capture/runtime/snapshot.ts | 32 +- .../runtime/interaction-snapshot-capture.ts | 11 +- .../interaction/runtime/selector-is.test.ts | 2 +- .../runtime/selector-read-shared.ts | 32 +- .../interaction/runtime/selector-read.test.ts | 29 +- .../interaction/runtime/wait-absent.test.ts | 6 +- 8 files changed, 245 insertions(+), 275 deletions(-) diff --git a/packages/contracts/src/backend.ts b/packages/contracts/src/backend.ts index ec47635b75..1499aa80da 100644 --- a/packages/contracts/src/backend.ts +++ b/packages/contracts/src/backend.ts @@ -23,11 +23,9 @@ import type { import type { Point, Rect, - SnapshotKeyboardBandFact, SnapshotNode, SnapshotOptions, SnapshotState, - SnapshotViewportSize, } from '@agent-device/kernel/snapshot'; // The backend's public leaf platform (approach b): backends distinguish iOS from @@ -45,25 +43,10 @@ export type BackendCommandContext = { }; export type BackendSnapshotResult = { - nodes?: SnapshotNode[]; - truncated?: boolean; - backend?: string; - snapshot?: SnapshotState; + snapshot: SnapshotState; appName?: string; appBundleId?: string; snapshotDiagnostics?: SnapshotDiagnosticsSummary; - /** - * The keyboard band this capture's producer measured (#2660), carried beside the tree it measured. - * A backend that publishes nothing measured nothing, and the tap-path keyboard guard keeps deriving - * the band from `nodes`. - */ - keyboard?: SnapshotKeyboardBandFact; - /** - * The box these rects are measured in, as the producer measured it (#3182). Absent means the - * producer measured no box — never a zero — since that absence is the only answer a producer with - * no screen to measure is allowed to give. - */ - viewport?: SnapshotViewportSize; } & SnapshotCaptureAnnotations; export type BackendSnapshotOptions = SnapshotOptions & { diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index 892001de73..b86847336b 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -230,9 +230,7 @@ test('runtime diff snapshot initializes baseline when no previous snapshot exist test('runtime snapshot emits filtered Android guidance from backend analysis', async () => { const device = createSnapshotOnlyDevice({ - nodes: [], - truncated: false, - backend: 'android', + snapshot: makeSnapshotState([], { truncated: false, backend: 'android' }), analysis: { rawNodeCount: 42, maxDepth: 6, @@ -253,12 +251,13 @@ test('runtime snapshot emits filtered Android guidance from backend analysis', a test('runtime snapshot renders the system-surface disclosure from Android annotations', async () => { const device = createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'FrameLayout', label: 'Quick settings' }, - { ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'Switch', label: 'Internet' }, - ], - truncated: false, - backend: 'android', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'FrameLayout', label: 'Quick settings' }, + { index: 1, depth: 1, parentIndex: 0, type: 'Switch', label: 'Internet' }, + ], + { truncated: false, backend: 'android' }, + ), androidSnapshot: { backend: 'android-helper', systemSurfaceOnly: true, @@ -274,9 +273,10 @@ test('runtime snapshot renders the system-surface disclosure from Android annota test('runtime snapshot warns when iOS interactive output is root-only', async () => { const device = createSnapshotOnlyDevice({ - nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Application' }], - truncated: false, - backend: 'xctest', + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Application' }], { + truncated: false, + backend: 'xctest', + }), }); const result = await device.capture.snapshot({ @@ -292,13 +292,14 @@ test('runtime snapshot warns when iOS interactive output is root-only', async () test('runtime snapshot flags a merged accessibility leaf and surfaces backend warnings', async () => { const mergedLabel = Array.from({ length: 30 }, (_, i) => `Row ${i}, Tap`).join(', '); const device = createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'App' }, - { ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'Other', label: mergedLabel }, - { ref: 'e3', index: 2, depth: 1, parentIndex: 0, type: 'Button', label: 'Ok' }, - ], - truncated: false, - backend: 'xctest', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'Application', label: 'App' }, + { index: 1, depth: 1, parentIndex: 0, type: 'Other', label: mergedLabel }, + { index: 2, depth: 1, parentIndex: 0, type: 'Button', label: 'Ok' }, + ], + { truncated: false, backend: 'xctest' }, + ), warnings: [ 'Recovered this snapshot with the fallback accessibility backend after sparse tree.', ], @@ -319,16 +320,17 @@ test('runtime snapshot flags a merged accessibility leaf and surfaces backend wa test('runtime snapshot does not flag prose text or labeled containers with children', async () => { const prose = Array.from({ length: 30 }, (_, i) => `clause ${i}`).join(', '); const device = createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'App' }, - // Long comma-joined prose on a text node: content, not a collapsed container. - { ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'StaticText', label: prose }, - // Same label shape on a container WITH children: not a merged leaf. - { ref: 'e3', index: 2, depth: 1, parentIndex: 0, type: 'Other', label: prose }, - { ref: 'e4', index: 3, depth: 2, parentIndex: 2, type: 'Button', label: 'Ok' }, - ], - truncated: false, - backend: 'xctest', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'Application', label: 'App' }, + // Long comma-joined prose on a text node: content, not a collapsed container. + { index: 1, depth: 1, parentIndex: 0, type: 'StaticText', label: prose }, + // Same label shape on a container WITH children: not a merged leaf. + { index: 2, depth: 1, parentIndex: 0, type: 'Other', label: prose }, + { index: 3, depth: 2, parentIndex: 2, type: 'Button', label: 'Ok' }, + ], + { truncated: false, backend: 'xctest' }, + ), }); const result = await device.capture.snapshot({ session: 'default' }); @@ -339,12 +341,22 @@ test('runtime snapshot does not flag prose text or labeled containers with child test('runtime snapshot renders the structured quality verdict and skips legacy detectors', async () => { const mergedLabel = Array.from({ length: 30 }, (_, i) => `Row ${i}, Tap`).join(', '); const device = createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'App' }, - { ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'Other', label: mergedLabel }, - ], - truncated: true, - backend: 'xctest', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'Application', label: 'App' }, + { index: 1, depth: 1, parentIndex: 0, type: 'Other', label: mergedLabel }, + ], + { + truncated: true, + backend: 'xctest', + snapshotQuality: { + state: 'recovered', + backend: 'queries', + reason: 'snapshot returned only structural application/window nodes', + collapsedLeafIndexes: [1], + }, + }, + ), quality: { state: 'recovered', backend: 'queries', @@ -372,13 +384,23 @@ test('runtime snapshot renders the structured quality verdict and skips legacy d test('runtime snapshot renders healthy-capture disclosures from the verdict alone', async () => { const tabBarLabel = Array.from({ length: 12 }, (_, i) => `Tab ${i}`).join(', '); const device = createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'Application', label: 'App' }, - { ref: 'e2', index: 1, depth: 1, parentIndex: 0, type: 'Other', label: tabBarLabel }, - { ref: 'e3', index: 2, depth: 1, parentIndex: 0, type: 'Button', label: 'Ok' }, - ], - truncated: false, - backend: 'xctest', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'Application', label: 'App' }, + { index: 1, depth: 1, parentIndex: 0, type: 'Other', label: tabBarLabel }, + { index: 2, depth: 1, parentIndex: 0, type: 'Button', label: 'Ok' }, + ], + { + truncated: false, + backend: 'xctest', + snapshotQuality: { + state: 'healthy', + backend: 'tree', + customActions: { read: 12, candidates: 19, truncated: 0, blocked: false }, + collapsedLeafIndexes: [1], + }, + }, + ), quality: { state: 'healthy', backend: 'tree', @@ -397,12 +419,13 @@ test('runtime snapshot renders healthy-capture disclosures from the verdict alon test('runtime snapshot does not warn for a normal iOS interactive output', async () => { const device = createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'Application' }, - { ref: 'e2', index: 1, depth: 1, type: 'Button', label: 'Continue' }, - ], - truncated: false, - backend: 'xctest', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'Application' }, + { index: 1, depth: 1, type: 'Button', label: 'Continue' }, + ], + { truncated: false, backend: 'xctest' }, + ), }); const result = await device.capture.snapshot({ @@ -415,13 +438,14 @@ test('runtime snapshot does not warn for a normal iOS interactive output', async test('runtime snapshot warns when Android hierarchy looks like a React Native overlay', async () => { const device = createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'Text', label: 'LogBox' }, - { ref: 'e2', index: 1, depth: 1, type: 'Text', label: 'Warnings' }, - { ref: 'e3', index: 2, depth: 1, type: 'Button', label: 'Dismiss' }, - ], - truncated: false, - backend: 'android', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'Text', label: 'LogBox' }, + { index: 1, depth: 1, type: 'Text', label: 'Warnings' }, + { index: 2, depth: 1, type: 'Button', label: 'Dismiss' }, + ], + { truncated: false, backend: 'android' }, + ), }); const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); @@ -431,17 +455,17 @@ test('runtime snapshot warns when Android hierarchy looks like a React Native ov test('runtime snapshot warns on collapsed Android React Native warning banners', async () => { const device = createSnapshotOnlyDevice({ - nodes: [ - { - ref: 'e1', - index: 0, - depth: 0, - type: 'android.widget.TextView', - label: 'Warning: Each child in a list should have a unique "key" prop.', - }, - ], - truncated: false, - backend: 'android', + snapshot: makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'android.widget.TextView', + label: 'Warning: Each child in a list should have a unique "key" prop.', + }, + ], + { truncated: false, backend: 'android' }, + ), }); const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); @@ -451,26 +475,25 @@ test('runtime snapshot warns on collapsed Android React Native warning banners', test('runtime snapshot does not suggest full-screen React Native warning parents', async () => { const result = await createSnapshotOnlyDevice({ - nodes: [ - { - ref: 'e1', - index: 0, - depth: 0, - type: 'XCUIElementTypeOther', - label: '!, Open debugger to view warnings.', - rect: { x: 0, y: 0, width: 402, height: 874 }, - }, - { - ref: 'e2', - index: 1, - depth: 1, - type: 'XCUIElementTypeOther', - label: '!, Open debugger to view warnings.', - rect: { x: 10, y: 786, width: 382, height: 68 }, - }, - ], - truncated: false, - backend: 'xctest', + snapshot: makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeOther', + label: '!, Open debugger to view warnings.', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + type: 'XCUIElementTypeOther', + label: '!, Open debugger to view warnings.', + rect: { x: 10, y: 786, width: 382, height: 68 }, + }, + ], + { truncated: false, backend: 'xctest' }, + ), }).capture.snapshot({ session: 'default', interactiveOnly: true }); assertReactNativeOverlayWarning(result.warnings); @@ -479,13 +502,14 @@ test('runtime snapshot does not suggest full-screen React Native warning parents test('runtime snapshot recognizes Android React Native stack overlays with Dismiss and Minimize controls', async () => { const result = await createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'android.widget.TextView', label: 'useOnyx.ts:80:43' }, - { ref: 'e2', index: 1, depth: 1, type: 'android.widget.Button', label: 'Dismiss' }, - { ref: 'e3', index: 2, depth: 1, type: 'android.widget.TextView', label: 'Minimize' }, - ], - truncated: false, - backend: 'android', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'android.widget.TextView', label: 'useOnyx.ts:80:43' }, + { index: 1, depth: 1, type: 'android.widget.Button', label: 'Dismiss' }, + { index: 2, depth: 1, type: 'android.widget.TextView', label: 'Minimize' }, + ], + { truncated: false, backend: 'android' }, + ), }).capture.snapshot({ session: 'default', interactiveOnly: true }); assertReactNativeOverlayWarning(result.warnings); @@ -493,12 +517,13 @@ test('runtime snapshot recognizes Android React Native stack overlays with Dismi test('runtime snapshot recognizes Android RedBox stacks without Minimize', async () => { const result = await createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'android.widget.TextView', label: 'useOnyx.ts:80:43' }, - { ref: 'e2', index: 1, depth: 1, type: 'android.widget.Button', label: 'Dismiss' }, - ], - truncated: false, - backend: 'android', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'android.widget.TextView', label: 'useOnyx.ts:80:43' }, + { index: 1, depth: 1, type: 'android.widget.Button', label: 'Dismiss' }, + ], + { truncated: false, backend: 'android' }, + ), }).capture.snapshot({ session: 'default', interactiveOnly: true }); assertReactNativeOverlayWarning(result.warnings); @@ -506,31 +531,29 @@ test('runtime snapshot recognizes Android RedBox stacks without Minimize', async test('runtime snapshot warns when iOS hierarchy looks like a React Native overlay', async () => { const device = createSnapshotOnlyDevice({ - nodes: [ - { - ref: 'e1', - index: 0, - depth: 0, - type: 'XCUIElementTypeOther', - label: 'React Native RedBox', - }, - { - ref: 'e2', - index: 1, - depth: 1, - type: 'XCUIElementTypeStaticText', - value: 'Runtime Error', - }, - { - ref: 'e3', - index: 2, - depth: 1, - type: 'XCUIElementTypeButton', - identifier: 'Reload JS', - }, - ], - truncated: false, - backend: 'xctest', + snapshot: makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeOther', + label: 'React Native RedBox', + }, + { + index: 1, + depth: 1, + type: 'XCUIElementTypeStaticText', + value: 'Runtime Error', + }, + { + index: 2, + depth: 1, + type: 'XCUIElementTypeButton', + identifier: 'Reload JS', + }, + ], + { truncated: false, backend: 'xctest' }, + ), }); const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); @@ -540,19 +563,19 @@ test('runtime snapshot warns when iOS hierarchy looks like a React Native overla test('runtime snapshot targets React Native LogBox close icon instead of warning body', async () => { const result = await createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'XCUIElementTypeOther', label: 'LogBox' }, - { - ref: 'e2', - index: 1, - depth: 1, - type: 'XCUIElementTypeStaticText', - label: 'Warning: Each child in a list should have a unique "key" prop.', - }, - { ref: 'e3', index: 2, depth: 1, type: 'XCUIElementTypeButton', label: '×' }, - ], - truncated: false, - backend: 'xctest', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'XCUIElementTypeOther', label: 'LogBox' }, + { + index: 1, + depth: 1, + type: 'XCUIElementTypeStaticText', + label: 'Warning: Each child in a list should have a unique "key" prop.', + }, + { index: 2, depth: 1, type: 'XCUIElementTypeButton', label: '×' }, + ], + { truncated: false, backend: 'xctest' }, + ), }).capture.snapshot({ session: 'default', interactiveOnly: true }); assertReactNativeOverlayWarning(result.warnings); @@ -560,13 +583,14 @@ test('runtime snapshot targets React Native LogBox close icon instead of warning test('runtime snapshot does not warn for ordinary Android validation errors', async () => { const device = createSnapshotOnlyDevice({ - nodes: [ - { ref: 'e1', index: 0, depth: 0, type: 'Text', label: 'Validation errors' }, - { ref: 'e2', index: 1, depth: 1, type: 'Text', label: 'Required' }, - { ref: 'e3', index: 2, depth: 1, type: 'Button', label: 'Submit order' }, - ], - truncated: false, - backend: 'android', + snapshot: makeSnapshotState( + [ + { index: 0, depth: 0, type: 'Text', label: 'Validation errors' }, + { index: 1, depth: 1, type: 'Text', label: 'Required' }, + { index: 2, depth: 1, type: 'Button', label: 'Submit order' }, + ], + { truncated: false, backend: 'android' }, + ), }); const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); @@ -590,9 +614,11 @@ test('runtime snapshot stale-drop warning uses the runtime clock', async () => { session.snapshot.createdAt = 1_000; const device = createAgentDevice({ backend: createSnapshotBackend(() => ({ - nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Text', label: 'After' }], - truncated: false, - backend: 'android', + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Text', label: 'After' }], { + truncated: false, + backend: 'android', + createdAt: 1_500, + }), })), artifacts: createLocalArtifactAdapter(), sessions: { @@ -740,9 +766,10 @@ function assertReactNativeOverlayWarning(warnings: string[] | undefined) { test('runtime snapshot discloses a cut capture the same way on every backend', async () => { for (const backend of ['android', 'xctest'] as const) { const device = createSnapshotOnlyDevice({ - nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Window', label: 'Home' }], - truncated: true, - backend, + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + truncated: true, + backend: backend, + }), }); const result = await device.capture.snapshot({ session: 'default' }); @@ -753,9 +780,10 @@ test('runtime snapshot discloses a cut capture the same way on every backend', a } const complete = createSnapshotOnlyDevice({ - nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Window', label: 'Home' }], - truncated: false, - backend: 'android', + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + truncated: false, + backend: 'android', + }), }); const result = await complete.capture.snapshot({ session: 'default' }); assert.equal( @@ -771,9 +799,10 @@ test('runtime snapshot discloses a cut capture the same way on every backend', a test('runtime snapshot publishes the keyboard band its capture measured', async () => { const band = { kind: 'visible', frame: { x: 0, y: 583, width: 402, height: 291 } } as const; const device = createSnapshotOnlyDevice({ - nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Window', label: 'Home' }], - backend: 'xctest', - keyboard: band, + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + backend: 'xctest', + keyboard: band, + }), }); const result = await device.capture.snapshot({ session: 'default' }); @@ -783,8 +812,9 @@ test('runtime snapshot publishes the keyboard band its capture measured', async test('runtime snapshot leaves the keyboard band off when the backend measured none', async () => { const device = createSnapshotOnlyDevice({ - nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Window', label: 'Home' }], - backend: 'xctest', + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + backend: 'xctest', + }), }); const result = await device.capture.snapshot({ session: 'default' }); @@ -817,9 +847,10 @@ test('runtime snapshot warns when its tree was read on a surface still moving af test('runtime snapshot publishes the viewport its producer measured', async () => { const device = createSnapshotOnlyDevice({ - nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Window', label: 'Home' }], - backend: 'android', - viewport: snapshotViewportSizeFrom({ x: 0, y: 0, width: 1080, height: 2400 }), + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + backend: 'android', + viewport: snapshotViewportSizeFrom({ x: 0, y: 0, width: 1080, height: 2400 }), + }), }); const result = await device.capture.snapshot({ session: 'default' }); @@ -829,8 +860,9 @@ test('runtime snapshot publishes the viewport its producer measured', async () = test('runtime snapshot leaves the viewport off when the producer measured no box', async () => { const device = createSnapshotOnlyDevice({ - nodes: [{ ref: 'e1', index: 0, depth: 0, type: 'Window', label: 'Home' }], - backend: 'android', + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Window', label: 'Home' }], { + backend: 'android', + }), }); const result = await device.capture.snapshot({ session: 'default' }); @@ -838,11 +870,10 @@ test('runtime snapshot leaves the viewport off when the producer measured no box assert.equal('viewport' in result, false); }); -// The Android helper's clickability evidence is retained by object identity, and this command -// merges the backend's facts into a copy of the state it hands over (#3182). A copy that forgets -// the evidence is silent until Maestro's clickable-first ordering finds no exact evidence, falls -// back to document order, and taps an inert duplicate of a tapped id (Android smoke lane). -test('runtime snapshot carries backend clickability evidence through the response merge', async () => { +// The Android helper's clickability evidence is retained by object identity. Presentation and +// public-response copies must carry it while projecting viewport from the canonical state (#3182). +// Losing it makes Maestro fall back to document order and tap an inert duplicate (Android smoke lane). +test('runtime snapshot carries backend clickability evidence through presentation and response copies', async () => { const clickability = { kind: 'exact', provider: 'android-helper', @@ -856,16 +887,30 @@ test('runtime snapshot carries backend clickability evidence through the respons { index: 0, depth: 0, type: 'TextView', label: 'Inert duplicate target' }, { index: 1, depth: 0, type: 'Button', label: 'Clickable duplicate target' }, ], - { backend: 'android', producer: 'android-uiautomator' }, + { + backend: 'android', + producer: 'android-uiautomator', + viewport: snapshotViewportSizeFrom({ x: 0, y: 0, width: 1080, height: 2400 }), + }, ); attachSnapshotClickabilityEvidence(state, clickability); - const device = createSnapshotOnlyDevice({ - snapshot: state, - viewport: snapshotViewportSizeFrom({ x: 0, y: 0, width: 1080, height: 2400 }), + let stored: Parameters[0] | undefined; + const device = createAgentDevice({ + backend: createSnapshotBackend(() => ({ snapshot: state })), + artifacts: createLocalArtifactAdapter(), + sessions: { + get: () => undefined, + set: (record) => { + stored = record; + }, + }, + policy: localCommandPolicy(), }); const result = await device.capture.snapshot({ session: 'default' }); + assert.deepEqual(readSnapshotClickabilityEvidence(stored?.snapshot), clickability); assert.deepEqual(readSnapshotClickabilityEvidence(result), clickability); + assert.equal(JSON.stringify(result).includes('clickability'), false); assert.deepEqual(result.viewport, { width: 1080, height: 2400 }); }); diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index 1a34e15744..9e1ef27f18 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -197,10 +197,9 @@ async function captureRuntimeSnapshot( ...(options.observeOnly === undefined ? {} : { observeOnly: options.observeOnly }), }, ); - const normalizedSnapshot = normalizeBackendSnapshot(result, runtime); const snapshot = copySnapshotClickabilityEvidence( - normalizedSnapshot, - ensureSnapshotPresentationKey(normalizedSnapshot, options), + result.snapshot, + ensureSnapshotPresentationKey(result.snapshot, options), ); const annotations = snapshotCaptureAnnotationsFrom(result); const warningTime = now(runtime); @@ -221,33 +220,6 @@ async function captureRuntimeSnapshot( }; } -function normalizeBackendSnapshot( - result: BackendSnapshotResult, - runtime: AgentDeviceRuntime, -): SnapshotState { - // A backend may hand over state it built itself, and the response-level facts it publishes beside - // that state travel with this capture: returning the state alone would drop a producer's own - // keyboard band or viewport box (#3182) on the one path where the backend named them itself. - // The copy must carry the private clickability evidence with it: that fact is retained by object - // identity, so a spread that forgets it makes Maestro's clickable-first ordering fall back to - // divergence and tap the document-order (inert) duplicate of a tapped id. - if (result.snapshot) { - return copySnapshotClickabilityEvidence(result.snapshot, { - ...result.snapshot, - ...(result.keyboard ? { keyboard: result.keyboard } : {}), - ...(result.viewport ? { viewport: result.viewport } : {}), - }); - } - return { - nodes: result.nodes ?? [], - truncated: result.truncated, - backend: result.backend as SnapshotState['backend'], - createdAt: now(runtime), - ...(result.keyboard ? { keyboard: result.keyboard } : {}), - ...(result.viewport ? { viewport: result.viewport } : {}), - }; -} - function nextSnapshotSession( requestedName: string | undefined, capture: SnapshotCapture, diff --git a/src/commands/interaction/runtime/interaction-snapshot-capture.ts b/src/commands/interaction/runtime/interaction-snapshot-capture.ts index 920c5abd2b..4ce1d1a89d 100644 --- a/src/commands/interaction/runtime/interaction-snapshot-capture.ts +++ b/src/commands/interaction/runtime/interaction-snapshot-capture.ts @@ -1,7 +1,7 @@ import { AppError } from '@agent-device/kernel/errors'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; -import { now, toBackendContext } from '../../runtime-common.ts'; +import { toBackendContext } from '../../runtime-common.ts'; /** * A leaf shared by every interaction runtime module that needs a fresh (or session-cached) tree: @@ -31,14 +31,7 @@ export async function captureInteractionSnapshot( interactiveOnly, includeRects: true, }); - const snapshot = - result.snapshot ?? - ({ - nodes: result.nodes ?? [], - truncated: result.truncated, - backend: result.backend as SnapshotState['backend'], - createdAt: now(runtime), - } satisfies SnapshotState); + const snapshot = result.snapshot; await runtime.sessions.set({ ...session, snapshot }); return { snapshot }; } diff --git a/src/commands/interaction/runtime/selector-is.test.ts b/src/commands/interaction/runtime/selector-is.test.ts index 429406a1de..5f1b38feeb 100644 --- a/src/commands/interaction/runtime/selector-is.test.ts +++ b/src/commands/interaction/runtime/selector-is.test.ts @@ -338,7 +338,7 @@ test('runtime is absent fails closed for a sparse capture and preserves the last test('runtime is absent fails closed for a truncated capture even with zero matches', async () => { const snapshot = makeSnapshotState([]); const device = createSelectorDevice(snapshot, { - captureSnapshot: async () => ({ snapshot, truncated: true }), + captureSnapshot: async () => ({ snapshot: { ...snapshot, truncated: true } }), }); await assert.rejects( diff --git a/src/commands/interaction/runtime/selector-read-shared.ts b/src/commands/interaction/runtime/selector-read-shared.ts index 6dfdc989a8..ca4fb75e9b 100644 --- a/src/commands/interaction/runtime/selector-read-shared.ts +++ b/src/commands/interaction/runtime/selector-read-shared.ts @@ -3,7 +3,6 @@ import type { CommandContext, CommandSessionRecord, } from '@agent-device/contracts/runtime-contract'; -import type { BackendSnapshotResult } from '@agent-device/contracts/backend'; import { AppError, discloseDispatch, @@ -27,7 +26,7 @@ import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-e import { elementMatchCandidateDetails } from '@agent-device/capture-kit/snapshot-lines'; import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import { extractReadableText } from '@agent-device/capture-kit/text-surface'; -import { now, toBackendContext } from '../../runtime-common.ts'; +import { toBackendContext } from '../../runtime-common.ts'; import { findNodeByLabel } from './selector-read-utils.ts'; import type { SelectorSnapshotInput } from '../../command-input.ts'; @@ -95,7 +94,7 @@ export async function captureSelectorSnapshot( ? { includeHiddenContentHints: captureOptions.includeHiddenContentHints } : {}), }); - const snapshot = snapshotStateFromResult(result, runtime); + const snapshot = result.snapshot; (options.signal ?? runtime.signal)?.throwIfAborted(); if ( captureOptions.updateSession && @@ -107,33 +106,6 @@ export async function captureSelectorSnapshot( return { sessionName, session, snapshot }; } -function snapshotStateFromResult( - result: BackendSnapshotResult, - runtime: AgentDeviceRuntime, -): SnapshotState { - if (result.snapshot) return mergeSnapshotAnnotations(result.snapshot, result); - return { - nodes: result.nodes ?? [], - truncated: result.truncated, - backend: result.backend as SnapshotState['backend'], - ...(result.quality ? { snapshotQuality: result.quality } : {}), - createdAt: now(runtime), - } satisfies SnapshotState; -} - -function mergeSnapshotAnnotations( - snapshot: SnapshotState, - result: BackendSnapshotResult, -): SnapshotState { - const merged = { ...snapshot }; - if (result.truncated === true || merged.truncated === true) merged.truncated = true; - else if (result.truncated !== undefined) merged.truncated = result.truncated; - if (result.quality && merged.snapshotQuality === undefined) { - merged.snapshotQuality = result.quality; - } - return merged; -} - export async function readText( runtime: AgentDeviceRuntime, capture: CapturedSnapshot, diff --git a/src/commands/interaction/runtime/selector-read.test.ts b/src/commands/interaction/runtime/selector-read.test.ts index 3f165f0a65..ec6a6fdd1d 100644 --- a/src/commands/interaction/runtime/selector-read.test.ts +++ b/src/commands/interaction/runtime/selector-read.test.ts @@ -237,24 +237,29 @@ test('runtime find wait reports sparse snapshot verdicts on the selector-read ro session.snapshot = record.snapshot ?? session.snapshot; }, } satisfies CommandSessionStore; - const sparseSnapshot = makeSnapshotState([ + const sparseSnapshot = makeSnapshotState( + [ + { + index: 0, + type: 'Application', + }, + ], { - index: 0, - type: 'Application', + backend: 'xctest', + snapshotQuality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, }, - ]); + ); const device = createAgentDevice({ backend: { platform: 'ios', captureSnapshot: async () => ({ - nodes: sparseSnapshot.nodes, - backend: 'xctest', - quality: { - state: 'sparse', - backend: 'private-ax', - reason: 'sparse tree', - reasonCode: 'sparse-tree', - }, + snapshot: sparseSnapshot, + quality: sparseSnapshot.snapshotQuality, }), } satisfies AgentDeviceBackend, artifacts: createLocalArtifactAdapter(), diff --git a/src/commands/interaction/runtime/wait-absent.test.ts b/src/commands/interaction/runtime/wait-absent.test.ts index 79ae16eabe..764db219b8 100644 --- a/src/commands/interaction/runtime/wait-absent.test.ts +++ b/src/commands/interaction/runtime/wait-absent.test.ts @@ -130,7 +130,7 @@ test('wait absent rides out sparse and truncated captures without counting them reasonCode: 'sparse-tree', }, }); - const truncated = { snapshot: makeSnapshotState([]), truncated: true }; + const truncated = snapshot(undefined, { truncated: true }); const device = absentDevice([sparse, truncated, snapshot()]); const result = await waitAbsent(device, 2000); @@ -163,7 +163,7 @@ test('wait absent excludes sparse and truncated polls from deadline readable-cap const sparse = snapshot(undefined, { snapshotQuality: { state: 'sparse', backend: 'private-ax', reasonCode: 'sparse-tree' }, }); - const truncated = { snapshot: makeSnapshotState([]), truncated: true }; + const truncated = snapshot(undefined, { truncated: true }); const device = absentDevice([snapshot('Removed'), sparse, truncated]); await assert.rejects(waitAbsent(device, 500), (error: unknown) => { @@ -200,7 +200,7 @@ test('wait absent preserves the final typed diagnostic when no valid capture arr }); test('wait absent preserves a final truncated diagnostic when no valid capture arrives', async () => { - const device = absentDevice([{ snapshot: makeSnapshotState([]), truncated: true }]); + const device = absentDevice([snapshot(undefined, { truncated: true })]); await assert.rejects(waitAbsent(device, 500), (error: unknown) => { assert.ok(error instanceof AppError); From b00861435086d40c78946e1083f01166ffa47236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 15 Oct 2026 15:57:09 +0200 Subject: [PATCH 2/2] test(capture): distinguish runtime and capture clocks --- src/commands/capture/runtime/snapshot.test.ts | 2 +- src/commands/capture/runtime/snapshot.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index b86847336b..6d73cec966 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -617,7 +617,7 @@ test('runtime snapshot stale-drop warning uses the runtime clock', async () => { snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Text', label: 'After' }], { truncated: false, backend: 'android', - createdAt: 1_500, + createdAt: 3_500, }), })), artifacts: createLocalArtifactAdapter(), diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index 9e1ef27f18..2c021b7138 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -214,7 +214,7 @@ async function captureRuntimeSnapshot( snapshot, options, session, - capturedAt: snapshot.createdAt ?? warningTime, + capturedAt: snapshot.createdAt, runtimeNow: warningTime, }), };