diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index 9852a26361..9df695c6ec 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -211,7 +211,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ min: 0, usageLabel: '--settle-quiet ', usageDescription: - 'Settle: quiet window the UI must hold to count as settled (default 500ms; 300ms when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the --timeout settle deadline', + 'Settle: quiet window the UI must hold to count as settled (default 500ms; 300ms when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the --timeout settle deadline; a replacement from a screen the pre-action capture read as presenting nothing, such as a tab switch, gets no 1500ms confirmation and keeps the asked window', projectConfig: true, recorded: false, }, diff --git a/packages/daemon/src/execution/generic-settle.ts b/packages/daemon/src/execution/generic-settle.ts index 86bf1d16a2..bd3b8fab84 100644 --- a/packages/daemon/src/execution/generic-settle.ts +++ b/packages/daemon/src/execution/generic-settle.ts @@ -17,10 +17,12 @@ import type { BoundContextFromFlags } from '../core/context.ts'; import type { RuntimeAdmissionBindings } from '../core/request-runtime-binding.ts'; import type { DaemonPlatformServices } from '../core/platform-services.ts'; import { resolveSettleObservationCapture } from '../core/settle-observation-capture.ts'; +import { isOutdatedObservation } from '../core/ref-frame.ts'; import { issueSettleRefs } from './session-snapshot.ts'; import type { SessionStore } from '../core/session-store.ts'; import type { DaemonRequest, DaemonResponse } from '../core/daemon-request.ts'; import type { SessionRef, SessionState } from '../core/session-state.ts'; +import type { SnapshotState } from '@agent-device/kernel/snapshot'; /** * `--settle` on the generic daemon route (#1638): `scroll` and `back` change @@ -84,11 +86,23 @@ export function planGenericSettleObservation( if (invalidSettleFlags) return { response: invalidSettleFlags }; const settle = readSettleRequest(params.flags); if (!settle) return {}; - const snapshot = params.session.snapshot; - const baseline = surfaceScopedNodes(snapshot); + const baseline = storedTreeBaseline(params.session.snapshot); return { observe: async () => await observeSettled(params, settle, baseline) }; } +/** + * The stored tree as the settle baseline. Its presented-surface fact describes the screen when the + * tree was captured, so a tree a later side effect outdated keeps its nodes for the diff but not + * the fact: an `absent` read taken before a sheet was presented must not rule out the confirmation + * owed to that sheet's dismissal (#3565). + */ +function storedTreeBaseline(snapshot: SnapshotState | undefined): SurfaceScopedNodes { + const baseline = surfaceScopedNodes(snapshot); + if (!snapshot || !isOutdatedObservation(snapshot)) return baseline; + const { presentedSurface: _outdated, ...current } = baseline; + return current; +} + /** * Best-effort like the settle engine itself: the action already succeeded, so a * runtime that cannot even be constructed (session evicted mid-request) diff --git a/src/__tests__/daemon-composition/generic-settle.test.ts b/src/__tests__/daemon-composition/generic-settle.test.ts index 3b948d3a6d..f472a548cb 100644 --- a/src/__tests__/daemon-composition/generic-settle.test.ts +++ b/src/__tests__/daemon-composition/generic-settle.test.ts @@ -2,7 +2,7 @@ import { composedPlatformServicesFixture } from './platform-services-fixture.ts' import { beforeEach, expect, test, vi } from 'vitest'; import type { CommandFlags } from '@agent-device/contracts/command'; import type { CommandClock } from '@agent-device/contracts/runtime-contract'; -import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { PresentedSurface, RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { APP_SURFACE, iosSystemSurfaceDisclosure, @@ -11,7 +11,16 @@ import { } from '@agent-device/contracts/ios-system-surface'; import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { activateCompleteRefFrame, refFrameState } from '@agent-device/daemon/testing/ref-frame'; +import { + PRESENTED_SHEET_FULL_NODES, + PRESENTED_SHEET_INTERACTIVE_NODES, + SHEET_DISMISSED_NODES, +} from './presented-sheet-trees.ts'; +import { + activateCompleteRefFrame, + markSessionSnapshotOutdated, + refFrameState, +} from '@agent-device/daemon/testing/ref-frame'; import { setSessionSnapshot } from '@agent-device/daemon/testing/session-snapshot'; import type { SessionStore } from '@agent-device/daemon/testing/session-store'; import type { DaemonRequest, DaemonResponse } from '@agent-device/daemon/daemon-request'; @@ -77,6 +86,7 @@ type SettlePayload = { captures: number; quietMs: number; timeoutMs: number; + waitedMs: number; refsGeneration?: number; diff?: { summary: { additions: number; removals: number; unchanged: number }; @@ -115,7 +125,7 @@ async function emulateCaptureSnapshotForSession( } function mockCommandDispatch( - snapshots: Array, + snapshots: RawSnapshotNode[][], producer: NonNullable = 'apple-runner', ) { let snapshotCalls = 0; @@ -181,6 +191,7 @@ type GenericSettleTree = { nodes: RawSnapshotNode[]; systemSurface?: IosSystemSurfaceProvenance; producer?: 'apple-runner' | 'simulator-ax-bridge'; + presentedSurface?: PresentedSurface; }; function snapshotPayload(tree: GenericSettleTree) { @@ -189,6 +200,7 @@ function snapshotPayload(tree: GenericSettleTree) { backend: 'xctest' as const, producer: tree.producer ?? ('apple-runner' as const), ...(tree.systemSurface ? { systemSurface: tree.systemSurface } : {}), + ...(tree.presentedSurface ? { presentedSurface: tree.presentedSurface } : {}), }; } @@ -556,6 +568,75 @@ test('back --settle attaches no diff across a sheet-to-app surface change and di expect(stored.snapshot?.iosSystemSurfaceBundleId).toBeUndefined(); }); +/** + * `back --settle` over a sheet left on the session by a capture of one given flavour, which is how + * the generic route picks its baseline (`generic-settle.ts` reads the stored tree directly). The + * settle loop's budget arithmetic runs on the injected clock, so no caller waits real time. + */ +async function dismissSheetFromBaseline( + sessionName: string, + baseline: GenericSettleTree, + options: { outdated?: boolean; captures?: RawSnapshotNode[][] } = {}, +): Promise { + const sessionStore = makeSessionStore(); + const session = seedSession(sessionName, sessionStore, baseline); + if (options.outdated) markSessionSnapshotOutdated(session); + // Rootless, as the first reads after a dismissal can be: only a presented baseline arms on them. + mockCommandDispatch(options.captures ?? [SHEET_DISMISSED_NODES.slice(1)]); + const response = await dispatchGeneric({ + sessionName, + sessionStore, + command: 'back', + flags: { settle: true, settleQuietMs: 25, timeoutMs: 30_000 }, + clock: fakeClock(), + }); + return expectOkData(response).settle as SettlePayload; +} + +test('the confirmation arms for one presented sheet whichever baseline flavour the route holds', async () => { + // The interactive capture projects the sheet as 4 nodes and the full read as 6, so a rule keyed on + // the projection disarms on the full one. The producer's fact is stamped before projection. + const interactive = await dismissSheetFromBaseline('generic-settle-dismiss-sheet-interactive', { + nodes: PRESENTED_SHEET_INTERACTIVE_NODES, + presentedSurface: 'presented', + }); + const full = await dismissSheetFromBaseline('generic-settle-dismiss-sheet-full', { + nodes: PRESENTED_SHEET_FULL_NODES, + presentedSurface: 'presented', + }); + + expect(interactive.settled).toBe(true); + expect(full.settled).toBe(true); + expect(interactive.waitedMs).toBeGreaterThanOrEqual(1_500); + expect(full.waitedMs).toBeGreaterThanOrEqual(1_500); +}); + +test('back --settle keeps the plain window when the stored tree presented nothing', async () => { + // The same broad replacement, stamped `absent` by its producer: the stored tree's fact reaches + // the loop, and a screen with nothing presented over it owes no dismissal confirmation (#3470). + const settle = await dismissSheetFromBaseline( + 'generic-settle-absent', + { nodes: PRESENTED_SHEET_FULL_NODES, presentedSurface: 'absent' }, + { captures: [SHEET_DISMISSED_NODES] }, + ); + + expect(settle.settled).toBe(true); + expect(settle.waitedMs).toBeLessThan(1_500); +}); + +test('back --settle keeps the confirmation when a side effect outdated an absent stored tree', async () => { + // The app screen was read `absent`, then a press presented the sheet without a capture. The + // stored fact predates the sheet, so it cannot rule out the confirmation its dismissal owes. + const settle = await dismissSheetFromBaseline( + 'generic-settle-outdated-absent', + { nodes: SHEET_DISMISSED_NODES, presentedSurface: 'absent' }, + { outdated: true, captures: [PRESENTED_SHEET_INTERACTIVE_NODES, SHEET_DISMISSED_NODES] }, + ); + + expect(settle.settled).toBe(true); + expect(settle.waitedMs).toBeGreaterThanOrEqual(1_500); +}); + test('the settle observation runs after the post-gesture stabilization marker', async () => { const sessionStore = makeSessionStore(); const sessionName = 'generic-settle-order'; diff --git a/src/__tests__/daemon-composition/presented-sheet-trees.ts b/src/__tests__/daemon-composition/presented-sheet-trees.ts new file mode 100644 index 0000000000..112821fbfc --- /dev/null +++ b/src/__tests__/daemon-composition/presented-sheet-trees.ts @@ -0,0 +1,81 @@ +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; + +/** + * One presented iOS `pageSheet`, in the two flavours a settle baseline can arrive as, plus the app + * screen a dismissal returns to. An interactive capture carries only the surface — the content + * underneath is not hittable, so it drops out of the projection (4 nodes on the iPhone 18 Pro sim, + * October 2026) — while a full read of the very same screen also reports the covered content (6 + * nodes). Arming the broad-transition confirmation must not depend on which flavour produced the + * baseline, which is what the caller that reads these asserts. + */ +export const PRESENTED_SHEET_INTERACTIVE_NODES: RawSnapshotNode[] = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Allow notifications?', + rect: { x: 10, y: 450, width: 370, height: 22 }, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + label: 'Continue', + rect: { x: 10, y: 600, width: 370, height: 44 }, + hittable: true, + }, + { + index: 3, + parentIndex: 0, + type: 'Button', + label: 'Cancel', + rect: { x: 10, y: 660, width: 370, height: 44 }, + hittable: true, + }, +]; + +export const PRESENTED_SHEET_FULL_NODES: RawSnapshotNode[] = [ + ...PRESENTED_SHEET_INTERACTIVE_NODES, + { + index: 4, + parentIndex: 0, + type: 'Other', + label: 'CoveredContent', + rect: { x: 0, y: 0, width: 390, height: 440 }, + }, + { + index: 5, + parentIndex: 0, + type: 'Button', + label: 'Open settings', + rect: { x: 10, y: 120, width: 370, height: 44 }, + }, +]; + +/** The app screen the sheet is dismissed from, which is what every settle capture then reads. */ +export const SHEET_DISMISSED_NODES: RawSnapshotNode[] = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Signed in as demo', + rect: { x: 10, y: 80, width: 200, height: 22 }, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + label: 'Home', + rect: { x: 10, y: 120, width: 370, height: 44 }, + hittable: true, + }, + { + index: 3, + parentIndex: 0, + type: 'TabBar', + label: 'Tab Bar', + rect: { x: 0, y: 800, width: 390, height: 44 }, + }, +]; diff --git a/src/commands/interaction/runtime/settle-presented-surface.test.ts b/src/commands/interaction/runtime/settle-presented-surface.test.ts new file mode 100644 index 0000000000..2226e1f063 --- /dev/null +++ b/src/commands/interaction/runtime/settle-presented-surface.test.ts @@ -0,0 +1,84 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { PresentedSurface, SnapshotState } from '@agent-device/kernel/snapshot'; +import { ref, selector } from './selector-read-utils.ts'; +import { createSettleDevice } from './__tests__/settle-device-fixtures.ts'; +import { + elementCatalogTabSnapshot, + elementSettingsTabSnapshot, +} from './stable-capture.fixtures.ts'; + +// Each settle route hands the loop its own baseline, and the producer's presented-surface fact has +// to arrive with it (#3565): a tab switch from a screen read `absent` keeps the plain window, while +// the same broad replacement without that reading still pays the confirmation floor. The tab +// switch is the transition #3470 measured; budgets run on the injected clock. + +function stamped(snapshot: SnapshotState, presentedSurface: PresentedSurface): SnapshotState { + return { ...snapshot, presentedSurface }; +} + +/** A press that switches from the catalog tab to the settings tab, with the given baseline fact. */ +function tabSwitchDevice(params: { stored: PresentedSurface; preActionCaptureFails?: boolean }) { + const catalog = elementCatalogTabSnapshot(); + let captures = 0; + return createSettleDevice({ + stored: stamped(catalog, params.stored), + captureSnapshot: () => { + captures += 1; + if (captures === 1 && params.preActionCaptureFails) throw new Error('capture unavailable'); + return { + snapshot: + captures === 1 + ? stamped(catalog, params.stored) + : stamped(elementSettingsTabSnapshot(), 'absent'), + }; + }, + }); +} + +function settingsTabRef(): string { + const node = elementCatalogTabSnapshot().nodes.find( + (candidate) => candidate.label === 'Settings', + ); + assert.ok(node?.ref); + return `@${node.ref}`; +} + +const ROUTES = [ + ['a selector press', () => selector('label=Settings')], + ['an @ref press', () => ref(settingsTabRef())], +] as const; + +for (const [label, target] of ROUTES) { + test(`${label} carries an absent baseline into the loop: a tab switch keeps the plain window`, async () => { + const device = tabSwitchDevice({ stored: 'absent' }); + + const result = await device.interactions.press(target(), { session: 'default', settle: {} }); + + assert.equal(result.settle?.settled, true); + assert.ok(result.settle.waitedMs < 1_500, `paid the floor: ${result.settle.waitedMs}ms`); + }); + + test(`${label} keeps the confirmation when the baseline's producer could not answer`, async () => { + const device = tabSwitchDevice({ stored: 'unknown' }); + + const result = await device.interactions.press(target(), { session: 'default', settle: {} }); + + assert.equal(result.settle?.settled, true); + assert.ok(result.settle.waitedMs >= 1_500, `lost the floor: ${result.settle.waitedMs}ms`); + }); +} + +test('a point press whose evidence capture failed does not trust the stored fact', async () => { + // With no capture of its own, the baseline is the stored tree, which may predate a side effect + // this layer cannot see: its `absent` must not rule the confirmation out. + const device = tabSwitchDevice({ stored: 'absent', preActionCaptureFails: true }); + + const result = await device.interactions.press( + { kind: 'point', x: 300, y: 820 }, + { session: 'default', settle: {} }, + ); + + assert.equal(result.settle?.settled, true); + assert.ok(result.settle.waitedMs >= 1_500, `trusted a stored fact: ${result.settle.waitedMs}ms`); +}); diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 8beefccf19..880e60acdc 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -145,7 +145,7 @@ async function settleAfterAction( ...(params.quietMs === undefined ? {} : { quietMs: params.quietMs }), timeoutMs, resetBudgetOnPrivateAxRecovery: true, - broadTransitionBaselineNodes: params.baseline.nodes, + broadTransitionBaseline: params.baseline, }); if (outcome.failure) return settleUnavailable(base, outcome.quietMs, outcome.failure.error); return await readSettledOutcome(runtime, options, params, base, outcome); @@ -271,8 +271,17 @@ function authorizedRefBaseline( return nonEmptyBaseline(surfaceScopedNodes(session?.refFrameSnapshot)); } +/** + * The latest stored tree, for a target no capture resolved. The diff can use its nodes, but its + * presented-surface fact may predate a side effect this layer cannot see, and an `absent` read + * taken before a sheet was presented must not rule out the confirmation owed to that sheet's + * dismissal (#3565). + */ function sessionBaseline(session: CommandSessionRecord | undefined): SurfaceScopedNodes { - return surfaceScopedNodes(session?.refFrameSnapshot ?? session?.snapshot); + const { presentedSurface: _unverified, ...baseline } = surfaceScopedNodes( + session?.refFrameSnapshot ?? session?.snapshot, + ); + return baseline; } function nonEmptyBaseline( diff --git a/src/commands/interaction/runtime/stable-capture.fixtures.ts b/src/commands/interaction/runtime/stable-capture.fixtures.ts index fae1eb69b1..9eab169147 100644 --- a/src/commands/interaction/runtime/stable-capture.fixtures.ts +++ b/src/commands/interaction/runtime/stable-capture.fixtures.ts @@ -1,4 +1,4 @@ -import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import type { RawSnapshotNode, Rect, SnapshotState } from '@agent-device/kernel/snapshot'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; export function elementSettingsSnapshot(offscreenRowYs: number[]): SnapshotState { @@ -133,3 +133,178 @@ export const elementSettledRoomSnapshot = makeSnapshotState( ], { backend: 'xctest', snapshotQuality: { state: 'healthy', backend: 'tree' } }, ); + +// Screen-replacement fixtures in the SHAPE a live iPhone 17 (iOS 27) session projects — an +// `Application` root, the screen's own content, and a tab bar whose buttons keep the same identities +// across tabs — kept compact on purpose: a live screen of this app reports 24-34 nodes and the same +// proportion here is 18 (`snapshot -i`, October 2026). What the tests need is the ratio, not the +// volume: each fixture is a BROAD replacement of its siblings — fewer than half the node identities +// survive — which is why the complete-viewport-projection arm alone treats an ordinary tab switch +// or push as a screen replacement (#3470). + +const TAB_BAR_LABELS = ['Home', 'Catalog', 'Form', 'Settings']; + +function iosScreenSnapshot( + content: Array<{ type: string; label: string; rect: Rect }>, +): SnapshotState { + const nodes: RawSnapshotNode[] = [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Agent Device Tester', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + ...content.map((node, offset) => ({ + index: offset + 1, + depth: 1, + parentIndex: 0, + ...node, + })), + ]; + const tabBarIndex = nodes.length; + nodes.push({ + index: tabBarIndex, + depth: 1, + parentIndex: 0, + type: 'TabBar', + label: 'Tab Bar', + rect: { x: 0, y: 791, width: 402, height: 83 }, + }); + TAB_BAR_LABELS.forEach((label, offset) => { + nodes.push({ + index: nodes.length, + depth: 2, + parentIndex: tabBarIndex, + type: 'Button', + label, + rect: { x: 12 + offset * 96, y: 795, width: 92, height: 54 }, + }); + }); + return makeSnapshotState(nodes, { + backend: 'xctest', + snapshotQuality: { state: 'healthy', backend: 'tree' }, + }); +} + +/** A tab's list screen: the shared tab bar plus that tab's own rows. */ +export function elementCatalogTabSnapshot(): SnapshotState { + return iosScreenSnapshot( + Array.from({ length: 12 }, (_, offset) => ({ + type: 'Cell', + label: `Catalog product ${offset + 1}`, + rect: { x: 0, y: 120 + offset * 90, width: 402, height: 84 }, + })), + ); +} + +/** The tab you switch to: different rows, same tab bar, same app root. */ +export function elementSettingsTabSnapshot(): SnapshotState { + return iosScreenSnapshot( + Array.from({ length: 12 }, (_, offset) => ({ + type: 'Button', + label: `Open settings fixture ${offset + 1}`, + rect: { x: 16, y: 120 + offset * 60, width: 370, height: 45 }, + })), + ); +} + +/** The screen a push opens: its own nav bar and content, and the tab bar's row is gone. */ +export function elementPushedDetailSnapshot(): SnapshotState { + return makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Agent Device Tester', + rect: { x: 0, y: 0, width: 402, height: 874 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'NavigationBar', + label: 'Automation', + rect: { x: 0, y: 62, width: 402, height: 54 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Button', + label: '(tabs)', + rect: { x: 16, y: 62, width: 44, height: 44 }, + }, + ...['Open automation sheet', 'Press canary', 'Long press canary'].map((label, offset) => ({ + index: 3 + offset, + depth: 1, + parentIndex: 0, + type: 'Button', + label, + rect: { x: 16, y: 240 + offset * 120, width: 370, height: 45 }, + })), + ], + { backend: 'xctest', snapshotQuality: { state: 'healthy', backend: 'tree' } }, + ); +} + +/** + * A system permission prompt's own projection: the runner roots it at an `Alert` node and the app + * content it presents over is not in the tree (#2660). Larger than `TINY_STABLE_TREE_NODE_COUNT`, so + * its size alone does not mark it as presented. + */ +export function systemPermissionPromptSnapshot(): SnapshotState { + return makeSnapshotState( + [ + { + index: 0, + depth: 0, + type: 'Alert', + label: 'Allow “Agent Device Tester” to access your microphone?', + rect: { x: 40, y: 309, width: 322, height: 283 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Allow Agent Device Tester to exercise microphone permissions.', + rect: { x: 60, y: 420, width: 282, height: 42 }, + }, + { + index: 2, + depth: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Microphone access lets the lab record a clip.', + rect: { x: 60, y: 470, width: 282, height: 38 }, + }, + { + index: 3, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Don’t Allow', + rect: { x: 60, y: 528, width: 130, height: 48 }, + }, + { + index: 4, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Allow', + rect: { x: 212, y: 528, width: 130, height: 48 }, + }, + { + index: 5, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Learn More', + rect: { x: 136, y: 580, width: 130, height: 48 }, + }, + ], + { backend: 'xctest', snapshotQuality: { state: 'healthy', backend: 'tree' } }, + ); +} diff --git a/src/commands/interaction/runtime/stable-capture.test.ts b/src/commands/interaction/runtime/stable-capture.test.ts index 31d2fceac2..05f6996643 100644 --- a/src/commands/interaction/runtime/stable-capture.test.ts +++ b/src/commands/interaction/runtime/stable-capture.test.ts @@ -9,17 +9,21 @@ import { createAgentDevice } from '../../../runtime.ts'; import { localCommandPolicy } from '@agent-device/contracts/command-policy'; import { runStableCaptureLoop } from './stable-capture.ts'; import { + elementCatalogTabSnapshot, + elementPushedDetailSnapshot, elementSettingsSnapshot, elementSettledRoomSnapshot, + elementSettingsTabSnapshot, elementThreadsNoticeSnapshot, elementTransientRoomSnapshot, + systemPermissionPromptSnapshot, } from './stable-capture.fixtures.ts'; import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; const BROAD_TRANSITION_PARAMS = { quietMs: 500, timeoutMs: 5_000, - broadTransitionBaselineNodes: elementThreadsNoticeSnapshot.nodes, + broadTransitionBaseline: { nodes: elementThreadsNoticeSnapshot.nodes }, }; // Each #1831 case below runs at BOTH windows. The confirmation used to switch itself off for a @@ -129,7 +133,7 @@ for (const requestedQuietMs of [500, 150]) { quietMs: requestedQuietMs, // Presented iOS modals can be complete interaction captures without retaining the // Application root in their projection. - broadTransitionBaselineNodes: elementThreadsNoticeSnapshot.nodes.slice(1), + broadTransitionBaseline: { nodes: elementThreadsNoticeSnapshot.nodes.slice(1) }, }, ); @@ -291,7 +295,7 @@ test('settle confirms an iOS broad replacement when capture omits snapshot backe ...BROAD_TRANSITION_PARAMS, // Presented iOS modals can be complete interaction captures without retaining the // Application root in their projection. - broadTransitionBaselineNodes: elementThreadsNoticeSnapshot.nodes.slice(1), + broadTransitionBaseline: { nodes: elementThreadsNoticeSnapshot.nodes.slice(1) }, }, ); @@ -326,6 +330,87 @@ test('settle confirms a tiny modal replacement through a partial post-action pro ); }); +test('settle confirms the dismissal of a presented surface larger than a tiny tree', async () => { + // A blocking system prompt projects 6 nodes — over `TINY_STABLE_TREE_NODE_COUNT` — and the + // captures after it arrive rootless. The producer's `presented` fact is what arms the + // confirmation; without it this is the larger partial projection below, which keeps the plain + // window. + const prompt = systemPermissionPromptSnapshot(); + const { runtime } = transitionRuntime({ + presented: prompt, + dismissesTo: elementCatalogTabSnapshot(), + omitViewportRoot: true, + settledAtMs: 1_200, + }); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + quietMs: 150, + timeoutMs: 5_000, + broadTransitionBaseline: { nodes: prompt.nodes, presentedSurface: 'presented' }, + }, + ); + + assert.equal(outcome.settled, true); + assert.ok(outcome.waitedMs >= 1_500, `settled prompt dismissal after ${outcome.waitedMs}ms`); +}); + +// The three navigation cases #3470 complains about. Each is a BROAD replacement by the identity +// ratio — both screens project a complete viewport and share under half their node identities. A +// baseline whose producer read no presented surface has no dismissal to confirm, so they keep the +// plain window; a producer that cannot answer keeps the parent's arming, which confirms them. +const NAVIGATION_TRANSITIONS = [ + ['a tab switch', elementCatalogTabSnapshot(), elementSettingsTabSnapshot()], + ['a push', elementCatalogTabSnapshot(), elementPushedDetailSnapshot()], + [ + // The screen a push LANDS ON can be tiny, and that must not arm the confirmation either: only + // the surface being dismissed is the trigger. + 'a push onto a nearly-empty screen', + elementCatalogTabSnapshot(), + elementThreadsNoticeSnapshot, + ], +] as const; +for (const [label, from, to] of NAVIGATION_TRANSITIONS) { + test(`settle keeps the quiet window across ${label} from a screen with nothing presented`, async () => { + const { runtime } = staticSnapshotRuntime(to); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + quietMs: 500, + timeoutMs: 5_000, + broadTransitionBaseline: { nodes: from.nodes, presentedSurface: 'absent' }, + }, + ); + + assert.equal(outcome.settled, true); + assert.ok( + outcome.waitedMs < 800, + `${label} settled after ${outcome.waitedMs}ms with ${outcome.captures} captures`, + ); + }); + + test(`settle still confirms ${label} when the producer cannot say what was presented`, async () => { + const { runtime } = staticSnapshotRuntime(to); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + quietMs: 500, + timeoutMs: 5_000, + broadTransitionBaseline: { nodes: from.nodes, presentedSurface: 'unknown' }, + }, + ); + + assert.equal(outcome.settled, true); + assert.ok(outcome.waitedMs >= 1_500, `${label} settled after ${outcome.waitedMs}ms`); + }); +} + test('a dismissal confirmation is not dropped by a later empty projection', async () => { // A confirmation is a fact about the transition, so once armed it holds the floor for the rest of // the run: later captures that cannot support the comparison (here, empty projections that match @@ -341,7 +426,7 @@ test('a dismissal confirmation is not dropped by a later empty projection', asyn { quietMs: 150, timeoutMs: 5_000, - broadTransitionBaselineNodes: elementThreadsNoticeSnapshot.nodes, + broadTransitionBaseline: { nodes: elementThreadsNoticeSnapshot.nodes }, }, ); @@ -380,7 +465,9 @@ for (const [label, midAnimation] of MID_ANIMATION_FIRST_CAPTURES) { timeoutMs: 5_000, // A full-screen settings table: the tiny-presented-baseline escape hatch does not apply, so // arming depends entirely on the captures below. - broadTransitionBaselineNodes: elementSettingsSnapshot([1_138, 1_423, 2_144, 2_434]).nodes, + broadTransitionBaseline: { + nodes: elementSettingsSnapshot([1_138, 1_423, 2_144, 2_434]).nodes, + }, }, ); @@ -404,7 +491,9 @@ test('settle keeps the default quiet window for a larger partial projection', as { session: 'default' }, { ...BROAD_TRANSITION_PARAMS, - broadTransitionBaselineNodes: elementSettingsSnapshot([1_138, 1_423, 2_144, 2_434]).nodes, + broadTransitionBaseline: { + nodes: elementSettingsSnapshot([1_138, 1_423, 2_144, 2_434]).nodes, + }, }, ); @@ -461,7 +550,7 @@ test('settle keeps the default quiet window for a backend-less overlapping local { quietMs: 500, timeoutMs: 5_000, - broadTransitionBaselineNodes: elementSettledRoomSnapshot.nodes, + broadTransitionBaseline: { nodes: elementSettledRoomSnapshot.nodes }, }, ); @@ -492,7 +581,7 @@ test('settle keeps the default quiet window when only keyboard chrome arrives', { quietMs: 500, timeoutMs: 5_000, - broadTransitionBaselineNodes: keyboardUp.nodes.filter((node) => node.index <= 30), + broadTransitionBaseline: { nodes: keyboardUp.nodes.filter((node) => node.index <= 30) }, }, ); @@ -534,10 +623,16 @@ function transitionRuntime( rootlessUntilMs?: number; /** 1-based capture indices that declare a non-XCTest capture channel. */ otherChannelAtCaptures?: number[]; + /** The surface the action dismisses; defaults to #1831's notice. */ + presented?: SnapshotState; + /** What the screen under it looks like once the dismissal completes. */ + dismissesTo?: SnapshotState; } = {}, ) { const captureBackend = options.captureBackend ?? 'tree'; const settledAtMs = options.settledAtMs ?? 800; + const presented = options.presented ?? elementThreadsNoticeSnapshot; + const dismissesTo = options.dismissesTo ?? elementSettledRoomSnapshot; let elapsedMs = 0; let captures = 0; const clock = { @@ -561,9 +656,9 @@ function transitionRuntime( }; } const snapshot = keepsBaseline - ? elementThreadsNoticeSnapshot + ? presented : withCaptureBackend( - elapsedMs < settledAtMs ? elementTransientRoomSnapshot : elementSettledRoomSnapshot, + elapsedMs < settledAtMs ? elementTransientRoomSnapshot : dismissesTo, captureBackend, ); const rootless = @@ -589,7 +684,7 @@ function transitionRuntime( } satisfies AgentDeviceBackend, artifacts: createLocalArtifactAdapter(), sessions: createMemorySessionStore([ - { name: 'default', snapshot: options.sessionSnapshot ?? elementThreadsNoticeSnapshot }, + { name: 'default', snapshot: options.sessionSnapshot ?? presented }, ]), policy: localCommandPolicy(), clock, diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index 05a4bac9f1..ca9670aa6f 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -7,6 +7,7 @@ import type { } from '@agent-device/kernel/snapshot'; import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '@agent-device/contracts/runtime-contract'; +import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; import { now, sleep } from '../../runtime-common.ts'; import { captureSelectorSnapshot, @@ -119,7 +120,7 @@ export async function runStableCaptureLoop( timeoutMs: number; resetBudgetOnPrivateAxRecovery?: boolean; /** Immutable pre-action projection; the stored session may advance before settle begins. */ - broadTransitionBaselineNodes?: SnapshotNode[]; + broadTransitionBaseline?: SurfaceScopedNodes; }, ): Promise { const { timeoutMs } = params; @@ -134,7 +135,7 @@ export async function runStableCaptureLoop( // A window the caller asked for is fixed. let quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS; const confirmation = broadTransitionConfirmation({ - baselineNodes: params.broadTransitionBaselineNodes, + baseline: params.broadTransitionBaseline, platform: runtime.backend.platform, appBundleId: session?.appBundleId, }); @@ -343,9 +344,14 @@ function createStableCapturePacing(start: number) { * compares each capture against the pre-action tree until one proves the replacement, and holds that * window for the rest of the run. * - * Two gates stand before the comparison. The pre-action projection is constant for the run and the + * Two gates stand before the comparison. The pre-action baseline is constant for the run and the * only input allowed to rule the confirmation out: a caller that passed none has nothing to confirm - * against. The capture gate — the capture proves an iOS XCTest read and carries a complete viewport + * against, and a baseline whose producer read no presented surface (`presentedSurface: 'absent'`) + * has no dismissal to confirm — a tab switch or a push replaces the screen broadly but presents + * nothing over it (#3470). The fact is stamped before projection, so every flavour of one screen's + * baseline (interactive, full, depth-limited, scoped, a stored ref frame) answers it alike; no fact, + * or `'unknown'`, keeps the arming below, so the fact only ever narrows the guard (#3565). The + * capture gate — the capture proves an iOS XCTest read and carries a complete viewport * projection — belongs to each capture, because the first read after a dismissal can still show the * dismissed surface, arrive rootless or empty mid-animation, or come from another backend than the * next poll will. A capture that fails it leaves the question open; the first capture that passes @@ -357,11 +363,12 @@ function createStableCapturePacing(start: number) { * caller's own window, floored at 1.5 s. */ function broadTransitionConfirmation(params: { - baselineNodes: SnapshotNode[] | undefined; + baseline: SurfaceScopedNodes | undefined; platform: AgentDeviceRuntime['backend']['platform']; appBundleId: string | undefined; }): StableCaptureConfirmation { - const { baselineNodes, platform, appBundleId } = params; + const { platform, appBundleId } = params; + const baseline = params.baseline?.presentedSurface === 'absent' ? undefined : params.baseline; // Open question, not a ruling: `undefined` is the state a capture that cannot support the // comparison leaves behind, so arming never depends on which capture happens to land first. let transitionBaseline: StableCaptureSignal | undefined; @@ -374,9 +381,9 @@ function broadTransitionConfirmation(params: { // omits `role`, parent links, and >1px-free geometry, all of which the chrome classifier reads, // and one pass costs tens of microseconds against the 300 ms poll cadence. requiredQuietMs(snapshot, currentQuietMs) { - if (transitionBaseline === undefined && baselineNodes) { + if (transitionBaseline === undefined && baseline) { transitionBaseline = broadTransitionBaseline({ - baselineNodes, + baseline, capture: snapshot, platform, appBundleId, @@ -415,25 +422,28 @@ type StableCaptureConfirmation = { * next capture, rather than ruling the transition out for the run. */ function broadTransitionBaseline(params: { - baselineNodes: SnapshotNode[]; + baseline: SurfaceScopedNodes; capture: Parameters[0]; platform: AgentDeviceRuntime['backend']['platform']; appBundleId: string | undefined; }): StableCaptureSignal | undefined { - const { baselineNodes, capture, platform, appBundleId } = params; + const { baseline, capture, platform, appBundleId } = params; if (!isBoundIosXCTestCapture(capture.backend, platform)) return undefined; // A broad *screen* replacement needs a complete post-action viewport projection. The baseline // comes from settle's authoritative pre-action capture, but a presented iOS modal can omit its // Application root and still be the complete interaction surface. Requiring the root only from // the candidate prevents scoped/synthetic post-action fragments from extending settle latency. const hasCompleteViewportProjection = capture.nodes.some(isViewportRootNode); - // Tiny pre-action surfaces are the other trustworthy completeness signal: iOS presents alerts - // and sheets as a handful of nodes, and their first post-dismissal capture can temporarily omit - // the viewport root. Treating that rootless replacement as an arbitrary scoped projection lets - // a coherent transitional tree win the default 500ms quiet window. - const hasTinyPresentedBaseline = baselineNodes.length <= TINY_STABLE_TREE_NODE_COUNT; - if (!hasCompleteViewportProjection && !hasTinyPresentedBaseline) return undefined; - return stableCaptureComparisonSignal(capture, baselineNodes, appBundleId); + // A presented pre-action surface is the other trustworthy completeness signal: the first capture + // after its dismissal can temporarily omit the viewport root, and treating that rootless + // replacement as an arbitrary scoped projection lets a coherent transitional tree win the plain + // quiet window. The producer's fact says so outright; without one, a tiny baseline stands in for + // it, because iOS presents alerts and sheets as a handful of nodes. + const hasPresentedBaseline = + baseline.presentedSurface === 'presented' || + baseline.nodes.length <= TINY_STABLE_TREE_NODE_COUNT; + if (!hasCompleteViewportProjection && !hasPresentedBaseline) return undefined; + return stableCaptureComparisonSignal(capture, baseline.nodes, appBundleId); } /** diff --git a/src/commands/post-action-observation-grammar.ts b/src/commands/post-action-observation-grammar.ts index b6f77aa0df..892cabed07 100644 --- a/src/commands/post-action-observation-grammar.ts +++ b/src/commands/post-action-observation-grammar.ts @@ -34,7 +34,7 @@ const settleFields = () => ({ 'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.', ), settleQuietMs: integerField( - 'Settle: quiet window in milliseconds (default 500; 300 when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the timeoutMs settle deadline.', + 'Settle: quiet window in milliseconds (default 500; 300 when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the timeoutMs settle deadline; a replacement from a screen the pre-action capture read as presenting nothing, such as a tab switch, gets no 1500ms confirmation and keeps the asked window.', { min: 0 }, ), timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }),