diff --git a/.device-evidence/CHECKLIST.md b/.device-evidence/CHECKLIST.md index b708fc77b8..63f61f0c85 100644 --- a/.device-evidence/CHECKLIST.md +++ b/.device-evidence/CHECKLIST.md @@ -103,7 +103,7 @@ cannot show a cache hit on command: `packages/daemon/src/execution/__tests__/sel drives a session-snapshot cache hit and pins that the consumed tree earns no repair proof and discloses nothing, `packages/daemon/src/core/__tests__/capture-disclosure-target-activation.test.ts` pins that the response builder never borrows a repair the stored snapshot happens to carry, and -`src/__tests__/daemon-composition/interaction/interaction-capture-disclosure.test.ts` pins that +`packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts` pins that a press which consumed no capture is not disclosed against an older tree. ## Why the live activation lane is manual diff --git a/packages/daemon/package.json b/packages/daemon/package.json index 32374c3b95..d3059b0bc7 100644 --- a/packages/daemon/package.json +++ b/packages/daemon/package.json @@ -184,18 +184,10 @@ "types": "./src/execution/interaction/internal/__tests__/*.ts", "default": "./src/execution/interaction/internal/__tests__/*.ts" }, - "./testing/interaction/internal/interaction-android-escape": { - "types": "./src/execution/interaction/internal/interaction-android-escape.ts", - "default": "./src/execution/interaction/internal/interaction-android-escape.ts" - }, "./testing/interaction/internal/interaction-touch-response": { "types": "./src/execution/interaction/internal/interaction-touch-response.ts", "default": "./src/execution/interaction/internal/interaction-touch-response.ts" }, - "./testing/interaction/internal/types": { - "types": "./src/core/interaction/internal/types.ts", - "default": "./src/core/interaction/internal/types.ts" - }, "./testing/interactor-resolution": { "types": "./src/core/interactor-resolution.ts", "default": "./src/core/interactor-resolution.ts" diff --git a/packages/daemon/src/__tests__/support/command-surface-override-fake.ts b/packages/daemon/src/__tests__/support/command-surface-override-fake.ts index 256754a6b2..790a3ade40 100644 --- a/packages/daemon/src/__tests__/support/command-surface-override-fake.ts +++ b/packages/daemon/src/__tests__/support/command-surface-override-fake.ts @@ -3,7 +3,7 @@ import type { DaemonCommandSurface, } from '@agent-device/contracts/daemon-command-surface'; import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; -import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import type { SnapshotPreferredBackend, SnapshotState } from '@agent-device/kernel/snapshot'; import { fakeCreateCommandSurface } from './command-surface-fake.ts'; export type CommandSurfaceMembers = { @@ -47,12 +47,17 @@ export async function captureAndStoreSnapshot( signal?: AbortSignal; interactiveOnly?: boolean; includeRects?: boolean; + preferredBackend?: SnapshotPreferredBackend; }, ): Promise { const name = params.session ?? 'default'; const captured = await config.backend.captureSnapshot?.( { session: name, requestId: params.requestId, signal: params.signal ?? config.signal }, - { interactiveOnly: params.interactiveOnly, includeRects: params.includeRects }, + { + interactiveOnly: params.interactiveOnly, + includeRects: params.includeRects, + preferredBackend: params.preferredBackend, + }, ); const snapshot = captured?.snapshot; if (!snapshot) throw new Error('captureAndStoreSnapshot: the backend returned no snapshot'); diff --git a/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts new file mode 100644 index 0000000000..f1694f3e37 --- /dev/null +++ b/packages/daemon/src/__tests__/support/touch-forwarding-surface-fake.ts @@ -0,0 +1,228 @@ +import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface'; +import type { + FillCommandResult, + InteractionTarget, + PressCommandResult, + SettleObservation, + SettleParams, +} from '@agent-device/contracts/interaction'; +import { buildSelectorChainForNode } from '@agent-device/selectors'; +import { AppError } from '@agent-device/kernel/errors'; +import { + centerOfRect, + type Point, + type SnapshotNode, + type SnapshotPreferredBackend, + type SnapshotQualityVerdict, + type SnapshotState, +} from '@agent-device/kernel/snapshot'; +import { + captureAndStoreSnapshot, + commandSurfaceWithMembers, + type CommandSurfaceMembers, +} from './command-surface-override-fake.ts'; +import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract'; + +type TouchResolution = { + point: Point; + target: InteractionTarget; + node?: SnapshotNode; + nodes?: SnapshotNode[]; + selectorChain?: string[]; + snapshotQuality?: SnapshotQualityVerdict; +}; + +export type TouchForwardingSurfaceOptions = { + /** + * Which node a selector expression names. The default reads `label="X"` / `label=X` and + * `id="X"` / `id=X`; a suite that needs another selector shape states its own matcher. + */ + matchSelector?: (nodes: SnapshotNode[], selector: string) => SnapshotNode | undefined; + /** + * The post-action observation a `--settle` touch owes. It runs after the touch with the + * settle parameters the daemon forwarded, the pre-action tree, and a capture taken through the + * config's backend and stored as the session snapshot; its answer rides on the result. + */ + observeSettle?: (params: { + settle: SettleParams; + baseline: SnapshotNode[] | undefined; + /** The quality verdict of the tree the touch resolved against, when it published one. */ + baselineQuality: SnapshotQualityVerdict | undefined; + capture: (options?: { + signal?: AbortSignal; + preferredBackend?: SnapshotPreferredBackend; + }) => Promise; + }) => Promise; + /** Members the suite answers itself, layered over the touch actions (selector reads, gestures). */ + members?: (config: AgentDeviceRuntimeConfig) => CommandSurfaceMembers; +}; + +function defaultMatchSelector(nodes: SnapshotNode[], selector: string): SnapshotNode | undefined { + const parsed = /^(label|id)="?([^"]*)"?$/.exec(selector.trim()); + if (!parsed) return undefined; + const [, key, value] = parsed; + return nodes.find((node) => (key === 'id' ? node.identifier : node.label) === value); +} + +function refBody(ref: string): string { + return ref.replace(/^@/, '').replace(/~s\d+$/, ''); +} + +/** + * A command surface whose touch actions do what the daemon depends on the real command family + * to do, and no more: locate the target (a stored-tree lookup for a `@ref`, one interactive + * capture through the config's backend for a selector), dispatch the touch through the + * config's backend, and answer with the resolved target and the tree it was read from. Selector + * disambiguation, hittability, the moving-target gate, and every other command-family decision + * are out of scope: the suites that use this surface pin what the daemon does around them. + */ +export function touchForwardingCreateCommandSurface( + options: TouchForwardingSurfaceOptions = {}, +): CreateDaemonCommandSurface { + const matchSelector = options.matchSelector ?? defaultMatchSelector; + return commandSurfaceWithMembers((config) => { + const locate = async ( + target: InteractionTarget, + context: { session?: string; requestId?: string; signal?: AbortSignal }, + action: 'click' | 'fill', + ): Promise => { + const chainFor = (node: SnapshotNode, nodes: SnapshotNode[]) => + buildSelectorChainForNode(node, config.backend.platform, { action, nodes }); + if (target.kind === 'point') return { point: { x: target.x, y: target.y }, target }; + if (target.kind === 'ref') { + const record = await config.sessions?.get(context.session ?? 'default'); + const tree = record?.refFrameSnapshot ?? record?.snapshot; + const nodes = tree?.nodes ?? []; + const node = nodes.find((candidate) => candidate.ref === refBody(target.ref)); + if (!node?.rect) { + throw new AppError('COMMAND_FAILED', `Ref ${target.ref} not found`, { + reason: 'ref_not_found', + dispatched: 'no', + }); + } + return { + point: centerOfRect(node.rect), + target, + node, + nodes, + selectorChain: chainFor(node, nodes), + snapshotQuality: tree?.snapshotQuality, + }; + } + const { nodes, snapshotQuality } = await captureAndStoreSnapshot(config, { + ...context, + interactiveOnly: true, + includeRects: true, + }); + const node = matchSelector(nodes, target.selector); + if (node?.rect) { + return { + point: centerOfRect(node.rect), + target, + node, + nodes, + selectorChain: chainFor(node, nodes), + snapshotQuality, + }; + } + throw new AppError('COMMAND_FAILED', `Selector did not match: ${target.selector}`, { + reason: 'selector_not_found', + dispatched: 'no', + }); + }; + + const resolvedTarget = (resolution: TouchResolution) => { + const { target, point, node, nodes, selectorChain } = resolution; + const preAction = nodes ? { nodes } : undefined; + if (target.kind === 'point') return { kind: 'point' as const, point, preAction }; + if (target.kind === 'ref') { + return { kind: 'ref' as const, point, target, node, selectorChain, preAction }; + } + return { + kind: 'selector' as const, + point, + target, + node: node as SnapshotNode, + selectorChain: selectorChain ?? [], + preAction, + }; + }; + + const observe = async ( + context: { session?: string; requestId?: string; signal?: AbortSignal }, + resolution: TouchResolution, + settle: SettleParams | undefined, + ): Promise => { + if (!settle || !options.observeSettle) return undefined; + return await options.observeSettle({ + settle, + baseline: resolution.nodes, + baselineQuality: resolution.snapshotQuality, + capture: (capture) => + captureAndStoreSnapshot(config, { + ...context, + signal: capture?.signal ?? context.signal, + preferredBackend: capture?.preferredBackend, + includeRects: true, + }), + }); + }; + + const press = async ( + target: InteractionTarget, + pressOptions?: { + session?: string; + requestId?: string; + signal?: AbortSignal; + settle?: SettleParams; + }, + ): Promise => { + const context = { + session: pressOptions?.session, + requestId: pressOptions?.requestId, + signal: pressOptions?.signal, + }; + const resolution = await locate(target, context, 'click'); + const backendResult = await config.backend.tap?.(context, resolution.point); + const settle = await observe(context, resolution, pressOptions?.settle); + return { + ...resolvedTarget(resolution), + ...(backendResult ? { backendResult } : {}), + ...(settle ? { settle } : {}), + }; + }; + + const fill = async ( + target: InteractionTarget, + text: string, + fillOptions?: { + session?: string; + requestId?: string; + signal?: AbortSignal; + settle?: SettleParams; + }, + ): Promise => { + const context = { + session: fillOptions?.session, + requestId: fillOptions?.requestId, + signal: fillOptions?.signal, + }; + const resolution = await locate(target, context, 'fill'); + const backendResult = await config.backend.fill?.(context, resolution.point, text); + const settle = await observe(context, resolution, fillOptions?.settle); + return { + ...resolvedTarget(resolution), + text, + ...(backendResult ? { backendResult } : {}), + ...(settle ? { settle } : {}), + }; + }; + + const own = options.members?.(config) ?? {}; + return { + capture: own.capture, + selectors: own.selectors, + interactions: { press, click: press, fill, ...own.interactions }, + }; + }); +} diff --git a/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts new file mode 100644 index 0000000000..da6bd3bafc --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; +import { + assertDispatchDisclosureDriversMatchRows, + DISPATCH_DISCLOSURE_TABLE_PATH, + dispatchDisclosureRowsOwnedBy, +} from '@agent-device/test-kit'; +import { makeAndroidSession } from '../../../__tests__/support/session-factories.ts'; +import { discloseRequestDispatch } from '../../request-dispatch-disclosure.ts'; +import { createRequestDispatchLedger } from '../../../core/request-dispatch-ledger.ts'; +import { assertAndroidPressStayedInApp } from './interaction-android-escape.ts'; + +// contracts/fixtures/dispatch-disclosure.json, post-action guard rows: the guard is driven itself, +// inside the router's dispatch seam. + +async function pressThatLeftTheApp(): Promise { + const session = makeAndroidSession('dispatch-disclosure-android', { + appBundleId: 'com.example.app', + }); + const observation = { + readAppState: async () => ({ package: 'com.android.settings' }), + isPermissionPackage: async () => false, + } as unknown as AndroidObservationAdapter; + return await discloseRequestDispatch( + { token: 't', session: session.name, command: 'press', positionals: ['@e1'] }, + createRequestDispatchLedger(), + async () => { + await assertAndroidPressStayedInApp(session, '@e1', observation); + return null; + }, + ); +} + +const DRIVERS: Record Promise> = { + 'post-action-guard.android-press-left-app': pressThatLeftTheApp, +}; + +const ROWS = dispatchDisclosureRowsOwnedBy( + import.meta.url, + fs.readFileSync(DISPATCH_DISCLOSURE_TABLE_PATH, 'utf8'), +); + +test('every post-action guard dispatch-disclosure row has exactly one driver', () => { + assertDispatchDisclosureDriversMatchRows(ROWS, Object.keys(DRIVERS)); +}); + +for (const row of ROWS) { + test(`${row.id}: ${row.trigger} → dispatched ${row.dispatched}`, async () => { + const drive = DRIVERS[row.id]; + assert.ok(drive, `no driver for ${row.id}`); + await assert.rejects(drive(), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.expectedPackage, 'com.example.app'); + assert.equal(error.details?.foregroundPackage, 'com.android.settings'); + assert.equal(error.details?.dispatched, row.dispatched); + return true; + }); + }); +} diff --git a/src/__tests__/daemon-composition/interaction/interaction-capture-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts similarity index 84% rename from src/__tests__/daemon-composition/interaction/interaction-capture-disclosure.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts index 4067928d7f..d6cacf552a 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-capture-disclosure.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-capture-disclosure.test.ts @@ -6,23 +6,17 @@ import { type SnapshotState, } from '@agent-device/kernel/snapshot'; import { iosTargetActivationDisclosure } from '@agent-device/contracts/ios-target-activation'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { getRuntimeBindings } from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { - contextFromFlags, - makeSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; -import { markDeferredInteractionOutcome } from '@agent-device/daemon/testing/deferred-interaction-outcome'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { handleInteractionCommands } from '../index.ts'; +import { getRuntimeBindings } from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { contextFromFlags, makeSession } from './__tests__/interaction-touch-fixtures.ts'; +import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; +import { markDeferredInteractionOutcome } from '../../../core/deferred-interaction-outcome.ts'; import { formatPostGestureOutcomeWarning } from '@agent-device/capture-kit/post-gesture-stability'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => { - const fixture = - await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'); - return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; -}); +installLegacyCaptureInteractorResolution(); const FACT: IosTargetActivation = { reason: 'stale_target', @@ -62,7 +56,7 @@ async function pressSelector(params: { const sessionStore = makeSessionStore(); sessionStore.publish(params.sessionName, makeSession(params.sessionName)); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: params.sessionName, @@ -115,7 +109,7 @@ test('a press that consumes no capture is not disclosed against an older tree', sessionStore.publish(sessionName, session); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: sessionName, @@ -175,7 +169,7 @@ async function pressAfterUnsettledScroll(selector: string) { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] }); let done = false; const pending = handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: 'default', command: 'press', positionals: [selector], flags: {} }, sessionName: 'default', sessionStore, diff --git a/src/__tests__/daemon-composition/interaction/interaction-common.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts similarity index 80% rename from src/__tests__/daemon-composition/interaction/interaction-common.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-common.test.ts index a29df237a2..7e3a513953 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-common.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-common.test.ts @@ -1,24 +1,22 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; +import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; import { beforeEach, expect, test } from 'vitest'; import { makeIosSession, makeAuthoringSession, authoringPublication, -} from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; +} from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { - finalizeTouchInteraction, - handleInteractionCommands, -} from '@agent-device/daemon/testing/interaction/index'; +import { activateCompleteRefFrame } from '../../../core/ref-frame.ts'; +import { finalizeTouchInteraction, handleInteractionCommands } from '../index.ts'; import { IOS_SIMULATOR } from '@agent-device/test-kit'; import { getRuntimeBindings, mockFillPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; const contextFromFlags = (_flags: CommandFlags | undefined) => ({}); @@ -165,7 +163,7 @@ test('parameterized fill scrubs concatenated backend values and object keys thro ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: sessionName, @@ -223,7 +221,7 @@ test('parameterized fill collapses whitespace-only backend echoes through the ha }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: touchForwardingCreateCommandSurface(), req: { token: 't', session: sessionName, @@ -298,3 +296,51 @@ test.each([ expect(session.actions[0]?.result).toEqual(response.data); }, ); + +test('a fill on a @ref acts on the node of the authorized ref frame, not on a newer observation', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ref-frame-authority-fill'; + const session = makeIosSession(sessionName, { appBundleId: 'com.example.app' }); + const field = (rect: { x: number; y: number; width: number; height: number }) => + attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect, + enabled: true, + hittable: true, + }, + ] as RawSnapshotNode[]); + session.snapshot = { + nodes: field({ x: 0, y: 0, width: 20, height: 20 }), + createdAt: Date.now(), + backend: 'xctest', + }; + activateCompleteRefFrame(session); + session.snapshot = { + nodes: field({ x: 200, y: 200, width: 20, height: 20 }), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.publish(sessionName, session); + mockFillPoint.mockResolvedValue({ filled: true }); + + const response = await handleInteractionCommands({ + createCommandSurface: touchForwardingCreateCommandSurface(), + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['@e1', 'hello'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok, JSON.stringify(response)).toBe(true); + expect(mockFillPoint).toHaveBeenCalledWith(expect.objectContaining({ point: { x: 10, y: 10 } })); +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts similarity index 79% rename from src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts index b3deb1ef0a..d885fcb40c 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts @@ -1,8 +1,8 @@ -import { composedPlatformServicesFixture } from '../platform-services-fixture.ts'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { beforeEach, test, vi } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; +import type { GesturePlan } from '@agent-device/contracts/gesture-plan-types'; import { AppError, type DispatchDisclosure, normalizeError } from '@agent-device/kernel/errors'; import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; import { @@ -10,44 +10,89 @@ import { DISPATCH_DISCLOSURE_TABLE_PATH, dispatchDisclosureRowsOwnedBy, } from '@agent-device/test-kit'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { makeAndroidSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { makeAndroidSession } from '../../../__tests__/support/session-factories.ts'; import { getRuntimeBindings, mockFillPoint, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { discloseRequestDispatch } from '@agent-device/daemon/testing/request-dispatch-disclosure'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; +import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; +import { discloseRequestDispatch } from '../../request-dispatch-disclosure.ts'; import { createRequestDispatchLedger, recordBoundMutations, -} from '@agent-device/daemon/testing/request-dispatch-ledger'; -import type { BindDeviceRuntime } from '@agent-device/daemon/testing/request-runtime-binding'; -import { clearAndroidObservationFixture } from '@agent-device/daemon/testing/__tests__/android-observation-fixture'; -import { activateCompleteRefFrame, expireRefFrame } from '@agent-device/daemon/testing/ref-frame'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import type { InteractionRouteInput } from '@agent-device/daemon/testing/interaction/internal/types'; -import { assertAndroidPressStayedInApp } from '@agent-device/daemon/testing/interaction/internal/interaction-android-escape'; -import { gestureRuntimeBindingsFixture } from '@agent-device/daemon/testing/interaction/internal/__tests__/gesture-runtime-bindings.fixtures'; +} from '../../../core/request-dispatch-ledger.ts'; +import type { BindDeviceRuntime } from '../../../core/request-runtime-binding.ts'; +import { clearAndroidObservationFixture } from '../../../__tests__/support/android-observation-fixture.ts'; +import { activateCompleteRefFrame, expireRefFrame } from '../../../core/ref-frame.ts'; +import { handleInteractionCommands } from '../index.ts'; +import type { InteractionRouteInput } from '../../../core/interaction/internal/types.ts'; +import { gestureRuntimeBindingsFixture } from './__tests__/gesture-runtime-bindings.fixtures.ts'; import { contextFromFlags, makeSession, makeStaleRefSession, -} from '@agent-device/daemon/testing/interaction/internal/__tests__/interaction-touch-fixtures'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from './__tests__/interaction-touch-fixtures.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; -// contracts/fixtures/dispatch-disclosure.json, daemon and post-action guard rows: the daemon rows -// drive a real `press` through the daemon interaction handler, inside the router's dispatch seam, -// with only the device touch mocked; the guard row drives the guard itself. +// contracts/fixtures/dispatch-disclosure.json, daemon rows: each drives a `press`, `fill`, `get`, +// or `swipe` through the daemon interaction handler, inside the router's dispatch seam, over a +// command surface that forwards the touch to the bound operations (only the device touch is +// faked). The post-action guard row is driven beside the guard, in +// interaction-android-escape-dispatch-disclosure.test.ts. -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); +const GESTURE_PLAN: GesturePlan = { + topology: 'single', + intent: 'fling', + executionProfile: 'timed-pan', + durationMs: 100, + viewport: { x: 0, y: 0, width: 400, height: 800 }, + pointers: [ + { + pointerId: 0, + samples: [ + { offsetMs: 0, point: { x: 100, y: 600 } }, + { offsetMs: 100, point: { x: 100, y: 200 } }, + ], + }, + ], +}; + +const createCommandSurface = touchForwardingCreateCommandSurface({ + members: (config) => ({ + selectors: { + get: async () => { + await config.backend.captureSnapshot?.({ session: 'dispatch-disclosure' }, {}); + throw new AppError('COMMAND_FAILED', 'No element matched label="Missing"', { + reason: 'selector_not_found', + }); + }, + }, + interactions: { + gesture: async (options) => { + await config.backend.performGesture?.({ session: options.session }, GESTURE_PLAN); + return { + kind: 'fling', + durationMs: GESTURE_PLAN.durationMs, + pointerCount: 1, + from: { x: 100, y: 600 }, + to: { x: 100, y: 200 }, + }; + }, + }, + }), +}); + +installLegacyCaptureInteractorResolution(); beforeEach(() => { resetGetRuntimeFixture(); + legacyDispatchCapture.mockReset(); + legacyDispatchCapture.mockResolvedValue({}); }); /** @@ -96,7 +141,7 @@ async function press({ }; sessionStore.publish(session.name, session); const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface, req: { token: 't', session: session.name, command, positionals, flags }, sessionName: session.name, sessionStore, @@ -133,24 +178,6 @@ async function pressAfterUnclassifiedTouchFailure( return await press({ positionals: ['@e1'] }); } -async function pressThatLeftTheApp(): Promise { - const session = makeAndroidSession('dispatch-disclosure-android', { - appBundleId: 'com.example.app', - }); - const observation = { - readAppState: async () => ({ package: 'com.android.settings' }), - isPermissionPackage: async () => false, - } as unknown as AndroidObservationAdapter; - return await discloseRequestDispatch( - { token: 't', session: session.name, command: 'press', positionals: ['@e1'] }, - createRequestDispatchLedger(), - async () => { - await assertAndroidPressStayedInApp(session, '@e1', observation); - return null; - }, - ); -} - /** `swipe --count 2` whose first repetition runs and whose second is refused before dispatch. */ async function swipeRefusedOnSecondRepetition(): Promise { const gestures = gestureRuntimeBindingsFixture(); @@ -165,8 +192,8 @@ async function swipeRefusedOnSecondRepetition(): Promise { const session = makeSession('dispatch-disclosure-swipe'); sessionStore.publish(session.name, session); const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: session.name, @@ -199,8 +226,8 @@ async function gestureDragRefusedBeforeDispatch(): Promise { expireRefFrame(session); sessionStore.publish(session.name, session); const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: session.name, @@ -258,8 +285,8 @@ async function refRefusedAfterAndroidRecovery(command: 'press' | 'gesture'): Pro openApp, }; const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: session.name, @@ -315,7 +342,7 @@ async function pressThenForegroundReadRefused(): Promise { }, }; const response = await routeInteraction({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface, req: { token: 't', session: session.name, command: 'press', positionals: ['50', '40'] }, sessionName: session.name, sessionStore, @@ -343,7 +370,6 @@ const DRIVERS: Record Promise> = { 'daemon.post-dispatch.press-then-foreground-read-refused': pressThenForegroundReadRefused, 'daemon.read-only-command': () => press({ command: 'get', positionals: ['text', 'label="Missing"'] }), - 'post-action-guard.android-press-left-app': pressThatLeftTheApp, }; const ROWS = dispatchDisclosureRowsOwnedBy( @@ -351,7 +377,7 @@ const ROWS = dispatchDisclosureRowsOwnedBy( fs.readFileSync(DISPATCH_DISCLOSURE_TABLE_PATH, 'utf8'), ); -test('every daemon and post-action guard dispatch-disclosure row has exactly one driver', () => { +test('every daemon dispatch-disclosure row has exactly one driver', () => { assertDispatchDisclosureDriversMatchRows(ROWS, Object.keys(DRIVERS)); }); @@ -378,9 +404,7 @@ test('the daemon keeps a producer verdict instead of inferring its own', async ( }); test('a read-only command discloses no over a producer verdict', async () => { - const capture = vi.mocked(captureSnapshotWithInteractor); - capture.mockClear(); - capture.mockRejectedValueOnce( + legacyDispatchCapture.mockRejectedValueOnce( new AppError('COMMAND_FAILED', 'runner capture lost', { dispatched: 'unknown' }), ); await assert.rejects( @@ -392,7 +416,7 @@ test('a read-only command discloses no over a producer verdict', async () => { return true; }, ); - assert.equal(capture.mock.calls.length, 1); + assert.equal(legacyDispatchCapture.mock.calls.length, 1); }); test('a read-only command discloses no over a producer verdict it throws', async () => { @@ -425,7 +449,7 @@ test('a plain Error a backend throws reaches the wire with dispatched unknown', }) as unknown as typeof bindings.bindDevice; await assert.rejects( routeInteraction({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface, req: { token: 't', session: session.name, command: 'press', positionals: ['@e1'], flags: {} }, sessionName: session.name, sessionStore, diff --git a/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts new file mode 100644 index 0000000000..9adbbd636b --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-settle-private-ax-route.test.ts @@ -0,0 +1,119 @@ +import { beforeEach, expect, test, vi } from 'vitest'; +import type { + Interactor, + SnapshotOptions, + SnapshotResult, +} from '@agent-device/contracts/interactor-types'; +import type { SettleObservation } from '@agent-device/contracts/interaction'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { contextFromFlags as buildDaemonContext } from '../../../core/context.ts'; +import { installInteractorResolution } from '../../../core/interactor-resolution.ts'; +import { handleInteractionCommands } from '../index.ts'; +import { getRuntimeBindings } from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; + +// The settle loop pins `private-ax` on its captures once the observation reports that producer; the +// daemon must carry that preference through its capture path to the interactor, which is what the +// runner request is built from. Where the loop decides to pin is pinned beside it, in +// src/commands/interaction/runtime/settle.test.ts, and how the Apple interactor turns the preference +// into a runner request in packages/platform-apple. + +const snapshotCalls: SnapshotOptions[] = []; + +const nodes = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + rect: { x: 20, y: 100, width: 120, height: 44 }, + hittable: true, + }, + ...Array.from({ length: 4 }, (_, offset) => ({ + index: offset + 2, + parentIndex: 0, + type: 'Button', + label: `Action ${offset + 1}`, + rect: { x: 20, y: 160 + offset * 50, width: 120, height: 44 }, + hittable: true, + })), +]; +const privateAxQuality = { + state: 'recovered' as const, + backend: 'private-ax' as const, + reasonCode: 'requested-backend' as const, +}; + +installInteractorResolution({ + resolve: async () => + ({ + snapshot: vi.fn(async (options: SnapshotOptions): Promise => { + snapshotCalls.push(options); + return { nodes, backend: 'xctest', quality: privateAxQuality } as SnapshotResult; + }), + }) as unknown as Interactor, +}); + +beforeEach(() => { + snapshotCalls.length = 0; +}); + +/** The pin the settle loop applies: every settle capture asks for `private-ax` once it is established. */ +const pinningSurface = touchForwardingCreateCommandSurface({ + observeSettle: async ({ settle, baselineQuality, capture }): Promise => { + let pinned = baselineQuality?.backend === 'private-ax'; + for (let captures = 0; captures < 3; captures += 1) { + const settled = await capture(pinned ? { preferredBackend: 'private-ax' } : undefined); + pinned ||= settled.snapshotQuality?.backend === 'private-ax'; + } + return { + settled: true, + waitedMs: 0, + captures: 3, + quietMs: settle.quietMs ?? 500, + timeoutMs: settle.timeoutMs ?? 10_000, + }; + }, +}); + +test('daemon press --settle pins private-ax on emitted snapshot runner requests', async () => { + const sessionName = 'press-settle-private-ax-route'; + const sessionStore = makeSessionStore(); + sessionStore.publish( + sessionName, + makeIosSession(sessionName, { + appBundleId: 'com.example.fixture', + snapshot: makeSnapshotState(nodes, { + backend: 'xctest', + snapshotQuality: { ...privateAxQuality, reasonCode: 'deferred' }, + }), + }), + ); + + const response = await handleInteractionCommands({ + createCommandSurface: pinningSurface, + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['label=Continue'], + flags: { settle: true, settleQuietMs: 25, timeoutMs: 1_000 }, + }, + sessionName, + sessionStore, + contextFromFlags: (flags, appBundleId, traceLogPath) => + buildDaemonContext('', flags, appBundleId, traceLogPath), + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(snapshotCalls.length).toBeGreaterThanOrEqual(3); + const preferredBackends = snapshotCalls.map((options) => options.preferredBackend); + expect(preferredBackends).toEqual([ + undefined, + ...Array.from({ length: preferredBackends.length - 1 }, () => 'private-ax' as const), + ]); +}); diff --git a/src/__tests__/daemon-composition/interaction/interaction-settle.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts similarity index 80% rename from src/__tests__/daemon-composition/interaction/interaction-settle.test.ts rename to packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts index ad26ed494e..2cbfe26249 100644 --- a/src/__tests__/daemon-composition/interaction/interaction-settle.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts @@ -1,24 +1,19 @@ -import { composedPlatformServicesFixture } from '../platform-services-fixture.ts'; import type { CommandFlags } from '@agent-device/contracts/command'; -import { legacyDispatchCapture } from '@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture'; +import { legacyDispatchCapture } from '../../../__tests__/support/legacy-snapshot-capture-fixture.ts'; import { test, expect, vi, beforeEach } from 'vitest'; -import { - createInteractionRuntime, - handleInteractionCommands, -} from '@agent-device/daemon/testing/interaction/index'; -import type { SessionStore } from '@agent-device/daemon/testing/session-store'; -import type { SessionRef, SessionState } from '@agent-device/daemon/testing/session-state'; +import { createInteractionRuntime, handleInteractionCommands } from '../index.ts'; +import type { SessionStore } from '../../../core/session-store.ts'; +import type { SessionRef, SessionState } from '../../../core/session-state.ts'; import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; -import { setSessionSnapshot } from '@agent-device/daemon/testing/session-snapshot'; +import { setSessionSnapshot } from '../../session-snapshot.ts'; import { activateCompleteRefFrame, expireRefFrame, refFrameState, refFrameTree, -} from '@agent-device/daemon/testing/ref-frame'; -import { captureSnapshotWithInteractor } from '@agent-device/daemon/testing/snapshot-interactor-capture'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; +} from '../../../core/ref-frame.ts'; +import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; import { clearRequestAbortRegistration, registerRequestAbort, @@ -29,8 +24,12 @@ import { mockFillPoint, mockTapPoint, resetGetRuntimeFixture, -} from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; +import { daemonPlatformServicesFixture } from '../../../__tests__/support/platform-services-fixture.ts'; +import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts'; +import { touchForwardingCreateCommandSurface } from '../../../__tests__/support/touch-forwarding-surface-fake.ts'; +import type { SettleObservation } from '@agent-device/contracts/interaction'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; // #1101 --settle daemon response shape: the settle payload (diff + settled + // refsGeneration) rides the wire response through the shared builder, and a @@ -38,11 +37,9 @@ import { testCreateCommandSurface } from '../command-surface-fixture.ts'; // Quiet windows are tuned down (--settle-quiet 25) so no test waits real time // beyond a few poll ticks. +installLegacyCaptureInteractorResolution(); + const mockCaptureSnapshotForSession = vi.hoisted(() => vi.fn()); -vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', () => ({ - captureSnapshotWithInteractor: vi.fn(), -})); -const nativeCapture = vi.mocked(captureSnapshotWithInteractor); const BEFORE_NODES = [ { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, @@ -106,6 +103,84 @@ function seedSession(sessionName: string, sessionStore: ReturnType `${node.type}|${node.label ?? ''}`; + +/** + * A stand-in for the command family's settled diff, keyed by role and label: enough for the daemon + * to have added lines that carry refs minted from the settled tree, removed lines that carry none, + * and, when nothing was added, the unchanged interactive tail. How the real diff is computed is + * pinned beside it, in src/commands/interaction/runtime/settle.test.ts. + */ +function settledDiff(baseline: SnapshotNode[] | undefined, settled: SnapshotNode[]) { + const before = baseline ?? []; + const beforeKeys = new Set(before.map(diffKey)); + const settledKeys = new Set(settled.map(diffKey)); + const added = settled.filter((node) => !beforeKeys.has(diffKey(node))); + const removed = before.filter((node) => !settledKeys.has(diffKey(node))); + const unchanged = settled.filter((node) => beforeKeys.has(diffKey(node))); + return { + diff: { + summary: { additions: added.length, removals: removed.length, unchanged: unchanged.length }, + lines: [ + ...removed.map((node) => ({ + kind: 'removed' as const, + text: `${node.type} "${node.label}"`, + })), + ...added.map((node) => ({ + kind: 'added' as const, + text: `${node.type} "${node.label}"`, + ref: node.ref, + })), + ], + }, + tail: + added.length === 0 + ? unchanged.flatMap((node) => + node.hittable && node.label && node.ref + ? [{ ref: node.ref, role: (node.type ?? '').toLowerCase(), label: node.label }] + : [], + ) + : undefined, + }; +} + +/** + * The touch-forwarding surface with the post-action observation `--settle` owes: it captures the + * settled tree through the config's backend under the settle budget's deadline and answers with the + * diff, or with the unavailable/stalled verdict the command family reports when that capture fails. + */ +const settlingSurface = touchForwardingCreateCommandSurface({ + observeSettle: async ({ settle, baseline, capture }): Promise => { + const quietMs = settle.quietMs ?? 500; + const timeoutMs = settle.timeoutMs ?? 10_000; + const deadline = AbortSignal.timeout(timeoutMs); + try { + const settled = await capture({ signal: deadline }); + const { diff, tail } = settledDiff(baseline, settled.nodes); + return { + settled: true, + waitedMs: 0, + captures: 1, + quietMs, + timeoutMs, + diff, + ...(tail ? { tail } : {}), + }; + } catch (error) { + return { + settled: false, + waitedMs: 0, + captures: 1, + quietMs, + timeoutMs, + hint: deadline.aborted + ? 'A snapshot capture stalled past the settle budget, so no settled diff is shown.' + : `Settle observation unavailable (${error instanceof Error ? error.message : String(error)}).`, + }; + } + }, +}); + test('interaction runtime inherits the registered daemon request signal', async () => { const sessionStore = makeSessionStore(); const sessionName = 'request-signal'; @@ -116,8 +191,8 @@ test('interaction runtime inherits the registered daemon request signal', async try { const runtime = await createInteractionRuntime({ - createCommandSurface: testCreateCommandSurface, - platformServices: composedPlatformServicesFixture(), + createCommandSurface: settlingSurface, + platformServices: daemonPlatformServicesFixture(), req: { token: 't', session: sessionName, @@ -175,7 +250,6 @@ beforeEach(() => { }); mockCaptureSnapshotForSession.mockReset(); mockCaptureSnapshotForSession.mockImplementation(emulateCaptureSnapshotForSession); - nativeCapture.mockReset(); }); const SETTLE_FLAGS = { settle: true, settleQuietMs: 25, timeoutMs: 2_000 }; @@ -221,7 +295,7 @@ test('press --settle responds with the settled diff, refsGeneration, and activat mockCommandDispatch({ snapshots: [BEFORE_NODES, AFTER_NODES, AFTER_NODES, AFTER_NODES] }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -271,7 +345,8 @@ test('held touch settle publishes refs into the current record of its scoped lif release = resolve; }); let captures = 0; - nativeCapture.mockImplementation(async () => { + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') return {}; captures += 1; if (captures === 2) { enter(); @@ -284,7 +359,7 @@ test('held touch settle publishes refs into the current record of its scoped lif }; }); const running = handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: address, @@ -359,7 +434,7 @@ test('press --settle on a removals-only diff attaches the unchanged interactive }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -396,7 +471,7 @@ test('press --settle rejects an expired-frame ref before dispatch or observation ); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -438,7 +513,7 @@ test('a settle observation without a diff leaves ref staleness untouched', async }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -504,7 +579,7 @@ test('a stalled settle capture receives its deadline signal and leaves the inter const startedAt = Date.now(); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -535,7 +610,7 @@ test('bare timeout without --settle stays compatible', async () => { mockCommandDispatch({ snapshots: [BEFORE_NODES] }); const compatible = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -561,7 +636,7 @@ test('settle-specific tuning flags without --settle are rejected', async () => { mockCommandDispatch({ snapshots: [BEFORE_NODES] }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, @@ -587,7 +662,7 @@ test('fill @ref --settle carries the settle payload on the ref wire shape', asyn mockCommandDispatch({ snapshots: [AFTER_NODES, AFTER_NODES] }); const response = await handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, + createCommandSurface: settlingSurface, req: { token: 't', session: sessionName, diff --git a/packages/test-kit/src/dispatch-disclosure.fixtures.ts b/packages/test-kit/src/dispatch-disclosure.fixtures.ts index 9bee01c77d..9916c98366 100644 --- a/packages/test-kit/src/dispatch-disclosure.fixtures.ts +++ b/packages/test-kit/src/dispatch-disclosure.fixtures.ts @@ -37,11 +37,12 @@ export const DISPATCH_DISCLOSURE_TABLE_PATH = path.join( * followed by `test(\`${row.id}`, which {@link DISPATCH_DISCLOSURE_ROW_LOOP} matches. */ export const DISPATCH_DISCLOSURE_DRIVER_OWNERS: Readonly> = { - 'daemon.': 'src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts', + 'daemon.': + 'packages/daemon/src/execution/interaction/internal/interaction-dispatch-disclosure.test.ts', 'daemon.scroll-no-progress': 'packages/daemon/src/execution/__tests__/scroll-movement.test.ts', 'daemon.route.': 'src/__tests__/daemon-composition/request-dispatch-disclosure.test.ts', 'post-action-guard.': - 'src/__tests__/daemon-composition/interaction/interaction-dispatch-disclosure.test.ts', + 'packages/daemon/src/execution/interaction/internal/interaction-android-escape-dispatch-disclosure.test.ts', 'ios-runner.': 'packages/platform-apple/src/runner/__tests__/index-dispatch-disclosure.test.ts', 'ios-runner.pre-send.': 'packages/platform-apple/src/runner/__tests__/runner-lifecycle-dispatch-disclosure.test.ts', diff --git a/src/__tests__/daemon-composition/interaction/interaction-settle-private-ax-route.test.ts b/src/__tests__/daemon-composition/interaction/interaction-settle-private-ax-route.test.ts deleted file mode 100644 index 8a7b0ca9af..0000000000 --- a/src/__tests__/daemon-composition/interaction/interaction-settle-private-ax-route.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { beforeEach, expect, test, vi } from 'vitest'; -import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; -import { makeIosSession } from '@agent-device/daemon/testing/__tests__/support/session-factories'; -import { makeSessionStore } from '@agent-device/daemon/testing/__tests__/support/store-factory'; -import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; -import { contextFromFlags as buildDaemonContext } from '@agent-device/daemon/testing/context'; -import { handleInteractionCommands } from '@agent-device/daemon/testing/interaction/index'; -import { getRuntimeBindings } from '@agent-device/daemon/testing/__tests__/interaction-get-runtime-fixture'; -import { getInteractor } from '../../../core/interactors.ts'; -import { installInteractorResolution } from '@agent-device/daemon/testing/interactor-resolution'; -import { testCreateCommandSurface } from '../command-surface-fixture.ts'; - -// Root composition hands the daemon its interactor resolution; this test drives the real -// settle capture, so it composes the real lookup the way the process root does. -installInteractorResolution({ resolve: getInteractor }); - -vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { - const actual = - await importOriginal(); - return { - ...actual, - runAppleRunnerCommand: vi.fn(async () => ({})), - }; -}); - -import { runAppleRunnerCommand } from '@agent-device/platform-apple/runner/operations'; - -const mockRunnerCommand = vi.mocked(runAppleRunnerCommand); -const nodes = [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, - { - index: 1, - parentIndex: 0, - type: 'Button', - label: 'Continue', - rect: { x: 20, y: 100, width: 120, height: 44 }, - hittable: true, - }, - ...Array.from({ length: 4 }, (_, offset) => ({ - index: offset + 2, - parentIndex: 0, - type: 'Button', - label: `Action ${offset + 1}`, - rect: { x: 20, y: 160 + offset * 50, width: 120, height: 44 }, - hittable: true, - })), -]; -const privateAxQuality = { - state: 'recovered' as const, - backend: 'private-ax' as const, - reasonCode: 'requested-backend' as const, -}; - -beforeEach(() => { - mockRunnerCommand.mockReset(); - mockRunnerCommand.mockImplementation(async (_device, command) => { - if (command.command === 'snapshot') return { nodes, snapshotQuality: privateAxQuality }; - return {}; - }); -}); - -test('daemon press --settle pins private-ax on emitted snapshot runner requests', async () => { - const sessionName = 'press-settle-private-ax-route'; - const sessionStore = makeSessionStore(); - sessionStore.publish( - sessionName, - makeIosSession(sessionName, { - appBundleId: 'com.example.fixture', - snapshot: makeSnapshotState(nodes, { - backend: 'xctest', - snapshotQuality: { ...privateAxQuality, reasonCode: 'deferred' }, - }), - }), - ); - - const response = await withAppleRunnerProvider(mockRunnerCommand, { deviceId: 'sim-1' }, () => - handleInteractionCommands({ - createCommandSurface: testCreateCommandSurface, - req: { - token: 't', - session: sessionName, - command: 'press', - positionals: ['label=Continue'], - flags: { settle: true, settleQuietMs: 25, timeoutMs: 1_000 }, - }, - sessionName, - sessionStore, - contextFromFlags: (flags, appBundleId, traceLogPath) => - buildDaemonContext('', flags, appBundleId, traceLogPath), - ...getRuntimeBindings(), - }), - ); - - expect(response?.ok).toBe(true); - const snapshotCommands = mockRunnerCommand.mock.calls - .map(([, command]) => command) - .filter((command) => command.command === 'snapshot'); - expect(snapshotCommands.length).toBeGreaterThanOrEqual(3); - const preferredBackends = snapshotCommands.map((command) => command.preferredBackend); - expect(preferredBackends).toEqual([ - undefined, - ...Array.from({ length: preferredBackends.length - 1 }, () => 'private-ax' as const), - ]); -}); diff --git a/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts b/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts new file mode 100644 index 0000000000..e144928e4a --- /dev/null +++ b/src/commands/interaction/runtime/selector-target-resolution-post-gesture.test.ts @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { PostGestureOutcome } from '@agent-device/kernel/snapshot'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { selector } from './selector-read-utils.ts'; +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; + +const UNSETTLED: PostGestureOutcome = { + kind: 'unsettled', + gesture: { action: 'scroll', positionals: ['down'] }, +}; + +const movingList = (label: string) => + makeSnapshotState([ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Cell', + label, + rect: { x: 16, y: 300, width: 370, height: 52 }, + hittable: true, + }, + ]); + +// The interactive capture that misses and the full-tree retry that follows it read the same +// moving surface, so the retry's tree carries the interactive capture's unsettled outcome: neither +// the refusal nor the session's stored tree may present the miss as settled. +test('a selector press retries a miss on the full tree and keeps the unsettled gesture outcome', async () => { + const interactive = { ...movingList('Wi-Fi'), postGestureOutcome: UNSETTLED }; + const full = movingList('Wi-Fi'); + const captures = [interactive, full]; + const taps: unknown[] = []; + const device = createInteractionDevice(movingList('Wi-Fi'), { + captureSnapshot: async () => ({ snapshot: captures.shift() ?? full }), + tap: async (_context, point) => { + taps.push(point); + }, + }); + + await assert.rejects( + () => device.interactions.press(selector('label="General"'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'selector_not_found'); + assert.equal(details?.dispatched, 'no'); + return true; + }, + ); + assert.deepEqual(taps, []); + assert.deepEqual(full.postGestureOutcome, UNSETTLED); +}); diff --git a/src/commands/interaction/runtime/settle-capture-deadline.test.ts b/src/commands/interaction/runtime/settle-capture-deadline.test.ts new file mode 100644 index 0000000000..4f3b7eeff6 --- /dev/null +++ b/src/commands/interaction/runtime/settle-capture-deadline.test.ts @@ -0,0 +1,80 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AgentDeviceBackend, BackendSnapshotResult } from '@agent-device/contracts/backend'; +import { localCommandPolicy } from '@agent-device/contracts/command-policy'; +import { createMemorySessionStore } from '../../../command-runtime/runtime-factory.ts'; +import { createLocalArtifactAdapter } from '../../../io.ts'; +import { createAgentDevice } from '../../../runtime.ts'; +import { ref, selector } from './selector-read-utils.ts'; +import { + buttonSnapshot, + createFakeClock, + createSettleDevice, + welcomeSnapshot, +} from './__tests__/settle-device-fixtures.ts'; + +test('a stalled settle capture receives its deadline signal and the press still succeeds', async () => { + const before = buttonSnapshot(); + let captureCalls = 0; + let observedAbort = false; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: (context) => { + captureCalls += 1; + if (captureCalls === 1) return Promise.resolve({ snapshot: before }); + return new Promise((_resolve, reject) => { + assert.ok(context.signal); + context.signal.addEventListener( + 'abort', + () => { + observedAbort = true; + reject(context.signal?.reason); + }, + { once: true }, + ); + }); + }, + tap: async () => ({ ok: true }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot: before }]), + policy: localCommandPolicy(), + clock: createFakeClock(), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + settle: { quietMs: 25, timeoutMs: 75 }, + }); + + assert.equal(result.kind, 'selector'); + assert.ok(result.settle); + assert.equal(result.settle.settled, false); + assert.equal(result.settle.diff, undefined); + assert.match(result.settle.hint ?? '', /capture stalled past the settle budget/i); + assert.equal(observedAbort, true); +}); + +test('fill on a @ref with --settle carries the settle observation on its result', async () => { + const before = buttonSnapshot(); + const after = welcomeSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : after }; + }, + }); + + const result = await device.interactions.fill(ref('@e1'), 'hello', { + session: 'default', + settle: {}, + }); + + assert.equal(result.kind, 'ref'); + assert.ok(result.settle); + assert.equal(result.settle.settled, true); + assert.ok(result.settle.diff); +});