Skip to content
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import type {
CreateDaemonCommandSurface,
DaemonCommandSurface,
} from '@agent-device/contracts/daemon-command-surface';
import type { AgentDeviceRuntimeConfig } from '@agent-device/contracts/runtime-contract';
import type { SnapshotState } from '@agent-device/kernel/snapshot';
import { fakeCreateCommandSurface } from './command-surface-fake.ts';

export type CommandSurfaceMembers = {
capture?: Partial<DaemonCommandSurface['capture']>;
selectors?: Partial<DaemonCommandSurface['selectors']>;
interactions?: Partial<DaemonCommandSurface['interactions']>;
};

/**
* The inert command surface with the members a suite names answered by that suite. The daemon
* suites that reach a selector read or a touch action use it to state what the command family
* answered, so every assertion stays about what the daemon does with the answer. `members` runs
* once per surface the daemon builds and receives that request's runtime config, which is how a
* member reaches the same `backend` and `sessions` seams the real command family reaches.
*/
export function commandSurfaceWithMembers(
members: (config: AgentDeviceRuntimeConfig) => CommandSurfaceMembers,
): CreateDaemonCommandSurface {
return async (config) => {
const inert = await fakeCreateCommandSurface(config);
const own = members(config);
return {
...inert,
signal: config.signal,
capture: { ...inert.capture, ...own.capture },
selectors: { ...inert.selectors, ...own.selectors },
interactions: { ...inert.interactions, ...own.interactions },
};
};
}

/**
* One capture the way the real command family takes it: through the config's backend, then
* written back as the session's stored snapshot through the config's session store.
*/
export async function captureAndStoreSnapshot(
config: AgentDeviceRuntimeConfig,
params: {
session?: string;
requestId?: string;
signal?: AbortSignal;
interactiveOnly?: boolean;
includeRects?: boolean;
},
): Promise<SnapshotState> {
const name = params.session ?? 'default';
const captured = await config.backend.captureSnapshot?.(
{ session: name, requestId: params.requestId, signal: params.signal ?? config.signal },
{ interactiveOnly: params.interactiveOnly, includeRects: params.includeRects },
);
const snapshot = captured?.snapshot;
if (!snapshot) throw new Error('captureAndStoreSnapshot: the backend returned no snapshot');
const current = await config.sessions?.get(name);
await config.sessions?.set({ ...(current ?? { name }), name, snapshot });
return snapshot;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import type { Interactor } from '@agent-device/contracts/interactor-types';
import { installInteractorResolution } from '../../core/interactor-resolution.ts';
import { captureSnapshotThroughLegacyDispatchFixture } from './legacy-snapshot-capture-fixture.ts';

/**
* Routes the daemon's legacy snapshot capture through the shared capture double
* (`legacyDispatchCapture`) by installing the interactor resolution the process root would
* compose. The resolved interactor answers `snapshot` only, so a suite that reaches any other
* interactor member fails loudly instead of touching a device.
*
* Installation is process-global and each vitest file runs in its own module graph, so a suite
* calls this once at module scope.
*/
export function installLegacyCaptureInteractorResolution(): void {
installInteractorResolution({
resolve: async (device, runnerContext) =>
({
snapshot: async (options) =>
await captureSnapshotThroughLegacyDispatchFixture({
device,
runnerContext,
options: options ?? {},
}),
}) as Interactor,
});
}
Original file line number Diff line number Diff line change
@@ -1,24 +1,38 @@
import { test, expect, vi, beforeEach } from 'vitest';

vi.mock('@agent-device/device-selection/dispatch-resolve', async (importOriginal) => {
const actual = (await importOriginal()) as Record<string, unknown>;
return {
...actual,
resolveTargetDevice: actual.resolveTargetDevice,
};
});
vi.mock('@agent-device/daemon/testing/snapshot-interactor-capture', async () => {
const fixture =
await import('@agent-device/daemon/testing/__tests__/legacy-snapshot-capture-fixture');
return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture };
});

import { test, expect, beforeEach } from 'vitest';
import type { FindReadResult } from '@agent-device/contracts/interaction';
import { refFrameScope, refFrameState } from '../../../core/ref-frame.ts';
import {
captureAndStoreSnapshot,
commandSurfaceWithMembers,
} from '../../../__tests__/support/command-surface-override-fake.ts';
import { fakeCreateCommandSurface } from '../../../__tests__/support/command-surface-fake.ts';
import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts';
import {
resetFindTouchRuntimeFixture,
runFindClickScenario,
} from '@agent-device/daemon/testing/interaction/internal/__tests__/find-touch-runtime-fixture';
import { testCreateCommandSurface } from '../command-surface-fixture.ts';
import { refFrameScope, refFrameState } from '@agent-device/daemon/testing/ref-frame';
} from './__tests__/find-touch-runtime-fixture.ts';

installLegacyCaptureInteractorResolution();

/**
* The command family's `find <q> list`: every node the locator matches, read from a capture it
* took through the config's backend and stored as the session snapshot, exactly as the daemon's
* ref issuance expects to find it.
*/
const listingSurface = (label: string) =>
commandSurfaceWithMembers((config) => ({
selectors: {
find: async (options): Promise<FindReadResult> => {
const snapshot = await captureAndStoreSnapshot(config, options);
return {
kind: 'list',
matches: snapshot.nodes
.filter((node) => node.label === label)
.map((node) => ({ ref: `@${node.ref}`, node })),
};
},
},
}));

beforeEach(() => {
resetFindTouchRuntimeFixture();
Expand All @@ -41,7 +55,7 @@ test('handleFindCommands ambiguous match lists snapshot-line candidates capped a
});

const { response } = await runFindClickScenario({
createCommandSurface: testCreateCommandSurface,
createCommandSurface: fakeCreateCommandSurface,
positionals: ['Follow', 'click'],
nodes: [
{ index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } },
Expand Down Expand Up @@ -75,7 +89,7 @@ test('handleFindCommands ambiguous match lists snapshot-line candidates capped a

test('handleFindCommands ambiguous match with few candidates lists them all uncapped', async () => {
const { response } = await runFindClickScenario({
createCommandSurface: testCreateCommandSurface,
createCommandSurface: fakeCreateCommandSurface,
positionals: ['Follow', 'click'],
nodes: [
{ index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } },
Expand Down Expand Up @@ -130,7 +144,7 @@ test('handleFindCommands list returns every match without acting', async () => {
});

const { response, invokeCalls, session } = await runFindClickScenario({
createCommandSurface: testCreateCommandSurface,
createCommandSurface: listingSurface('Follow'),
positionals: ['text', 'Follow', 'list'],
nodes: [
{ index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } },
Expand All @@ -157,7 +171,7 @@ test('handleFindCommands list returns every match without acting', async () => {

test('handleFindCommands list on a unique match still lists instead of tapping', async () => {
const { response, invokeCalls } = await runFindClickScenario({
createCommandSurface: testCreateCommandSurface,
createCommandSurface: listingSurface('Dictionary'),
positionals: ['Dictionary', 'list'],
nodes: [
{ index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } },
Expand Down Expand Up @@ -196,7 +210,7 @@ test('handleFindCommands selector-shaped find rejects multiple matches with cand
});

const { response, invokeCalls } = await runFindClickScenario({
createCommandSurface: testCreateCommandSurface,
createCommandSurface: fakeCreateCommandSurface,
positionals: ['role=button', 'click'],
nodes: [
{ index: 0, ref: 'e1', type: 'Application', rect: { x: 0, y: 0, width: 800, height: 1200 } },
Expand Down Expand Up @@ -226,7 +240,7 @@ test('handleFindCommands selector-shaped find honors the explicit --first opt-ou
});

const { response, invokeCalls } = await runFindClickScenario({
createCommandSurface: testCreateCommandSurface,
createCommandSurface: fakeCreateCommandSurface,
positionals: ['role=button', 'click'],
flags: { findFirst: true },
nodes: [
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
import type { DaemonResponse } from '@agent-device/kernel/contracts';
import type { CommandFlags } from '@agent-device/contracts/command';
import type { FillCommandResult, PressCommandResult } from '@agent-device/contracts/interaction';
import { test, expect, vi, beforeEach } from 'vitest';
import { handleInteractionCommands } from '../index.ts';
import type { DaemonRequest } from '../../../core/daemon-request.ts';
import { makeSessionStore } from '../../../__tests__/support/store-factory.ts';
import { makeIosSession as makeSession } from '../../../__tests__/support/session-factories.ts';
import { commandSurfaceWithMembers } from '../../../__tests__/support/command-surface-override-fake.ts';
import { installLegacyCaptureInteractorResolution } from '../../../__tests__/support/legacy-capture-interactor-resolution.ts';
import {
findTouchRuntimeBindings,
mockDispatch,
resetFindTouchRuntimeFixture,
} from './__tests__/find-touch-runtime-fixture.ts';
import { invokeFindHandler } from './__tests__/find-handler-fixture.ts';

installLegacyCaptureInteractorResolution();

beforeEach(() => {
resetFindTouchRuntimeFixture();
});

// --- #1654: the production route, find handler through the real interaction handler ---
//
// find.test.ts's tests stub `invoke`, so they prove what find SENDS. These drive the real
// `handleInteractionCommands` on the other end, so they prove the resolved node survives the hop:
// the command surface receives the node find selected as its `preresolvedTarget`. They fail if
// either producer (click or fill) stops attaching the channel, or the handler stops forwarding it,
// which a hand-built request cannot catch. That the command family then acts on the carried node
// rather than re-resolving the ref is pinned beside it in
// src/commands/interaction/runtime/interactions-find-carried-target.test.ts.

const findRouteContextFromFlags = (flags: CommandFlags | undefined) => ({
count: flags?.count,
intervalMs: flags?.intervalMs,
delayMs: flags?.delayMs,
holdMs: flags?.holdMs,
jitterPx: flags?.jitterPx,
doubleTap: flags?.doubleTap,
clickButton: flags?.clickButton,
});

/** The tree find's capture returns: "Save" sits at a distinctive point. */
function freshFindTree() {
return [
{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } },
{
index: 1,
parentIndex: 0,
type: 'XCUIElementTypeButton',
label: 'Save',
rect: { x: 300, y: 500, width: 20, height: 20 },
enabled: true,
hittable: true,
},
];
}

type ForwardedTarget = { ref?: string; label?: string };

/**
* A command surface whose `press`/`fill` record the `preresolvedTarget` the handler forwarded and
* answer with the carried node, the way the command family does for a carried target.
*/
function recordingTouchSurface(forwarded: ForwardedTarget[]) {
const answer = (
options: { preresolvedTarget?: { ref: string; node: { label?: string } } } | undefined,
) => {
forwarded.push({
ref: options?.preresolvedTarget?.ref,
label: options?.preresolvedTarget?.node.label,
});
return {
kind: 'ref' as const,
target: { kind: 'ref' as const, ref: options?.preresolvedTarget?.ref ?? '' },
point: { x: 310, y: 510 },
};
};
return commandSurfaceWithMembers(() => ({
interactions: {
click: vi.fn(async (_target, options) => answer(options) as PressCommandResult),
press: vi.fn(async (_target, options) => answer(options) as PressCommandResult),
fill: vi.fn(
async (_target, text, options) => ({ ...answer(options), text }) as FillCommandResult,
),
},
}));
}

async function runFindThroughHandler(positionals: string[], forwarded: ForwardedTarget[]) {
const sessionStore = makeSessionStore();
const sessionName = 'default';
sessionStore.publish(sessionName, makeSession(sessionName));
mockDispatch.mockImplementation(async (_device, command) =>
command === 'snapshot' ? { nodes: freshFindTree(), backend: 'xctest' } : {},
);

const createCommandSurface = recordingTouchSurface(forwarded);
const invokeCalls: DaemonRequest[] = [];
const response = await invokeFindHandler({
createCommandSurface,
sessionName,
sessionStore,
positionals,
invoke: async (req) => {
invokeCalls.push(req);
return (
(await handleInteractionCommands({
createCommandSurface,
req,
sessionName,
sessionStore,
contextFromFlags: findRouteContextFromFlags,
...findTouchRuntimeBindings(),
})) ??
({
ok: false,
error: { code: 'COMMAND_FAILED', message: 'no interaction handler' },
} satisfies DaemonResponse)
);
},
});
return { response, invokeCalls };
}

test('#1654 production route: find click carries its resolved node into the interaction handler', async () => {
const forwarded: ForwardedTarget[] = [];
const { response, invokeCalls } = await runFindThroughHandler(
['text', 'Save', 'click'],
forwarded,
);

expect(response?.ok).toBe(true);
// The producer attaches the channel — fails if handleFindClick stops doing so.
expect(invokeCalls[0]?.internal?.findResolvedTarget?.node?.label).toBe('Save');
expect(invokeCalls[0]?.internal?.findResolvedTarget?.ref).toBe('@e2');
expect(forwarded).toEqual([{ ref: '@e2', label: 'Save' }]);
});

test('#1654 production route: find fill carries its resolved node into the interaction handler', async () => {
const forwarded: ForwardedTarget[] = [];
const { response, invokeCalls } = await runFindThroughHandler(
['text', 'Save', 'fill', 'hello'],
forwarded,
);

expect(response?.ok).toBe(true);
// Fill has its own producer and its own forwarding hop; assert it separately.
expect(invokeCalls[0]?.internal?.findResolvedTarget?.node?.label).toBe('Save');
expect(invokeCalls[0]?.internal?.findResolvedTarget?.ref).toBe('@e2');
expect(forwarded).toEqual([{ ref: '@e2', label: 'Save' }]);
});
Loading
Loading