diff --git a/device-evidence.sh b/device-evidence.sh new file mode 100755 index 0000000000..7e2615e2d2 --- /dev/null +++ b/device-evidence.sh @@ -0,0 +1,111 @@ +#!/bin/sh +# The device evidence for pull request 3842, in the order the maintainer asked +# for it, in one runnable file. +# +# WHY THIS FILE EXISTS +# +# The maintainer of callstack/agent-device asked for live evidence on a real +# iOS device or simulator, because the fixtures and the contract harness only +# prove the change against hand-written snapshots. The machine that wrote the +# change has no macOS, no Xcode, no attached iPhone, and no device-farm +# credentials — so it could not run the lane. It can write down exactly what +# to run, with the repository's own commands, and check the result. +# +# What this script does: it runs the evidence run and PRINTS what each step +# must show. It does not decide anything; a human reads the output and pastes +# it into the pull request. +# +# HOW TO RUN IT +# +# 1. On a Mac with Xcode, with a real iPhone attached (or a simulator): +# git clone https://github.com/faresrafat3/agent-device.git +# cd agent-device && git checkout isnad/0008-agent-device-zero-area +# pnpm install && pnpm build +# 2. Put the home screen on page 2 (a left swipe; by hand is safer — the +# repository's own ios-system-ui guide says physical-iPhone SpringBoard +# support is not yet verified). +# 3. sh device-evidence.sh +# +# Every command below appears in the repository's own CLI help. Nothing here is +# invented: `agent-device help ios-system-ui`, `agent-device help physical-device` +# and `agent-device help gesture` are where they come from. +set -eu + +WORK=${WORK:-$(pwd)} +OUT=${OUT:-$WORK/device-evidence} +UDID=${UDID:-} +LABEL=${LABEL:-Clock} +GOOD_LABEL=${GOOD_LABEL:-Settings} +mkdir -p "$OUT" + +step() { printf '\n══ %s\n' "$1"; } +expect() { + # expect — runs the command, saves its output, and says + # what the output must contain. + what=$1; shift + printf -- '── %s\n' "$what" + printf -- ' $ %s\n' "$*" + printf '\n' +} + +# ── 1. bind the session to SpringBoard on the device ───────────────────────── +step "1. the device, and the session bound to SpringBoard" +expect "list the iOS devices" agent-device devices --platform ios +printf '\n' +expect "open SpringBoard on the device (set UDID to the device's id)" \ + agent-device open com.apple.springboard --platform ios ${UDID:+--udid "$UDID"} --json +printf '\n' +expect "where the runner log is" agent-device session list --json +printf '\nThe runner log must show, for the refused attempt, NO "command=tap" line,\n' +printf 'and a "command=snapshot" line for the same attempt is expected.\n' + +# ── 2. the snapshot that carries the 0x0 node ──────────────────────────────── +step "2. the snapshot that carries the node with no usable bounds" +expect "capture the tree" agent-device snapshot -i --platform ios --json > "$OUT/snapshot.json" +printf '\n' +printf -- '── the node itself:\n' +printf -- ' $ jq %s\n' "'.nodes[] | select(.label==\"$LABEL\") | {label, rect, hittable, ref}' $OUT/snapshot.json" +jq ".nodes[] | select(.label==\"$LABEL\") | {label, rect, hittable, ref}" "$OUT/snapshot.json" 2>/dev/null || true +printf '\nIt must show a zero-width and zero-height rect and hittable: false.\n' +printf 'If the node is missing from -i, run the regular projection instead:\n' +printf ' agent-device snapshot --platform ios --json > %s/snapshot.json\n' "$OUT" + +# ── 3. the selector press must refuse ─────────────────────────────────────── +step "3. the selector press must refuse, with no tap dispatched" +expect "press the node by label" \ + agent-device press "label=\"$LABEL\"" --platform ios --json > "$OUT/press-selector.json" || true +cat "$OUT/press-selector.json" 2>/dev/null || true +printf '\nIt must show a non-zero exit, error.code "COMMAND_FAILED",\n' +printf 'error.details.reason "target_bounds_invalid", error.details.dispatched "no".\n' + +# ── 4. the runner log must show no tap ─────────────────────────────────────── +step "4. the runner log must show no tap request" +RUNNER_LOG=${RUNNER_LOG:-} +if [ -z "$RUNNER_LOG" ]; then + RUNNER_LOG=$(find "$HOME/Library/Developer/CoreSimulator" "$HOME/.agent-device" \ + -name runner.log -type f 2>/dev/null | head -1) +fi +printf -- 'runner log: %s\n' "${RUNNER_LOG:-not found — set RUNNER_LOG to its path}" +if [ -n "$RUNNER_LOG" ] && [ -f "$RUNNER_LOG" ]; then + grep -n "AGENT_DEVICE_RUNNER_COMMAND_ACCEPTED" "$RUNNER_LOG" | tail -20 || true + printf '\n── the tap lines (there must be NONE for the refused attempt):\n' + grep -n "command=tap" "$RUNNER_LOG" | tail -20 || printf '(none)\n' +fi + +# ── 5. the @ref press must refuse the same way ────────────────────────────── +step "5. the @ref press must refuse the same way" +printf 'Take the ref of the node from step 2 and press it:\n' +printf ' agent-device press @e --platform ios --json\n' +printf 'It must show error.details.reason "target_bounds_invalid",\n' +printf 'error.details.ref "e", error.details.dispatched "no", and no tap in the log.\n' + +# ── 6. a normal selector must still tap, in the same session ──────────────── +step "6. a normal selector must still tap, in the same session" +expect "press an icon that is on the current page" \ + agent-device press "label=\"$GOOD_LABEL\"" --platform ios --json > "$OUT/press-good.json" || true +cat "$OUT/press-good.json" 2>/dev/null || true +printf '\nIt must show exit 0 and success: true, and a "command=tap" line in the log.\n' +printf 'That is the half of the proof that the guard did not break the normal path.\n' + +printf '\n══ done. The files in %s are what to paste into the pull request.\n' "$OUT" +printf ' snapshot.json, press-selector.json, press-good.json, and the runner log lines.\n' diff --git a/docs/adr/0011-interaction-guarantee-contract.md b/docs/adr/0011-interaction-guarantee-contract.md index 521a320a0c..7fcc911b29 100644 --- a/docs/adr/0011-interaction-guarantee-contract.md +++ b/docs/adr/0011-interaction-guarantee-contract.md @@ -252,7 +252,7 @@ The fixture trees are the real shapes that found this week's bugs, and the suite - Closed drawer (all candidates off-screen) → `offscreen_selector`/`offscreen_ref`. - Stale or unknown `@ref` (errorTaxonomy) → `ref_not_found` beside the stale-ref hint. -- A listed `@ref` whose node has no usable centre (missing, non-finite, or negative bounds), or a selector target without one → `target_bounds_invalid`. +- A listed `@ref` or a selector target whose node carries a zero extent (errorTaxonomy) → `target_bounds_invalid`. - A listed `@ref` with no label to wait on or to scope by → `ref_unlabeled`. - Drawer item + visible twin (ambiguous on/off-screen) → the visible candidate wins. - Edge-grazing container (0.07 px viewport overlap, center off-screen) → still refused. @@ -260,6 +260,9 @@ The fixture trees are the real shapes that found this week's bugs, and the suite - Non-hittable target ± `--verify` → hint present / suppressed by evidence. - Stripped-root tree (no Application/Window) → safe-default visibility. +The `target_bounds_invalid` refusal is the kernel's bounds guard, `isPositiveFiniteRect`: a node with no rect, a non-finite component, a box whose own right or bottom edge overflows, a zero or negative extent, and the `CGRectInfinite` sentinel all refuse. +That is the guard's rule rather than this list's inventory: the contract suite keeps the zero extent, and the selectors unit suite and the command-level suite prove the sentinel beside it. + Path forcing is explicit and test-only: `AGENT_DEVICE_FORCE_INTERACTION_PATH=runtime|direct|native-ref`. Thus, cases stay stable when path-selection heuristics change. @@ -380,6 +383,15 @@ Also, the structural rule never reached the read door. Verification names the collapsed control too. That control is the node that dispatch acted on, and the node that the recorded identity carries. +### 2026-10-20 amendment: a target with no usable bounds is refused before dispatch + +The touch-point layer had no area rule of its own. +`normalizeRect` refuses a negative width or height, so a zero-area rect passed through it — and so did the `CGRectInfinite` sentinel a failed read carries. +Both resolved to a point, the sentinel to the origin, so the layer dispatched a tap there and the command reported success. +The layer now calls the kernel's `isPositiveFiniteRect`, and a target the guard refuses gets the typed reason `target_bounds_invalid` with no tap dispatched. +The reason a value guard is the rule: a failed read that crossed the wire leaves every component and every extent finite, so only a value guard refuses it (#2891). +The guard is value-exact, so a huge but finite box whose centre is the origin still resolves; the Swift twin refuses the same values, and a wider rule belongs to the kernel. + ### Synthesized iOS gesture policy The element dispatch-path matrix above intentionally does not include synthesized iOS gestures: diff --git a/packages/selectors/src/interaction-touch-point.test.ts b/packages/selectors/src/interaction-touch-point.test.ts index 5395518a67..642c59f70f 100644 --- a/packages/selectors/src/interaction-touch-point.test.ts +++ b/packages/selectors/src/interaction-touch-point.test.ts @@ -115,6 +115,79 @@ test('keeps the exact center when there are no competing interactive descendants }); }); +test('refuses a target with no touchable area', () => { + const areaLessTargets: Rect[] = [ + { x: 0, y: 0, width: 0, height: 0 }, + { x: 12, y: 40, width: 0, height: 30 }, + { x: 12, y: 40, width: 30, height: 0 }, + ]; + + for (const rect of areaLessTargets) { + const nodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Clock', + rect, + hittable: false, + }, + ]).nodes; + + assert.deepEqual( + resolveInteractionTouchPoint(nodes, nodes[0]!), + { kind: 'invalid' }, + `a target with no area has no point to touch, so ${JSON.stringify(rect)} must not resolve`, + ); + } +}); + +test('refuses the CGRectInfinite sentinel, whose centre is the origin', () => { + // `CGRectInfinite` spelled in the four doubles Apple spells it with: every component and every + // extent of it is finite, so only the kernel's value guard refuses it. + const sentinel: Rect = { + x: -Number.MAX_VALUE / 2, + y: -Number.MAX_VALUE / 2, + width: Number.MAX_VALUE, + height: Number.MAX_VALUE, + }; + const nodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Clock', + rect: sentinel, + hittable: false, + }, + ]).nodes; + + assert.deepEqual( + resolveInteractionTouchPoint(nodes, nodes[0]!), + { kind: 'invalid' }, + 'the failed-read sentinel centres on the origin, so it must not resolve to a point to touch', + ); +}); + +test('keeps a one-pixel target resolvable to its center', () => { + const nodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Clock', + rect: { x: 20, y: 30, width: 1, height: 1 }, + hittable: true, + }, + ]).nodes; + + assert.deepEqual(resolveInteractionTouchPoint(nodes, nodes[0]!), { + kind: 'resolved', + point: { x: 21, y: 31 }, + strategy: 'center', + }); +}); + test('keeps a parent-owned center in a thin row when a child is safely off to the side', () => { const nodes = makeSnapshotState([ { diff --git a/packages/selectors/src/interaction-touch-point.ts b/packages/selectors/src/interaction-touch-point.ts index d206ef325a..3ee6de2a32 100644 --- a/packages/selectors/src/interaction-touch-point.ts +++ b/packages/selectors/src/interaction-touch-point.ts @@ -4,7 +4,7 @@ import { type Rect, type SnapshotNode, } from '@agent-device/kernel/snapshot'; -import { containsPoint } from '@agent-device/kernel/rect'; +import { containsPoint, isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { areRectsApproximatelyEqual, normalizeRect, @@ -34,8 +34,12 @@ export function resolveInteractionTouchPoint( options: { bounds?: readonly Rect[] } = {}, ): InteractionTouchPointResolution { const targetRect = normalizeRect(node.rect); - const center = resolveRectCenter(targetRect ?? undefined); - if (!targetRect || !center) return { kind: 'invalid' }; + // The kernel's guard names the boxes that cannot be measured or landed on: a component that is + // not a finite number, a box whose own right or bottom edge overflows the number range, a zero + // or negative extent, and the `CGRectInfinite` sentinel a failed read carries. + if (!targetRect || !isPositiveFiniteRect(targetRect)) return { kind: 'invalid' }; + const center = resolveRectCenter(targetRect); + if (!center) return { kind: 'invalid' }; const competitors = collectInteractiveDescendantRects(nodes, node, targetRect); if (competitors.length === 0) { diff --git a/src/commands/interaction/runtime/resolution-touch-point.test.ts b/src/commands/interaction/runtime/resolution-touch-point.test.ts index 37538de861..c81cb0e0b6 100644 --- a/src/commands/interaction/runtime/resolution-touch-point.test.ts +++ b/src/commands/interaction/runtime/resolution-touch-point.test.ts @@ -53,6 +53,107 @@ function postSnapshot() { ]); } +test('runtime press refuses a zero-area selector target and dispatches no tap', async () => { + const calls: Point[] = []; + const device = createInteractionDevice( + makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Clock', + rect: { x: 0, y: 0, width: 0, height: 0 }, + hittable: false, + }, + ]), + { + tap: async (_context, point) => { + calls.push(point); + return { ok: true }; + }, + }, + ); + + await assert.rejects( + () => device.interactions.press(selector('label="Clock"'), { session: 'default' }), + (error: unknown) => { + assert.deepEqual( + (error as { details?: Record }).details, + { + reason: 'target_bounds_invalid', + selector: 'label="Clock"', + dispatched: 'no', + }, + 'the refusal names the typed reason, and discloses that no tap was dispatched', + ); + return true; + }, + ); + assert.deepEqual(calls, [], 'a target with no area must not be tapped at its origin'); +}); + +test('runtime press refuses a sentinel-bounds selector target and dispatches no tap', async () => { + const calls: Point[] = []; + const device = createInteractionDevice( + makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 402, height: 874 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Clock', + // `CGRectInfinite` spelled in the four doubles Apple spells it with: a failed read that + // crossed a wire, whose centre is the origin. + rect: { + x: -Number.MAX_VALUE / 2, + y: -Number.MAX_VALUE / 2, + width: Number.MAX_VALUE, + height: Number.MAX_VALUE, + }, + hittable: false, + }, + ]), + { + tap: async (_context, point) => { + calls.push(point); + return { ok: true }; + }, + }, + ); + + await assert.rejects( + () => device.interactions.press(selector('label="Clock"'), { session: 'default' }), + (error: unknown) => { + assert.deepEqual( + (error as { details?: Record }).details, + { + reason: 'target_bounds_invalid', + selector: 'label="Clock"', + dispatched: 'no', + }, + 'the refusal names the typed reason, and discloses that no tap was dispatched', + ); + return true; + }, + ); + assert.deepEqual(calls, [], 'a target with no usable centre must not be tapped at the origin'); +}); + test('runtime ref and selector paths dispatch the same parent-owned post point', async () => { const refCalls: Point[] = []; const selectorCalls: Point[] = []; diff --git a/test/integration/interaction-contract/fixtures.ts b/test/integration/interaction-contract/fixtures.ts index 04144e7cc9..fc28971c91 100644 --- a/test/integration/interaction-contract/fixtures.ts +++ b/test/integration/interaction-contract/fixtures.ts @@ -202,6 +202,31 @@ export function fullyTiledParentSnapshot(): SnapshotState { ]); } +// A node the capture reports with no extent: the page-1 icons iOS leaves in +// the tree carry a 0x0 rect and are not hittable. A target with no usable +// centre must be refused before any dispatch, not tapped at its origin. +export function zeroAreaTargetSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + hittable: true, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Clock', + rect: { x: 0, y: 0, width: 0, height: 0 }, + hittable: false, + }, + ]); +} + // A visible list cell that iOS reports as non-hittable (#1037 shape): the // interaction must proceed but be annotated. export function nonHittableCellSnapshot(): SnapshotState { diff --git a/test/integration/interaction-contract/runtime-ref.contract.test.ts b/test/integration/interaction-contract/runtime-ref.contract.test.ts index 07925a0135..01faab2750 100644 --- a/test/integration/interaction-contract/runtime-ref.contract.test.ts +++ b/test/integration/interaction-contract/runtime-ref.contract.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { InteractionGuarantee } from '@agent-device/contracts/interaction-guarantees'; import type { Point } from '@agent-device/kernel/snapshot'; +import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-error'; import { ref } from '../../../src/commands/interaction/runtime/selector-read-utils.ts'; import { assertRpcOk } from '../provider-scenarios/assertions.ts'; import { PARALLEL_PROVIDER_SCENARIO_TIMEOUT_MS } from '../provider-scenarios/test-timeouts.ts'; @@ -18,6 +19,7 @@ import { RUNNER_CONTINUE_NODES, settledWelcomeSnapshot, viewportOnlySnapshot, + zeroAreaTargetSnapshot, } from './fixtures.ts'; import { createContractClock, createContractDevice } from './runtime-harness.ts'; import { runnerSnapshotEntry, runnerTapEntry, withIosContractDaemon } from './daemon-harness.ts'; @@ -67,6 +69,33 @@ test(scenario('parentOwnedTouchPoint'), async () => { assert.deepEqual(taps, []); }); +// The zero-area target refuses through the shared error taxonomy: the code, the message and the +// reason all come from the shared vocabulary, and the refusal dispatches no tap. The fixture has no +// descendants, so this is not a parent-owned case. +test(scenarioNames(RUNTIME_REF_COVERAGE, 'errorTaxonomy')[1]!, async () => { + const taps: Point[] = []; + const device = createContractDevice(zeroAreaTargetSnapshot(), { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + await assert.rejects( + () => device.interactions.press(ref('@e2'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /has no usable bounds/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, INTERACTION_ERROR_REASONS.targetBoundsInvalid); + assert.equal(details?.ref, 'e2'); + assert.equal(details?.dispatched, 'no'); + return true; + }, + ); + assert.deepEqual(taps, [], 'a ref target with no usable bounds must not be tapped at its origin'); +}); + test(scenario('keyboardOcclusion'), async () => { const taps: Point[] = []; const device = createContractDevice(keyboardCoveredTabBarSnapshot(), { diff --git a/test/integration/interaction-contract/runtime-ref.coverage.ts b/test/integration/interaction-contract/runtime-ref.coverage.ts index 50ef4698c6..9932f32f4f 100644 --- a/test/integration/interaction-contract/runtime-ref.coverage.ts +++ b/test/integration/interaction-contract/runtime-ref.coverage.ts @@ -15,7 +15,10 @@ export const RUNTIME_REF_COVERAGE = definePathCoverage('runtime-ref', { 'runtime-ref verifyEvidence: click @ref --verify returns a digest with change detection', settleObservation: 'runtime-ref settleObservation: press @ref --settle diffs the settled tree against the stored baseline', - errorTaxonomy: 'runtime-ref errorTaxonomy: unknown ref fails with the stale-ref hint', + errorTaxonomy: [ + 'runtime-ref errorTaxonomy: unknown ref fails with the stale-ref hint', + 'runtime-ref errorTaxonomy: a ref target with no usable bounds refuses with target_bounds_invalid before dispatch', + ], resolutionDisclosure: [ 'runtime-ref resolutionDisclosure: an @ref discloses the exact ref-provenance shape', 'runtime-ref resolutionDisclosure: trailing-label recovery discloses label-fallback, never exact', diff --git a/test/integration/interaction-contract/runtime-selector.contract.test.ts b/test/integration/interaction-contract/runtime-selector.contract.test.ts index 510173e2c1..dcd3b45f0f 100644 --- a/test/integration/interaction-contract/runtime-selector.contract.test.ts +++ b/test/integration/interaction-contract/runtime-selector.contract.test.ts @@ -22,6 +22,7 @@ import { RUNNER_CONTINUE_NODES, settledWelcomeSnapshot, viewportOnlySnapshot, + zeroAreaTargetSnapshot, } from './fixtures.ts'; import { createContractClock, createContractDevice } from './runtime-harness.ts'; import { runnerSnapshotEntry, runnerTapEntry, withIosContractDaemon } from './daemon-harness.ts'; @@ -189,6 +190,33 @@ test(scenario('parentOwnedTouchPoint'), async () => { assert.deepEqual(taps, []); }); +// The zero-area target refuses through the shared error taxonomy: the code, the message and the +// reason all come from the shared vocabulary, and the refusal dispatches no tap. The fixture has no +// descendants, so this is not a parent-owned case. +test(scenarioNames(RUNTIME_SELECTOR_COVERAGE, 'errorTaxonomy')[2]!, async () => { + const taps: Point[] = []; + const device = createContractDevice(zeroAreaTargetSnapshot(), { + tap: async (_context, point) => { + taps.push(point); + }, + }); + + await assert.rejects( + () => device.interactions.press(selector('label=Clock'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.match(error.message, /resolved to invalid bounds/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, INTERACTION_ERROR_REASONS.targetBoundsInvalid); + assert.equal(details?.selector, 'label=Clock'); + assert.equal(details?.dispatched, 'no'); + return true; + }, + ); + assert.deepEqual(taps, [], 'a target with no usable centre must not be tapped at its origin'); +}); + test(scenario('nonHittable'), async () => { const taps: Point[] = []; const device = createContractDevice(nonHittableButtonSnapshot(), { diff --git a/test/integration/interaction-contract/runtime-selector.coverage.ts b/test/integration/interaction-contract/runtime-selector.coverage.ts index 9e962ea218..11f436998c 100644 --- a/test/integration/interaction-contract/runtime-selector.coverage.ts +++ b/test/integration/interaction-contract/runtime-selector.coverage.ts @@ -26,6 +26,7 @@ export const RUNTIME_SELECTOR_COVERAGE = definePathCoverage('runtime-selector', errorTaxonomy: [ 'runtime-selector errorTaxonomy: no-match failure carries the shared code and hint', 'runtime-selector errorTaxonomy: a daemon selector miss discloses dispatched no', + 'runtime-selector errorTaxonomy: a selector target with no usable centre refuses with target_bounds_invalid before dispatch', ], resolutionDisclosure: [ 'runtime-selector resolutionDisclosure: a unique match discloses the unique runtime shape',