Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 111 additions & 0 deletions device-evidence.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/bin/sh
# The device evidence for pull request 3842, in the order the maintainer asked
# for it, in one runnable file.
#
# WHY THIS FILE EXISTS
#
# The maintainer of callstack/agent-device asked for live evidence on a real
# iOS device or simulator, because the fixtures and the contract harness only
# prove the change against hand-written snapshots. The machine that wrote the
# change has no macOS, no Xcode, no attached iPhone, and no device-farm
# credentials — so it could not run the lane. It can write down exactly what
# to run, with the repository's own commands, and check the result.
#
# What this script does: it runs the evidence run and PRINTS what each step
# must show. It does not decide anything; a human reads the output and pastes
# it into the pull request.
#
# HOW TO RUN IT
#
# 1. On a Mac with Xcode, with a real iPhone attached (or a simulator):
# git clone https://github.com/faresrafat3/agent-device.git
# cd agent-device && git checkout isnad/0008-agent-device-zero-area
# pnpm install && pnpm build
# 2. Put the home screen on page 2 (a left swipe; by hand is safer — the
# repository's own ios-system-ui guide says physical-iPhone SpringBoard
# support is not yet verified).
# 3. sh device-evidence.sh
#
# Every command below appears in the repository's own CLI help. Nothing here is
# invented: `agent-device help ios-system-ui`, `agent-device help physical-device`
# and `agent-device help gesture` are where they come from.
set -eu

WORK=${WORK:-$(pwd)}
OUT=${OUT:-$WORK/device-evidence}
UDID=${UDID:-}
LABEL=${LABEL:-Clock}
GOOD_LABEL=${GOOD_LABEL:-Settings}
mkdir -p "$OUT"

step() { printf '\n══ %s\n' "$1"; }
expect() {
# expect <what> <command...> — runs the command, saves its output, and says
# what the output must contain.
what=$1; shift
printf -- '── %s\n' "$what"
printf -- ' $ %s\n' "$*"
printf '\n'
}

# ── 1. bind the session to SpringBoard on the device ─────────────────────────
step "1. the device, and the session bound to SpringBoard"
expect "list the iOS devices" agent-device devices --platform ios
printf '\n'
expect "open SpringBoard on the device (set UDID to the device's id)" \
agent-device open com.apple.springboard --platform ios ${UDID:+--udid "$UDID"} --json
printf '\n'
expect "where the runner log is" agent-device session list --json
printf '\nThe runner log must show, for the refused attempt, NO "command=tap" line,\n'
printf 'and a "command=snapshot" line for the same attempt is expected.\n'

# ── 2. the snapshot that carries the 0x0 node ────────────────────────────────
step "2. the snapshot that carries the node with no usable bounds"
expect "capture the tree" agent-device snapshot -i --platform ios --json > "$OUT/snapshot.json"
printf '\n'
printf -- '── the node itself:\n'
printf -- ' $ jq %s\n' "'.nodes[] | select(.label==\"$LABEL\") | {label, rect, hittable, ref}' $OUT/snapshot.json"
jq ".nodes[] | select(.label==\"$LABEL\") | {label, rect, hittable, ref}" "$OUT/snapshot.json" 2>/dev/null || true
printf '\nIt must show a zero-width and zero-height rect and hittable: false.\n'
printf 'If the node is missing from -i, run the regular projection instead:\n'
printf ' agent-device snapshot --platform ios --json > %s/snapshot.json\n' "$OUT"

# ── 3. the selector press must refuse ───────────────────────────────────────
step "3. the selector press must refuse, with no tap dispatched"
expect "press the node by label" \
agent-device press "label=\"$LABEL\"" --platform ios --json > "$OUT/press-selector.json" || true
cat "$OUT/press-selector.json" 2>/dev/null || true
printf '\nIt must show a non-zero exit, error.code "COMMAND_FAILED",\n'
printf 'error.details.reason "target_bounds_invalid", error.details.dispatched "no".\n'

# ── 4. the runner log must show no tap ───────────────────────────────────────
step "4. the runner log must show no tap request"
RUNNER_LOG=${RUNNER_LOG:-}
if [ -z "$RUNNER_LOG" ]; then
RUNNER_LOG=$(find "$HOME/Library/Developer/CoreSimulator" "$HOME/.agent-device" \
-name runner.log -type f 2>/dev/null | head -1)
fi
printf -- 'runner log: %s\n' "${RUNNER_LOG:-not found — set RUNNER_LOG to its path}"
if [ -n "$RUNNER_LOG" ] && [ -f "$RUNNER_LOG" ]; then
grep -n "AGENT_DEVICE_RUNNER_COMMAND_ACCEPTED" "$RUNNER_LOG" | tail -20 || true
printf '\n── the tap lines (there must be NONE for the refused attempt):\n'
grep -n "command=tap" "$RUNNER_LOG" | tail -20 || printf '(none)\n'
fi

# ── 5. the @ref press must refuse the same way ──────────────────────────────
step "5. the @ref press must refuse the same way"
printf 'Take the ref of the node from step 2 and press it:\n'
printf ' agent-device press @e<N> --platform ios --json\n'
printf 'It must show error.details.reason "target_bounds_invalid",\n'
printf 'error.details.ref "e<N>", error.details.dispatched "no", and no tap in the log.\n'

# ── 6. a normal selector must still tap, in the same session ────────────────
step "6. a normal selector must still tap, in the same session"
expect "press an icon that is on the current page" \
agent-device press "label=\"$GOOD_LABEL\"" --platform ios --json > "$OUT/press-good.json" || true
cat "$OUT/press-good.json" 2>/dev/null || true
printf '\nIt must show exit 0 and success: true, and a "command=tap" line in the log.\n'
printf 'That is the half of the proof that the guard did not break the normal path.\n'

printf '\n══ done. The files in %s are what to paste into the pull request.\n' "$OUT"
printf ' snapshot.json, press-selector.json, press-good.json, and the runner log lines.\n'
14 changes: 13 additions & 1 deletion docs/adr/0011-interaction-guarantee-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,14 +252,17 @@ The fixture trees are the real shapes that found this week's bugs, and the suite

- Closed drawer (all candidates off-screen) → `offscreen_selector`/`offscreen_ref`.
- Stale or unknown `@ref` (errorTaxonomy) → `ref_not_found` beside the stale-ref hint.
- A listed `@ref` whose node has no usable centre (missing, non-finite, or negative bounds), or a selector target without one → `target_bounds_invalid`.
- A listed `@ref` or a selector target whose node carries a zero extent (errorTaxonomy) → `target_bounds_invalid`.
- A listed `@ref` with no label to wait on or to scope by → `ref_unlabeled`.
- Drawer item + visible twin (ambiguous on/off-screen) → the visible candidate wins.
- Edge-grazing container (0.07 px viewport overlap, center off-screen) → still refused.
- Covered node → occlusion refusal.
- Non-hittable target ± `--verify` → hint present / suppressed by evidence.
- Stripped-root tree (no Application/Window) → safe-default visibility.

The `target_bounds_invalid` refusal is the kernel's bounds guard, `isPositiveFiniteRect`: a node with no rect, a non-finite component, a box whose own right or bottom edge overflows, a zero or negative extent, and the `CGRectInfinite` sentinel all refuse.
That is the guard's rule rather than this list's inventory: the contract suite keeps the zero extent, and the selectors unit suite and the command-level suite prove the sentinel beside it.

Path forcing is explicit and test-only: `AGENT_DEVICE_FORCE_INTERACTION_PATH=runtime|direct|native-ref`.
Thus, cases stay stable when path-selection heuristics change.

Expand Down Expand Up @@ -380,6 +383,15 @@ Also, the structural rule never reached the read door.
Verification names the collapsed control too.
That control is the node that dispatch acted on, and the node that the recorded identity carries.

### 2026-10-20 amendment: a target with no usable bounds is refused before dispatch

The touch-point layer had no area rule of its own.
`normalizeRect` refuses a negative width or height, so a zero-area rect passed through it — and so did the `CGRectInfinite` sentinel a failed read carries.
Both resolved to a point, the sentinel to the origin, so the layer dispatched a tap there and the command reported success.
The layer now calls the kernel's `isPositiveFiniteRect`, and a target the guard refuses gets the typed reason `target_bounds_invalid` with no tap dispatched.
The reason a value guard is the rule: a failed read that crossed the wire leaves every component and every extent finite, so only a value guard refuses it (#2891).
The guard is value-exact, so a huge but finite box whose centre is the origin still resolves; the Swift twin refuses the same values, and a wider rule belongs to the kernel.

### Synthesized iOS gesture policy

The element dispatch-path matrix above intentionally does not include synthesized iOS gestures:
Expand Down
73 changes: 73 additions & 0 deletions packages/selectors/src/interaction-touch-point.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,79 @@ test('keeps the exact center when there are no competing interactive descendants
});
});

test('refuses a target with no touchable area', () => {
const areaLessTargets: Rect[] = [
{ x: 0, y: 0, width: 0, height: 0 },
{ x: 12, y: 40, width: 0, height: 30 },
{ x: 12, y: 40, width: 30, height: 0 },
];

for (const rect of areaLessTargets) {
const nodes = makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Button',
label: 'Clock',
rect,
hittable: false,
},
]).nodes;

assert.deepEqual(
resolveInteractionTouchPoint(nodes, nodes[0]!),
{ kind: 'invalid' },
`a target with no area has no point to touch, so ${JSON.stringify(rect)} must not resolve`,
);
}
});

test('refuses the CGRectInfinite sentinel, whose centre is the origin', () => {
// `CGRectInfinite` spelled in the four doubles Apple spells it with: every component and every
// extent of it is finite, so only the kernel's value guard refuses it.
const sentinel: Rect = {
x: -Number.MAX_VALUE / 2,
y: -Number.MAX_VALUE / 2,
width: Number.MAX_VALUE,
height: Number.MAX_VALUE,
};
const nodes = makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Button',
label: 'Clock',
rect: sentinel,
hittable: false,
},
]).nodes;

assert.deepEqual(
resolveInteractionTouchPoint(nodes, nodes[0]!),
{ kind: 'invalid' },
'the failed-read sentinel centres on the origin, so it must not resolve to a point to touch',
);
});

test('keeps a one-pixel target resolvable to its center', () => {
const nodes = makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Button',
label: 'Clock',
rect: { x: 20, y: 30, width: 1, height: 1 },
hittable: true,
},
]).nodes;

assert.deepEqual(resolveInteractionTouchPoint(nodes, nodes[0]!), {
kind: 'resolved',
point: { x: 21, y: 31 },
strategy: 'center',
});
});

test('keeps a parent-owned center in a thin row when a child is safely off to the side', () => {
const nodes = makeSnapshotState([
{
Expand Down
10 changes: 7 additions & 3 deletions packages/selectors/src/interaction-touch-point.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import {
type Rect,
type SnapshotNode,
} from '@agent-device/kernel/snapshot';
import { containsPoint } from '@agent-device/kernel/rect';
import { containsPoint, isPositiveFiniteRect } from '@agent-device/kernel/rect';
import {
areRectsApproximatelyEqual,
normalizeRect,
Expand Down Expand Up @@ -34,8 +34,12 @@ export function resolveInteractionTouchPoint(
options: { bounds?: readonly Rect[] } = {},
): InteractionTouchPointResolution {
const targetRect = normalizeRect(node.rect);
const center = resolveRectCenter(targetRect ?? undefined);
if (!targetRect || !center) return { kind: 'invalid' };
// The kernel's guard names the boxes that cannot be measured or landed on: a component that is
// not a finite number, a box whose own right or bottom edge overflows the number range, a zero
// or negative extent, and the `CGRectInfinite` sentinel a failed read carries.
if (!targetRect || !isPositiveFiniteRect(targetRect)) return { kind: 'invalid' };
const center = resolveRectCenter(targetRect);
if (!center) return { kind: 'invalid' };

const competitors = collectInteractiveDescendantRects(nodes, node, targetRect);
if (competitors.length === 0) {
Expand Down
101 changes: 101 additions & 0 deletions src/commands/interaction/runtime/resolution-touch-point.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,107 @@ function postSnapshot() {
]);
}

test('runtime press refuses a zero-area selector target and dispatches no tap', async () => {
const calls: Point[] = [];
const device = createInteractionDevice(
makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Application',
rect: { x: 0, y: 0, width: 402, height: 874 },
hittable: true,
},
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'Button',
label: 'Clock',
rect: { x: 0, y: 0, width: 0, height: 0 },
hittable: false,
},
]),
{
tap: async (_context, point) => {
calls.push(point);
return { ok: true };
},
},
);

await assert.rejects(
() => device.interactions.press(selector('label="Clock"'), { session: 'default' }),
(error: unknown) => {
assert.deepEqual(
(error as { details?: Record<string, unknown> }).details,
{
reason: 'target_bounds_invalid',
selector: 'label="Clock"',
dispatched: 'no',
},
'the refusal names the typed reason, and discloses that no tap was dispatched',
);
return true;
},
);
assert.deepEqual(calls, [], 'a target with no area must not be tapped at its origin');
});

test('runtime press refuses a sentinel-bounds selector target and dispatches no tap', async () => {
const calls: Point[] = [];
const device = createInteractionDevice(
makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Application',
rect: { x: 0, y: 0, width: 402, height: 874 },
hittable: true,
},
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'Button',
label: 'Clock',
// `CGRectInfinite` spelled in the four doubles Apple spells it with: a failed read that
// crossed a wire, whose centre is the origin.
rect: {
x: -Number.MAX_VALUE / 2,
y: -Number.MAX_VALUE / 2,
width: Number.MAX_VALUE,
height: Number.MAX_VALUE,
},
hittable: false,
},
]),
{
tap: async (_context, point) => {
calls.push(point);
return { ok: true };
},
},
);

await assert.rejects(
() => device.interactions.press(selector('label="Clock"'), { session: 'default' }),
(error: unknown) => {
assert.deepEqual(
(error as { details?: Record<string, unknown> }).details,
{
reason: 'target_bounds_invalid',
selector: 'label="Clock"',
dispatched: 'no',
},
'the refusal names the typed reason, and discloses that no tap was dispatched',
);
return true;
},
);
assert.deepEqual(calls, [], 'a target with no usable centre must not be tapped at the origin');
});

test('runtime ref and selector paths dispatch the same parent-owned post point', async () => {
const refCalls: Point[] = [];
const selectorCalls: Point[] = [];
Expand Down
25 changes: 25 additions & 0 deletions test/integration/interaction-contract/fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,31 @@ export function fullyTiledParentSnapshot(): SnapshotState {
]);
}

// A node the capture reports with no extent: the page-1 icons iOS leaves in
// the tree carry a 0x0 rect and are not hittable. A target with no usable
// centre must be refused before any dispatch, not tapped at its origin.
export function zeroAreaTargetSnapshot(): SnapshotState {
return makeSnapshotState([
{
index: 0,
depth: 0,
type: 'Application',
label: 'Example',
rect: { x: 0, y: 0, width: 400, height: 800 },
hittable: true,
},
{
index: 1,
depth: 1,
parentIndex: 0,
type: 'Button',
label: 'Clock',
rect: { x: 0, y: 0, width: 0, height: 0 },
hittable: false,
},
]);
}

// A visible list cell that iOS reports as non-hittable (#1037 shape): the
// interaction must proceed but be annotated.
export function nonHittableCellSnapshot(): SnapshotState {
Expand Down
Loading