From ebb60f25b5d1667097f5c7af0469d781135d17f8 Mon Sep 17 00:00:00 2001 From: Hur Ali Date: Wed, 14 Oct 2026 12:37:39 +0500 Subject: [PATCH 01/26] feat: add app-flow --- docs/mcp-tools.md | 15 + docs/telemetry.md | 4 +- package-lock.json | 204 +- package.json | 1 + scripts/build.mjs | 2 +- skills/mobile-dev/SKILL.md | 15 + src/app-flow/ad-script.ts | 241 + src/app-flow/app-flow-normalize.ts | 104 + src/app-flow/armed-recording.ts | 104 + src/app-flow/capture.ts | 3222 ++++++++++++ src/app-flow/cdp/agent-runtime.ts | 1216 +++++ src/app-flow/cdp/connection.ts | 814 +++ src/app-flow/cdp/discovery.ts | 221 + src/app-flow/cdp/runtime-source.ts | 132 + src/app-flow/cli-args.ts | 12 + src/app-flow/decision-provider.ts | 85 + src/app-flow/exploration.ts | 1178 +++++ src/app-flow/hedge.ts | 98 + src/app-flow/host.ts | 44 + src/app-flow/index.ts | 203 + src/app-flow/launcher-deferral.ts | 35 + src/app-flow/manual/commands.ts | 29 + src/app-flow/manual/context.ts | 27 + src/app-flow/manual/index.ts | 181 + src/app-flow/manual/locators.ts | 56 + src/app-flow/manual/path-commands.ts | 88 + src/app-flow/manual/session.ts | 491 ++ src/app-flow/manual/validate.ts | 65 + src/app-flow/nav-route-target.ts | 39 + src/app-flow/nav-state-gate.ts | 37 + src/app-flow/nav-state.ts | 375 ++ src/app-flow/orchestrator.ts | 4413 +++++++++++++++++ src/app-flow/param-store.ts | 964 ++++ src/app-flow/parser.ts | 1667 +++++++ src/app-flow/path-minimize.ts | 200 + src/app-flow/path-provenance.ts | 86 + src/app-flow/path-run.ts | 199 + src/app-flow/reach/adapters.ts | 413 ++ src/app-flow/reach/arrival.ts | 33 + src/app-flow/reach/auth-gate-policy.ts | 138 + src/app-flow/reach/chrome-inventory.ts | 753 +++ src/app-flow/reach/chrome-path.ts | 136 + src/app-flow/reach/deep-link-policy.ts | 227 + src/app-flow/reach/delivery.ts | 272 + src/app-flow/reach/exploration-control.ts | 119 + src/app-flow/reach/frame-stability.ts | 198 + src/app-flow/reach/ladder.ts | 106 + src/app-flow/reach/not-found-signal.ts | 218 + src/app-flow/reach/ports.ts | 259 + src/app-flow/reach/probe-route.ts | 184 + src/app-flow/reach/prompts.ts | 62 + src/app-flow/reach/queue.ts | 80 + src/app-flow/reach/runtime-nodes.ts | 197 + src/app-flow/reach/seed.ts | 133 + src/app-flow/reach/strategies/deep-link.ts | 563 +++ src/app-flow/reach/strategies/render.ts | 153 + src/app-flow/reach/strategies/saved-replay.ts | 153 + .../strategies/verify-with-diagnostic.ts | 267 + src/app-flow/reach/structure-key.ts | 54 + src/app-flow/reach/sweep-order.ts | 349 ++ src/app-flow/reach/types.ts | 217 + src/app-flow/reach/walk-edges.ts | 143 + src/app-flow/replay-divergence.ts | 153 + src/app-flow/runtime-navigator.ts | 144 + src/app-flow/runtime-render.ts | 386 ++ src/app-flow/screen-keys.ts | 34 + src/app-flow/screen-signals.ts | 195 + src/app-flow/shell.ts | 62 + src/app-flow/start-request.ts | 103 + src/app-flow/storage.ts | 337 ++ src/app-flow/timing.ts | 252 + src/app-flow/walk-commands.ts | 167 + src/server/app-flow-deps/ios/runner-busy.ts | 65 + .../app-flow-deps/ios/scheme-approval.ts | 167 + src/server/app-flow-deps/metro/logs.ts | 124 + src/server/app-flow-deps/metro/port.ts | 119 + .../app-flow-deps/native-build-store.ts | 4 + src/server/app-flow-deps/native-build.ts | 284 ++ src/server/app-flow-deps/pi/usage.ts | 31 + .../profiling/agent-device-command.ts | 35 + src/server/app-flow-deps/profiling/cdp.ts | 37 + .../app-flow-deps/profiling/snapshot.ts | 35 + .../profiling/transition-target.ts | 37 + src/server/app-flow-host.ts | 684 +++ src/server/app-flow-tools.ts | 261 + src/server/plugin.ts | 6 +- src/server/telemetry.ts | 2 +- src/shared/app-flow.ts | 733 +++ src/shared/telemetry.ts | 3 +- src/ui/app-flow-panel.ts | 216 + src/ui/app-flow-recording.ts | 71 + src/ui/app.tsx | 1 + src/ui/components/app-flow-graph.tsx | 129 + src/ui/components/app-flow-layout.ts | 164 + src/ui/components/app-flow-view.tsx | 108 + src/ui/components/manual-recorder-panel.tsx | 77 + src/ui/components/manual-wireframe.ts | 40 + src/ui/components/workspace.tsx | 31 +- src/ui/workspace-app.tsx | 7 +- tests/app-flow-host.test.ts | 138 + tests/app-flow-layout.test.ts | 59 + tests/app-flow-normalize.test.ts | 100 + tests/app-flow-panel.test.ts | 64 + tests/app-flow-parser.test.ts | 95 + tests/app-flow-screen-keys.test.ts | 29 + tests/app-flow-walk-commands.test.ts | 102 + tests/react-ui.test.ts | 11 +- tests/telemetry.test.ts | 12 +- tests/ui-telemetry.test.ts | 5 + 109 files changed, 28162 insertions(+), 51 deletions(-) create mode 100644 src/app-flow/ad-script.ts create mode 100644 src/app-flow/app-flow-normalize.ts create mode 100644 src/app-flow/armed-recording.ts create mode 100644 src/app-flow/capture.ts create mode 100644 src/app-flow/cdp/agent-runtime.ts create mode 100644 src/app-flow/cdp/connection.ts create mode 100644 src/app-flow/cdp/discovery.ts create mode 100644 src/app-flow/cdp/runtime-source.ts create mode 100644 src/app-flow/cli-args.ts create mode 100644 src/app-flow/decision-provider.ts create mode 100644 src/app-flow/exploration.ts create mode 100644 src/app-flow/hedge.ts create mode 100644 src/app-flow/host.ts create mode 100644 src/app-flow/index.ts create mode 100644 src/app-flow/launcher-deferral.ts create mode 100644 src/app-flow/manual/commands.ts create mode 100644 src/app-flow/manual/context.ts create mode 100644 src/app-flow/manual/index.ts create mode 100644 src/app-flow/manual/locators.ts create mode 100644 src/app-flow/manual/path-commands.ts create mode 100644 src/app-flow/manual/session.ts create mode 100644 src/app-flow/manual/validate.ts create mode 100644 src/app-flow/nav-route-target.ts create mode 100644 src/app-flow/nav-state-gate.ts create mode 100644 src/app-flow/nav-state.ts create mode 100644 src/app-flow/orchestrator.ts create mode 100644 src/app-flow/param-store.ts create mode 100644 src/app-flow/parser.ts create mode 100644 src/app-flow/path-minimize.ts create mode 100644 src/app-flow/path-provenance.ts create mode 100644 src/app-flow/path-run.ts create mode 100644 src/app-flow/reach/adapters.ts create mode 100644 src/app-flow/reach/arrival.ts create mode 100644 src/app-flow/reach/auth-gate-policy.ts create mode 100644 src/app-flow/reach/chrome-inventory.ts create mode 100644 src/app-flow/reach/chrome-path.ts create mode 100644 src/app-flow/reach/deep-link-policy.ts create mode 100644 src/app-flow/reach/delivery.ts create mode 100644 src/app-flow/reach/exploration-control.ts create mode 100644 src/app-flow/reach/frame-stability.ts create mode 100644 src/app-flow/reach/ladder.ts create mode 100644 src/app-flow/reach/not-found-signal.ts create mode 100644 src/app-flow/reach/ports.ts create mode 100644 src/app-flow/reach/probe-route.ts create mode 100644 src/app-flow/reach/prompts.ts create mode 100644 src/app-flow/reach/queue.ts create mode 100644 src/app-flow/reach/runtime-nodes.ts create mode 100644 src/app-flow/reach/seed.ts create mode 100644 src/app-flow/reach/strategies/deep-link.ts create mode 100644 src/app-flow/reach/strategies/render.ts create mode 100644 src/app-flow/reach/strategies/saved-replay.ts create mode 100644 src/app-flow/reach/strategies/verify-with-diagnostic.ts create mode 100644 src/app-flow/reach/structure-key.ts create mode 100644 src/app-flow/reach/sweep-order.ts create mode 100644 src/app-flow/reach/types.ts create mode 100644 src/app-flow/reach/walk-edges.ts create mode 100644 src/app-flow/replay-divergence.ts create mode 100644 src/app-flow/runtime-navigator.ts create mode 100644 src/app-flow/runtime-render.ts create mode 100644 src/app-flow/screen-keys.ts create mode 100644 src/app-flow/screen-signals.ts create mode 100644 src/app-flow/shell.ts create mode 100644 src/app-flow/start-request.ts create mode 100644 src/app-flow/storage.ts create mode 100644 src/app-flow/timing.ts create mode 100644 src/app-flow/walk-commands.ts create mode 100644 src/server/app-flow-deps/ios/runner-busy.ts create mode 100644 src/server/app-flow-deps/ios/scheme-approval.ts create mode 100644 src/server/app-flow-deps/metro/logs.ts create mode 100644 src/server/app-flow-deps/metro/port.ts create mode 100644 src/server/app-flow-deps/native-build-store.ts create mode 100644 src/server/app-flow-deps/native-build.ts create mode 100644 src/server/app-flow-deps/pi/usage.ts create mode 100644 src/server/app-flow-deps/profiling/agent-device-command.ts create mode 100644 src/server/app-flow-deps/profiling/cdp.ts create mode 100644 src/server/app-flow-deps/profiling/snapshot.ts create mode 100644 src/server/app-flow-deps/profiling/transition-target.ts create mode 100644 src/server/app-flow-host.ts create mode 100644 src/server/app-flow-tools.ts create mode 100644 src/shared/app-flow.ts create mode 100644 src/ui/app-flow-panel.ts create mode 100644 src/ui/app-flow-recording.ts create mode 100644 src/ui/components/app-flow-graph.tsx create mode 100644 src/ui/components/app-flow-layout.ts create mode 100644 src/ui/components/app-flow-view.tsx create mode 100644 src/ui/components/manual-recorder-panel.tsx create mode 100644 src/ui/components/manual-wireframe.ts create mode 100644 tests/app-flow-host.test.ts create mode 100644 tests/app-flow-layout.test.ts create mode 100644 tests/app-flow-normalize.test.ts create mode 100644 tests/app-flow-panel.test.ts create mode 100644 tests/app-flow-parser.test.ts create mode 100644 tests/app-flow-screen-keys.test.ts create mode 100644 tests/app-flow-walk-commands.test.ts diff --git a/docs/mcp-tools.md b/docs/mcp-tools.md index 63c185c5..4b22c4ad 100644 --- a/docs/mcp-tools.md +++ b/docs/mcp-tools.md @@ -41,5 +41,20 @@ | `mobile_read_display_fps` | Read FPS intervals and connection status | | `mobile_read_performance_frames` | Read Android jank/pacing statistics and page through exact display frames | | `mobile_display_fps_close` | Stop FPS collection and release the tracing connection | +| `mobile_appflow_start` | Start a non-blocking structure build or agent-assisted screen sweep | +| `mobile_appflow_status` | Read current or persisted App Flow status | +| `mobile_appflow_result` | Read the current or persisted screen graph | +| `mobile_appflow_pause` | Pause a running sweep at a safe boundary (app only) | +| `mobile_appflow_cancel` | Cancel a run and any parked agent decision | +| `mobile_appflow_launch_app` | Launch the selected project without starting a sweep (app only) | +| `mobile_appflow_resolve_prompt` | Resolve a human-action gate after the user acts (app only) | +| `mobile_appflow_manual_start` | Start recording a path to one screen (app only) | +| `mobile_appflow_manual_act` | Perform and record one manual path action (app only) | +| `mobile_appflow_manual_stop` | Save the active manual path recording (app only) | +| `mobile_appflow_manual_cancel` | Discard the active manual path recording (app only) | +| `mobile_appflow_manual_replay` | Replay a saved path to one screen (app only) | +| `mobile_appflow_next_decision` | Long-poll for the next active-agent sweep decision (model only) | +| `mobile_appflow_submit_decision` | Submit one role-valid JSON decision and wait briefly for the next (model only) | +| `mobile_appflow_screenshot` | Read an authorized App Flow PNG (app only) | When reactivated, the `agent-device` MCP server exposes the pinned runtime's official operations through compact, validated schemas, including `open`, `snapshot`, `press`, `fill`, `type`, `scroll`, `wait`, `find`, `get`, `is`, `close`, and debugging tools. Their input schemas describe each command. The source [control skill](../skills/agent-device/SKILL.md), currently excluded from the package, explains session ordering and links to the version-matched guide. diff --git a/docs/telemetry.md b/docs/telemetry.md index f2e8e6e8..47e8251d 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -2,6 +2,8 @@ [Back to README](../README.md) · [Contributing](../CONTRIBUTING.md) +App Flow uses the `app-flow` UI surface. `ui.app_flow.start` measures the start request, `ui.app_flow.start_to_progress` ends on the first progress event, and `ui.app_flow.graph_process` measures local graph layout/derivation. Fixed complete/error counters record terminal outcomes. `app_flow.decision_wait.duration` measures time parked for the active Codex agent, with a fixed outcome; this time is excluded from the engine's exploration budget. Status and decision polling are excluded from MCP trace sampling. Prompts, responses, screenshots, project paths, run/request IDs, graph content, device identifiers, and token data are never attached to these measurements. UI timing windows flush on surface changes and disposal; provider timers are cleared on submit, cancellation, suspension, and server teardown. + Since 0.1.140, `android.emulator.adb_reconnects` counts adb transport reconnects for emulators that stay unauthorized, with a fixed `trigger` (`boot` while waiting for an AVD, `selection` for an already running emulator). It uses the simulator surface and Android emulator context and sends no serials, AVD names or adb output. Existing Android startup diagnostics and MCP tool errors cover the boot outcome. Since 0.1.139, `ios.accessibility_bridge.restarts` counts restarts of the simulator's CoreSimulatorBridge after a describe-UI read reported no accessibility data, with a fixed `outcome` (`recovered`, `unavailable` or `failed`). Failed restarts also report a handled `simulator.accessibility_bridge` error. The existing `inspection.tool` errors, sampled MCP traces and `ui.annotations.inspection` timings cover the retried read. No device identifiers, accessibility content or command output are sent. @@ -272,7 +274,7 @@ Agent Device telemetry is inactive while its MCP entry is disabled; the active M | Native operations | Bounded timing windows for connection, physical iOS input acknowledgement and video packet processing, iOS log processing, Android CPU sampling, and FPS read/processing. Filter by `component`, `runtime_platform` and `surface` to identify the responsible helper. Baguette currently records resources and crashes. | | Node responsiveness | Automatic event-loop delay, utilization and process uptime. | | UI responsiveness | Browser tracing captures available web vitals. Custom metrics record visible animation-frame intervals, intervals over 50 ms, Event Timing interaction durations, long tasks and long animation frames where supported. | -| Product surfaces | Metrics carry `surface=simulator`, `logs`, `performance`, `recording` or `comparison`, plus view, visible device layout and monitoring state. Log filter time, buffered/filtered rows, performance batch processing, canvas draw time and time to first video frame help explain slow surfaces. | +| Product surfaces | Metrics carry `surface=app-flow`, `simulator`, `logs`, `performance`, `recording` or `comparison`, plus view, visible device layout and monitoring state. App Flow readiness/graph work, log filter time, buffered/filtered rows, performance batch processing, canvas draw time and time to first video frame help explain slow surfaces. | | Frame capture | `ui.screenshot.capture` measures synchronous canvas PNG encoding and base64 extraction in milliseconds for Select captures and physical iOS screenshots. Screenshot tools retain sampled MCP traces and report handled capture, attachment, and clipboard failures without image content. | | Storage | Every five minutes, the agent-device launcher measures its own session state directory and the shared Apple runner cache in bytes. It skips symlinks and sends only the storage kind and size. | | Usage | Surface views and visible time, tool action outcomes, log searches, attachments and send-to-chat actions are counted without their content. | diff --git a/package-lock.json b/package-lock.json index 6edfeeeb..12cc439d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@openai/mcp-extensions": "0.1.0", "@sentry/node": "11.2.0", "@sentry/react": "11.2.0", + "@xyflow/react": "^12.12.0", "ajv": "8.20.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -2007,7 +2008,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3944,7 +3944,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3958,7 +3957,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3972,7 +3970,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3986,7 +3983,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4000,7 +3996,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4014,7 +4009,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4028,7 +4022,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4042,7 +4035,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4056,7 +4048,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4070,7 +4061,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4084,7 +4074,6 @@ "cpu": [ "loong64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4098,7 +4087,6 @@ "cpu": [ "loong64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4112,7 +4100,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4126,7 +4113,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4140,7 +4126,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4154,7 +4139,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4168,7 +4152,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4182,7 +4165,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4196,7 +4178,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4210,7 +4191,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4224,7 +4204,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4238,7 +4217,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4252,7 +4230,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4266,7 +4243,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4280,7 +4256,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -5137,6 +5112,15 @@ "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", "license": "MIT" }, + "node_modules/@types/d3-drag": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", + "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, "node_modules/@types/d3-ease": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", @@ -5167,6 +5151,12 @@ "@types/d3-time": "*" } }, + "node_modules/@types/d3-selection": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.12.tgz", + "integrity": "sha512-Qe/KWYhEiIIxGs7HrAAjMfShxKldx19SJtr5zu53f3afPsdZNz7HHtdTLXo/kqeiWNXVycI24kSnfzBYkTzpgw==", + "license": "MIT" + }, "node_modules/@types/d3-shape": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.2.0.tgz", @@ -5188,6 +5178,25 @@ "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", "license": "MIT" }, + "node_modules/@types/d3-transition": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.9.tgz", + "integrity": "sha512-0sE1406XBYJGiqD3AusTl9ZqC//2mIXix51tbom25gDCA8ri4xnSZg28CaSE8Srl6FClqABUKfsn0qghgdepMA==", + "license": "MIT", + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -5269,6 +5278,48 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/@xyflow/react": { + "version": "12.12.0", + "resolved": "https://registry.npmjs.org/@xyflow/react/-/react-12.12.0.tgz", + "integrity": "sha512-74oXI0Rgm1Eu33cHQAd74J6oVgCH9mbFyTVQbsGgp2kmWwh0HdtBr2VP04z8mbPD+M/xO69nqG5dDEt7/EN2rQ==", + "license": "MIT", + "dependencies": { + "@xyflow/system": "0.0.83", + "classcat": "^5.0.3", + "zustand": "^4.4.0" + }, + "peerDependencies": { + "@types/react": ">=17", + "@types/react-dom": ">=17", + "react": ">=17", + "react-dom": ">=17" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@xyflow/system": { + "version": "0.0.83", + "resolved": "https://registry.npmjs.org/@xyflow/system/-/system-0.0.83.tgz", + "integrity": "sha512-a//eDjez3WZF47aaPTfVUDD0OINOq+WVwBZCRTqG+m0a5euX8EwlJ4jMY1Wxz0Ocj4xOeekJ70V22Pq0ll8RJw==", + "license": "MIT", + "dependencies": { + "@types/d3-drag": "^3.0.7", + "@types/d3-interpolate": "^3.0.4", + "@types/d3-selection": "^3.0.10", + "@types/d3-transition": "^3.0.8", + "@types/d3-zoom": "^3.0.8", + "d3-drag": "^3.0.0", + "d3-interpolate": "^3.0.1", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0" + } + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -5654,6 +5705,12 @@ "url": "https://polar.sh/cva" } }, + "node_modules/classcat": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.5.tgz", + "integrity": "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==", + "license": "MIT" + }, "node_modules/cli-cursor": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", @@ -6060,6 +6117,28 @@ "node": ">=12" } }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/d3-ease": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", @@ -6115,6 +6194,15 @@ "node": ">=12" } }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/d3-shape": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", @@ -6160,6 +6248,41 @@ "node": ">=12" } }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/data-urls": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", @@ -6861,7 +6984,6 @@ "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -10635,6 +10757,34 @@ "peerDependencies": { "zod": "^3.25.28 || ^4" } + }, + "node_modules/zustand": { + "version": "4.5.7", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.7.tgz", + "integrity": "sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw==", + "license": "MIT", + "dependencies": { + "use-sync-external-store": "^1.2.2" + }, + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "@types/react": ">=16.8", + "immer": ">=9.0.6", + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + } + } } } } diff --git a/package.json b/package.json index 75b131d8..cf86b0c1 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@openai/mcp-extensions": "0.1.0", "@sentry/node": "11.2.0", "@sentry/react": "11.2.0", + "@xyflow/react": "^12.12.0", "ajv": "8.20.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", diff --git a/scripts/build.mjs b/scripts/build.mjs index 7270d743..e205da2a 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -138,7 +138,7 @@ const server = await build({ format: "esm", platform: "node", target: "node22", minify: false, legalComments: "eof", metafile: true, sourcemap: "external", plugins: [sentryEsbuildPlugin({ project: "codex-mobile-dev-server", telemetry: false, sourcemaps: { disable: true }, release: { inject: false, create: false, finalize: false } })], - banner: { js: "import { createRequire as mobileDevBundleRequire } from 'node:module'; const require = mobileDevBundleRequire(import.meta.url);" }, + banner: { js: "import { createRequire as mobileDevBundleRequire } from 'node:module'; import { fileURLToPath as mobileDevFileURLToPath } from 'node:url'; import { dirname as mobileDevDirname } from 'node:path'; const require = mobileDevBundleRequire(import.meta.url); const __filename = mobileDevFileURLToPath(import.meta.url); const __dirname = mobileDevDirname(__filename);" }, }); await mkdir(".sentry/server", { recursive: true }); for (const name of ["server.mjs", "agent-device-server.mjs", "ios-mirror-service.mjs"]) { diff --git a/skills/mobile-dev/SKILL.md b/skills/mobile-dev/SKILL.md index f641f231..43b91152 100644 --- a/skills/mobile-dev/SKILL.md +++ b/skills/mobile-dev/SKILL.md @@ -71,6 +71,21 @@ Selecting a log and clicking Attach to chat puts its message, stack, source, lev The host needs an Apple Silicon Mac and Xcode 26 or later with an iOS simulator runtime. The plugin uses Codex's bundled Node runtime. If the backend fails, read its error before retrying `mobile_start_baguette`. Keep logs on stderr because stdout carries MCP messages. +### App Flow + +Use App Flow to map an Expo Router or React Navigation project. `mobile_appflow_start` is non-blocking. Pass the project folder and either `structure-only` for an unattended route graph or `screen-ui-and-structure` for a simulator sweep with captures and runtime navigation evidence. + +For `screen-ui-and-structure`, keep servicing the same run until it reaches a terminal or human-action state: + +1. Call `mobile_appflow_next_decision` with the returned `runId`. A `waiting` result means local build or device work is still progressing; poll again. +2. For a `pending` request, follow its role instructions and history, inspect its inline screenshot when present, and return exactly one strict JSON object through `mobile_appflow_submit_decision` with the same `runId` and `requestId`. +3. A successful submit may contain the next request. Continue directly from it; call `next_decision` again only when the submit reports `waiting`. +4. Stop when status is completed, failed, cancelled, or suspended, or when the run reports `needs-user-action`. + +Decision JSON is role-specific. Auth classification uses `{ "verdict": "auth" | "clear" | "uncertain", "reason": string }`; arrival and navigation requests must follow their supplied schema; runtime-screen classification uses `{ "distinct": boolean, "key"?: string, "info"?: string }`; state-trigger planning returns actions, a state name, destination locator, and nullable note. Do not wrap the object in Markdown. + +An auth/review prompt is deliberately separate from the agent's screenshot classification. When the run needs user action, tell the user to log in or review the simulator and stop the decision loop. Never claim that the human action happened and never call `mobile_appflow_resolve_prompt` on the user's behalf. The App Flow panel owns Stop, Continue anyway, and Continue controls. A paused or inactivity-suspended sweep can be resumed by starting the same project again; its checkpoint preserves completed screens. + Build and launch the user's app with that app project's own tools. These tools do not build apps or install Xcode runtimes. Do not run type checks, lint, visual checks, or React Doctor unless the user asks. Before starting an app dev server, check for an existing server from that project. ## Android diff --git a/src/app-flow/ad-script.ts b/src/app-flow/ad-script.ts new file mode 100644 index 00000000..cfe5ee71 --- /dev/null +++ b/src/app-flow/ad-script.ts @@ -0,0 +1,241 @@ +/** + * Locating and editing a single step inside a recorded `.ad` script. + * + * Pure and separate from `capture.ts` for the same reason `replay-divergence.ts` is: `capture.ts` + * reaches `electron` through `./storage` and cannot be imported under the test runner. What lives + * here decides which line of a verified recording gets rewritten, so it is worth testing. + * + * Step numbers in agent-device's divergence reports are 1-based over the lines it *executes*, which + * is every line except `context`, blanks and comments — the `# agent-device:target-v1` identity + * records among them. Step 5 of the measured social-app seeds is therefore script line 8, and + * nothing may assume the two numberings agree. + */ + +/** The `# agent-device:target-v1 {...}` comment that records the identity of the step below it. */ +const TARGET_COMMENT = '# agent-device:target-v1' + +function isExecutable(line: string): boolean { + const trimmed = line.trim() + return trimmed.length > 0 && !trimmed.startsWith('#') && !trimmed.startsWith('context ') +} + +/** The 0-based file line index of a 1-based execution step, or `undefined` if the script is shorter. */ +export function lineIndexOfStep(lines: readonly string[], step: number): number | undefined { + let seen = 0 + for (const [index, line] of lines.entries()) { + if (!isExecutable(line)) continue + seen += 1 + if (seen === step) return index + } + return undefined +} + +/** + * The selector of a `press` step, unescaped, or `undefined` for any other kind of step. + * + * Deliberately narrow: a press is the only step the recovery paths know how to re-perform or + * re-bind, and every divergence measured on social-app is one. + */ +export function pressSelectorAtStep(lines: readonly string[], step: number): string | undefined { + const index = lineIndexOfStep(lines, step) + if (index === undefined) return undefined + const match = /^press\s+"(.*)"$/.exec(lines[index].trim()) + return match ? match[1].replace(/\\(["\\])/g, '$1') : undefined +} + +/** + * Drop the recorded identity comment above a press step, leaving the press itself untouched. + * + * This is the last lever available for the failure that costs this sweep every seed but Home. The + * recorded identity is *correct about the button* — `matchCount 1`, no competing candidate, every + * field matching except where the button sits — and wrong only about the tree around it, which + * reads `application/"Blue Sky"` on one snapshot and `other/"Home"` on the next of the same screen. + * Record-and-heal cannot win that: it re-recorded the identical ancestry and the healed script + * failed at the same step. Removing the annotation removes the guard built from it, so the step + * resolves on its selector alone — which already matched one element and the right one. + * + * What is given up is narrow and worth naming: that press no longer re-verifies *which* element it + * found. The destination `wait` after it keeps its own identity record, so arrival is still proved; + * only the press's own structural assertion goes. Callers must therefore strip on the evidence of + * an ancestry-only mismatch and never as a way to quiet a `SELECTOR_MISS` or an ambiguous match, + * where it would turn a caught mistake into a silent wrong tap. + * + * Returns `undefined` when there is nothing to strip — the step is not a press, or it carries no + * identity comment — so a caller can tell "already stripped" from "stripped now" and stop looping. + */ +export function stripRecordedIdentity( + lines: readonly string[], + step: number +): string[] | undefined { + const index = lineIndexOfStep(lines, step) + if (index === undefined) return undefined + if (!/^press\s+"/.test(lines[index].trim())) return undefined + if (index === 0 || !lines[index - 1].trim().startsWith(TARGET_COMMENT)) return undefined + return [...lines.slice(0, index - 1), ...lines.slice(index)] +} + +/** The timeout recorded on agent-device's own selector waits, reused for the ones we insert. */ +const INSERTED_WAIT_TIMEOUT_MS = 15_000 + +/** The recorded `target-v1` payload above a step, or `undefined` when there is none to read. */ +export function identityAtStep( + lines: readonly string[], + step: number +): Record | undefined { + const index = lineIndexOfStep(lines, step) + if (index === undefined || index === 0) return undefined + const previous = lines[index - 1].trim() + if (!previous.startsWith(TARGET_COMMENT)) return undefined + try { + const parsed: unknown = JSON.parse(previous.slice(TARGET_COMMENT.length).trim()) + return typeof parsed === 'object' && parsed !== null + ? (parsed as Record) + : undefined + } catch { + return undefined + } +} + +/** + * The recorded ancestry of an identity, as comparable `role/label` keys. + * + * Keys rather than objects because this exists to answer one question — do two recorded identities + * hang off the same node — and the divergence that motivates it names that node by exactly these + * two fields (`recorded=application/"Blue Sky"`). + */ +function ancestryKeys(identity: Record | undefined): string[] { + const ancestry = identity?.ancestry + if (!Array.isArray(ancestry)) return [] + return ancestry.flatMap((raw) => { + if (typeof raw !== 'object' || raw === null) return [] + const entry = raw as Record + const role = typeof entry.role === 'string' ? entry.role : '' + const label = typeof entry.label === 'string' ? entry.label : '' + return [`${role}/${label}`] + }) +} + +/** The leading `id="…"` alternative of a press selector, or `undefined` when it does not lead with one. */ +function idClause(selector: string): string | undefined { + const first = selector.split('||')[0].trim() + return /^id="[^"]+"$/.test(first) ? first : undefined +} + +/** The `wait "" ` line agent-device would have recorded for this id clause. */ +function waitLineFor(clause: string): string { + return `wait "${clause.replace(/"/g, '\\"')}" ${INSERTED_WAIT_TIMEOUT_MS}` +} + +/** The previous executable line before a file index, or `undefined` at the start of the script. */ +function previousExecutable(lines: readonly string[], index: number): string | undefined { + for (let i = index - 1; i >= 0; i--) if (isExecutable(lines[i])) return lines[i].trim() + return undefined +} + +export interface UnguardedPresses { + readonly lines: string[] + /** 1-based steps, in the *input* numbering, whose recorded identity was dropped. */ + readonly stripped: number[] + /** 1-based steps, in the *input* numbering, that gained a selector wait in front of them. */ + readonly guarded: number[] +} + +/** + * Drop every recorded identity that hangs off the node a divergence just proved unstable, and give + * each unguarded press a selector wait in its place. + * + * Two changes to `stripRecordedIdentity`'s one-step-at-a-time behaviour, both measured on the + * 15-53-19 sweep, where this path cost 17s across two replays and still published nothing. + * + * **All at once, not one per replay.** The caller used to strip the reported step, replay (~8s), + * read the next report, strip again. But the instability is a property of a *node*, not of a step: + * `ProfileFollowers` recorded `application/"Blue Sky"` in the ancestry of both its presses and the + * replay rejected both, the second only after a round trip that could have been predicted from the + * first. So every press sharing an ancestry entry with the proven-unstable one is stripped in the + * same pass. Steps whose ancestry is disjoint keep their guard: the evidence is about one node, and + * widening past it would start swallowing the moved-or-relabelled control that record-and-heal is + * the right answer for. An identity with no recorded ancestry at all proves nothing shareable, so + * only the reported step is touched. + * + * **A wait where the guard was.** `stripRecordedIdentity`'s doc names what a strip gives up — the + * press stops re-verifying *which* element it found — and treats the destination wait as enough to + * cover it. For `ProfileFollowers` step 7 it was not: that press had no wait in front of it at all, + * so dropping its identity turned `IDENTITY_MISMATCH` into `COMMAND_FAILED`, the bare selector + * evaluated against a tree whose header was still `other/"Profile banner placeholder"` — the same + * placeholder the earlier mismatch had named. The identity record had been doing double duty as a + * *temporal* guard. Replacing it with the selector wait agent-device records for its own presses + * restores that, for the better reason: the press now waits for the control to exist rather than + * asserting the shape of the tree around it. + * + * Only a press leading with an `id="…"` alternative gets a wait, and only when the step before it + * is not already that wait — an id is the one part of a recorded selector that is not derived from + * volatile display text (`label="1 follower"`), and a wait on the rest would be a guess. + * + * Inserting a wait renumbers every later step, so the result is not a base for a second round of + * report-driven stripping: the caller replays it once and restores the original if it still fails. + * + * Returns `undefined` when `step` is not a press carrying an identity — the same "nothing to do" + * signal `stripRecordedIdentity` gives, so an already-stripped script cannot be processed twice. + */ +export function unguardPressSteps( + lines: readonly string[], + step: number +): UnguardedPresses | undefined { + if (!stripRecordedIdentity(lines, step)) return undefined + + const unstable = new Set(ancestryKeys(identityAtStep(lines, step))) + + // Collected as (step, lineIndex) pairs up front, because editing invalidates both numberings. + const targets: { step: number; index: number }[] = [] + let seen = 0 + for (const [index, line] of lines.entries()) { + if (!isExecutable(line)) continue + seen += 1 + if (!/^press\s+"/.test(line.trim())) continue + if (index === 0 || !lines[index - 1].trim().startsWith(TARGET_COMMENT)) continue + const shares = ancestryKeys(identityAtStep(lines, seen)).some((key) => unstable.has(key)) + if (seen === step || shares) targets.push({ step: seen, index }) + } + + // Descending, so each edit leaves the indices of the ones not yet applied untouched. + let next = [...lines] + const stripped: number[] = [] + const guarded: number[] = [] + for (const target of [...targets].reverse()) { + const selector = pressSelectorAtStep(lines, target.step) + const clause = selector ? idClause(selector) : undefined + const wait = clause ? waitLineFor(clause) : undefined + const alreadyGuarded = previousExecutable(lines, target.index - 1) === wait + + next = [ + ...next.slice(0, target.index - 1), + ...(wait && !alreadyGuarded ? [wait] : []), + ...next.slice(target.index) + ] + stripped.unshift(target.step) + if (wait && !alreadyGuarded) guarded.unshift(target.step) + } + + return { lines: next, stripped, guarded } +} + +/** + * The selector a published script's trailing destination guard waits on, or `undefined`. + * + * `saveAndReplayRecording` appends that guard as the last thing it records, so for any script that + * reached publication the final executable line is `wait "" 15000` on a non-chrome + * landmark of the destination. Reading it back is what lets a caller that did not record the + * script — the saved-replay rung, verifying it for the first time — hand the same selector to the + * recovery tiers, which need somewhere to wait after they rewrite a step. + * + * Deliberately only the *last* line. A script is full of intermediate waits, and any of them would + * be accepted by a looser search while proving nothing about arrival. + */ +export function destinationGuardSelector(lines: readonly string[]): string | undefined { + for (let i = lines.length - 1; i >= 0; i--) { + if (!isExecutable(lines[i])) continue + const match = /^wait\s+"(.*)"\s+\d+$/.exec(lines[i].trim()) + return match ? match[1].replace(/\\(["\\])/g, '$1') : undefined + } + return undefined +} diff --git a/src/app-flow/app-flow-normalize.ts b/src/app-flow/app-flow-normalize.ts new file mode 100644 index 00000000..9c8f42a0 --- /dev/null +++ b/src/app-flow/app-flow-normalize.ts @@ -0,0 +1,104 @@ +/** + * Back-compat normalization for a document read from `app-flow.json` — factored out of + * `readAppFlow` (`storage.ts`) purely so it has a unit test: `storage.ts` value-imports `electron`, + * which loads the instant anything imports that module, so this logic could not otherwise run under + * plain `node --test`. + * + * Every new field Step 4 added is defaulted here value-driven — following the pre-existing + * `node.urlPath ?? null` pattern this function also carries forward — rather than gated on + * `manifest.formatVersion`. A build that is interrupted mid-write can leave a document whose + * manifest already claims a v4 `formatVersion` but whose nodes/edges predate this round; defaulting + * on whether each field is actually present is correct in that case, where branching on the version + * number would not be. A v3 document normalized here becomes a v4 document with empty runtime + * state, which is exactly right: a v3 sweep observed none of it. + */ +import type { AppFlowResult } from '../shared/app-flow.ts' +import { + inferChromeKind, + looksLikeChromeCaption, + navigatorChrome, + rebuildChromeEdges +} from './reach/chrome-inventory.ts' + +export function normalizeAppFlow(result: AppFlowResult): AppFlowResult { + // Two repairs beyond the `kind` back-fill, both applied on read so a document written by an + // earlier sweep heals instead of carrying its damage into every consumer that opens it. + // + // Items are re-filtered through `looksLikeChromeCaption`. That guard did not exist when these + // documents were written, and without it a real sweep recorded 97 items for one tab bar — whole + // post bodies matched against node vocabulary by token overlap, so that a feed caption about + // budget legislation was persisted as the control that opens `Support`. Re-running the guard + // here costs a string length check per item and removes every one of them. + // + // Records are deduped by id because a chrome id doubles as a graph node id now + // (`chromeVirtualNodes`), and a document written before `sameChrome` settled identical ids can + // hold two records under one — a real sweep produced two `chrome:tabs:MessagesConversation`. + // First wins: the earlier sighting is the one whose provenance the id already describes. + const healed = dedupeById( + (result.chrome ?? []).map((chrome) => ({ + ...chrome, + kind: chrome.kind ?? inferChromeKind(chrome.reveal), + items: chrome.items.filter((item) => looksLikeChromeCaption(item.label)) + })) + ) + const allNodes = result.nodes.map((node) => ({ + ...node, + urlPath: node.urlPath ?? null, + reach: node.reach ?? (node.urlPath ? 'deep-link' : 'navigation-only'), + deepLinkRejection: node.deepLinkRejection ?? null, + origin: node.origin ?? 'parsed' + })) + // Which of the recorded surfaces the screens' own captures corroborate — see `navigatorChrome`. + // The `chrome` array itself keeps every record it ever had, honest observation history and all; + // this is only about which of them earn a node and edges, and it has to be recomputed on read + // because a document written before `navigatorChrome` existed has virtual nodes for surfaces that + // were never navigator chrome at all — a conversation's overflow menu persisted as "Tab bar". + const navigators = navigatorChrome(healed, allNodes) + const navigatorIds = new Set(navigators.map((chrome) => chrome.id)) + const nodes = allNodes.filter((node) => node.origin !== 'chrome' || navigatorIds.has(node.id)) + return { + ...result, + manifest: { + ...result.manifest, + formatVersion: Math.max(result.manifest.formatVersion ?? 1, 1) + }, + nodes, + edges: result.edges.map((edge) => ({ + ...edge, + origin: edge.origin ?? 'parsed' + })), + // Filtered to the surfaces the captures corroborate, exactly as the sweep's own assembly does + // (`orchestrator.ts`) — a document written before `navigatorChrome` existed carries records for + // context menus that were never navigator chrome, and they must not survive a read. + runtimeEdges: rebuildChromeEdges(result.runtimeEdges ?? [], navigators, nodes), + // Per-item, like `node.origin` above, not just defaulting the whole array to `[]`: a document + // can have a real, non-empty `chrome` array and still predate the `kind` field on each entry. + // `inferChromeKind` is the same reveal-length rule a fresh detection already uses, so a + // back-filled record compares identically under `sameChrome` (`reach/chrome-inventory.ts`) to one + // that carried `kind` from the start. + // Two repairs beyond the `kind` back-fill, both applied on read so a document written by an + // earlier sweep heals instead of carrying its damage into every consumer that opens it. + // + // Items are re-filtered through `looksLikeChromeCaption`. That guard did not exist when these + // documents were written, and without it a real sweep recorded 97 items for one tab bar — + // whole post bodies matched against node vocabulary by token overlap, so that a feed caption + // about budget legislation was persisted as the control that opens `Support`. Re-running the + // guard here costs a string length check per item and removes every one of them. + // + // Records are deduped by id because a chrome id doubles as a graph node id now + // (`chromeVirtualNodes`), and a document written before `sameChrome` settled identical ids can + // hold two records under one — a real sweep produced two `chrome:tabs:MessagesConversation`. + // First wins: the earlier sighting is the one whose provenance the id already describes. + chrome: healed, + navigation: result.navigation ?? null + } +} + +function dedupeById(records: T[]): T[] { + const seen = new Set() + return records.filter((record) => { + if (seen.has(record.id)) return false + seen.add(record.id) + return true + }) +} diff --git a/src/app-flow/armed-recording.ts b/src/app-flow/armed-recording.ts new file mode 100644 index 00000000..5ec22b2e --- /dev/null +++ b/src/app-flow/armed-recording.ts @@ -0,0 +1,104 @@ +/** + * The one `--saveScript` transaction a device may have open at a time, and the rule every other + * `apps.open` in this module has to obey while it is open. + * + * agent-device publishes a recording only if exactly one successful open belongs to it. A second + * one — any `apps.open` issued between the arming open and `sessions.saveScript` / + * `sessions.close({ saveScript })` — kills the transaction, and the publish that follows fails + * with `This script recording was aborted by a second successful open and cannot be published`. + * Nothing in the device API asks before opening, so the only way to keep that from happening is + * for the sites that can open the app to know a recording is armed. + * + * Measured, session_2026-10-13_11-35-39: six walks armed a recording and six of them landed on the + * dev-client launcher, because the arming open is `relaunch: true` and a dev client restarted + * cold comes up on its launcher. `waitForAppReady` then cleared the launcher the only way it can, + * with an `apps.open` of the bundle URL, and every one of those walks died at the publish: + * + * [ 355.322] ProfileFollowers explore seeded at /profile/:name <- recording armed here + * [ 357.987] dev-client launcher on screen with a recording armed — waiting up to 20.0s + * [ 379.530] apps.open — xyz.blueskyweb.app bluesky://expo-development-client/?url=... + * [ 403.843] walked edge — Profile -> ProfileFollowers <- arrival verified + * [ 406.114] ProfileFollowers ... aborted by a second successful open and cannot be published + * + * The same shape at t=430.6/456.2, t=481.0, t=566.7/623.8, t=649.3/679.7 and t=704.3/743.1. + * `MyProfile` — the one walk whose arming open did not land on the launcher, so the one walk with + * no recovery open in its armed window — is the only node in that sweep that published + * (`flows/myprofile.ad`, t=153.4). The `go home` lines sitting just ahead of five of those + * failures are *downstream* of them, not the cause: they are `discardRecordingCandidate`'s soft + * home, reached from the catch that the publish failure throws into (the log line carries no + * `recorded steps kept at …` suffix, which is the branch where the candidate file was never + * written because the transaction had already been aborted). + * + * Two different answers follow, because the two openers want different things: + * + * - `goHome` is suppressed outright while armed (`suppressesSoftHome`). It is a convenience — + * put the app back on a known screen after an abandoned walk — and its own doc comment already + * records that the next caller to need a transaction restarts the app anyway. Skipping it costs + * a cosmetically wrong starting screen; issuing it costs the script. + * + * - the dev-client launcher recovery cannot be suppressed: the app is not on screen, and waiting + * it out is already tried first and already measured as hopeless on this kind of device (see + * `launcher-deferral.ts`). So it opens, marks the transaction dead on its way past + * (`noteOpenWhileArmed`), and `startReplayRecording` re-arms afterwards (`needsRearm`) on top + * of the app the recovery has just loaded — a soft open this time, so there is no cold start, + * no launcher and therefore no third open to abort the replacement. + * + * Carried on `CaptureDevice` rather than threaded through arguments for the same reason + * `pendingEntryGate` is: arming happens in the orchestrator's exploration setup and the opens that + * can abort it happen several frames down inside `waitForAppReady`, and there is exactly one + * recording per device at a time. + */ +export type ArmedRecording = { + /** Candidate path the live transaction will publish to. Carried for the log line only. */ + readonly scriptPath: string + /** Set once an `apps.open` has been issued behind this transaction's back. */ + aborted: boolean +} + +/** A freshly armed transaction. Nothing has opened the app behind it yet, by construction. */ +export function armRecording(scriptPath: string): ArmedRecording { + return { scriptPath, aborted: false } +} + +/** + * Whether a soft `goHome` must be skipped right now. + * + * Only while a transaction is armed *and still alive*: once something has already aborted it there + * is no publication left to protect, and the cosmetic value of putting the app back on its launch + * screen is the only thing left on the table — so the home is allowed through again rather than + * suppressed for the rest of a doomed walk. + */ +export function suppressesSoftHome(armed: ArmedRecording | null | undefined): boolean { + return armed !== null && armed !== undefined && !armed.aborted +} + +/** + * Mark the armed transaction as aborted by an open that is about to be issued, and report whether + * this call is the one that did it. + * + * Called *before* the open rather than after, because the thing being recorded is a decision the + * caller has already made and the open itself may never return — a recovery that throws has still + * opened the app as far as agent-device is concerned. + * + * The return value drives one log line, which is why it is "did this call change anything" rather + * than "is it aborted": the launcher recovery can issue two opens (reconnect, then relaunch) for + * one armed recording, and saying so twice would read as two lost scripts. + */ +export function noteOpenWhileArmed(armed: ArmedRecording | null | undefined): boolean { + if (!armed || armed.aborted) return false + armed.aborted = true + return true +} + +/** + * Whether `startReplayRecording` has to replace the transaction it just armed before handing the + * walk over. + * + * True exactly when the readiness wait that follows the arming open had to open the app itself. + * The transaction is already unpublishable at that point, and the walk about to be recorded into + * it would run to a verified arrival and then throw it away — which is the 5-in-6 failure the + * measured sweep spent 683s producing. + */ +export function needsRearm(armed: ArmedRecording | null | undefined): boolean { + return armed !== null && armed !== undefined && armed.aborted +} diff --git a/src/app-flow/capture.ts b/src/app-flow/capture.ts new file mode 100644 index 00000000..be65fa8d --- /dev/null +++ b/src/app-flow/capture.ts @@ -0,0 +1,3222 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import { existsSync } from 'node:fs' +import { readdir, readFile, writeFile } from 'node:fs/promises' +import { homedir } from 'node:os' +import { basename, isAbsolute, join } from 'node:path' +import type { AppFlowNode, AppFlowPressable, AppFlowStep } from '../shared/app-flow.ts' +import { + acceptOpenInAppPrompt, + isUrlSchemePreapproved, + preapproveIosUrlScheme, + shouldPreapproveUrlScheme +} from '../server/app-flow-deps/ios/scheme-approval.ts' +import { retryWhileRunnerBusy } from '../server/app-flow-deps/ios/runner-busy.ts' +import { + bundlerCliArgs, + findAvailablePort, + isExpoProject, + isMetroReady, + isReactNativeCliScript, + parseMetroPortFromOutput +} from '../server/app-flow-deps/metro/port.ts' +import { beginSessionMetroLog, endMetroLogFile, pipeChildToMetroLog } from '../server/app-flow-deps/metro/logs.ts' +import { + devClientReconnectUrl, + fingerprint, + isDevClientLauncher, + isFrameUnchangedSinceAction, + isLoadingScreen, + needsDevClientReconnect, + similarity +} from './screen-signals.ts' +import { quoteCliArg } from './cli-args.ts' +import { agentDeviceOpenCommand } from './walk-commands.ts' +import { launcherDeferralMs, type LauncherDeferralOutcome } from './launcher-deferral.ts' +import { + armRecording, + needsRearm, + noteOpenWhileArmed, + suppressesSoftHome, + type ArmedRecording +} from './armed-recording.ts' +import type { Phase, PhaseLedger } from './timing.ts' +import { + indexDeepLinkPath, + isAmbiguousSelectorMessage, + describeSessionLoss, + isSessionLostMessage, + serializeLocator +} from './exploration.ts' +import { PROBE_FRAME_ELEMENT_FLOOR } from './reach/probe-route.ts' +import { describeReplayDivergence, parseStructuralOnlyMismatchStep } from './replay-divergence.ts' +import { unguardPressSteps } from './ad-script.ts' +import { getProjectDir } from './storage.ts' +import { computeNativeBuildFingerprint } from '../server/app-flow-deps/native-build.ts' +import { createAppFlowNativeBuildStore } from '../server/app-flow-deps/native-build-store.ts' +import type { NativeBuildStore } from '../server/app-flow-deps/native-build.ts' +import { parseInteractionTarget, toAgentDeviceTarget } from '../server/app-flow-deps/profiling/transition-target.ts' +import { appFlowAgentDeviceStateDir, appFlowDataDir, getAppFlowHost, resolveAgentDeviceCommand } from './host.ts' + +export interface CaptureContext { + projectPath: string + projectId: string + scheme: string | null + appName: string | null + bundleId: string | null + slug: string | null + platform: 'ios' + sessionName: string + signal?: AbortSignal + /** + * Where this sweep's wall clock is tallied, or absent to measure nothing. + * + * Optional, and carried on the context rather than held in a module-level singleton, because a + * manual capture session runs alongside a sweep with its own device and its own clock — a global + * would silently bill one run's reads to the other's summary. Every device primitive worth timing + * already takes `ctx`, so this reaches all of them without threading a new parameter. + */ + timing?: PhaseLedger +} + +const LAUNCH_TIMEOUT_MS = 10 * 60 * 1000 +/** + * How long the frame from before a launch is disbelieved — and, only when there is no such frame to + * compare against, a flat sleep before the first readiness poll. + * + * Deliberately shorter than the launch itself takes: it was never sized to cover a cold start, only + * to stop the very first accessibility read landing on a process that has not drawn a single frame + * yet. Everything beyond that is `waitForAppReady`'s stability poll, which has the full + * `APP_READY_TIMEOUT_MS` to work with and catches a slow start by construction. + * + * It is no longer usually spent at all. A relaunch is nearly always preceded by a successful read, + * so the poll can tell a frozen pre-launch frame from the app's first real one by comparing them, + * and a warm relaunch — where the first read is already the new screen — pays nothing. This value + * then bounds only how long an *unchanged* frame is held, which is the same window the sleep was + * approximating. Measured on a warm simulator run: the first poll after this wait already read a + * fully drawn screen, i.e. the whole thing was dead time on the common path. + */ +const APP_LOAD_WAIT_MS = 1_500 +const APP_READY_TIMEOUT_MS = 90 * 1000 +/** + * How long a frame that has *settled below* `minInteractiveElements` is given to grow before the + * wait gives up on the floor and returns. + * + * Separate from `timeoutMs` because the two bound different failures. `timeoutMs` is sized for a + * cold 6130-module bundle that has not drawn anything yet; this is sized for an app that has drawn + * something and the caller does not believe it. Collapsing them would mean a genuinely sparse + * first screen — a login form is three controls — holding the sweep for the full 90s to learn + * what the first settle already said. + * + * Twenty seconds, from two launches of the same app that disagreed: the first went substantive + * 12.5s after the poll first settled on a splash, the second about 18s after. Fifteen was set from + * the first alone and the second missed it by seconds — the wait gave up at 2 elements and the + * auth gate's own wait, running straight afterwards, found 20 elements in 1.7s. + * + * That near-miss is also why this is sized generously rather than tightly. Expiring here is a + * handoff, not a failure: `AUTH_GATE_FRAME_WAIT_MS` holds the identical floor immediately + * downstream, so the cost of being slightly short is a re-paid settle handshake, while the cost of + * being long is a genuinely sparse first screen — a login form is three controls — waiting this + * out before anyone looks at it. Both are small; the second is the one that scales with nothing, + * so the number sits above the measurements rather than between them. + */ +export const SUBSTANTIVE_FRAME_GRACE_MS = 20 * 1000 +/** Quiet window `startReplayRecording`'s recorded `wait stable` requires before it calls the UI + * settled — agent-device's own default, which is tuned for exactly this open/relaunch case. */ +const APP_STABLE_QUIET_MS = 500 +/** Ceiling for that same wait. Far below `APP_READY_TIMEOUT_MS` because this one is best-effort: + * when it gives up, `waitForAppReady` is still ahead of it with the full budget, so spending the + * whole 90s here would only delay the wait that can actually rescue a slow start. */ +const APP_STABLE_TIMEOUT_MS = 20 * 1000 +/** + * How long the recorded entry gate waits for the walk's first press target — see + * `pendingEntryGate` on `CaptureDevice`. + * + * Two clocks share this one number, which is the point of it. Live it resolves on the first check, + * because the gate is only ever issued against an element the planner just read off the screen, so + * it costs the walk nothing. At *replay* time the same line becomes the budget for a cold app to + * draw the screen that element is on, which is the entire reason the gate exists, so it cannot be + * short. 15s matches the destination guard `saveAndReplayRecording` appends at the other end of the + * script: the two are the same mechanism pointed at opposite ends of a journey, and there is no + * reason for them to disagree about how long an app gets to render a screen. + */ +const ENTRY_GATE_TIMEOUT_MS = 15 * 1000 +/** + * How long a dev-client launcher recovery is left to take effect before `waitForAppReady` + * escalates to the next move. + * + * Sized for the slow half of what the first move has to do: reconnecting tells the dev client to + * fetch and evaluate a debug bundle from Metro, which on a cold Metro is a full transform of the + * app, not a reload. Too short and the poll relaunches the app out from under a reconnect that was + * working; the cost of too long is only that a hopeless launcher is declared hopeless later, and + * `waitForAppReady`'s own deadline still bounds that. + */ +const LAUNCHER_RECOVERY_GRACE_MS = 30 * 1000 +/** + * Sleep *between* readiness polls — not the interval between them. + * + * The distinction is the whole reason this is as low as it is: a poll is a `capturePressables`, + * which measured 0.9–1.7s against a simulator, so the real spacing between two consecutive reads + * is this value plus that. At the old 1_500 the effective spacing was ~3.1s and the cheapest + * possible settle — two matching reads — cost ~4.6s on every screen, of which only the two reads + * were doing any work. The question a second read answers is "is this still the same screen", and + * the read's own latency already provides most of the separation that question needs. + */ +const APP_READY_POLL_MS = 400 +/** + * How alike two consecutive polls must be for the screen to count as settled. Matches the + * exploration loop's own "nothing changed" bar, so the two agree on what one screen looks like + * over time. + */ +export const APP_READY_SETTLE_SIMILARITY = 0.98 +/** + * How alike a settled frame must be to the frame from *before* the action for the settle to be + * disbelieved as stale. Deliberately far below `APP_READY_SETTLE_SIMILARITY`, because the two + * comparisons ask different questions and 0.98 answers the wrong one. + * + * 0.98 asks "is this the identical render". The stale guard's question is "is this still the + * screen we came from" — and a frame caught *mid-transition* is neither: it is a composite, the + * outgoing screen with some of the incoming one drawn over it. Measured on + * `session_2026-10-13_14-33-40.log`, ProfileFollowers: the pre-open frame was Profile's own + * delivery read at t=235.945 (30 controls), the settle at t=241.663 agreed twice on **46** + * controls, and the delivery read 0.9s later found **9** — the real Followers screen. A 46-control + * composite sitting over a 30-control profile clears 0.98 trivially (it is 16 controls richer), so + * the guard was live, compared, and waved the straddle through; the screenshot written to + * `screens/profilefollowers.png` is the MyProfile screen, verified by eye. + * + * 0.88 is set between the two things that have to stay apart. Above it: a composite that still + * carries the old screen's controls, which is what this must catch. Below it: two genuinely + * different screens, which must not be held — the same run's ProfileFollowers pair (46 mid-flight + * vs 9 settled) and PostLikedBy's (52 vs 8) share almost no vocabulary at all. Being wrong in the + * "hold" direction costs at most `ROUTE_STALE_GRACE_MS` of polling on a screen that had in fact + * arrived; being wrong the other way costs a whole wrong capture, which is what this run paid + * eight times. + */ +export const APP_READY_STALE_SIMILARITY = 0.88 +/** Consecutive failing accessibility reads after which `waitForAppReady` stops waiting. */ +const APP_READY_MAX_READ_FAILURES = 3 +/** + * The router's own transition beat: how long the pre-open frame is disbelieved after a deep link. + * + * Was a flat sleep in front of the settle poll, and that is the only thing it ever covered. It used + * to be sized as if it were the entire wait — 2.5s — which stopped being true once + * `settleAfterRoute` put the stability poll behind it. Trimming it to 800ms left the last of the + * problem in place: a deep link that has already landed by the time the poll starts still paid it, + * which was 11 of these per sweep in an eight-screen run, ~9s of nothing. + * + * Now the window is measured instead. The poll compares what it reads against the frame from + * before the open (`isFrameUnchangedSinceAction`), so an arrived screen settles on the first two + * agreeing reads and only a screen that still looks like where it came from waits — after which + * an unchanged frame is accepted, because a deep link to the screen the app is already on is a + * legitimate no-op. + * + * What is left here is only the flat post-*tap* sleep `waitForRoute` pays (see that function). The + * deep-link guard's ceiling moved to `ROUTE_STALE_GRACE_MS`, because 800ms turned out to be too + * short for the guard and raising it here would have charged every tap in the sweep for it. + */ +const ROUTE_WAIT_MS = 800 +/** + * The ceiling on the *stale-frame* guard after a deep link — `ROUTE_WAIT_MS`'s other job, split + * off from it because the two numbers are now sized by different measurements. + * + * `ROUTE_WAIT_MS` is still the flat sleep `waitForRoute` pays after a tap (~30 per sweep), where + * raising it is a pure cost. This is the budget `settleAfterRoute` gives the guard to disbelieve a + * frame that still looks like where we came from, where 800ms was demonstrably too short. + * Measured on `session_2026-10-13_14-33-40.log`, PostLikedBy: `apps.open` at t=458.609, + * `waitForAppReady` entered at t=459.188, and the first agreeing pair landed at t=460.369 — 1.18s + * into the poll, past an 800ms grace that had therefore already expired before the guard had a + * single pair to judge. The guard was not wrong there; it was never consulted. That settle agreed + * on 52 controls, which is exactly PostThread's own count from its delivery read at t=454.565 — + * the previous screen, photographed as `screens/postlikedby.png` while the real Liked By screen + * (8 controls) arrived 0.8s later. + * + * 2000ms covers that case with room: every one of the eight straddles in that run had its real + * screen in hand within ~1.0s of the first (wrong) agreeing pair. The cost is bounded and paid + * only by a screen that genuinely has not moved, and only until it does — see + * `ROUTE_STALE_MIN_REJECTED_PAIRS`, which is the half of this budget a fast machine cannot + * silently opt out of. + */ +const ROUTE_STALE_GRACE_MS = 2_000 +/** + * Agreeing pairs the stale guard must reject before the wall-clock grace above is allowed to run + * out on it. + * + * This exists because a millisecond budget alone is a guard that quietly disables itself on fast + * hardware: the faster the accessibility read, the more likely the first agreeing pair lands after + * the grace rather than inside it, which is precisely how PostLikedBy's guard came to be dead on + * arrival at 1.18s against 800ms. Counting rejections instead states the intent directly — "do not + * accept a frame that looks like where we came from until you have disbelieved it at least twice" + * — and is invariant to how quick the device is. + * + * Two, not one: one rejection plus a poll interval is ~0.6s on the measured device, which is + * inside the window the eight straddles landed in. Two takes the guard past it while costing a + * genuinely-unchanged screen (a deep link to the screen the app is already on) two extra reads at + * ~0.18s each before the grace clock even starts counting. + */ +const ROUTE_STALE_MIN_REJECTED_PAIRS = 2 +/** + * How recently a URL must have been opened for `settleAfterRoute` to measure the stale guard's + * grace from it rather than from the wait itself. Generously longer than the ~0.5s gap it exists + * to recover (see `staleClockStartedAt`) and far shorter than anything the sweep does between two + * screens, so the stamp is either this settle's own open or ignored. + */ +const ROUTE_OPEN_CLOCK_MAX_AGE_MS = 5_000 +/** + * The quiet window agent-device must observe before it calls a post-action UI settled. + * + * Its default is 500ms and it is a flat cost on every tap and every text entry — ~30 of them per + * sweep, which is where the `press` phase's 3.60s average largely comes from. 200ms is still + * several frames of stillness at 60Hz, which is what the window is for: distinguishing "the screen + * stopped changing" from "the screen is mid-transition". It is shortened rather than removed + * because the thing it guards against, reading the pre-action frame as the result of the action, + * is the failure that costs a whole wrong capture. + */ +const SETTLE_QUIET_MS = 200 +/** Bound on the post-deep-link settle poll — see `settleAfterRoute`. Far below + * `APP_READY_TIMEOUT_MS` because a deep link that has not resolved within this long is a failed + * deep link, not a slow one, and the ladder has an exploration rung waiting behind it. */ +const ROUTE_SETTLE_TIMEOUT_MS = 30 * 1000 +/** Budget for `openDeepLink`'s one-read liveness probe — see its call site. Short because the only + * answer it is waiting for is "the process is drawing", and anything slower than this is the cold + * start the full `waitForAppReady` behind it exists to handle. */ +const APP_ALIVE_PROBE_TIMEOUT_MS = 5 * 1000 +/** + * How old a read may be and still describe the frame the app was on *before* the caller's action. + * + * Deliberately far more generous than `APP_ALIVE_REUSE_MS`, which this was first written to share. + * That one answers "is the process alive right now", where a ten-second-old yes is worthless. This + * one answers "what was on screen before we acted", and nothing between the two reads touches the + * device — the gap is almost always a model call — so age does not make the answer wrong. + * + * The asymmetry is what makes a loose bound safe. A before-frame that has gone stale can only + * cause a *false hold*, it can never cause a false settle, and a false hold requires the stale + * frame to still match what is on screen — which means it is the same screen after all. The bound + * exists only to rule out a read from so long ago that the app may have been relaunched out from + * under it. + * + * Sharing the 10s bar cost two launch sleeps per sweep in a measured run: every wait that followed + * a model call found its read "too old" and fell back to sleeping, which is exactly the case the + * comparison handles best. + */ +const FRAME_COMPARISON_MAX_AGE_MS = 60 * 1000 + +/** + * How recently `capturePressables` must have succeeded for `openDeepLink` to take its word for it + * instead of spending a snapshot on the liveness probe. + * + * The probe asks one question — "is this process alive and drawing" — and a read that came back + * with interactive elements a moment ago has already answered it. On the sweep's hot path the + * previous screen's delivery read is always that answer, so the probe was re-asking something the + * harness had just been told, at ~0.9s of accessibility snapshot per screen. + * + * Bounded rather than unbounded because the thing it guards against is a *cold* process, and the + * gap this covers is local bookkeeping — choosing delivery sources, copying a frame, writing the + * checkpoint — not anything that can start the app over. Ten seconds is comfortably longer than + * that gap and far shorter than a launch, so a reused answer is never one that had time to stop + * being true; anything slower falls through and pays for the real read. + */ +const APP_ALIVE_REUSE_MS = 10 * 1000 +// Match post-install CLI output only. xcodebuild prints "BUILD SUCCEEDED" before +// simctl install; treating that as done kills `expo run:ios` too early. +const NATIVE_INSTALL_SUCCESS_RE = + /successfully launched the app|installed the app|opening on |opening .+ on /i +const EXPO_GO_APP_NAME = 'Expo Go' +const EXPO_GO_VERSION = '57.0.9' + +function log(step: string, detail?: string): void { + console.log(detail ? `[app-flow] ${step} — ${detail}` : `[app-flow] ${step}`) +} + +/** + * Times `work` against the sweep's ledger when one is attached, and otherwise just runs it. + * + * The `?? work()` fallback is what keeps `ctx.timing` optional at every call site instead of making + * each primitive branch on it: a standalone or test context measures nothing and pays nothing, and + * the primitive reads the same either way. + */ +export function measurePhase( + ctx: CaptureContext, + phase: Phase, + work: () => Promise +): Promise { + return ctx.timing?.measure(phase, work) ?? work() +} + +function agentDeviceBootCommand(): string { + return 'agent-device boot --platform ios --target mobile' +} + +function agentDeviceInstallCommand(app: string, appPath: string): string { + return `agent-device install ${quoteCliArg(app)} ${quoteCliArg(appPath)} --platform ios` +} + +// Moved to `walk-commands.ts`, the pure module the sweep's own `pathToReach` emission is built +// from — this line is one of the two command forms both producers write, and a second copy here is +// exactly how the two would drift. Re-exported so every existing `capture.ts` importer is +// unchanged. +export { agentDeviceOpenCommand } from './walk-commands.ts' + +type AgentDeviceClient = Awaited> +type LaunchMode = 'native-run' | 'expo-go' + +export interface CaptureDevice { + client: AgentDeviceClient + deviceId: string + deviceName: string + launchProcess: ChildProcess | null + metroPort: number + metroLogPath: string | null + launchTarget: string + launchUrl: string | null + /** + * The URL that tells an Expo dev client which Metro server to load, or `null` when this app is + * not one (Expo Go, which carries its bundle URL in `launchUrl`, or a plain React Native build). + * + * Only ever used to recover from the dev client's launcher screen. It is deliberately *not* + * `launchUrl`: `launchUrl` is the URL every ordinary launch and relaunch goes through, and + * routing normal launches through the dev-client URL would reload the bundle on every session + * recovery instead of just resuming the app that is already running. + */ + devClientUrl: string | null + useCustomScheme: boolean + setupCommands: string[] + schemePromptChecked: boolean + /** + * The most recent accessibility read that came back from this device, and when. Written by every + * successful `capturePressables`; read only by `openDeepLink`'s liveness probe, which is a + * question a recent enough read already answers — see `APP_ALIVE_REUSE_MS`. + * + * Deliberately not a cache of the *screen*: nothing may use this to decide what the app is + * showing, only whether its process is up and drawing. The sweep moves the app constantly, so by + * the time anything reads this the contents are usually a screen the app has already left. + */ + lastRead: { at: number; pressables: AppFlowPressable[] } | null + /** + * When this device last had a deep-link URL handed to it (`apps.open` returning, in + * `openDeepLink`), or null before the first one. + * + * Read by `settleAfterRoute` alone, as the moment the stale-frame guard's grace should be + * measured from. It is on the device rather than passed as an argument because the open and the + * settle are two separate calls made by two different modules — `openDeepLink` is a `DevicePort` + * method and `waitForRoute` is another, with the deep-link strategy in between — so there is no + * call frame that holds both. See `staleClockStartedAt` for the ~0.5s this recovers. + */ + lastRouteOpenAt: number | null + /** + * Set by `startReplayRecording`, consumed by the next `pressWithLocator`: the recording has just + * relaunched the app, so the press about to happen is the one that will run against a cold app + * when this script is replayed, and it gets a recorded `wait` on its own target first. + * + * Carried on the device rather than passed down because the two functions are several frames + * apart — recording starts in the orchestrator's exploration setup and the press happens inside + * whatever `executeNavigationActions` call the planner's first decision produces — and there is + * exactly one recording per device at a time, which is what makes a single flag sufficient. + */ + pendingEntryGate: boolean + /** + * Whether a launcher deferral has already run its full length on this device without the + * launcher clearing. Written by `waitForAppReady`, read by it on every later call to decide + * whether deferring again is still worth the wait — see `launcher-deferral.ts`. + */ + launcherDeferral?: LauncherDeferralOutcome + /** + * The `--saveScript` transaction currently open on this device, or null/absent while there is + * none. Written by `startReplayRecording` and cleared by every site that finalises or discards + * one; read by `goHome` and by `waitForAppReady`'s launcher recovery, which are the two places + * in this module that can issue the "second successful open" agent-device refuses to publish + * after — see `armed-recording.ts` for the measured sweep that made this necessary. + */ + armedRecording?: ArmedRecording | null +} + +/** + * The action shape the navigation model exchanges during exploration. This is exactly `AppFlowStep` + * (`shared/app-flow.ts`) — an alias, not a copy, so the persisted counterpart the reach pipeline writes + * into `app-flow.json` (a recorded chrome's `reveal` steps, plan Step 4d) and the in-flight action + * this module executes stay one type. Aliased here, rather than renamed at every call site, because + * `AppFlowStep` had to move to `shared/app-flow.ts` (the renderer reads persisted steps too) while this + * module's many call sites — `exploration.ts` imports `AppFlowNavigationAction` from here, not from + * `shared/app-flow.ts` — did not need to change for that move. + */ +export type AppFlowNavigationAction = AppFlowStep + +/** + * A locator good enough to guard a saved recording with. Narrower than `AppFlowNavigationAction`'s + * locator on purpose: it excludes `by: 'text'`, because agent-device only accepts a destination + * guard whose *resolved node* carries an id or an accessibility label — see + * `AppFlowNavigationDecision` below and `saveAndReplayRecording`. + */ +export interface AppFlowDestinationLocator { + by: 'label' | 'id' + value: string +} + +export type AppFlowNavigationDecision = + | { decision: 'act'; action: AppFlowNavigationAction; reason: string } + /** + * `destinationLocator` names a durable landmark on the screen the walk landed on, so the + * recording of that walk can be published. agent-device refuses to write a script that has no + * "portable destination guard" after its last mutating action: a `wait` on a selector whose + * matched element has a recorded identity it can re-verify at replay time. Without one, + * `sessions.saveScript` throws and the whole journey — already judged correct — is unpublishable, + * which is exactly what happened to every exploration-based capture in the device run this field + * was added for. It is optional because a missing or unusable locator must never cost us the + * arrival itself; the caller falls back to a landmark read off the destination's own controls, + * and failing that publishes nothing and keeps the arrival. + */ + | { decision: 'arrived'; reason: string; destinationLocator?: AppFlowDestinationLocator } + | { decision: 'blocked'; reason: string } + +interface LoadAgentDeviceResult { + createAgentDeviceClient: (config?: Record) => AgentDeviceClient +} + +interface PackageJson { + scripts?: Record + dependencies?: Record + devDependencies?: Record +} + +/** + * Pins every agent-device daemon this app starts to one state directory. + * + * The sweep talks to agent-device two different ways: in-process through + * `createAgentDeviceClient`, and by shelling out to the `agent-device` CLI for boot/install/open + * (`spawnCommand` below). Each of those starts its own daemon, and a daemon holds the iOS runner + * lease — `sessions.close()` deliberately does not release it ("close keeps a healthy iOS runner + * warm by default"), so the lease outlives the session that took it. + * + * agent-device allows exactly two ways past a lease another daemon holds: a proxy logical-lease + * takeover, which needs a `proxy` lease provider we do not use, and a same-state-dir takeover, + * which compares `AGENT_DEVICE_STATE_DIR` against the owner's `ownerStateDir` (`runner-lease.js`). + * With the variable unset that comparison reads an undefined current dir and can never match, so + * *every* cross-daemon lease request is refused — the takeover path is not merely unlikely, it is + * unreachable. + * + * That was the first of two things stopping replay scripts from being saved. On the sweep before + * this fix every explored screen failed `saveAndReplayRecording`, HomeTab/Home naming the cause + * outright — "iOS runner for is already owned by another agent-device daemon" — and the + * rest surfacing it one level down as the replay's pre-action snapshot failing (`capture-failed`). + * The sweep after it confirmed the lease is no longer the obstacle: HomeTab/Home published a + * verified script, and the remaining failures had moved on to a different, later error inside a + * replay that now actually ran (see `startReplayRecording`'s recorded stable wait for that one). + * + * Setting one dir for the whole app makes the in-process client and every spawned CLI daemon + * same-state-dir peers, which is the takeover agent-device's own docs point at (its guidance for + * the opposite fix — stop the owning daemon — assumes a CI shape where the two are meant to be + * strangers; here they are the same app and should share). `spawnCommand` passes `process.env` + * straight through, so assigning it here covers the CLI path too. + */ +function pinAgentDeviceStateDir(): void { + if (process.env.AGENT_DEVICE_STATE_DIR?.trim()) return + process.env.AGENT_DEVICE_STATE_DIR = appFlowAgentDeviceStateDir() +} + +async function loadAgentDevice(): Promise { + // Before the import, not after: the module reads AGENT_DEVICE_STATE_DIR when it initialises, and + // a daemon started by an earlier CLI spawn must already agree with it. + pinAgentDeviceStateDir() + // agent-device is ESM-only; electron-vite emits CJS for the main process + // with dynamicImportInCjs enabled so this import() is preserved at runtime. + return getAppFlowHost().loadAgentDevice() as Promise +} + +let nativeStore: { dataDir: string; store: NativeBuildStore } | null = null +function nativeBuildStore(): NativeBuildStore { + const dataDir = appFlowDataDir() + if (!nativeStore || nativeStore.dataDir !== dataDir) { + nativeStore = { dataDir, store: createAppFlowNativeBuildStore(dataDir) } + } + return nativeStore.store +} + +async function closeExistingSession(client: AgentDeviceClient): Promise { + try { + await client.sessions.close() + } catch { + // No leftover session for this name. + } +} + +function detectPackageManager(projectPath: string): 'yarn' | 'pnpm' | 'bun' | 'npm' { + if (existsSync(join(projectPath, 'yarn.lock'))) return 'yarn' + if (existsSync(join(projectPath, 'pnpm-lock.yaml'))) return 'pnpm' + if (existsSync(join(projectPath, 'bun.lockb')) || existsSync(join(projectPath, 'bun.lock'))) { + return 'bun' + } + return 'npm' +} + +function runScriptCommand( + packageManager: string, + script: string, + extraArgs: string[] = [] +): string { + const extras = extraArgs.join(' ') + if (packageManager === 'npm') { + return extras ? `npm run ${script} -- ${extras}` : `npm run ${script}` + } + if (packageManager === 'bun') { + return extras ? `bun run ${script} -- ${extras}` : `bun run ${script}` + } + return extras ? `${packageManager} ${script} ${extras}` : `${packageManager} ${script}` +} + +async function readPackageJson(projectPath: string): Promise { + const pkgPath = join(projectPath, 'package.json') + if (!existsSync(pkgPath)) { + throw new Error(`No package.json found in ${projectPath}`) + } + return JSON.parse(await readFile(pkgPath, 'utf-8')) as PackageJson +} + +async function readPlatformScript( + projectPath: string, + platform: 'ios' | 'android' +): Promise { + const pkg = await readPackageJson(projectPath) + const script = pkg.scripts?.[platform] + if (!script) { + throw new Error( + `package.json is missing an "${platform}" script. Add one (for example "ios": "expo run:ios") and retry.` + ) + } + return script +} + +function classifyLaunchMode(script: string): LaunchMode { + if ( + /\b(expo\s+run:|react-native\s+run-|run-ios|run:ios|run-android|run:android)\b/.test(script) + ) { + return 'native-run' + } + return 'expo-go' +} + +function nativeDeviceArgs(script: string, deviceId?: string): string[] { + const reactNativeCli = isReactNativeCliScript(script) + if (!deviceId || /\s--(?:device|deviceId|udid)(?:\s|=|$)/.test(` ${script} `)) return [] + return [reactNativeCli ? '--udid' : '--device', deviceId] +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function attachAbort(child: ChildProcess, signal?: AbortSignal): () => void { + const abort = (): void => { + if (child.pid) child.kill('SIGTERM') + } + signal?.addEventListener('abort', abort, { once: true }) + return (): void => signal?.removeEventListener('abort', abort) +} + +async function spawnCommand( + command: string, + cwd: string, + signal?: AbortSignal +): Promise<{ child: ChildProcess; getOutput: () => string; getExitCode: () => number | null }> { + // The CLI starts its own daemon, which must land in the same state dir as the in-process + // client's — see `pinAgentDeviceStateDir`. `env: process.env` carries it through. + pinAgentDeviceStateDir() + const child = spawn('/bin/zsh', ['-lc', resolveAgentDeviceCommand(command)], { + cwd, + env: process.env, + stdio: ['ignore', 'pipe', 'pipe'] + }) + + let output = '' + const append = (chunk: Buffer): void => { + output = (output + chunk.toString('utf-8')).slice(-12_000) + } + child.stdout?.on('data', append) + child.stderr?.on('data', append) + + let exitCode: number | null = null + child.on('exit', (code) => { + exitCode = code + }) + + const detachAbort = attachAbort(child, signal) + child.once('exit', detachAbort) + + return { + child, + getOutput: () => output, + getExitCode: () => exitCode + } +} + +async function waitWhile( + signal: AbortSignal | undefined, + predicate: () => Promise | boolean, + timeoutMessage: () => string +): Promise { + const deadline = Date.now() + LAUNCH_TIMEOUT_MS + log('waitWhile', `poll 1s, timeout ${LAUNCH_TIMEOUT_MS / 1000}s`) + while (Date.now() < deadline) { + if (signal?.aborted) throw new Error('Cancelled') + if (await predicate()) return + await new Promise((resolve) => setTimeout(resolve, 1000)) + } + throw new Error(timeoutMessage()) +} + +async function installNativeApp( + ctx: CaptureContext, + script: string, + deviceId?: string +): Promise<{ child: ChildProcess | null; metroPort: number | null; metroLogPath: string }> { + const reactNativeCli = isReactNativeCliScript(script) + let metroPort: number | null = reactNativeCli ? await findAvailablePort() : null + const extraArgs = [ + ...bundlerCliArgs({ + reactNativeCli, + startBundler: true, + port: reactNativeCli ? metroPort! : 0 + }), + ...nativeDeviceArgs(script, deviceId) + ] + const command = runScriptCommand(detectPackageManager(ctx.projectPath), ctx.platform, extraArgs) + log('install native app', `${command} (timeout ${LAUNCH_TIMEOUT_MS / 1000}s)`) + const metroLogPath = await beginSessionMetroLog({ + userDataDir: appFlowDataDir(), + projectId: ctx.projectId, + sessionName: ctx.sessionName + }) + const { child, getOutput, getExitCode } = await spawnCommand(command, ctx.projectPath, ctx.signal) + pipeChildToMetroLog(metroLogPath, child) + + let launched = false + const watch = (chunk: Buffer): void => { + const text = chunk.toString('utf-8') + if (NATIVE_INSTALL_SUCCESS_RE.test(text)) launched = true + if (metroPort == null) { + const parsed = parseMetroPortFromOutput(text) + if (parsed) metroPort = parsed + } + } + child.stdout?.on('data', watch) + child.stderr?.on('data', watch) + + try { + await waitWhile( + ctx.signal, + async () => { + const exitCode = getExitCode() + if (exitCode !== null && exitCode !== 0) { + throw new Error( + `"${command}" exited with code ${exitCode}.\n${getOutput().trim() || '(no output)'}` + ) + } + if (!(exitCode === 0 || launched)) return false + if (metroPort == null) metroPort = parseMetroPortFromOutput(getOutput()) + return metroPort != null && (await isMetroReady(metroPort)) + }, + () => + `"${command}" timed out waiting for the native ${ctx.platform} install.\n${getOutput().trim() || '(no output)'}` + ) + } catch (error) { + if (child.pid) child.kill('SIGTERM') + throw error + } + + log('install native app', `app launched, Metro on ${metroPort} pid=${child.pid}`) + return { child, metroPort, metroLogPath } +} + +async function runUntilExit(command: string, cwd: string, signal?: AbortSignal): Promise { + const { child, getOutput, getExitCode } = await spawnCommand(command, cwd, signal) + try { + await waitWhile( + signal, + () => getExitCode() !== null, + () => `"${command}" timed out.\n${getOutput().trim() || '(no output)'}` + ) + } catch (error) { + if (child.pid) child.kill('SIGTERM') + throw error + } + + const exitCode = getExitCode() + if (exitCode !== 0) { + throw new Error( + `"${command}" exited with code ${exitCode}.\n${getOutput().trim() || '(no output)'}` + ) + } + return getOutput() +} + +function stripAnsi(text: string): string { + return text.replace(new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g'), '') +} + +function resolveMaybeHomePath(filePath: string, cwd: string): string { + if (filePath === '~') return homedir() + if (filePath.startsWith('~/')) return join(homedir(), filePath.slice(2)) + if (!isAbsolute(filePath)) return join(cwd, filePath) + return filePath +} + +function parseExpoGoDownloadPath(output: string, cwd: string): string | null { + const lines = stripAnsi(output) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + const marker = 'Expo Go downloaded to' + for (let i = 0; i < lines.length; i++) { + if (!lines[i].startsWith(marker)) continue + const rest = lines[i].slice(marker.length).trim() + const candidate = rest || lines[i + 1] + if (candidate) return resolveMaybeHomePath(candidate, cwd) + } + return null +} + +async function findDownloadedExpoGo( + cwd: string, + platform: 'ios' | 'android' +): Promise { + const entries = await readdir(cwd).catch(() => []) + const suffix = platform === 'android' ? '.apk' : '.app' + const hit = entries.find((entry) => entry.toLowerCase().endsWith(suffix)) + return hit ? join(cwd, hit) : null +} + +async function downloadExpoGo(ctx: CaptureContext): Promise { + const projectDir = getProjectDir(ctx.projectId) + const existing = await findDownloadedExpoGo(projectDir, ctx.platform) + if (existing && existsSync(existing)) return existing + + const command = `npx -y expo-go download ${ctx.platform} ${EXPO_GO_VERSION}` + log('download Expo Go', `${command} (timeout ${LAUNCH_TIMEOUT_MS / 1000}s)`) + const output = await runUntilExit(command, projectDir, ctx.signal) + const fromLog = parseExpoGoDownloadPath(output, projectDir) + if (fromLog && existsSync(fromLog)) return fromLog + + const fromDir = await findDownloadedExpoGo(projectDir, ctx.platform) + if (fromDir) return fromDir + + throw new Error( + `Could not determine the Expo Go download path after "${command}".\n${output.trim() || '(no output)'}` + ) +} + +async function ensureMetro( + ctx: CaptureContext +): Promise<{ child: ChildProcess; metroPort: number; metroLogPath: string }> { + const pkg = await readPackageJson(ctx.projectPath) + const packageManager = detectPackageManager(ctx.projectPath) + const expo = isExpoProject(pkg) + let metroPort = expo ? null : await findAvailablePort() + const portArgs = ['--port', expo ? '0' : String(metroPort)] + const command = pkg.scripts?.start + ? runScriptCommand(packageManager, 'start', portArgs) + : expo + ? 'npx expo start --port 0' + : `npx react-native start --port ${metroPort}` + log('start Metro', `${command} (poll 1s, timeout ${LAUNCH_TIMEOUT_MS / 1000}s)`) + + const metroLogPath = await beginSessionMetroLog({ + userDataDir: appFlowDataDir(), + projectId: ctx.projectId, + sessionName: ctx.sessionName + }) + const { child, getOutput, getExitCode } = await spawnCommand(command, ctx.projectPath, ctx.signal) + pipeChildToMetroLog(metroLogPath, child) + const watch = (chunk: Buffer): void => { + if (metroPort != null) return + const parsed = parseMetroPortFromOutput(chunk.toString('utf-8')) + if (parsed) metroPort = parsed + } + child.stdout?.on('data', watch) + child.stderr?.on('data', watch) + + try { + await waitWhile( + ctx.signal, + async () => { + const exitCode = getExitCode() + if (exitCode !== null && exitCode !== 0) { + throw new Error( + `"${command}" exited with code ${exitCode}.\n${getOutput().trim() || '(no output)'}` + ) + } + if (metroPort == null) metroPort = parseMetroPortFromOutput(getOutput()) + return metroPort != null && (await isMetroReady(metroPort)) + }, + () => `"${command}" timed out waiting for Metro.\n${getOutput().trim() || '(no output)'}` + ) + log('metro status', `running on ${metroPort}`) + return { child, metroPort: metroPort!, metroLogPath } + } catch (error) { + if (child.pid) child.kill('SIGTERM') + throw error + } +} + +function matchInstalledApp(apps: string[], candidates: Array): string | null { + const installed = apps.map((app) => app.trim()).filter(Boolean) + for (const candidate of candidates) { + if (!candidate) continue + const hit = installed.find( + (app) => app === candidate || app.toLowerCase() === candidate.toLowerCase() + ) + if (hit) return installedAppLaunchTarget(hit) + } + for (const candidate of candidates) { + if (!candidate) continue + const needle = candidate.toLowerCase() + const hit = installed.find((app) => app.toLowerCase().includes(needle)) + if (hit) return installedAppLaunchTarget(hit) + } + return null +} + +function installedAppLaunchTarget(app: string): string { + const formattedApp = app.match(/^.+\s+\(([A-Za-z0-9_.-]+\.[A-Za-z0-9_.-]+)\)$/) + return formattedApp?.[1]?.trim() || app +} + +async function listInstalledApps( + client: AgentDeviceClient, + platform: 'ios' | 'android' +): Promise { + return (await client.apps.list({ platform }).catch(() => [])) as string[] +} + +async function ensureExpoGoInstalled( + client: AgentDeviceClient, + ctx: CaptureContext +): Promise<{ launchTarget: string; installPath: string | null }> { + const apps = await listInstalledApps(client, ctx.platform) + const existing = matchInstalledApp(apps, [EXPO_GO_APP_NAME, 'host.exp.Exponent']) + if (existing) { + log('Expo Go already installed', existing) + return { launchTarget: existing, installPath: null } + } + + const appPath = await downloadExpoGo(ctx) + log('apps.install', appPath) + const result = await client.apps.install({ + appPath, + platform: ctx.platform + }) + return { + launchTarget: result.app || result.appId || EXPO_GO_APP_NAME, + installPath: appPath + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +/** + * agent-device's client throws its `AppError` for every failed call, and that error's `.message` + * is not the only place session-loss wording can land. `replay.run` in particular wraps a divergent + * step in "Replay failed at step N (...): ", and when the divergence handler's own + * post-failure snapshot attempt is what actually hits the dead session, the cause it records is a + * generic `Could not capture a fresh snapshot to verify the recorded target before acting + * (capture-failed)` — the literal "no active session" wording `isSessionLostMessage` matches never + * reaches `.message` at all, only `AppError`'s `details.hint` ("Post-failure snapshot capture + * failed ()"), one level down, where the real wording is interpolated. + * + * This is NOT, however, what was stopping replay scripts from being saved — an earlier version of + * this comment claimed it was, and the 2026-09-23 sweep disproved it: not one replay recovered + * here, because the underlying message was "iOS runner ... is already owned by another + * agent-device daemon", a lease conflict rather than a dead session. That cause is addressed at + * its source by `pinAgentDeviceStateDir`. The hint check stays because a genuinely lost session + * still surfaces through this same wrapper and is worth one recovery attempt, but it is a + * secondary path, not the fix for the lease problem. + */ +function agentDeviceErrorHint(error: unknown): string | undefined { + const details = (error as { details?: unknown } | null | undefined)?.details + const hint = (details as { hint?: unknown } | undefined)?.hint + return typeof hint === 'string' ? hint : undefined +} + +/** + * What agent-device knows about a divergent replay step, flattened into one log line. + * + * `AppError.message` for a `REPLAY_DIVERGENCE` says which step failed and which of the two target + * codes it was, and that is all it says. The reason sits one level down in `details.divergence`, + * which agent-device's own error hint points at — "Read details.divergence (screen/suggestions) + * for repair context" — and which this project has been throwing away at every catch site. + * + * That cost two rounds of inference on the same failure. `IDENTITY_MISMATCH` on + * `press id="bottomBarProfileBtn"` was read as a readiness race, on the reasoning that the + * recorded identity is positional (`"sibling":7,"viewportOrder":0`) and a half-rendered tree would + * index differently; the fix was a recorded `wait stable` after the entry gate, and the 14-09-09 + * sweep recorded that wait, honoured it, and failed on the identical step with the identical code. + * Timing is therefore ruled out and the recorded settle has been reverted. `targetBinding` is the + * field that would have said so in one run rather than two: `mismatches` names the identity + * components that differed and `candidates` lists what else in the tree carried the recorded + * identity, so a positional drift and a genuinely duplicated control are no longer the same + * observation. + * + * Returns undefined rather than throwing or logging on anything unexpected: this runs inside catch + * blocks whose job is to report the original failure, and a diagnostic that can mask the error it + * was added to explain is worse than no diagnostic. + */ +function isMissingSessionError(error: unknown): boolean { + if (isSessionLostMessage(errorMessage(error))) return true + const hint = agentDeviceErrorHint(error) + return hint !== undefined && isSessionLostMessage(hint) +} + +async function openAppSession( + client: AgentDeviceClient, + app: string, + options?: { + url?: string | null + relaunch?: boolean + udid?: string + /** The sweep's ledger, when the caller has one. See `CaptureContext.timing`. */ + timing?: PhaseLedger + } +): Promise { + const launchTarget = installedAppLaunchTarget(app) + const url = options?.url?.trim() || null + // The scheme preapproval is inside the measurement on purpose. It is a `simctl` call of its own, + // it happens only on the open path, and billing it anywhere else would leave the sweep's single + // most repeated device action under-reported by however long it takes. + const open = async (): Promise => { + if (url && options?.udid && shouldPreapproveUrlScheme(url)) { + log('preapprove url scheme', `${url} → ${launchTarget}`) + await preapproveIosUrlScheme({ udid: options.udid, url, bundleId: launchTarget }) + } + log( + 'apps.open', + `${launchTarget}${url ? ` ${url}` : ''}${options?.relaunch ? ' --relaunch' : ''}` + ) + await client.apps.open({ + app: launchTarget, + platform: 'ios', + ...(url ? { url } : {}), + ...(options?.relaunch ? { relaunch: true } : {}) + }) + } + await (options?.timing?.measure('open-url', open) ?? open()) +} + +/** + * Clears the simulator's first-open "Open in ?" confirmation, when there is one to clear. + * + * Skipped outright once `preapproveIosUrlScheme` has written the scheme's launch-services approval + * for this simulator: there is then no prompt to find, and looking for one is not free. The probe + * runs an `alert` command, whose main-thread block enumerates SpringBoard while the app it just + * launched is still cold — the runner abandons that block when the enumeration outlives its budget, + * and then refuses every snapshot behind it with RUNNER_BUSY until it drains. On this sweep that + * cost the first deep link its whole route-settle window and every one after it too, until the + * runner passed its wedge threshold and was recycled. The pre-approval is what actually suppresses + * the prompt; this is only the fallback for when writing it failed. + */ +async function dismissOpenInAppPrompt(device: CaptureDevice, url?: string | null): Promise { + if (device.schemePromptChecked) return + if (!url || !shouldPreapproveUrlScheme(url)) return + if (isUrlSchemePreapproved(device.deviceId, url)) { + device.schemePromptChecked = true + log('ios open-in prompt', 'skipped — scheme preapproved') + return + } + const result = await acceptOpenInAppPrompt(device.client) + if (result === 'accepted' || result === 'absent') { + device.schemePromptChecked = true + } + log('ios open-in prompt', result) +} + +/** + * Runs one `agent-device open` line from a stored `pathToReach`, prompt handling included. + * + * Exported for the manual recorder's Replay button, which walks a path against a `CaptureDevice` + * rather than the `ProfilingDevice` `reachStartScreen` expects. It exists so `openAppSession` and + * `dismissOpenInAppPrompt` can both stay private: the two always go together at the call sites + * inside this module, and a caller that took only the first would reintroduce the first-open + * "Open in ?" sheet that the second is there to clear. + */ +export async function openPathToReachApp( + device: CaptureDevice, + app: string, + url: string | null +): Promise { + await openAppSession(device.client, app, { url, udid: device.deviceId }) + await dismissOpenInAppPrompt(device, url) +} + +/** + * Brings the bound app back after a lost session, and does not return until it is usable again. + * + * The wait is the point. Closing an agent-device session tears down the XCUITest runner, and the + * app under test goes with it, so this is a *cold* launch however cheap the `apps.open` looks: the + * command returns in about two seconds while the JS bundle is still loading and no navigator is + * mounted yet. Both callers hand the app something to do the moment this resolves — one re-opens + * the deep link it was in the middle of, the other screenshots — and a URL delivered into that gap + * is not slow, it is *lost*: React Navigation rejects it ("the 'navigation' object hasn't been + * initialized yet"), the app finishes launching on its default screen, and the arrival judge + * truthfully reports a mismatch for a route that works perfectly well. A device run shows all three + * of its deep links failing exactly this way, each one 1.4s after this relaunch, which was enough + * to trip the breaker and send all 84 routes into full exploration walks. + * + * `waitForAppReady` returns as soon as the app settles, so a warm relaunch pays almost nothing and + * only a genuinely cold start spends the timeout it needs. + */ +export async function reopenBoundApp( + ctx: CaptureContext, + device: CaptureDevice, + timeoutMs = APP_READY_TIMEOUT_MS +): Promise { + await launchBoundApp(device, ctx.timing) + await waitForAppReady(ctx, device, ctx.signal, timeoutMs) +} + +/** + * The launch half of `reopenBoundApp`, without the wait that follows it. + * + * Split out for `waitForAppReady`, which needs to relaunch a stalled app from *inside* its own + * poll loop: calling `reopenBoundApp` there would re-enter `waitForAppReady` and nest one poll + * loop inside another for as long as the app kept stalling. + */ +/** + * Reconnect a dev client that is sitting on its launcher, by opening the bundle URL for it. + * + * This exists because `launchBoundApp` cannot do it. That function opens `device.launchTarget` + * with `relaunch: Boolean(device.launchUrl)`, and a native dev-client build has no `launchUrl` — + * `openInstalledApp` returns `launchUrl: null` for every `native-run` app, since its deep links + * are fired individually rather than at launch. So the recovery reduced to `apps.open` with no URL + * and no relaunch, against a process that was already frontmost: it brought the launcher to the + * front of itself. The 2026-09-23 sweep logged the result exactly — sixteen detections, sixteen + * `dev-client launcher still on screen after a relaunch`, no recoveries. + * + * A plain relaunch would not have fixed it either. The launcher is not a transient state the app + * recovers from on restart; it is what the dev client shows when it has no bundle to run, so + * restarting it shows the launcher again. What clears it is being told where Metro is, which is + * the one thing a person does by hand and the sweep never did. + * + * Returns whether a reconnect was attempted, so the caller can fall back to an ordinary relaunch + * for an app that has no dev client. + */ +async function reconnectDevClient(device: CaptureDevice, timing?: PhaseLedger): Promise { + if (!device.devClientUrl) return false + log('reconnect dev client', device.devClientUrl) + await openAppSession(device.client, device.launchTarget, { + timing, + url: device.devClientUrl, + // The dev client is already running and showing its launcher; the URL is what it needs, not a + // restart. Relaunching first would only throw away the process it is about to load into. + relaunch: false, + udid: device.deviceId + }) + await dismissOpenInAppPrompt(device, device.devClientUrl) + return true +} + +/** + * Restarts the already-running bound app so it comes up in a fresh state, without touching the + * installed build or Metro. + * + * No longer on the adopt path, and the reason is a warning for any future caller: this opens + * `device.launchUrl`, which is `null` for every `native-run` dev-client build (see + * `openInstalledApp`), so on those builds it is a bare `--relaunch` with no bundle URL — and a + * dev client restarted with nothing to load comes up on its *launcher*. The 2026-10-09 social-app + * sweep is the record: a warm, settled Bluesky build was relaunched here at t=65.1s and the sweep + * did not have a real screen again until t=93.6s, having burned the entry-chrome pass, two + * `arrival` model calls against the Expo launcher, and a cached auth verdict read off a dead app. + * `AppFlowBuilder`'s adopt branch now probes readiness and soft-homes instead. + * + * Kept because a caller that genuinely needs the process thrown away still has nothing else, and + * because it is correct for any build that does carry a launch URL (Expo Go). + */ +export async function relaunchBoundApp(ctx: CaptureContext, device: CaptureDevice): Promise { + log('relaunch bound app', device.launchTarget) + await openAppSession(device.client, device.launchTarget, { + url: device.launchUrl, + relaunch: true, + udid: device.deviceId + }) + await dismissOpenInAppPrompt(device, device.launchUrl) + await waitForAppReady(ctx, device, ctx.signal) +} + +/** + * Re-roots the already-running bound app at its launch URL *without* restarting the process — the + * cheapest way back to a known starting screen that exists on an agent-device harness. + * + * Identical to `relaunchBoundApp` above with the one flag flipped, and the flag is the whole + * point. `openAppSession` only passes `relaunch: true` when asked, so this is an `apps.open` that + * hands the running app a URL to navigate to instead of throwing its process away: no bundle + * reload, no splash, no `iOS Bundled 16079ms (6130 modules)` on a cold cache. The app ends up + * where a fresh launch would have put it, which is all a caller that just wants to stop being on + * some half-explored screen actually needs. + * + * Measured through the ledger like the other opens (`open-url`, supplied by `openAppSession`), + * because a soft home that turns out *not* to be cheap is exactly the kind of thing a sweep should + * be able to read off its own timings rather than argue about. + * + * It is not a substitute for `relaunchBoundApp` where state must genuinely be thrown away: an app + * carrying modal stacks, a dirty Redux store or a logged-in-as-someone-else session comes back + * from this with all of it intact. Callers that need a *clean* app still relaunch. + * + * @param options.reconnectDevClientWhenUnloaded + * For a build with no `launchUrl` of its own — which is every `native-run` dev client — re-root + * through `reconnectDevClient` instead. Without it this function is silently a no-op on exactly + * those builds: `openAppSession` with a null URL and no `--relaunch`, against a process that is + * already frontmost, brings the app to the front of itself (`reconnectDevClient` records sixteen + * consecutive recoveries that did nothing at all for this reason). + * + * Opt-in rather than automatic, because the two situations want opposite things from the same + * URL. Opening `devClientUrl` makes the dev client re-fetch and re-evaluate the bundle — that is + * a reload, with the splash and the `6130 modules` build behind it, which on the 2026-10-09 + * social-app sweep killed a warm app and cost 28s to climb back from. Only a caller that has + * already established the app is *not* loaded (the adopt path in `orchestrator.ts`, after a + * floored `waitForAppReady` came back false) may ask for it. `discardRecordingCandidate`, the + * other caller, is soft-homing a live app mid-sweep and must keep the no-op. + */ +export async function goHome( + ctx: CaptureContext, + device: CaptureDevice, + options?: { reconnectDevClientWhenUnloaded?: boolean } +): Promise { + // Suppressed, not reordered or removed. A soft home is an `apps.open` like any other, so issuing + // one inside an armed `--saveScript` window is the "second successful open" that makes the + // recording unpublishable — and this function's only caller mid-sweep + // (`discardRecordingCandidate`) wants a tidy starting screen, which is worth strictly less than + // the script. Its own doc comment already records why skipping is free here: the next site to + // need a transaction opens with `relaunch: true` of its own, so the app is going back to its + // launch screen regardless, just later and at nobody's extra expense. + // + // Only while the transaction is still *alive* — see `suppressesSoftHome`. A recording something + // has already aborted has no publication left to protect, and holding the home back for the rest + // of that walk would cost the tidy screen for nothing. + if (suppressesSoftHome(device.armedRecording)) { + log( + 'go home', + `deferred — a recording is armed (${device.armedRecording?.scriptPath}) and opening the app here would abort it` + ) + return + } + if (options?.reconnectDevClientWhenUnloaded && needsDevClientReconnect(device)) { + // Delegated rather than reimplemented with `url: device.devClientUrl`: that function also + // preapproves the scheme and clears the iOS "Open in?" sheet for this exact URL, and a second + // copy of that sequence is a second thing to keep in step with the dev client. + await reconnectDevClient(device, ctx.timing) + await waitForAppReady(ctx, device, ctx.signal) + return + } + log('go home', `${device.launchTarget}${device.launchUrl ? ` ${device.launchUrl}` : ''}`) + await openAppSession(device.client, device.launchTarget, { + timing: ctx.timing, + url: device.launchUrl, + relaunch: false, + udid: device.deviceId + }) + await dismissOpenInAppPrompt(device, device.launchUrl) + await waitForAppReady(ctx, device, ctx.signal) +} + +/** + * Ends whatever agent-device session is live, swallowing the "there wasn't one" case. + * + * Every `--save-script` call opens a transaction, and agent-device refuses to arm a second one + * over a live first — `stripIdentityAndReplay` documents the refusal it had to work around + * ("would refuse to be replaced"). Since `saveAndReplayRecording` stopped closing the session it + * publishes from (see the `else` branch there), a recording transaction can still be live when the + * next node arms its own, so every site that arms one now ends the previous one here first. + * + * It is free wherever it is called from, which is why it can be unconditional: each of those sites + * continues with a `replay.run` or an `apps.open --relaunch`, and every recorded `.ad` opens with + * `open --relaunch` of its own — so the app was going to be restarted by the very next call + * regardless of whether this closed it. + */ +export async function endLiveSession(device: CaptureDevice): Promise { + await device.client.sessions.close().catch(() => undefined) + // Whatever `--saveScript` transaction that session owned went with it, unfinalized. Clearing the + // marker is what lets `goHome` open the app again — holding the suppression past the end of the + // recording it was protecting would cost every later soft home for nothing. + device.armedRecording = null +} + +async function launchBoundApp(device: CaptureDevice, timing?: PhaseLedger): Promise { + log( + 'reopen bound app', + `${device.launchTarget}${device.launchUrl ? ` ${device.launchUrl}` : ''} --relaunch=${Boolean(device.launchUrl)}` + ) + await openAppSession(device.client, device.launchTarget, { + timing, + url: device.launchUrl, + relaunch: Boolean(device.launchUrl), + udid: device.deviceId + }) + await dismissOpenInAppPrompt(device, device.launchUrl) +} + +async function openInstalledApp( + client: AgentDeviceClient, + ctx: CaptureContext, + mode: LaunchMode, + options: { + expoGoTarget?: string + udid: string + metroPort: number + /** + * The dev-client reconnect URL to launch the native app with, or null to launch it bare. + * + * Only set when the native build was *skipped*. A build that actually ran has already launched + * the app against the Metro it started, so opening it again with a URL would only throw that + * process away. A skipped build has not: the dev client would cold-launch having never been + * told where Metro is this session, land on its own launcher, and pay the recovery round trip + * (reconnect, then relaunch) that the skip was meant to save. + */ + devClientUrl?: string | null + } +): Promise<{ launchTarget: string; launchUrl: string | null; useCustomScheme: boolean }> { + if (mode === 'expo-go') { + const launchTarget = options.expoGoTarget ?? EXPO_GO_APP_NAME + const launchUrl = `exp://127.0.0.1:${options.metroPort}` + await openAppSession(client, launchTarget, { + url: launchUrl, + udid: options.udid, + timing: ctx.timing + }) + return { launchTarget, launchUrl, useCustomScheme: false } + } + + const apps = await listInstalledApps(client, ctx.platform) + const nativeTarget = matchInstalledApp(apps, [ctx.bundleId, ctx.appName, ctx.slug]) + if (!nativeTarget) { + const lookedFor = + [ctx.bundleId, ctx.appName, ctx.slug].filter(Boolean).join(', ') || 'an app name' + const installed = apps.filter(Boolean).slice(0, 20).join(', ') || '(none)' + throw new Error( + `Native ${ctx.platform} app was not installed after running the ${ctx.platform} script. Looked for ${lookedFor}. Installed: ${installed}.` + ) + } + + await openAppSession(client, nativeTarget, { + url: options.devClientUrl ?? null, + udid: options.udid, + timing: ctx.timing + }) + return { launchTarget: nativeTarget, launchUrl: null, useCustomScheme: Boolean(ctx.scheme) } +} + +/** + * The launch target of an installed native build that still matches the project's native sources, + * or `null` when the build has to be run. + * + * `npm run ios` is a full native build, install and launch on every invocation with no + * short-circuit of its own, and on a warm machine it was still the single most expensive step in + * the whole sweep — a measured 25-30s to reinstall a binary that was already on the simulator and + * byte-identical to the one it replaced. It is only genuinely required when the *native* inputs + * changed: a debug dev client fetches its JS bundle from Metro at runtime, which is why editing a + * screen does not invalidate the installed app. So when the fingerprint still matches, Metro is + * started directly and the app is opened against it, which is the state the build would have + * handed back anyway. + * + * Both halves have to hold. The fingerprint says the binary would come out the same; the device + * listing says that binary is still there — a simulator can be erased, or the record can describe + * an app the user deleted by hand, and a stale record is worse than a slow build because every + * later step would fail against an app that is not installed. + * + * `AI_HARNESS_FORCE_NATIVE_BUILD=1` opts out, for the case where the fingerprint is right and the + * binary still is not — a half-finished install, or a native input this does not know to look at. + * + * Ported from the profiling pipeline, which has run this exact check for a while; the shared + * fingerprint/record module now lives in `../native-build` so neither pipeline owns it. The records + * are shared too, in one `userData/native-builds.json` keyed by project path, platform and device — + * so a build this sweep pays for is reused by profiling and debugging, and clearing one pipeline's + * results no longer throws away the other's install. See `createNativeBuildStore`. + */ +async function resolveReusableNativeBuild( + client: AgentDeviceClient, + ctx: CaptureContext, + deviceId: string +): Promise { + if (process.env.AI_HARNESS_FORCE_NATIVE_BUILD === '1') { + log('checkInstalledBuild', 'AI_HARNESS_FORCE_NATIVE_BUILD=1 — rebuilding') + return null + } + const fingerprint = await computeNativeBuildFingerprint(ctx.projectPath, ctx.platform) + if (!fingerprint) { + log('checkInstalledBuild', `no ${ctx.platform}/ sources to fingerprint — rebuilding`) + return null + } + const known = await nativeBuildStore().has(ctx.projectPath, ctx.platform, deviceId) + const recorded = await nativeBuildStore().read(ctx.projectPath, ctx.platform, deviceId, fingerprint) + if (!recorded) { + log( + 'checkInstalledBuild', + known + ? 'native sources changed since the last install — rebuilding' + : 'no install recorded for this device yet — building once' + ) + return null + } + const apps = await listInstalledApps(client, ctx.platform) + const installed = matchInstalledApp(apps, [recorded]) + if (!installed) { + log('checkInstalledBuild', `${recorded} is no longer installed — rebuilding`) + return null + } + return installed +} + +/** + * Boots the simulator this project was last built on, falling back to the daemon's own choice. + * + * Asking for a specific device is what makes the build record worth keeping: the record is keyed by + * device, so letting the daemon pick afresh each run means reuse holds only while that pick happens + * to be stable. It is a preference and never a requirement — the remembered simulator can have been + * deleted, renamed or be missing from a different Xcode, and none of that is a reason to fail a + * sweep that an unpinned boot would have run fine. + */ +async function bootPreferredDevice( + client: AgentDeviceClient, + ctx: CaptureContext +): Promise<{ device: string; id: string }> { + const preferred = await nativeBuildStore().preferredDevice(ctx.projectPath, ctx.platform) + if (preferred) { + log('devices.boot', `platform=ios target=mobile udid=${preferred} (last built here)`) + try { + return (await client.devices.boot({ + platform: 'ios', + target: 'mobile', + udid: preferred + })) as { device: string; id: string } + } catch (error) { + log( + 'devices.boot', + `${preferred} unavailable (${errorMessage(error)}) — letting the daemon pick` + ) + } + } else { + log('devices.boot', 'platform=ios target=mobile') + } + return (await client.devices.boot({ + platform: 'ios', + target: 'mobile' + })) as { device: string; id: string } +} + +export async function bootDevice(ctx: CaptureContext): Promise { + const script = await readPlatformScript(ctx.projectPath, ctx.platform) + const mode = classifyLaunchMode(script) + log('bootDevice', `mode=${mode} session=${ctx.sessionName}`) + + const { createAgentDeviceClient } = await loadAgentDevice() + const client = createAgentDeviceClient({ + session: ctx.sessionName, + cwd: ctx.projectPath, + responseLevel: 'digest' + }) + + log('sessions.close', 'drop leftover session if any') + await closeExistingSession(client) + + const bootResult = await bootPreferredDevice(client, ctx) + log('devices.boot', `device=${bootResult.device} id=${bootResult.id}`) + + const setupCommands = [agentDeviceBootCommand()] + let expoGoTarget: string | undefined + let launchProcess: ChildProcess | null = null + let metroPort: number + let metroLogPath: string | null = null + let reusedNativeBuild = false + if (mode === 'native-run') { + const reusable = await resolveReusableNativeBuild(client, ctx, bootResult.id) + if (reusable) { + log('installNativeApp', `skipped — ${reusable} already installed and unchanged`) + reusedNativeBuild = true + const metro = await ensureMetro(ctx) + launchProcess = metro.child + metroPort = metro.metroPort + metroLogPath = metro.metroLogPath + } else { + const installed = await installNativeApp(ctx, script, bootResult.id) + launchProcess = installed.child + metroPort = installed.metroPort ?? 0 + metroLogPath = installed.metroLogPath + if (!metroPort) { + throw new Error('Native install finished but Metro did not report a port.') + } + } + } else { + const expoGo = await ensureExpoGoInstalled(client, ctx) + expoGoTarget = expoGo.launchTarget + if (expoGo.installPath) { + setupCommands.push(agentDeviceInstallCommand(expoGo.launchTarget, expoGo.installPath)) + } + const metro = await ensureMetro(ctx) + launchProcess = metro.child + metroPort = metro.metroPort + metroLogPath = metro.metroLogPath + log('ensureMetro', `started pid=${metro.child.pid} port=${metroPort}`) + } + const devClientUrl = devClientReconnectUrl(ctx.scheme, mode === 'native-run', metroPort) + const { launchTarget, launchUrl, useCustomScheme } = await openInstalledApp(client, ctx, mode, { + expoGoTarget, + udid: bootResult.id, + metroPort, + devClientUrl: reusedNativeBuild ? devClientUrl : null + }) + if (mode === 'native-run' && !reusedNativeBuild) { + // Hash the sources *after* the build, not before it. `expo run:ios` runs `pod install` and can + // rewrite `Podfile.lock` and the Xcode project as part of the build, so a fingerprint taken up + // front never matches again and no later session would ever skip. + // + // `openInstalledApp` has just proved the app is installed — it throws otherwise — so this + // records a launch target that is known to be on the device. + await nativeBuildStore().record( + ctx.projectPath, + ctx.platform, + bootResult.id, + await computeNativeBuildFingerprint(ctx.projectPath, ctx.platform), + launchTarget + ) + } + log( + 'bootDevice done', + `launchTarget=${launchTarget} launchUrl=${launchUrl ?? 'none'} devClientUrl=${devClientUrl ?? 'none'} metroPort=${metroPort} metroLog=${metroLogPath ?? 'none'} useCustomScheme=${useCustomScheme} nativeBuild=${mode === 'native-run' ? (reusedNativeBuild ? 'reused' : 'built') : 'n/a'}` + ) + + const device: CaptureDevice = { + client, + deviceId: bootResult.id, + deviceName: bootResult.device, + launchProcess, + metroPort, + metroLogPath, + launchTarget, + launchUrl, + devClientUrl, + useCustomScheme, + setupCommands, + schemePromptChecked: false, + lastRead: null, + lastRouteOpenAt: null, + pendingEntryGate: false + } + await dismissOpenInAppPrompt(device, launchUrl) + return device +} + +function buildUrlPath(node: AppFlowNode, params: Record): string { + if (!node.urlPath) throw new Error(`Screen ${node.id} has no deep link`) + const pattern = node.urlPath + /** + * A stand-in here opens whatever record the app happens to have at that id, and the screen that + * comes back is photographed and judged as if it were the one that was asked for — or the 'not + * found' that follows is indistinguishable from a route this build withholds. That is the one + * failure a URL can produce that looks exactly like a finding, and it is why the param store's + * rule is that a value is observed or it does not exist. + * + * Callers already decline to deep-link a route whose params are unfilled (`unresolvedParams`), so + * reaching this is a caller bug, and throwing is what makes it one. A thrown deep link is a + * failed rung, which the ladder falls through exactly as it falls through a skipped one. + */ + const fill = (key: string): string => { + const value = params[key] + if (value === undefined) { + throw new Error( + `Screen ${node.id} has no observed value for :${key} — ${pattern} cannot be opened by URL` + ) + } + return value + } + const urlPath = pattern + .split('/') + .map((seg) => { + if (seg.startsWith(':')) return fill(seg.slice(1)) + if (seg.startsWith('[') && seg.endsWith(']')) { + return fill(seg.slice(1, -1).replace(/^\.\.\./, '')) + } + return seg + }) + .join('/') + + // "/" does not switch the current tab, so a root-path screen needs a stand-in route — and + // `indexDeepLinkPath` only ever returns a grounded one (see its doc comment for the invented + // `bluesky://src` this used to produce). Null means this screen has no URL to open, which is + // `deepLinkStrategy`'s cue to skip; reaching here with null means a caller bypassed that check. + if (urlPath === '/' || urlPath === '') { + const indexPath = indexDeepLinkPath(node) + if (!indexPath) { + throw new Error( + `Screen ${node.id} sits at "/" and this app declares no route group to open instead — it has to be reached through the UI` + ) + } + return indexPath + } + + return urlPath +} + +async function wait(signal: AbortSignal | undefined, ms: number, label: string): Promise { + log(label, `${ms / 1000}s`) + const deadline = Date.now() + ms + while (Date.now() < deadline) { + if (signal?.aborted) throw new Error('Cancelled') + await sleep(Math.min(500, deadline - Date.now())) + } +} + +export async function waitForAppLoaded(signal?: AbortSignal): Promise { + await wait(signal, APP_LOAD_WAIT_MS, 'wait for app to load') +} + +/** + * Waits until the first real screen has mounted, rather than sleeping for a fixed time. + * `APP_LOAD_WAIT_MS` alone is far too short for a cold dev-client launch: the app is still + * bundling, so the screenshot (and any screenshot we hand an agent) is a splash screen. + * + * Polls the accessibility tree and returns once it holds interactive elements that are not + * loading chrome and that stopped changing between two polls. On timeout it returns instead + * of throwing — a slow app should still be captured, just with a warning in the log. + * + * Returns whether the app actually settled. Most callers ignore it and capture regardless, which is + * the right default. It matters to callers that are about to draw a *conclusion* from the frame: + * "the app never settled" and "the app settled on the wrong screen" look identical in a screenshot, + * and only the second is evidence about anything. See the deep-link probe in `orchestrator.ts`, + * where conflating the two switched deep links off for an entire 84-route sweep. + */ +export async function waitForAppReady( + ctx: CaptureContext, + device: CaptureDevice, + signal?: AbortSignal, + timeoutMs = APP_READY_TIMEOUT_MS, + options?: { + /** + * How long to sleep before the first poll *when there is no recent read to compare against*. + * Defaults to `APP_LOAD_WAIT_MS`, which is right after a launch the caller just issued. + * + * No longer the usual path: a read from moments ago makes this sleep measurable instead of + * assumed — see `staleGraceMs` below — and the sleep is then skipped entirely. It remains the + * fallback for the one case nothing can be measured in, a first launch with no prior read, and + * a caller that knows the app is already up (`settleAfterRoute`) still passes 0. + */ + initialWaitMs?: number + /** + * How long an unchanged frame is disbelieved before being accepted, when a read from just + * before the caller's action is available to compare against. Defaults to `APP_LOAD_WAIT_MS`, + * the launch case; `settleAfterRoute` passes `ROUTE_WAIT_MS`, the router-transition case. + * + * This is the budget that replaced both flat sleeps. The sleeps were never waiting for the app + * to load — the poll below does that, with a real deadline — they were waiting out the window + * in which the poll could read the *previous* screen twice and call it the result of the + * action. That window is now detected rather than slept through, so a screen that has already + * arrived pays nothing and only a screen that genuinely has not moved pays anything at all. + * + * A floor, and a live one. Reaching the check at all costs two accessibility reads, and a read + * is cheap: the phase ledger of `session_2026-10-09_17-22-18.log` measures 0.18s each (120 + * calls, 21.9s total). Two agreeing reads therefore take roughly 0.4–0.5s, which is well under + * either grace, so the comparison is not dormant — an unchanged frame genuinely is disbelieved + * for the full grace before it is accepted. That is the intent: it is the only thing standing + * between a fast accessibility read and the bug the sleeps existed for, and it means a screen + * that legitimately does not change, such as the four identical 404s this app's gated routes + * render, pays the whole grace. An earlier version of this comment put a read at 0.9–1.7s and + * concluded the check never fired; that figure predates removing `customActions` from the + * snapshot (see the note in `waitForAppReady`) and was wrong by roughly 8x, as was the + * conclusion drawn from it. Raising either grace raises that cost on every unchanged screen. + */ + staleGraceMs?: number + /** + * The overlap at which a settled frame counts as "still the frame from before the action". + * Defaults to `APP_READY_SETTLE_SIMILARITY`, which is what every caller but `settleAfterRoute` + * wants: after a launch there is no mid-transition composite to catch, only a screen that has + * or has not been replaced. + * + * `settleAfterRoute` passes `APP_READY_STALE_SIMILARITY` instead — see that constant for the + * measured straddle 0.98 let through. + */ + staleSimilarityFloor?: number + /** + * When the stale guard's grace clock starts. Defaults to this call's own start, which is wrong + * by exactly the gap between the action and the wait for the one caller that has an action to + * point at: `apps.open` returned at t=240.461 in `session_2026-10-13_14-33-40.log` and this + * function was entered at t=240.932, so ~0.5s of an 800ms grace was spent before the first + * read. `settleAfterRoute` passes the moment the URL was actually opened + * (`device.lastRouteOpenAt`), so the grace covers the router's transition rather than + * whatever the harness did between the open and the wait. + */ + staleClockStartedAt?: number + /** + * Agreeing pairs the stale guard must reject before `staleGraceMs` is allowed to expire. + * Defaults to 0 — the grace is a pure wall-clock budget, as it was for every caller before + * this. `settleAfterRoute` passes `ROUTE_STALE_MIN_REJECTED_PAIRS`; see that constant for why + * a wall clock alone is a guard that disables itself on fast hardware. + */ + staleMinRejectedPairs?: number + /** + * Interactive-element count a settled frame must reach before it counts as ready. Absent for + * every caller that only wants "two reads agree"; set by a caller that has already decided a + * small frame is not an answer (the deep-link probe's `PROBE_FRAME_ELEMENT_FLOOR`). + * + * This exists because the alternative — calling this function in a loop and checking the count + * between calls — is the wrong shape for a function built to return the moment two reads + * agree. A dev client's loading screen is perfectly stable, so every call returned in about a + * second with the same 2-element frame, and the caller immediately re-entered. Measured on the + * social-app cold start: the same three log lines (`poll 400ms`, `frame is still the one from + * before this action`, `settled with 2 interactive elements after 3 reads in 1.0s`) repeated + * thirteen times between t=45.9s and t=59.6s, 13.7s in all. Each restart discarded the poll + * state and re-paid the stale-frame handshake, and none of the thirteen could ever return + * anything but the answer the caller had just rejected. One call that keeps polling past a + * too-small settle has the same deadline, the same reads and none of that overhead. + * + * The frame still has to settle as well as be large enough: a count that is high only because + * the screen is mid-render is not ready either. And it is a bound, not a promise — on timeout, + * or when `minInteractiveElementsGraceMs` runs out, this returns false like any other unsettled + * wait, and the caller decides what a frame that never grew means. + */ + minInteractiveElements?: number + /** + * How long to keep polling once the frame has *settled* below `minInteractiveElements`, + * measured from that first undersized settle rather than from the call. + * + * Defaults to no limit beyond `timeoutMs`, which is what every caller before this one wanted: + * the deep-link probe would rather spend its whole 30s than judge the transport off a splash. + * A caller that cannot distinguish "still loading" from "legitimately sparse" passes this + * instead — see `SUBSTANTIVE_FRAME_GRACE_MS` and the launch-readiness call in + * `orchestrator.ts`, where the alternative is a login screen holding the sweep for 90s. + */ + minInteractiveElementsGraceMs?: number + /** + * How long to tolerate the dev-client launcher on screen before recovering from it. + * + * Recovery opens the app again, and agent-device aborts an armed recording's publication on + * "a second successful open" — so for the one caller that has a recording armed, the recovery + * below is not a repair, it is what guarantees the script can never be published. Measured on + * social-app: `startReplayRecording` armed at t=67.2s, the relaunch landed on the launcher, + * this wait reconnected at t=69.6s, and the publish 16s later failed with exactly that + * message. Every sweep since has shipped an empty `flows/` directory for the same reason. + * + * Deferring rather than disabling, because the launcher genuinely does not clear itself in the + * case this recovery was written for (a dev client pointed at no Metro, where nobody is going + * to tap the server URL). Spending this long first costs a slow recording the delay and + * nothing else: when the deferral expires the recovery runs exactly as it does today, so the + * worst case is the publication failure we already have, reached later. + */ + deferLauncherRecoveryMs?: number + } +): Promise { + let deadline = Date.now() + timeoutMs + const initialWaitMs = options?.initialWaitMs ?? APP_LOAD_WAIT_MS + // Taken before the first poll, because the poll's own reads overwrite `device.lastRead`. Absent + // only on a first launch, where there is nothing to compare against and the flat sleep below is + // all that is left. + const recentRead = device.lastRead + const frameBeforeAction = + recentRead && Date.now() - recentRead.at <= FRAME_COMPARISON_MAX_AGE_MS + ? fingerprint(recentRead.pressables) + : null + const staleGraceMs = options?.staleGraceMs ?? APP_LOAD_WAIT_MS + const staleSimilarityFloor = options?.staleSimilarityFloor ?? APP_READY_SETTLE_SIMILARITY + const staleMinRejectedPairs = options?.staleMinRejectedPairs ?? 0 + const minInteractiveElements = options?.minInteractiveElements ?? 0 + const minInteractiveElementsGraceMs = options?.minInteractiveElementsGraceMs ?? Infinity + const requestedLauncherDeferralMs = options?.deferLauncherRecoveryMs ?? 0 + const deferLauncherRecoveryMs = launcherDeferralMs( + requestedLauncherDeferralMs, + device.launcherDeferral ?? 'untested' + ) + // Said once per call rather than once per poll: the launcher is read every 400ms and the fact + // being reported ("we are choosing to wait this one out") does not change between reads. + let loggedLauncherDeferral = false + // Tracked apart from `loggedLauncherDeferral` so that a call which defers and *then* runs out + // says both things. Sharing one flag would swallow the second line on exactly the path the + // deferral was built for. + let loggedLauncherRecoveryCause = false + const startedAt = Date.now() + // The stale guard's clock, which is the caller's action rather than this call — see + // `staleClockStartedAt`. Clamped to `startedAt` so a stamp from the future (or from a previous + // sweep, if a caller ever passed a stale one) can only shorten nothing and lengthen nothing. + const staleClockStartedAt = Math.min(options?.staleClockStartedAt ?? startedAt, startedAt) + let heldStaleFrame = false + // Agreeing pairs the stale guard has rejected on this call. Counted separately from + // `sleeps.staleFrame` even though the two move together today, because one is the guard's own + // state and the other is a diagnostic tally, and a later sleep added to this branch would + // silently change what the guard means. + let staleRejectedPairs = 0 + // The element count last reported while holding a settled frame below the floor, so the log says + // each distinct thing once (2, then 2 again, then 15) instead of once per poll. + let lastUndersizedCount: number | null = null + // When the frame first settled below the floor, so `minInteractiveElementsGraceMs` runs from the + // moment there was something to disbelieve rather than from the call — a cold start that spends + // 20s drawing nothing at all has not used any of the grace up. + let undersizedSettledAt: number | null = null + // Only on a first launch, where there is nothing to compare against and a flat sleep is all that + // is left. With a frame in hand the app was alive moments ago, so the poll below starts reading + // immediately and the stale-frame guard — not a sleep — is what keeps it off the pre-action frame. + // + // A one-poll-interval delay was tried here on the strength of the pre-`customActions` sweep, where + // settles landed on exactly 3 reads and 5.7s, matching `1.56 x 3 + 0.4 x 2` to the tenth: the + // signature of a first read that agreed with nothing. Skipping that read looked worth ~1.96s for + // 0.4s. Removing `customActions` from the snapshot then took a read from 1.56s to 0.31s, and the + // measured distribution did not move — 25 of 38 settles still took 3 reads — so the delay bought + // no read at all, while its share of a settle grew from 7% of 5.7s to 20% of 2.0s. The premise was + // specific to reads being expensive, and it is not worth re-deriving from a sweep older than that. + if (!frameBeforeAction && initialWaitMs > 0) { + await wait(signal, Math.min(initialWaitMs, timeoutMs), 'wait for app to load') + } + log('waitForAppReady', `poll ${APP_READY_POLL_MS}ms, timeout ${timeoutMs / 1000}s`) + let previous: ReadonlySet | null = null + // A read that *errors* is not the same as a read that returns nothing: an empty tree means the + // app has not drawn yet and waiting is exactly right, while a failing read means the device is + // not answering us at all (a lost session most often), and no amount of further waiting will + // change that — the caller's next device call is what will surface the real error. Polling on + // regardless would spend this whole timeout per call, which is the difference between a deep + // link costing a second and costing half a minute once a session has dropped. + let consecutiveReadFailures = 0 + // Reads, not polls: the settle's cost is dominated by the accessibility snapshot, and "settled + // with 9 elements" alone never said whether that took the cheapest possible two reads or twenty. + // The two cases want opposite fixes — a faster read versus a screen that will not hold still — + // and the line could not tell them apart. + let reads = 0 + // What each inter-read sleep was waiting on. The `settle-poll` row of the phase table is exactly + // these sleeps (one per read but the last), so a sweep whose row doubled — 51.6s to 160.6s, with + // 245 calls — could say how long it waited and never what it waited *for*. Four causes, four + // different remedies: a launcher still clearing, a pre-action frame still on screen, a screen + // too small to be the app, or one that simply has not stopped changing. + const sleeps = { launcher: 0, staleFrame: 0, undersized: 0, unsettled: 0 } + const sleepSummary = (): string => + `sleeps: ${sleeps.staleFrame} stale-frame, ${sleeps.undersized} undersized, ${sleeps.launcher} launcher, ${sleeps.unsettled} unsettled` + let launcherRecoveries = 0 + let lastLauncherRecoveryAt = 0 + while (Date.now() < deadline) { + if (signal?.aborted) throw new Error('Cancelled') + const pressables = await capturePressables( + ctx, + device, + Math.max(1_000, Math.min(30_000, deadline - Date.now())), + // The poll already clamps each read to what is left; this makes the busy-retry backoff + // respect the same deadline instead of sleeping 42s of waits inside a 30s window. + { deadlineAt: deadline } + ).catch(() => { + consecutiveReadFailures += 1 + return null + }) + reads += 1 + if (pressables !== null) consecutiveReadFailures = 0 + else if (consecutiveReadFailures >= APP_READY_MAX_READ_FAILURES) { + log('waitForAppReady', `device stopped answering after ${consecutiveReadFailures} reads`) + return false + } + // The dev client's launcher is a settled, interactive screen, so every check below would + // happily call it the app and hand it to a judge. It is also the one screen that will never + // resolve itself by waiting — nobody is going to tap the server URL — so the only useful + // response is to put the app back on screen ourselves. + // + // Two different moves, in the order a person would try them. First tell the dev client where + // Metro is, which is what actually clears this screen and what the human fix (tapping a + // "Recently Opened" row) does. Only then fall back to relaunching the app, which is all that + // is available for a launcher-*like* screen this detector caught in an app with no dev client + // to address. A launcher still on screen after both is a genuine failure to report, not retry. + // + // Each move is given `LAUNCHER_RECOVERY_GRACE_MS` before the next one is considered, because + // neither takes effect by the time the very next poll reads the screen: a reconnect has to + // fetch and evaluate the bundle, and until it does the launcher is still what is drawn. + // Escalating on that first read would relaunch the app out from under the reconnect that was + // working, and then report a failure caused by the escalation itself. + if (pressables?.length && isDevClientLauncher(pressables)) { + const deferringRecovery = Date.now() - startedAt < deferLauncherRecoveryMs + const waitingOnRecovery = Date.now() - lastLauncherRecoveryAt < LAUNCHER_RECOVERY_GRACE_MS + if (deferringRecovery) { + if (!loggedLauncherDeferral) { + loggedLauncherDeferral = true + log( + 'waitForAppReady', + `dev-client launcher on screen with a recording armed — waiting up to ${( + deferLauncherRecoveryMs / 1000 + ).toFixed( + 1 + )}s for it to clear, because opening the app again here would abort publication` + ) + } + // The caller's budget was not sized for a bundle build any more than it was for a + // recovery, so the deferral extends the deadline the same way issuing one does. + deadline = Math.max(deadline, startedAt + deferLauncherRecoveryMs) + } else if (!waitingOnRecovery) { + if (requestedLauncherDeferralMs > 0 && !loggedLauncherRecoveryCause) { + loggedLauncherRecoveryCause = true + // Two different facts, and the log has to tell them apart: a deferral that has just run + // out on this call, versus one that was never spent because an earlier call already + // proved this launcher does not clear itself. + log( + 'waitForAppReady', + deferLauncherRecoveryMs > 0 + ? `dev-client launcher did not clear in ${(deferLauncherRecoveryMs / 1000).toFixed(1)}s — recovering, which will abort the armed recording` + : 'dev-client launcher on screen with a recording armed — recovering immediately, an earlier deferral on this device already ran out without it clearing' + ) + } + // Recorded whether or not this call spent the wait: the point is that this device's + // launcher has been observed not to clear on its own, and nothing about the next call + // makes that less true. + if (requestedLauncherDeferralMs > 0) device.launcherDeferral = 'expired' + // Marked before the open, not after: as far as agent-device is concerned the transaction + // is dead the moment the open succeeds, and a recovery that throws on the way back has + // still opened the app. `startReplayRecording` reads this flag the moment this wait + // returns and re-arms on top of whatever the recovery loaded, so the walk that follows is + // recorded into a transaction that can actually publish — see `armed-recording.ts`. + if (noteOpenWhileArmed(device.armedRecording)) { + log( + 'waitForAppReady', + `the recovery below aborts the armed recording (${device.armedRecording?.scriptPath}); it will be re-armed once the app is back` + ) + } + if (launcherRecoveries >= 2) { + log( + 'waitForAppReady', + `dev-client launcher still on screen after ${launcherRecoveries} recovery attempts` + ) + return false + } + const reconnected = + launcherRecoveries === 0 && (await reconnectDevClient(device, ctx.timing)) + if (reconnected) { + log('waitForAppReady', 'dev-client launcher on screen — reconnecting to Metro') + launcherRecoveries = 1 + } else { + log('waitForAppReady', 'dev-client launcher on screen — relaunching the bound app') + await launchBoundApp(device, ctx.timing) + // A relaunch is the last move available, whether it ran first (no dev-client URL to try) + // or second: counting it out here stops the loop asking for a third. + launcherRecoveries = 2 + } + lastLauncherRecoveryAt = Date.now() + // The caller's budget was sized for whatever it was originally waiting on — a router + // transition is 30s, and `settleAfterRoute` passes exactly that. None of those budgets + // were sized for a bundle build, so without this the call would issue the recovery and + // then time out while it was still working, handing the caller a launcher frame to judge + // and a walk to spend on the launcher's own controls. Extended only once a recovery has + // actually been issued, and only far enough to see it through. + deadline = Math.max(deadline, lastLauncherRecoveryAt + LAUNCHER_RECOVERY_GRACE_MS) + } + previous = null + // Deliberately not `APP_READY_POLL_MS`. That interval is tuned for the settle case, where + // the answer can arrive on the very next read and polling fast is what makes it cheap. Here + // the answer cannot arrive for `LAUNCHER_RECOVERY_GRACE_MS` by construction — the recovery + // above has just been issued and the loop is only waiting it out — so every read in that + // window is known-useless, and the runner is documented as fragile under back-to-back + // accessibility snapshot loops (see `openDeepLink`). Pinned to the cadence this branch had + // before the settle poll was sped up. + sleeps.launcher += 1 + await measurePhase(ctx, 'settle-poll', () => sleep(Math.max(APP_READY_POLL_MS, 1_500))) + continue + } + if (pressables?.length && !isLoadingScreen(pressables)) { + const current = fingerprint(pressables) + // Compared by token overlap, not by exact equality of the control list. Exact equality asks + // the screen to be byte-identical between two polls a second and a half apart, which a static + // settings page manages and a *feed* never does: a post loads, a like count ticks, an avatar + // resolves, and the list differs forever. Every content screen therefore ran the full 90s and + // logged "still loading" while sitting fully rendered in front of the poll. Overlap answers + // the question actually being asked — is this still the same screen — and the same 0.98 bar + // the exploration loop already uses for "that action changed nothing" applies here. + if (previous && similarity(previous, current) >= APP_READY_SETTLE_SIMILARITY) { + // Two reads agreeing is necessary but not sufficient: the screen from *before* the action + // is the most stable thing there is, precisely because nothing is happening on it. See + // `isFrameUnchangedSinceAction`. + if ( + frameBeforeAction && + isFrameUnchangedSinceAction({ + before: frameBeforeAction, + settled: current, + // The grace clock is not allowed to run at all until the guard has actually rejected + // `staleMinRejectedPairs` agreeing pairs — see that option. Reporting 0 elapsed is how + // that is said through a function whose contract is "elapsed versus grace", and it + // keeps `isFrameUnchangedSinceAction` free of a second, device-shaped concept. + elapsedMs: + staleRejectedPairs >= staleMinRejectedPairs ? Date.now() - staleClockStartedAt : 0, + graceMs: staleGraceMs, + similarityFloor: staleSimilarityFloor + }) + ) { + if (!heldStaleFrame) { + heldStaleFrame = true + log( + 'waitForAppReady', + `frame is still the one from before this action — waiting up to ${staleGraceMs / 1000}s for it to change` + ) + } + staleRejectedPairs += 1 + previous = current + sleeps.staleFrame += 1 + await measurePhase(ctx, 'settle-poll', () => sleep(APP_READY_POLL_MS)) + continue + } + // Stable but too small to be the app — see `minInteractiveElements`. Checked after the + // stale-frame guard so the two holds compose: a frame must be both new and substantial. + // `previous` is kept, so the moment the count clears the floor the very next read settles + // without a fresh agreeing pair. + if (pressables.length < minInteractiveElements) { + undersizedSettledAt ??= Date.now() + const heldForMs = Date.now() - undersizedSettledAt + if (heldForMs >= minInteractiveElementsGraceMs) { + // Out of grace with a settled frame in hand. Returning false rather than true keeps + // the floor honest: the caller asked whether it got a substantive frame, and it did + // not. Callers that capture regardless are unaffected — they ignore the return. + log( + 'waitForAppReady', + `stable at ${pressables.length} interactive elements for ${(heldForMs / 1000).toFixed(1)}s without reaching ${minInteractiveElements} — treating this as the app's own screen` + ) + return false + } + if (lastUndersizedCount !== pressables.length) { + lastUndersizedCount = pressables.length + log( + 'waitForAppReady', + `stable at ${pressables.length} interactive elements, below the ${minInteractiveElements} required — still waiting for the app to finish loading` + ) + } + previous = current + sleeps.undersized += 1 + await measurePhase(ctx, 'settle-poll', () => sleep(APP_READY_POLL_MS)) + continue + } + log( + 'waitForAppReady', + `settled with ${pressables.length} interactive elements after ${reads} reads in ${((Date.now() - startedAt) / 1000).toFixed(1)}s (${sleepSummary()})` + ) + return true + } + previous = current + } else { + previous = null + } + sleeps.unsettled += 1 + await measurePhase(ctx, 'settle-poll', () => sleep(APP_READY_POLL_MS)) + } + log( + 'waitForAppReady', + `still loading after ${timeoutMs / 1000}s and ${reads} reads, capturing anyway (${sleepSummary()})` + ) + return false +} + +/** + * A flat post-action sleep. Deliberately still a sleep: no caller of this function is followed by + * the before/after frame comparison that `settleAfterRoute` relies on to do without one. + * + * `ROUTE_WAIT_MS` is a *bound* only for `settleAfterRoute`, because that path alone goes through + * `waitForAppReady`, which captures `device.lastRead` as the pre-action frame. Every caller of + * this function skips that: `executeNavigationActions` sleeps after each tap, input and swipe and + * then returns to the next action or the next screenshot with no readiness poll in between; + * `probeUnmatchedRoute` follows it with a single `capturePressables`; and the manual-mode replay + * and session recorders do the same. Removing the sleep there would not hand the decision to a + * comparison, it would hand it to nothing, so the screenshot or read after a tap could land on the + * frame from before the tap and the vision judge would report a wrong screen — the failure that + * costs a full exploration, against 0.8s per tap saved. + * + * Even for `settleAfterRoute` the comparison is live (see `staleGraceMs`: a read costs 0.18s per + * `session_2026-10-09_17-22-18.log`, so two reads finish well inside the grace), but it is + * evidence that a *deep link* does not need the sleep, not that a tap does. The ~30 `wait for route — 0.8s` lines per sweep are all this + * function, from `executeNavigationActions`. Replacing them safely needs a post-tap settle that + * compares against the pre-tap frame, which is a behavioural change in the exploration loop and + * not something this sleep can be shortened into. + */ +export async function waitForRoute(signal?: AbortSignal, timeoutMs = ROUTE_WAIT_MS): Promise { + await wait(signal, Math.min(ROUTE_WAIT_MS, timeoutMs), 'wait for route') +} + +/** + * What to wait for after opening a deep link, before judging where it landed. + * + * A flat sleep is only ever right when the app was already on screen and settled — then the router + * has one transition to run and a couple of seconds covers it. It is wrong in the case this sweep + * hits constantly: a reach that lost its session leaves the next node's deep link firing into an + * app the recovery path has just relaunched, so when the sleep ends the frame is the splash screen + * or the home feed the cold start landed on, the judge honestly reports a mismatch, and a route + * that works perfectly from a terminal gets recorded as broken. So this is the stability poll + * `waitForAppReady` owns, with no launch sleep in front of it. + * + * The router beat is still accounted for, but as a bound rather than a cost: the settled *previous* + * screen is exactly what the poll would otherwise lock onto, and `waitForAppReady` now recognises + * it by comparison instead of being slept past it. + * + * Returns `waitForAppReady`'s own verdict — whether the app settled — for callers about to judge + * the frame that follows. See that function's doc comment. + */ +export async function settleAfterRoute( + ctx: CaptureContext, + device: CaptureDevice, + signal?: AbortSignal, + timeoutMs = ROUTE_SETTLE_TIMEOUT_MS, + options?: { minInteractiveElements?: number } +): Promise { + // Only an open from moments ago can be what this settle is waiting on. Not every caller of this + // function follows an `openDeepLink` — the deep-link probe's launch-readiness wait runs *before* + // its open — and anchoring the guard's clock to a URL opened a minute ago would start the grace + // already expired, which is the exact failure this change exists to remove. + const openedAt = device.lastRouteOpenAt + const staleClockStartedAt = + openedAt !== null && Date.now() - openedAt <= ROUTE_OPEN_CLOCK_MAX_AGE_MS ? openedAt : undefined + return waitForAppReady(ctx, device, signal, timeoutMs, { + minInteractiveElements: options?.minInteractiveElements, + // No sleep in front of the poll, and none needed: the pre-open frame is what the sleep was + // guarding against being mistaken for the destination, and `waitForAppReady` now compares + // against it directly. `ROUTE_WAIT_MS` survives as the ceiling on that comparison — the + // router beat it was always sized for — rather than as a cost every route pays. + initialWaitMs: 0, + staleGraceMs: ROUTE_STALE_GRACE_MS, + // The three things `session_2026-10-13_14-33-40.log` showed the guard needed and did not have: + // a floor that recognises a mid-transition composite (0.98 did not — see + // `APP_READY_STALE_SIMILARITY`), a clock that starts at the open rather than at this wait + // (~0.5s of the old 800ms went to the gap between them), and a minimum number of rejections so + // a fast read cannot settle before the guard has judged anything at all (PostLikedBy's first + // agreeing pair landed at 1.18s, past the whole grace). + staleSimilarityFloor: APP_READY_STALE_SIMILARITY, + staleClockStartedAt, + staleMinRejectedPairs: ROUTE_STALE_MIN_REJECTED_PAIRS + }) +} + +async function openUrlInSession( + ctx: CaptureContext, + device: CaptureDevice, + app: string, + url: string | null, + options?: { relaunch?: boolean } +): Promise { + try { + await openAppSession(device.client, app, { + url, + relaunch: options?.relaunch, + udid: device.deviceId, + timing: ctx.timing + }) + } catch (error) { + if (!isMissingSessionError(error)) throw error + log( + 'openApp', + `no active session, recovering — while opening ${app}${url ? ' with a URL' : ''} — ${describeSessionLoss(errorMessage(error), agentDeviceErrorHint(error))}` + ) + // `reopenBoundApp` waits for the app to be ready before returning, which is what makes the + // retry below a delivery rather than a URL thrown at a launching process — see its comment. + await reopenBoundApp(ctx, device) + await openAppSession(device.client, app, { + url, + relaunch: options?.relaunch, + udid: device.deviceId, + timing: ctx.timing + }) + } + await dismissOpenInAppPrompt(device, url) +} + +export function resolveDeepLink( + ctx: CaptureContext, + device: CaptureDevice, + node: AppFlowNode, + params: Record +): { app: string; url: string } { + const urlPath = buildUrlPath(node, params) + if (device.useCustomScheme && ctx.scheme) { + return { + app: device.launchTarget, + url: `${ctx.scheme}://${urlPath.replace(/^\//, '')}` + } + } + if (!device.launchUrl) { + throw new Error(`Screen ${node.id} has no usable native URL scheme`) + } + return { + app: device.launchTarget, + url: `exp://127.0.0.1:${device.metroPort}/--${urlPath}` + } +} + +export function buildAgentDevicePathToReach( + ctx: CaptureContext, + device: CaptureDevice, + node: AppFlowNode, + params: Record, + opened?: { app: string; url: string } +): string[] { + const target = opened ?? resolveDeepLink(ctx, device, node, params) + return [...device.setupCommands, agentDeviceOpenCommand(target.app, target.url)] +} + +/** + * What one deep-link open produced: where it was sent, and what was on screen just before it went. + * + * `preOpenPressables` is not new work — it is the liveness probe's own read (or the recent read + * that stood in for it), which this function already had in hand and threw away. Returning it is + * what lets the caller ask the question nothing in the sweep could previously answer: whether the + * frame it is about to photograph is the destination or the screen the app was already on. See + * `reach/frame-stability.ts`. Null when the probe read failed, which is not evidence either way. + */ +export interface DeepLinkOpenResult { + app: string + url: string + preOpenPressables?: AppFlowPressable[] | null +} + +export async function openDeepLink( + ctx: CaptureContext, + device: CaptureDevice, + node: AppFlowNode, + params: Record +): Promise { + const urlPath = buildUrlPath(node, params) + log('openDeepLink', `${node.id} → ${urlPath}`) + + // Make sure the app is actually up before handing it a URL. A deep link delivered to a process + // that is still cold-starting is the one the app drops: it finishes its own launch and lands on + // whatever it opens by default, which is why these routes open correctly from a terminal (where + // the app is already running) and mismatch here. The sweep reaches this state routinely — a lost + // session makes the delivery capture relaunch the app, and the very next node's deep link is + // then the first thing to touch it. + // + // One read, not a settle loop. The question here is only "is this process alive and drawing", + // and a single snapshot that comes back with interactive elements answers it — settling is the + // job of the `settleAfterRoute` poll that runs *after* the URL is opened, which is the one whose + // answer anything depends on. Running a full `waitForAppReady` here as well made this the most + // expensive thing in the sweep that nothing reads: two back-to-back polling loops around every + // deep link, each firing accessibility snapshots every 1.5s. That is precisely the load that + // blows the iOS runner's execution watchdog, so the harness was reliably wedging the runner just + // before the read it actually cares about — on device, every single deep link logged + // `runner busy` retries on the second poll and then gave up with "still loading after 30s". + // + // The cold-start case this exists for is still covered: an empty or failed read falls through to + // the full wait, which is the only situation that ever needed it. + // + // `retries: 0` is load-bearing, and its absence undid everything the paragraph above describes. + // A read inherits the busy-retry schedule by default, so this 5s probe was quietly sleeping + // 5 + 12 + 25 = 42s whenever the runner was busy — which, on the path this exists for, is + // *always*: the previous screen's activity is what wedged the runner in the first place. The + // probe then fell through to the 30s wait anyway, so the cheap pre-check cost 72s. A device log + // shows it eating a 60s probe budget whole before the URL was even opened, leaving the arrival + // judge no time to run and the deep-link probe permanently "inconclusive". + // + // Zero retries is the right number rather than a smaller schedule: a busy runner already means + // "not ready", which is exactly the fallback branch below, and that branch's own poll retries + // against a real deadline. + // The dev client's launcher is fully rendered and interactive — "Development Servers", + // "Recently Opened", "Enter URL manually" are all real pressables — so a bare "did we get + // anything back" check calls it alive and hands it the deep link below. The launcher has no + // router behind it to receive that link, so the app then "opens" straight into whatever the + // launcher itself does with an arbitrary URL tap, which is not the route this call asked for. + // `alive` is already in hand from the probe above, so this costs no extra device call, and + // falling through to `waitForAppReady` is correct because that is the function that owns + // launcher recovery (reconnect to Metro, then relaunch — see its comment above). + // + // The read itself is skipped when one from moments ago is already in hand — see + // `APP_ALIVE_REUSE_MS`. The checks below then run against that read instead, unchanged: a reused + // answer is held to exactly the same bar as a fresh one, so a stale launcher or loading screen + // still falls through to the full `waitForAppReady`. + const recent = device.lastRead + const reusable = recent && Date.now() - recent.at <= APP_ALIVE_REUSE_MS ? recent.pressables : null + if (reusable) log('openDeepLink', 'app was alive moments ago, skipping the liveness probe') + const alive = + reusable ?? + (await capturePressables(ctx, device, APP_ALIVE_PROBE_TIMEOUT_MS, { + retries: 0 + }).catch(() => null)) + if (!alive?.length || isLoadingScreen(alive) || isDevClientLauncher(alive)) { + await waitForAppReady(ctx, device, ctx.signal, ROUTE_SETTLE_TIMEOUT_MS, { initialWaitMs: 0 }) + } + // The last thing read before the URL goes out, which is the probe's own frame on the hot path + // and `waitForAppReady`'s final poll on the cold one. Taken here rather than reconstructed by + // the caller, because by the time the caller sees anything the app has already moved. + const preOpenPressables = device.lastRead?.pressables ?? alive ?? null + // Stamped on the way out of each `apps.open` below rather than once up here: the grace this + // feeds is the *router's* transition beat, and the probe and launcher recovery above can spend + // seconds before the URL is ever handed over. See `CaptureDevice.lastRouteOpenAt`. + + if (device.useCustomScheme && ctx.scheme) { + const deepPath = urlPath.replace(/^\//, '') + const url = `${ctx.scheme}://${deepPath}` + try { + await openUrlInSession(ctx, device, device.launchTarget, url) + device.lastRouteOpenAt = Date.now() + return { app: device.launchTarget, url, preOpenPressables } + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + if (!/failed to open/i.test(message)) throw error + } + } + + if (!device.launchUrl) { + throw new Error(`Screen ${node.id} has no usable native URL scheme`) + } + + const metroUrl = `exp://127.0.0.1:${device.metroPort}/--${urlPath}` + const app = device.launchTarget + await openUrlInSession(ctx, device, app, metroUrl) + device.lastRouteOpenAt = Date.now() + return { app, url: metroUrl, preOpenPressables } +} + +/** + * One bounded read of the current screen, answering only "is this the dev-client launcher right + * now" — nothing more. This exists because `waitForAppReady`'s boolean return conflates two + * different failures: "the app never came up" and "the app came up on the wrong screen". A caller + * outside `waitForAppReady` that gets `false` back cannot tell which one happened, and the states + * loop in orchestrator.ts needs exactly that distinction to decide whether to escalate to the user + * (a stuck launcher, which a person can clear in seconds by tapping into the app) or to treat it as + * an ordinary readiness timeout. + * + * A failed read is reported as "not the launcher", never as "is the launcher": a device that will + * not answer at all is a different, unrelated problem (a lost session, most often), and the + * caller's next device call is what should surface that error rather than this helper papering + * over it with a launcher diagnosis it has no evidence for. + */ +export async function isOnDevClientLauncher( + ctx: CaptureContext, + device: CaptureDevice, + timeoutMs = APP_ALIVE_PROBE_TIMEOUT_MS +): Promise { + const pressables = await capturePressables(ctx, device, timeoutMs, { retries: 0 }).catch( + () => null + ) + return !!pressables?.length && isDevClientLauncher(pressables) +} + +/** + * How long `startReplayRecording` waits for the dev-client launcher to clear on its own before + * letting `waitForAppReady` recover from it the usual way. + * + * Sized off the bundle load the recovery exists to avoid racing: the cold-cache figure + * `PROBE_LAUNCH_READY_BUDGET_MS` is built from is `iOS Bundled 16079ms (6130 modules)`, and a warm + * relaunch on the measured run cleared in about 2.5s. Twenty seconds covers the cold case with + * room over it, and is spent only when the launcher is actually on screen. + */ +const RECORDING_LAUNCHER_PATIENCE_MS = 20 * 1000 + +/** + * Start a clean agent-device recording. Exactly one app open is recorded. + * + * **One caller now: the manual recorder** (`manual/session.ts`). The sweep used to open every + * exploration walk with this, purely so the recorded script's first line would be + * `open --relaunch` — a cold start, a bundle reload and the launcher deferral below, paid once per + * walk for the artefact rather than for the walk, which on the 2026-10-13 sweep cost ~173s of 683s + * and published one script out of seven explorations. Exploration now writes down the commands it + * walked instead (`walk-commands.ts`) and resets with a soft re-root (`DevicePort.resetToLaunch`), + * so nothing in the sweep arms a recording any more. + * + * Everything below — the relaunch, the recorded `wait stable`, `RECORDING_LAUNCHER_PATIENCE_MS`, + * the re-arm, `armed-recording.ts` and `launcher-deferral.ts` — therefore survives for the manual + * recorder alone, which still publishes a real `.ad` script and still needs every one of them. + * None of it is reachable from a sweep. + */ +export async function startReplayRecording( + ctx: CaptureContext, + device: CaptureDevice, + scriptPath: string, + launchUrl: string | null = device.launchUrl, + timeoutMs = APP_READY_TIMEOUT_MS +): Promise { + try { + // Load-bearing, and more so now that the sweep leaves sessions open: a publish + // (`saveAndReplayRecording`'s non-repair branch) and a discard (`discardRecordingCandidate`) + // both hand this function a live, still-armed recording, and agent-device will not arm the + // `saveScript` open below over an existing transaction. This close is what discards it — + // unfinalized, so nothing is written — and it is also the only place the abandoned recordings + // those two paths leave behind are reliably cleaned up. + // + // The `relaunch: true` that follows is equally load-bearing, and for a reason that has nothing + // to do with getting back to a known state: it is the *recorded* first line of the candidate. + // A script whose opening step is `open --relaunch` replays standalone from any device state, + // which is the property every seed and every saved-replay rung depends on. Replacing it with a + // soft `goHome` would record a script that only replays when the app happens to already be + // running, so the cold start here is the price of the artefact, not of this walk. + await device.client.sessions.close() + } catch { + // The first recording attempt may not have an active session yet. + } + // The relaunch restarts the app cold, so the first screen is a splash again: the navigation + // actions that follow would tap into a loading screen and fail every attempt. + // + // Two waits, because they solve the same problem on two different clocks. `waitForAppReady` + // below polls accessibility from this process and is what makes the *live* walk safe — but it + // is our code, not a session command, so agent-device never sees it and it leaves no trace in + // the recorded script. At replay time the script therefore went straight from `open --relaunch` + // to the walk's first `press`, into an app that was still starting, and every explored journey + // died with "the recorded target no longer matches any current element" on its first action. + // + // `command.wait({ stable: true })` is a session command, so it *is* recorded, emitting a + // `wait stable ` line right behind the open — which is precisely what + // agent-device documents it for ("the fallback for open/relaunch/navigation"). It deliberately + // does not replace the destination guard `saveAndReplayRecording` appends: a stable wait is not + // a portable destination guard, and the guard has to be a selector wait on an identified + // landmark. This one only has to keep the replay from acting on a splash screen. + // + // Best-effort on purpose. A UI that never goes quiet within the budget — an indefinite spinner, + // a looping splash animation — throws `wait_stable_timeout` here, and letting that propagate + // would abandon a recording that `waitForAppReady` may still be able to rescue. Swallowing it + // costs nothing: the line is simply absent from the script, which is exactly where we were. + async function recordStableWait(): Promise { + try { + await measurePhase(ctx, 'device-wait', () => + device.client.command.wait({ + platform: ctx.platform, + stable: true, + quietMs: APP_STABLE_QUIET_MS, + timeoutMs: Math.max(1_000, Math.min(timeoutMs, APP_STABLE_TIMEOUT_MS)) + }) + ) + } catch (error) { + log('startReplayRecording', `stable wait not recorded — ${errorMessage(error)}`) + } + } + /** + * The open that arms the transaction, plus the recorded stable wait that has to sit behind it. + * + * A closure only so the re-arm below can reuse it without a second copy of the + * `saveScript`/`force` call and the `wait stable` line every recorded script needs in front of + * its first press. `relaunch` is the one thing that differs between the two uses, and the one + * thing that matters — see the re-arm's own comment. + */ + const arm = async (relaunch: boolean): Promise => { + await measurePhase(ctx, 'open-url', () => + device.client.apps.open({ + app: installedAppLaunchTarget(device.launchTarget), + platform: ctx.platform, + udid: device.deviceId, + ...(relaunch ? { relaunch: true } : {}), + saveScript: scriptPath, + force: true, + ...(launchUrl ? { url: launchUrl } : {}) + }) + ) + // Set the moment the open returns, so every `apps.open` this module issues from here on knows + // it would be the second one — `goHome` suppresses itself and the launcher recovery marks the + // transaction dead on its way past. See `armed-recording.ts`. + device.armedRecording = armRecording(scriptPath) + await dismissOpenInAppPrompt(device, launchUrl) + await recordStableWait() + } + await arm(true) + // The one caller that must not let this wait recover from the dev-client launcher: the open + // above armed the recording, and the recovery's own `apps.open` is the "second successful open" + // that aborts publication. See `deferLauncherRecoveryMs`. + // + // The substantive-frame floor is here for the reason it is everywhere else — stable is not + // loaded — but this was the last open in the sweep still without it, and the cost was visible in + // the 14-33-20 log: the arming open `settled with 3 interactive elements after 2 reads in 0.6s`, + // the planner was handed that splash as its first screenshot, and it spent a model call + // (1.5s, 1653 tokens) answering `{"type":"wait","duration":2000}` — buying with a round trip + // exactly what this wait is supposed to provide for free. Every recorded walk paid it. + // + // Graced rather than held to `timeoutMs` for the same reason as `manual/replay.ts`: a walk may + // legitimately begin on a sparse screen, and refusing to record one at all is worse than + // recording one that starts thin. + await waitForAppReady(ctx, device, ctx.signal, timeoutMs, { + deferLauncherRecoveryMs: RECORDING_LAUNCHER_PATIENCE_MS, + minInteractiveElements: PROBE_FRAME_ELEMENT_FLOOR, + minInteractiveElementsGraceMs: SUBSTANTIVE_FRAME_GRACE_MS + }) + // The deferral above buys time; it does not buy the outcome. On a dev client pointed at no Metro + // the launcher never clears on its own, so the wait runs out and recovers with an `apps.open` of + // the bundle URL — and from that moment the transaction armed a few lines up is dead, whatever + // the walk that follows manages to prove. That is the whole of the 2026-10-13 sweep's failure: + // six armed walks, six launcher recoveries, five verified arrivals thrown away at the publish + // with `aborted by a second successful open` (t=406.1, 456.2, 623.8, 679.7, 743.1), and the one + // node whose arming open happened to skip the launcher (`MyProfile`, t=153.4) the only one that + // produced a script. + // + // So the dead transaction is replaced rather than walked into. The close discards it without + // writing anything (the same discard this function opens with), and a second `saveScript` open + // arms a fresh one on top of the app the recovery has just finished loading. + // + // `relaunch: false` for that second open, and it is the only honest choice available: a + // relaunch is what put the dev client on its launcher in the first place, so re-arming with one + // would land on the launcher again, recover again, and abort again — the loop this is here to + // break. The cost is the property the opening comment of this function calls load-bearing: the + // re-armed script starts `open ` rather than `open --relaunch `, so it replays against + // a running app rather than from any device state. That is a weaker artefact than the cold-start + // script, and a strictly better one than the empty `flows/` directory this path has shipped on + // every sweep so far. It is also confined to exactly the devices that cannot produce the strong + // one: a launcher that clears within the deferral never reaches this branch at all. + if (needsRearm(device.armedRecording)) { + log( + 'startReplayRecording', + `re-arming ${scriptPath} after the launcher recovery — its open aborted the first transaction; the replacement records "open" rather than "open --relaunch"` + ) + try { + await device.client.sessions.close() + } catch { + // Same tolerance as the opening close: an aborted transaction may have taken the session + // with it, and there is nothing to discard if it did. + } + await arm(false) + // Without a relaunch there is no cold start to wait out, so this is only the substantive-frame + // floor the walk's first screenshot needs — and the launcher deferral is deliberately not + // requested again: the app is already loaded, and a launcher appearing here would mean the + // re-arm itself failed, which the walk's own first observation reports better than a second + // 20s wait would. + await waitForAppReady(ctx, device, ctx.signal, timeoutMs, { + minInteractiveElements: PROBE_FRAME_ELEMENT_FLOOR, + minInteractiveElementsGraceMs: SUBSTANTIVE_FRAME_GRACE_MS + }) + } + // The walk's first press is the one that has to survive a cold replay — see `pendingEntryGate`. + device.pendingEntryGate = true +} + +/** + * Press one locator, narrowing and retrying once if agent-device reports it ambiguous. + * + * agent-device refuses an ambiguous selector rather than guessing, and a caller deriving locators + * from `capturePressables` cannot see the ambiguity coming: that list deduplicates by the exact + * `{text,label,identifier,role}` object it builds, so two controls identical in all four — which + * Bluesky's two `label="Open drawer menu"` buttons are — collapse into one entry. Every locator + * derived from it therefore looks unique to us and is ambiguous on device. `hittable=true` is the + * narrowing that matches how the list was built in the first place (one of a rendered-but-off-canvas + * pair is not tappable), and it is tried only after the plain locator has actually failed this way, + * so the common single-match tap keeps its existing one-call cost and behaviour. + * + * Shared by `executeNavigationActions` (the sweep) and the manual recorder, which issues one press + * per user click and hits the same ambiguity on the same screens. + */ +export async function pressWithLocator( + ctx: CaptureContext, + device: CaptureDevice, + locator: { by: 'text' | 'label' | 'id'; value: string }, + timeoutMs = 10_000 +): Promise { + if (device.pendingEntryGate) await recordEntryGate(ctx, device, locator) + const pressWith = (qualifiers?: Readonly>): Promise => + measurePhase(ctx, 'press', () => + device.client.interactions.press({ + platform: ctx.platform, + ...toAgentDeviceTarget(parseInteractionTarget(serializeLocator(locator, qualifiers))), + settle: true, + settleQuietMs: SETTLE_QUIET_MS, + timeoutMs: Math.max(1, Math.min(10_000, timeoutMs)) + }) + ) + try { + return didSettle(await pressWith()) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + if (!isAmbiguousSelectorMessage(message)) throw error + log( + 'pressWithLocator', + `${serializeLocator(locator)} matched several elements, retrying with hittable=true` + ) + return didSettle(await pressWith({ hittable: true })) + } +} + +/** + * Whether agent-device actually observed the UI go quiet after the action, rather than merely + * having been asked to. + * + * `settle` is documented as best-effort and never fails the action, so a `settle: true` request is + * a request and the result is the answer. The distinction is the whole basis for skipping the flat + * post-action sleep in `executeNavigationActions`: a settle that happened has already waited for + * exactly what the sleep was approximating, and a settle that did not leaves nothing behind it, so + * the sleep is still the only thing standing between the next screenshot and the pre-tap frame. + */ +function didSettle(result: unknown): boolean { + return (result as { settle?: { settled?: boolean } } | null)?.settle?.settled === true +} + +/** + * Writes a `wait` on this press's own target into the recording, immediately ahead of the press. + * + * `startReplayRecording` already explains why `waitForAppReady` leaves no trace in a recorded + * script and why a `wait stable` line was put behind the relaunch to stand in for it. The measured + * run says that stand-in is not enough: three of four explored journeys still died at their + * script's *first* press with "the recorded target no longer matches any current element", and + * `profile.ad` — a script that had just replayed successfully — failed the same way twenty seconds + * later when the next screen tried to seed from it. An unchanged script failing intermittently + * against an unchanged app is a readiness race, not locator drift. + * + * Stability is simply the wrong question, and `probe-route.ts` already paid to learn it: a splash + * screen is perfectly stable, and this app's first readiness check settled on a dev-client loading + * frame carrying 2 interactive elements. `wait stable` is satisfied by that frame, the recorded + * press fires into it, and there is no tab bar there to press. A wait on the press's own target + * cannot be satisfied early, because the element it names is not on a splash screen. + * + * Why the press's own target and not a landmark of the entry screen: a landmark has to be *chosen*, + * and the obvious choice — the first identified control on the settled frame — is routinely a piece + * of feed content whose id does not exist on the next launch. That gate would expire on every + * replay and cost the full timeout before pressing anyway. The press target needs no choosing and + * is exactly the element the replay is about to require. + * + * Only the first press of a recording is gated. It is the one that runs against a cold app, and + * the measured failures are all on it; every later press happens on a screen the previous action + * already brought up. Gating all of them would also put this timeout in front of every genuinely + * stale locator during a flailing walk, which is a real cost for no measured benefit. + * + * Best-effort, like the stable wait it backs up: when the wait itself fails the press still runs, + * because the press's own error is the more informative one and swallowing this leaves the script + * exactly where it already was. + */ +async function recordEntryGate( + ctx: CaptureContext, + device: CaptureDevice, + locator: { by: 'text' | 'label' | 'id'; value: string } +): Promise { + // Cleared before the attempt, not after it: this is the first press either way, and a failed + // gate must not leave the flag set for the next press on a screen that never needed one. + device.pendingEntryGate = false + const selector = serializeLocator(locator) + try { + await measurePhase(ctx, 'device-wait', () => + device.client.command.wait({ + platform: ctx.platform, + selector, + timeoutMs: ENTRY_GATE_TIMEOUT_MS + }) + ) + log('pressWithLocator', `entry gate ${selector}`) + } catch (error) { + log('pressWithLocator', `entry gate ${selector} not recorded — ${errorMessage(error)}`) + } +} + +export async function executeNavigationActions( + ctx: CaptureContext, + device: CaptureDevice, + actions: readonly AppFlowNavigationAction[], + timeoutMs?: number +): Promise { + const deadline = Date.now() + (timeoutMs ?? 12_500 * Math.max(1, actions.length)) + for (let index = 0; index < actions.length; index++) { + if (ctx.signal?.aborted) throw new Error('Cancelled') + if (Date.now() >= deadline) throw new Error('Navigation action budget exhausted') + const action = actions[index] + // Whether the action itself waited for the UI to go quiet. `press` and `fill` ask for a settle + // and report back; `scroll` has no settle option at all, so a swipe always falls through to the + // sleep below. + let settled = false + switch (action.type) { + case 'tap': { + if (!action.locator) throw new Error('Tap action requires a locator') + settled = await pressWithLocator( + ctx, + device, + action.locator, + Math.max(1, Math.min(10_000, deadline - Date.now())) + ) + break + } + case 'input': + if (!action.locator || action.value == null) + throw new Error('Input action requires locator and value') + settled = didSettle( + await device.client.interactions.fill({ + platform: ctx.platform, + ...toAgentDeviceTarget(parseInteractionTarget(serializeLocator(action.locator))), + text: action.value, + settle: true, + settleQuietMs: SETTLE_QUIET_MS, + timeoutMs: Math.max(1, Math.min(10_000, deadline - Date.now())) + }) + ) + break + case 'swipe': + await device.client.interactions.scroll({ + platform: ctx.platform, + direction: action.direction ?? 'up' + }) + break + case 'wait': + await measurePhase(ctx, 'device-wait', () => + device.client.command.wait({ + platform: ctx.platform, + durationMs: Math.max( + 0, + Math.min(action.duration ?? 1000, 10_000, deadline - Date.now()) + ) + }) + ) + break + } + // The sleep is what stops the next screenshot or accessibility read from landing on the frame + // from *before* this action — a mistake that costs a wrong capture and a full exploration, + // against 0.8s saved. So it is skipped only on the evidence that something equivalent already + // happened: agent-device reporting that it watched the UI go quiet after the action + // (`settleQuietMs` above). A settle that was requested but not observed, and a swipe, which + // cannot request one, both still pay it. + if (action.type !== 'wait' && !settled) { + await waitForRoute(ctx.signal, Math.max(0, deadline - Date.now())) + } + } +} + +// `verifyAndSaveReplay` lived here: save the live recording, close the session and replay the file +// to prove it. Its one caller was the state-variant pass in `orchestrator.ts`, which now composes +// a state's journey out of command lines (the base screen's `pathCommands` plus the lines for the +// actions that open the variant) instead of recording a second `.ad` for it. + +/** + * Drop the recorded identity of every press that hangs off the node a divergence proved unstable, + * give each unguarded press a selector wait in its place, and replay the result once. + * + * The only recovery tier there is. It used to be the second of two, reached once agent-device's + * record-and-heal had declined, and the heal is gone because every firing of it this repo recorded + * was a failure — the 20-05-23 evidence below, and the same three-seed result on the 15-07-13 + * sweep (`replay-divergence.ts`). There was never a fallback chain to lose, either: the two tiers + * were selected by complementary predicates, so at most one of them could ever run on a given + * divergence. See `unguardPressSteps` for why re-recording provably cannot fix this class, what the + * strip gives up and what the inserted wait puts back. + * + * One pass, where this used to loop a strip and a replay per diverging step. The loop was charging + * a full replay for information the first report already carried: instability belongs to a *node*, + * and `ProfileFollowers` recorded that same node in the ancestry of both its presses. Measured on + * the 15-53-19 sweep it spent 269-286s on two rounds, published nothing, and left the screen that + * depended on it to walk itself from a cold launch. `unguardPressSteps` now acts on every press + * sharing the proven ancestry at once, so there is one replay to pay for and nothing to iterate. + * + * Still evidence-gated, never optimistic. `parseStructuralOnlyMismatchStep` returns a step only + * for an `IDENTITY_MISMATCH` with `matchCount === 1`, no competing candidate and no mismatched + * field outside `ancestry[...]` and `position:` — the two that describe where the node sits rather + * than what it is, and the two a replay cannot influence. Anything else — a `SELECTOR_MISS`, an + * ambiguous match, a label that moved — declines with the original failure intact, as does + * `unguardPressSteps` finding nothing to do, which is what stops a script already stripped from + * being processed twice. + * + * `position:` is why this tier sees the walks that used to die in the heal. On the 20-05-23 sweep + * `MyProfile`, `Profile` and `ProfileFollowers` all failed their tab-bar press with a position + * mismatch reported alongside the ancestry one, which left the divergence looking heal-shaped: + * all three were healed, all three heals reproduced the failure, and all three candidates were + * discarded — 117s, 39% of that sweep, that published nothing. Counting position as structural + * routes them here instead, because dropping the annotation stops pre-action verification running + * and takes the position check with it. + * + * Only a failure carrying no divergence at all is retried, and only once: a divergence is the app + * disagreeing with the script and repeating it changes nothing, whereas a bare `Daemon request + * timed out` is the plumbing, and discarding a candidate over it reports a failure the app never + * had. Measured: `Profile` lost its seed to exactly that, after a strip nothing suggests was wrong. + * + * The original bytes are restored on failure. A preserved `.failed` candidate is read to find out + * what the walk recorded, and leaving an edited script there would misreport that. + */ +async function stripIdentityAndReplay( + ctx: CaptureContext, + device: CaptureDevice, + scriptPath: string, + destinationSelector: string, + error: unknown, + deadline: number +): Promise { + const step = parseStructuralOnlyMismatchStep(error) + if (step === undefined) return false + + const original = await readFile(scriptPath, 'utf8').catch(() => undefined) + if (original === undefined) return false + + const unguarded = unguardPressSteps(original.split('\n'), step) + if (!unguarded) return false + + // The failed replay above was armed with `saveScript`, so it is holding a repair transaction and + // would refuse to be replaced. A bare close ends the transaction without writing anything, which + // is exactly right here: the script on disk is the one being edited, not one to be published. + await device.client.sessions.close().catch(() => undefined) + + const attempt = async (): Promise => { + await runRepairedReplay(ctx, device, scriptPath, deadline) + await measurePhase(ctx, 'device-wait', () => + device.client.command.wait({ + platform: ctx.platform, + selector: destinationSelector, + timeoutMs: 15_000 + }) + ) + } + + log( + 'saveAndReplayRecording', + `dropped recorded identity at step(s) ${unguarded.stripped.join(', ')} of ${scriptPath}` + + (unguarded.guarded.length > 0 + ? `; inserted a selector wait before step(s) ${unguarded.guarded.join(', ')}` + : '') + ) + await writeFile(scriptPath, unguarded.lines.join('\n'), 'utf8') + + let transientRetry = true + while (Date.now() < deadline) { + try { + await attempt() + log( + 'saveAndReplayRecording', + `verified ${scriptPath} with ${unguarded.stripped.length} recorded identity(ies) dropped` + ) + return true + } catch (replayError) { + const divergence = describeReplayDivergence(replayError) + log( + 'saveAndReplayRecording', + `${scriptPath} still diverges after unguarding — ${errorMessage(replayError)}${ + divergence ? ` — divergence: ${divergence}` : '' + }` + ) + if (divergence !== undefined || !transientRetry) break + transientRetry = false + log('saveAndReplayRecording', `no divergence reported — retrying ${scriptPath} once`) + } + } + + await writeFile(scriptPath, original, 'utf8').catch(() => undefined) + log('saveAndReplayRecording', `restored ${scriptPath}; stripping did not make it replayable`) + return false +} + +async function runRepairedReplay( + ctx: CaptureContext, + device: CaptureDevice, + scriptPath: string, + deadline: number +): Promise { + const replay = await measurePhase(ctx, 'replay', () => + device.client.replay.run({ + path: scriptPath, + platform: ctx.platform, + udid: device.deviceId, + keepSession: true, + timeoutMs: Math.max(1_000, deadline - Date.now()) + }) + ) as { sessionActive: boolean } + if (!replay.sessionActive) + throw new Error(`Repaired replay ${scriptPath} did not keep an active session`) + await waitForAppReady( + ctx, + device, + ctx.signal, + Math.max(0, Math.min(APP_READY_TIMEOUT_MS, deadline - Date.now())) + ) +} + +/** + * Save a recorded journey and replay it, leaving the verified session available to inspect. + * + * `destinationSelector` is not optional and not a nicety: agent-device's `sessions.saveScript` + * refuses to publish an active-session script unless, after the last action it classifies as + * mutating (every tap, fill and gesture the walk made), there is a `wait` on a selector whose + * matched element it captured a re-verifiable identity for. That is the "portable destination + * guard" in its error message, and `force: true` does not bypass it — force only governs + * overwriting an existing file, while the guard scan runs before anything is written. So the wait + * below is the only reason a saved exploration script can exist at all; without it every call here + * throws and an already-verified journey is lost. + * + * The selector must resolve to an element bearing an id or an accessibility label, because + * agent-device declines to record landmark evidence for a node with neither and then rejects the + * guard for having no evidence. `serializeLocator` on a `by: 'label' | 'id'` locator guarantees + * that by construction; `by: 'text'` does not, which is why callers must not pass one. + * + * `repairArmed` selects which of agent-device's two mutually exclusive publication paths to use, + * and getting it wrong does not degrade gracefully — it throws. A session opened by + * `replay.run({ saveScript })` is *repair-armed* (ADR 0012 decision 6: that option "arms + * agent-supervised re-record repair from this replay attempt onward"), and such a session refuses + * `sessions.saveScript` outright with "This session has an active .ad repair transaction and + * cannot use ordinary active-session publication." It publishes only through its own finalize, + * which is `sessions.close({ saveScript })` — the case agent-device's own `SessionCloseResult` + * doc calls "a repair-armed session's finalize". + * + * This mattered far more than one failed script. A sweep's seeded walks all start with + * `replay.run({ saveScript })` on a parent's verified replay, so every one of them was repair-armed + * and every one failed to publish. A screen with no saved replay cannot become a seed for the next + * screen, so the seed pool froze at whatever had been captured before the first seeded walk: in the + * measured run that was three screens at t=246s, and all twelve subsequent bounded hops seeded from + * the same stale `Profile` script, each paying a ~10s replay plus a full multi-step walk for a hop + * that should have been one tap from its real parent. + */ +export async function saveAndReplayRecording( + ctx: CaptureContext, + device: CaptureDevice, + scriptPath: string, + destinationSelector: string, + timeoutMs = 120_000, + { + repairArmed = false, + verifyByReplay = true + }: { repairArmed?: boolean; verifyByReplay?: boolean } = {} +): Promise { + const deadline = Date.now() + timeoutMs + await measurePhase(ctx, 'device-wait', () => + device.client.command.wait({ + platform: ctx.platform, + selector: destinationSelector, + timeoutMs: Math.max(1_000, Math.min(15_000, deadline - Date.now())) + }) + ) + // Publication is the end of the transaction either way — it writes the file on one path and + // finalizes the session on the other — and a publication that *throws* has ended it just as + // surely (an abort is what the throw usually says). So the marker comes down before the attempt + // rather than after a successful one: from here on there is nothing left for `goHome` to + // protect, and the discard the caller runs on failure wants its soft home back. + device.armedRecording = null + if (repairArmed) { + // One call, not two: the close *is* the publication here, so there is no `saveScript` step to + // precede it. `force` carries the same meaning on both paths — overwrite an existing target + // rather than refuse — and the candidate path is passed again even though the arming + // `replay.run` already named it, because close's own `saveScript` is what decides where the + // healed script lands and leaving it to the documented `.healed.ad` default would put + // the file somewhere `publishScript` is not looking for it. + await device.client.sessions.close({ saveScript: scriptPath, force: true }) + } else { + // `saveScript` writes the file; nothing after it is publication. The bare `sessions.close()` + // that used to follow was pure teardown — and on this harness teardown is not free: closing an + // agent-device session tears down the XCUITest runner and takes the app under test with it + // (see `reopenBoundApp`), so every successful walk ended by killing the app it had just proven + // a journey through. The next node then opened on a dead process: a deep-link rung paying a + // full cold start (`iOS Bundled 16079ms (6130 modules)` on a cold cache, ~2.5s warm) for a + // navigation the live app would have answered immediately. + // + // Leaving the session open costs nothing it did not already cost. The script is on disk, so + // nothing further is owed to it; the recording stays armed and keeps absorbing whatever device + // commands come next, and that buffer is never read — the next walk's `startReplayRecording` + // opens with its own `sessions.close()` (which discards an unfinalized transaction without + // writing), and every other site that arms a `--save-script` transaction now ends this one + // first through `endLiveSession`, which is what keeps agent-device from refusing the new + // arming over a live one. + // + // The close is only skipped on this path. `repairArmed` above still closes because there the + // close *is* the publication, and the `verifyByReplay` path below still closes because + // `verifyAndRepairReplay` arms a repair transaction of its own — it calls `endLiveSession` + // itself, where a close is free because the replay it is about to run relaunches the app from + // the script's leading `open --relaunch` anyway. + await device.client.sessions.saveScript({ path: scriptPath, force: true }) + } + + // `verifyByReplay: false` stops here, with the script on disk, nothing replayed, and — since + // the save above no longer closes — the app still running on the destination screen. + // + // The replay below re-runs the journey that has just been walked, and its script opens with + // `open --relaunch`, so every verification is a cold app start plus the whole walk again. On the + // 20-05-23 sweep that was 11 replays, 54.4s, 18% of the run — and not one of the scripts it + // verified was ever used as a seed, because nothing downstream of those screens was reached + // before the sweep ended. + // + // What the replay proves is real and is not reproduced anywhere else: the live walk dispatched + // its presses against the tree it was reading, with no identity verification, so a replay is the + // first time the recorded `target-v1` evidence is re-read against a fresh tree. That is the + // entire IDENTITY_MISMATCH class, and on the same sweep it caught three genuinely broken + // scripts. It is not redundant — it is early. The caller that skips it is not giving the check + // up, it is moving it to the first replay that was going to happen anyway: a seeded walk opens + // with `replay.run({ path: seed, saveScript })`, which replays the seed and arms the next + // recording in one call, so the seed's verification is already paid for there. A script nothing + // ever seeds off is published unverified and says so (`replayVerified: false`), which is a + // better artefact than the verified one we were paying a cold relaunch to produce and then + // never reading. + // + // The recovery tier goes with it, necessarily — `stripIdentityAndReplay` is driven by a + // divergence report, and there is no report without a replay. It runs unchanged at the deferred + // verification instead. + if (!verifyByReplay) return + + await verifyAndRepairReplay(ctx, device, scriptPath, destinationSelector, deadline) +} + +/** + * Replay a script that is already on disk, and repair it in place if it diverges. + * + * The second half of what `saveAndReplayRecording` used to do inline, lifted out because it is no + * longer only that function's business. Verification now happens at first use rather than at + * publication, so the caller that discovers a divergence is usually the saved-replay rung, and it + * needs the strip — otherwise deferring verification would quietly mean giving repair up, and the + * one script the strip rescued on the 20-05-23 sweep (`SearchTab/Search`) would have stayed broken + * for every sweep that followed. + * + * `destinationSelector` is where the strip waits after rewriting a step. A caller that recorded the + * script has it in hand; one that did not reads it back off the script's trailing guard with + * `destinationGuardSelector`. + * + * `replayError` is the divergence the caller is already holding, and passing it is what skips the + * opening replay below. The callers genuinely differ here, so it is optional rather than required: + * `saveAndReplayRecording`, `savedReplayStrategy` and the manual replay button have *not* replayed + * the script — for them this call is the replay, and its success is what leaves the app on the + * destination with a live session. Only `replaySeedBase` arrives after a failed armed replay of + * this same script, and for it the opening replay re-derived, at a measured ~6.2s, a divergence it + * already had in hand. Nothing downstream needs the session that replay opened: the strip's first + * act is to close whatever session is live (it must, to release the caller's armed repair + * transaction) and its `runRepairedReplay` relaunches from the script's leading `open --relaunch`, + * carrying its own `waitForAppReady`. + */ +export async function verifyAndRepairReplay( + ctx: CaptureContext, + device: CaptureDevice, + scriptPath: string, + destinationSelector: string, + deadline: number, + replayError?: unknown +): Promise { + // Whatever session the caller was holding ends here, before the armed replay below asks for one + // of its own. Two callers can arrive with a live session now: `saveAndReplayRecording`'s + // non-repair path, which stopped closing after `saveScript` so the app survives into the next + // node, and the saved-replay rung, which may be reached with the previous node's recording still + // armed. Either way agent-device will not arm a second `--save-script` transaction over a live + // one — `stripIdentityAndReplay` names the refusal — and the close is free here regardless, + // because `runReplay` immediately replays a script whose first line is `open --relaunch`. + await endLiveSession(device) + // The script is on disk by this point, so recovery below only has to redo the replay, never the + // walk that produced it: `runReplay` is what a retry re-runs after the app is relaunched. + // + // `saveScript` here is not a second publication — the script was already written above — it is + // what arms the repair transaction, and arming is the whole reason a divergence is survivable: + // an unarmed replay reports the divergence and is done with it, while an armed one holds the + // session open at the failed step. On the clean path the arming costs nothing: nothing finalizes + // the transaction with `sessions.close({ saveScript })`, and a transaction nobody finalizes + // simply never writes. + const runReplay = async (): Promise => { + const replayTimeoutMs = Math.max(1_000, deadline - Date.now()) + const replay = await measurePhase(ctx, 'replay', () => + device.client.replay.run({ + path: scriptPath, + platform: ctx.platform, + udid: device.deviceId, + keepSession: true, + saveScript: scriptPath, + force: true, + timeoutMs: replayTimeoutMs + }) + ) as { sessionActive: boolean } + if (!replay.sessionActive) + throw new Error(`Replay ${scriptPath} did not keep an active session`) + await waitForAppReady( + ctx, + device, + ctx.signal, + Math.max(0, Math.min(APP_READY_TIMEOUT_MS, deadline - Date.now())) + ) + } + + let error = replayError + if (error === undefined) { + try { + await runReplay() + return + } catch (firstFailure) { + error = firstFailure + } + } + + if (await stripIdentityAndReplay(ctx, device, scriptPath, destinationSelector, error, deadline)) + return + // See `isMissingSessionError`'s doc comment: a daemon replaced mid-walk kills whatever session + // `replay.run` opens for itself, and that death can surface as this generic "could not capture + // a fresh snapshot" wrapper instead of the literal "no active session" wording — the same + // underlying cause `captureScreenshot` above recovers from. One retry only: if the replay still + // cannot get a session back after `reopenBoundApp`, this is a real replay failure and must + // propagate rather than loop. + if (!isMissingSessionError(error)) throw error + log( + 'saveAndReplayRecording', + `no active session mid-replay, recovering — ${describeSessionLoss(errorMessage(error), agentDeviceErrorHint(error))}` + ) + await reopenBoundApp(ctx, device) + await runReplay() +} + +// `recordStateFromReplay` lived here: replay a base screen's `.ad` armed with `saveScript`, run a +// fixed action list, and finalize the repair transaction into a state-variant script. Same caller, +// same reason it is gone — see the note on `verifyAndSaveReplay` above. + +/** The one-line `pathToReach` form for a published `.ad`. Only the manual recorder produces one + * now; `buildAgentDeviceScreens` still reads it for a capture that carries a manual script. */ +export function replayPathToReach(device: CaptureDevice, scriptPath: string): string[] { + return [...device.setupCommands, `agent-device replay ${quoteCliArg(scriptPath)} --keep-session`] +} + +/** + * `relaunched: true` means the missing-session recovery branch fired: `reopenBoundApp` relaunches + * the app at its launch URL before the screenshot is taken, so the frame this call returns is the + * *launch screen*, not whatever the caller had navigated to. Callers must not record that frame as + * the target's capture — see `decideDelivery` in `reach/delivery.ts`, which is what this flag exists + * to feed. The other recovery call sites (`openUrlInSession`) re-open the original URL afterwards + * and so never need this signal. + */ +export async function captureScreenshot( + ctx: CaptureContext, + device: CaptureDevice, + path: string +): Promise<{ relaunched: boolean }> { + log('screenshot', path) + try { + // A screenshot is a read: re-running it whole is safe, and a busy runner is the normal + // aftermath of dozens of taps and accessibility snapshots on an animating screen — exactly + // what exploration does right before the screenshot that confirms arrival, making this the + // call in the sweep most likely to hit the watchdog and the one that least deserves to lose a + // verified arrival over it. Mirrors `capturePressables` below. + await measurePhase(ctx, 'screenshot', () => + retryWhileRunnerBusy(() => device.client.capture.screenshot({ path }), { + signal: ctx.signal, + onRetry: (attempt, waitMs) => + log('screenshot', `runner busy; retry ${attempt} in ${Math.round(waitMs / 1000)}s`) + }) + ) + log('screenshot', 'saved') + return { relaunched: false } + } catch (error) { + if (!isMissingSessionError(error)) throw error + // `path` names the caller's frame (`verify-deep-link-check.png`, `explore.png`, a delivery + // capture), which is the only record of what the sweep was in the middle of when the session + // turned out to be gone. + log( + 'screenshot', + `no active session, recovering — while saving ${basename(path)} — ${describeSessionLoss(errorMessage(error), agentDeviceErrorHint(error))}` + ) + // Waits for the relaunched app to settle. The frame itself is still reported as `relaunched` + // and must not be recorded as the target's capture, but everything the sweep does *after* this + // recovery — the next deep link, above all — now finds an app that can act on it. + await reopenBoundApp(ctx, device) + await measurePhase(ctx, 'screenshot', () => device.client.capture.screenshot({ path })) + return { relaunched: true } + } +} + +// A path no router can match. Long and self-describing so it is recognisable in Metro logs. +const UNMATCHED_PROBE_PATH = '__app-flow-probe-no-such-route' + +/** + * Opens a deep link that cannot match any route and reports what the app renders for it. + * + * That screen is the reference for "this capture never arrived": a route whose capture looks the + * same as an impossible route's either did not resolve, or never got the chance because something + * intercepted the navigation. This is screenmap's bogus-param probe one level up — route instead + * of param — and it needs to know nothing about what the intercepting screen *is*. + * + * Returns null when the app has no usable URL scheme, in which case arrival cannot be checked and + * callers must treat captures as unverified rather than guess. + */ +export async function probeUnmatchedRoute( + ctx: CaptureContext, + device: CaptureDevice +): Promise { + const url = + device.useCustomScheme && ctx.scheme + ? `${ctx.scheme}://${UNMATCHED_PROBE_PATH}` + : device.launchUrl + ? `exp://127.0.0.1:${device.metroPort}/--/${UNMATCHED_PROBE_PATH}` + : null + if (!url) return null + log('probeUnmatchedRoute', url) + try { + await openUrlInSession(ctx, device, device.launchTarget, url) + await waitForRoute(ctx.signal) + return await capturePressables(ctx, device) + } catch (error) { + log('probeUnmatchedRoute', `failed: ${error instanceof Error ? error.message : String(error)}`) + return null + } +} + +export async function capturePressables( + ctx: CaptureContext, + device: CaptureDevice, + timeoutMs = 30_000, + options?: { + /** + * Absolute timestamp past which this read is of no use to the caller, so the busy-retry backoff + * can stop short instead of sleeping into it — see `RunnerBusyRetryOptions.deadlineAt`. Callers + * that are themselves on a clock (`waitForAppReady`) pass theirs; a one-off read has no deadline + * beyond `timeoutMs` and omits it, keeping the full retry schedule. + */ + deadlineAt?: number + /** + * How many times a busy runner is retried. Defaults to the full schedule. A caller whose read + * is a *question about whether the device is answering at all* passes 0: for it, `RUNNER_BUSY` + * is not an obstacle in front of the answer, it is the answer. + */ + retries?: number + } +): Promise { + // A snapshot is a read: re-running it whole is safe, and a busy runner is the normal aftermath + // of a watchdogged capture on a heavy or animating screen rather than a failure of this screen. + const snapshot = await measurePhase(ctx, 'device-read', () => + retryWhileRunnerBusy( + () => + device.client.capture.snapshot({ + platform: ctx.platform, + // The client defaults to the compact digest response level for routine App + // Flow commands. Snapshot extraction needs the typed node payload instead. + responseLevel: 'default', + interactiveOnly: true, + // No `customActions`. It was requested on every read in this module and consumed by + // none of them: the node loop below keeps `value`, `label`, `identifier` and `role`, + // and nothing downstream of `AppFlowPressable` has ever had a field to put an + // accessibility custom action in. On the iOS simulator the flag is not free — it + // selects a capability the snapshot backend then has to be admitted for, and enumerates + // the custom actions of every merged element — so this was per-element device work on + // all 114 reads of the reference sweep in service of a field that was dropped on the + // floor. How much of the measured 1.56s it was buying is exactly what the next sweep + // will say; the case for removing it does not depend on the answer. + forceFull: true, + noRecord: true, + timeoutMs + }), + { + signal: ctx.signal, + deadlineAt: options?.deadlineAt, + retries: options?.retries, + onRetry: (attempt, waitMs) => + log('capturePressables', `runner busy; retry ${attempt} in ${Math.round(waitMs / 1000)}s`) + } + ) + ) as { nodes?: unknown } + + if (!Array.isArray(snapshot.nodes)) { + throw new Error('agent-device snapshot did not return a nodes array') + } + + const seen = new Set() + const pressables: AppFlowPressable[] = [] + + for (const nodeValue of snapshot.nodes) { + if (!nodeValue || typeof nodeValue !== 'object') continue + const node = nodeValue as { + visibleToUser?: boolean + enabled?: boolean + role?: string + type?: string + value?: string + label?: string + identifier?: string + } + // The snapshot is already scoped to interactive elements. React Native Pressable + // children are commonly exposed by iOS as StaticText without a button role or a + // truthy hittable field, even though agent-device can press them by text/label. + // Filtering those fields here would discard the exact selector candidates we need. + if (node.visibleToUser === false || node.enabled === false) continue + + const semanticType = `${node.role ?? node.type ?? ''}` + .toLowerCase() + .replace(/^xcuielementtype/, '') + if (['application', 'window', 'scrollview', 'scroll-view'].includes(semanticType)) continue + + const pressable: AppFlowPressable = { + ...(node.value ? { text: node.value } : {}), + ...(node.label ? { label: node.label } : {}), + ...(node.identifier ? { identifier: node.identifier } : {}), + ...(node.role || node.type ? { role: node.role ?? node.type } : {}) + } + if (![pressable.text, pressable.label, pressable.identifier].some((value) => value?.trim())) { + continue + } + + const identity = JSON.stringify(pressable) + if (seen.has(identity)) continue + seen.add(identity) + pressables.push(pressable) + } + + // Every successful read, not just the ones a caller keeps. This is the only funnel for + // accessibility reads in this module, so stamping it here is what lets `openDeepLink` answer + // "is the process alive and drawing" without spending a snapshot of its own. + device.lastRead = { at: Date.now(), pressables } + return pressables +} + +function stopLaunchProcess(child: ChildProcess | null): void { + if (!child?.pid) return + try { + child.kill('SIGTERM') + } catch { + // Process already exited. + } +} + +export async function closeDevice(device: CaptureDevice): Promise { + log('closeDevice', 'stop Metro if we started it, sessions.close --shutdown') + stopLaunchProcess(device.launchProcess) + await endMetroLogFile(device.metroLogPath) + try { + await device.client.sessions.close({ shutdown: true }) + log('sessions.close', 'done') + } catch (error) { + console.error('Failed to close agent-device session:', error) + } +} diff --git a/src/app-flow/cdp/agent-runtime.ts b/src/app-flow/cdp/agent-runtime.ts new file mode 100644 index 00000000..5d2294f7 --- /dev/null +++ b/src/app-flow/cdp/agent-runtime.ts @@ -0,0 +1,1216 @@ +/* eslint-disable @typescript-eslint/no-explicit-any, no-var, @typescript-eslint/no-unused-vars, prefer-rest-params, @typescript-eslint/no-this-alias -- + `any`: everything below reads objects owned by the app under test (fibers, navigation state, + props). None of it has a type we control, and a fake hierarchy would only pretend otherwise. + `var`: deliberate, see the syntax constraint below; the lint rule's advice is the very thing the + Hermes debugger evaluator is not trusted with. Unused catch bindings: `catch (error)` is kept + over the newer binding-less form for the same reason. `arguments` and `this` aliasing: the + patch wrappers forward whatever the app passed, of any arity, with its own receiver; rest + parameters are banned syntax here. */ +/** + * The in-app agent: the code that runs inside the app under test's JS runtime, as real typed + * source. + * + * Until now every expression we evaluated in the app was a template-string constant + * (`COLLECT_ROOTS_JS`, `FIND_NAVIGATOR_JS`, `NAV_STATE_EXPR`, `readinessExpr`). Those cannot be + * type-checked, linted, or unit-tested, and the cost of that was paid for in real bugs: the + * `hidden()` style-array prune in `runtime-render.ts` was found by running against a live app + * because no test could reach it. Here the same logic is ordinary exported functions. They are + * tested directly against a hand-built fake fiber tree (`agent-runtime.test.ts`), and at runtime + * `runtime-source.ts` injects them by calling `fn.toString()` on each. + * + * ## CRITICAL SYNTAX CONSTRAINT -- read before "modernising" anything in this file + * + * The source below is evaluated by Hermes' **debugger** evaluator (`Runtime.evaluate`), which does + * not lower code the way the app's bundler does. The reference implementation this transport is + * modelled on runs an esbuild `target: 'es2015'` pass over its agent for exactly that reason. We + * avoid that build step by writing the agent in conservative syntax by hand, which is also what the + * existing ES5-style expressions already did. Inside every function in `AGENT_FUNCTIONS`: + * + * - no `async` / `await` (and so no promises-as-control-flow); + * - no optional chaining and no nullish coalescing: write the explicit null check; + * - no spread or rest syntax, in object literals, arrays or calls; + * - no `for...of` (use an indexed loop; `for...in` over a plain object is fine); + * - no classes, no destructuring, no default parameters, no template literals, no getters; + * - `var` and plain `function` only. `Set` and `Map` exist in Hermes and are already used by the + * expressions this replaces. + * + * A violation does not fail at the edit: it type-checks, passes every test that runs the functions + * under Node, and then arrives as a Hermes compile error against a live device. So `runtime-source.ts` + * scans the assembled source for these tokens and refuses to build it, and the test suite runs that + * scan. Do not add comments inside a function body that contain those tokens spelled out; the scan + * is textual and cannot tell a comment from code. + * + * ## Other rules that follow from "this is toString()'d" + * + * - **A function may reference only other `AGENT_FUNCTIONS` and the app's globals.** It is + * re-created in a fresh scope in the app, so a module constant (`WALK_BUDGET`) or an imported + * helper does not exist there and fails as a `ReferenceError` on first use. Configuration crosses + * the boundary as an argument (`AgentLimits`), and the test suite evaluates the assembled source + * in an empty `node:vm` context so such a reference cannot hide. + * - **Names are load-bearing.** Functions call each other by their declared name, so a bundler that + * renames a top-level declaration (to dodge a collision) would silently break the call. Every + * name carries an `agent` prefix to make a collision unlikely, and `runtime-source.ts` checks + * `fn.name` against the registry and throws rather than inject a renamed function. + * - **No recursion by name, no closures over module state.** The helpers that would naturally + * recurse (`agentStyleFlags`, `agentComponentName`) are iterative. + * + * ## One navigator search, shared + * + * `runtime-navigator.ts` explains that the reader and the writer must share a single search so they + * can never disagree about which navigator is real. That property is kept here by construction: + * `agentFindNavigator` is the only place a navigator is found, and `nav-state` and `readiness` + * (and, later, a navigate command) all call it. Until the old string path is retired there are + * still two copies of that search in the repo; `agent-runtime.test.ts` runs both against the same + * fake tree so they cannot drift apart unnoticed. + * + * ## The lease + * + * `agentInstall` arms a watchdog that is refreshed by every command. If the host stops talking for + * `leaseMs` (a crashed Electron process, a dropped socket nobody noticed) the watchdog calls + * `restore()` and deletes the agent global, so a dead harness cannot leave the app permanently + * instrumented. This is the safety property the rest of the transport exists to serve, so it is + * deliberately fail-closed: with no working timer the install refuses rather than run without a + * watchdog, and a failure to re-arm the timer restores immediately instead of continuing. + * + * `restore()` runs the `undo` list in reverse order, and every entry is isolated so one failing undo + * cannot strand the others or the global. Since readiness moved into the app (below) the list is no + * longer empty: it holds the reversal of every patch the agent applies to the app's own globals, + * which is exactly why the lease exists. If the host dies, the 10s watchdog un-patches `fetch`, + * `XMLHttpRequest.prototype.send` and the DevTools hook before it deletes the agent. + * + * ## Readiness lives in the app + * + * The old settle loop sampled a stateless snapshot from Node and called the screen settled after two + * identical samples (`runtime-render.ts`, `reach/adapters.ts`). That was sound only because a CLI + * evaluation cost ~570ms: two samples were >500ms apart, six times the 80ms quiet window the + * reference implementation waits for, with frames certainly painted in between. The socket transport + * measured **13.8ms median round-trip (11.7-23.5ms over 20 samples)** against a real device: two + * polls are now less than one 60Hz frame apart, and two identical samples prove neither a quiet + * window nor a painted frame. Keeping that heuristic would make the render rung confidently wrong, + * which is worse than slow. So the decision moves to where the evidence is: + * + * - **React commits** are timestamped by chaining `__REACT_DEVTOOLS_GLOBAL_HOOK__.onCommitFiberRoot`. + * - **In-flight network** is counted by wrapping `fetch` and `XMLHttpRequest.prototype.send`. + * - **A quiet window** is the time since the last commit and the last network completion, compared + * by the agent against thresholds the host passes in (`quietMs`, `idleMs`). + * - **A paint check** is two nested `requestAnimationFrame` callbacks, with no commit in between. + * + * The reply carries the verdict plus the evidence behind it, and `confident`/`missing` say which + * signals could not be installed, so a verdict that rests on fewer signals is distinguishable from + * one that does not. What no longer has parity with the old string path is listed on + * `agentReadiness`'s sibling `agentVerdict`. + * + * ## Patching the app is the risky part + * + * Each patch goes through `agentPatchProperty`, which owns the rules: pass arguments, `this` and + * results through untouched; keep bookkeeping in its own try block so it cannot reach the app; + * register the reversal on `undo`; refuse (rather than throw) when the target is missing or not + * writable; and, if something else wrapped the property after us, neutralise our wrapper instead of + * overwriting theirs on restore. + */ + +import type { NavRoute } from '../nav-state.ts' +import type { ReadinessSnapshot } from '../runtime-render.ts' + +/** Node ceiling for the fiber walk. Generous by design; the deadline below is the real bound. */ +export const WALK_BUDGET = 60_000 +/** How long the walk may occupy the app's JS thread before abandoning the search. */ +export const WALK_DEADLINE_MS = 1_000 +/** Ceiling for the walk beneath the focused screen when sampling readiness. */ +export const INNER_WALK_BUDGET = 40_000 +/** How long the host may go quiet before the app restores itself. Matches the reference's 10s. */ +export const AGENT_LEASE_MS = 10_000 + +/** + * Default time with no React commit before the tree counts as quiet. + * + * 500ms is ~30 frames at 60Hz. It has to cover the gap between a response landing and the commit + * that renders it (a few frames) and the gap from that commit to a dependent request starting (a + * waterfall: the next request bumps the in-flight count, so only the commit-to-request hop needs + * covering). It is shorter than the reference implementation's 1500ms on purpose: every capture in + * a sweep pays this wait, hundreds of times, and the old path accepted a 570ms sample spacing. The + * margin the 1500ms buys is mostly sampling error, which an exact in-app timestamp does not have. + * The cost of being too short is a premature capture; the host's own settle budget and the arrival + * judge are the backstops. Not measured on a device: tune it from capture results, and note a screen + * that commits more often than this (a ticking clock, a live feed) never reports quiet. + */ +export const QUIET_MS = 500 +/** Default time with no request in flight and none completed before the network counts as idle. */ +export const IDLE_MS = 500 +/** + * How long to wait for the two animation frames before reporting `painted: false`. A frame that has + * not arrived in 500ms (~30 frames) means the app is backgrounded or its UI thread is stalled, and + * waiting longer only burns the host's 1500ms command timeout. + */ +export const PAINT_TIMEOUT_MS = 500 + +/** The walk bounds, passed into the app because module constants do not survive `toString()`. */ +export interface AgentLimits { + walkBudget: number + walkDeadlineMs: number + innerWalkBudget: number + /** Default `quietMs` for a readiness command that does not carry its own. */ + quietMs: number + /** Default `idleMs` for a readiness command that does not carry its own. */ + idleMs: number + paintTimeoutMs: number +} + +export const DEFAULT_AGENT_LIMITS: AgentLimits = { + walkBudget: WALK_BUDGET, + walkDeadlineMs: WALK_DEADLINE_MS, + innerWalkBudget: INNER_WALK_BUDGET, + quietMs: QUIET_MS, + idleMs: IDLE_MS, + paintTimeoutMs: PAINT_TIMEOUT_MS +} + +/** Commands the host may send. `restore` is also what the lease watchdog does on its own. */ +export type AgentCommand = + | { type: 'heartbeat' } + | { type: 'restore' } + | { type: 'nav-state' } + | { + type: 'readiness' + route: string + expectedParams: Record | null + /** Override the commit quiet window for this call. */ + quietMs?: number + /** Override the network idle window for this call. */ + idleMs?: number + } + +export interface HeartbeatReply { + alive: true +} +export interface RestoreReply { + restored: true +} +/** `routes` is null when no navigator is mounted (release build, still booting, no DevTools hook). */ +export interface NavStateReply { + routes: NavRoute[] | null +} +/** + * `ReadinessSnapshot` as the app computes it, plus the in-app verdict. Unlike the pipe-delimited + * string it replaces, nothing here has to fit in 200 characters, so `active` is the whole focused + * path rather than its last three segments. + */ +export interface ReadinessReply extends ReadinessSnapshot { + /** + * The app's own verdict: the screen was found, shows no loading affordance, the tree has been + * quiet, and frames painted. Computed over the signals that are available; read `confident`. + */ + settled: boolean + /** No commit for `quietMs` and no request in flight or recently completed for `idleMs`. */ + quiet: boolean + /** Two frames ran with no commit between them; null when there is no `requestAnimationFrame`. */ + painted: boolean | null + /** True only when every signal (commits, fetch, xhr, paint) was installed. */ + confident: boolean + /** Signals that could not be installed. Non-empty means `settled` rests on less evidence. */ + missing: string[] + /** Milliseconds since the last observed commit; null when commits cannot be observed. */ + commitQuietMs: number | null + /** 0 while a request is in flight, else ms since the last completion; null when untracked. */ + networkQuietMs: number | null + /** Requests started and not yet completed; null when no network signal is available. */ + inflight: number | null +} +/** What the agent says about a command it will not run. Passed through to the caller as data. */ +export interface AgentErrorReply { + error: 'invalid-command' | 'unknown-command' | 'command-failed' + detail?: string +} +export interface AgentStoppedReply { + stopped: true +} + +/** The shape of the object installed at `globalThis[key]`. */ +export interface InstalledAgent { + invoke(commandJson: string, reply: (value: unknown) => void): void + restore(): void +} + +/** + * The fibers to start a walk from: one `current` per root of every React Native renderer. + * + * Renderers are enumerated and filtered by `rendererPackageName` rather than assuming React Native + * is renderer id 1. `hook.renderers` is keyed by registration order, so a bundle that registers + * anything first would leave `getFiberRoots(1)` empty. Id 1 stays as the fallback because on the + * measured reference app it is in fact the React Native renderer. + */ +export function agentCollectFiberRoots(): any[] { + var hook = (globalThis as any).__REACT_DEVTOOLS_GLOBAL_HOOK__ + if (!hook || typeof hook.getFiberRoots !== 'function') return [] + var rendererIds: any[] = [] + try { + if (hook.renderers && typeof hook.renderers.forEach === 'function') { + hook.renderers.forEach(function (renderer: any, id: any) { + if (!renderer || renderer.rendererPackageName === 'react-native-renderer') { + rendererIds.push(id) + } + }) + } + } catch (error) { + rendererIds = [] + } + if (!rendererIds.length) rendererIds.push(1) + var stack: any[] = [] + for (var r = 0; r < rendererIds.length; r++) { + var roots: any = null + try { + roots = hook.getFiberRoots(rendererIds[r]) + } catch (error) { + roots = null + } + if (!roots || typeof roots.forEach !== 'function') continue + roots.forEach(function (root: any) { + if (root && root.current) stack.push(root.current) + }) + } + return stack +} + +/** + * Finds the app's React Navigation root. Returns `{ nav, state }` -- the object a caller can read + * or dispatch through, and the root state already read from it -- or null when none is mounted. + * + * Two shapes are probed because which one is reachable is a property of the app under test. A + * **container ref** exposes `getRootState` directly but is not guaranteed to sit in the fiber tree + * as a prop or a `stateNode`. A **screen's `navigation` prop** is in the tree on every screen + * React Navigation renders and carries `getState` and `getParent`, but knows only its own + * navigator, so it is climbed to the root: reading it directly would report one stack and miss + * every sibling tab. The climb discriminates on those two read-only methods rather than on + * `dispatch`, which keeps the reading commands from ever naming a mutating one. + * + * Bounded by `limits.walkBudget` nodes and `limits.walkDeadlineMs` of wall clock, and gives up + * rather than finishing at any cost: this runs on the app's JS thread. The deadline is the real + * protection. The node cap is generous because a measured live tree ran to 17,822 fibers and the + * comparison project records a busy feed exceeding 16,000, so a tight cap would fail on exactly the + * screens that matter. A `seen` set guards cycles, which a child/sibling push cannot do alone. + * + * A carrier that answers badly does not end the search: each `getRootState()` is called in its own + * try block and its result must have `routes`, so a still-mounting container ref is skipped rather + * than accepted as a successful read of an empty sweep. It is called on the carrier, not detached, + * because a container ref reads its internals through its receiver. + */ +export function agentFindNavigator(limits: AgentLimits): { nav: any; state: any } | null { + var stack = agentCollectFiberRoots() + if (!stack.length) return null + var deadline = Date.now() + limits.walkDeadlineMs + var budget = limits.walkBudget + var seen = new Set() + var climb = function (value: any): { nav: any; state: any } | null { + if (!value || typeof value !== 'object') return null + if (typeof value.getState !== 'function' || typeof value.getParent !== 'function') return null + try { + var nav = value + var climbed = new Set() + while (typeof nav.getParent === 'function' && !climbed.has(nav)) { + climbed.add(nav) + var parent = nav.getParent() + if (!parent) break + nav = parent + } + var state = typeof nav.getRootState === 'function' ? nav.getRootState() : nav.getState() + return state && state.routes ? { nav: nav, state: state } : null + } catch (error) { + return null + } + } + var found: { nav: any; state: any } | null = null + while (stack.length && budget-- > 0 && !found) { + if ((budget & 255) === 0 && Date.now() > deadline) return null + var fiber = stack.pop() + if (!fiber || seen.has(fiber)) continue + seen.add(fiber) + var memo = fiber.memoizedState + var props = fiber.memoizedProps + var carriers = [ + fiber.stateNode, + props && props.value, + props && props.navigationRef, + memo && memo.memoizedState && memo.memoizedState.current + ] + for (var at = 0; at < carriers.length && !found; at++) { + var carrier = carriers[at] + if (carrier && typeof carrier.getRootState === 'function') { + try { + var direct = carrier.getRootState() + if (direct && direct.routes) found = { nav: carrier, state: direct } + } catch (error) { + found = null + } + } + } + if (!found) found = climb(props && props.navigation) + if (!found) found = climb(props && props.value) + if (!found) found = climb(fiber.stateNode) + if (fiber.child) stack.push(fiber.child) + if (fiber.sibling) stack.push(fiber.sibling) + } + return found +} + +/** + * Flattens a navigation state into the routes `nav-state.ts` has always returned, depth first. + * + * Only values that can honestly become a URL segment survive: strings and finite numbers (as + * strings). A route object holding a whole post or a callback neither crosses the wire nor arrives + * as `[object Object]` pretending to be an id. Depth is capped at 12, as before. + */ +export function agentFlattenRoutes(state: any): NavRoute[] { + var flat: NavRoute[] = [] + var walk = function (node: any, depth: number): void { + if (!node || depth > 12 || !Array.isArray(node.routes)) return + for (var i = 0; i < node.routes.length; i++) { + var route = node.routes[i] + if (!route || typeof route.name !== 'string') continue + var entry: NavRoute = { name: route.name } + if (typeof route.path === 'string' && route.path) entry.path = route.path + var raw = route.params + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + var kept: Record = {} + var any = false + for (var key in raw) { + var value = raw[key] + if (typeof value === 'string') { + kept[key] = value + any = true + } else if (typeof value === 'number' && isFinite(value)) { + kept[key] = String(value) + any = true + } + } + if (any) entry.params = kept + } + flat.push(entry) + if (route.state) walk(route.state, depth + 1) + } + } + walk(state, 0) + return flat +} + +/** The `nav-state` command: the flattened navigator state, or `{ routes: null }`. */ +export function agentNavState(limits: AgentLimits): NavStateReply { + var found = agentFindNavigator(limits) + if (!found || !found.state) return { routes: null } + return { routes: agentFlattenRoutes(found.state) } +} + +/** + * Reduces a React Native `style` prop to the two flags that decide visibility. + * + * Style props are routinely **arrays**, and nested ones, so reading only a plain object misses most + * of the hidden subtrees an app actually produces. Flattened with the array's own merge semantics: + * left to right, a later entry overrides an earlier one. Depth is capped at 5. Iterative, because + * an injected function cannot rely on calling itself by name (see the file doc). + */ +export function agentStyleFlags(style: any): { display?: unknown; opacity?: unknown } { + var out: { display?: unknown; opacity?: unknown } = {} + // Entries are processed in reading order; a stack of [node, depth] pairs is reversed on push so + // the first array element is popped first. + var stack: Array<{ node: any; depth: number }> = [{ node: style, depth: 0 }] + while (stack.length) { + var item = stack.pop() as { node: any; depth: number } + var node = item.node + if (!node || item.depth > 5) continue + if (Array.isArray(node)) { + for (var index = node.length - 1; index >= 0; index--) { + stack.push({ node: node[index], depth: item.depth + 1 }) + } + continue + } + if (typeof node !== 'object') continue + if (node.display !== undefined) out.display = node.display + if (node.opacity !== undefined) out.opacity = node.opacity + } + return out +} + +/** + * Whether the app has hidden this subtree, so it should not count as on screen. + * + * Counting a hidden subtree reports content and stability for a screen the user cannot see. + * Measured on a real app: a stopped `ActivityIndicator` sat under + * `style: [{display:"none",opacity:0}]` and was counted as a live loading affordance, so every + * readiness poll on that screen reported a skeleton and the settle loop burned its whole 8s budget + * on a fully rendered screen. That is why the style is flattened with array merge semantics + * (`agentStyleFlags`) rather than "any entry hides it": `[{display:"none"},{display:"flex"}]` is + * visible, and pruning a visible screen is the more damaging way to be wrong. + * + * Deliberately more thorough than the reference implementation, which reads a plain-object style + * only and so misses every array-valued one. + */ +export function agentHidden(props: any): boolean { + if (!props) return false + if (props.focused === false || props.isFocused === false || props.screenFocused === false) { + return true + } + if (props.style) { + var flags = agentStyleFlags(props.style) + if (flags.display === 'none' || flags.opacity === 0) return true + } + return props.accessibilityElementsHidden === true || props['aria-hidden'] === true +} + +/** + * A component's display name. It may sit on a function, or on the object `memo` and `forwardRef` + * wrap it in; the latter is routine in React Native, and reading only functions would miss most of + * the loading indicators worth detecting. Follows `type` wrappers a bounded number of times. + */ +export function agentComponentName(value: any): string { + for (var hops = 0; hops < 8; hops++) { + if (!value) return '' + if (typeof value === 'function') return value.displayName || value.name || '' + if (typeof value !== 'object') return '' + var direct = + value.displayName || + (typeof value.render === 'function' + ? value.render.displayName || value.render.name || '' + : '') + if (direct) return direct + value = value.type + } + return '' +} + +/** + * The `readiness` command: one stateless snapshot of how settled a screen is. + * + * Ported from `readinessExpr`; see `runtime-render.ts` for why this decides *when to photograph* + * rather than *whether we arrived*. This function is still the stateless sample (the result is an + * object, not an `AIHR1|` string); `agentSettle` adds the stateful evidence around it, and + * `agentVerdict` folds the two together. + * + * Two traversal details were each found by measurement and are kept exactly: + * + * - **Siblings are pushed before the hidden check.** A fiber's later siblings are reachable only + * through it, so skipping the push together with the subtree severs the rest of the sibling list. + * One hidden scene cut a walk from 1425 nodes to 937 and hid the focused screen entirely, so every + * poll reported `found: false`. + * - **A parameter mismatch makes `found` false**, not merely a different signature. A dispatch still + * in flight lands with the previous instance's params for a poll or two, and settling on that + * would photograph the wrong screen. + */ +export function agentReadiness( + routeName: string, + expectedParams: Record | null, + limits: AgentLimits +): ReadinessSnapshot { + var empty: ReadinessSnapshot = { + found: false, + content: 0, + loading: null, + signature: '', + active: [], + paramMismatch: [] + } + var found = agentFindNavigator(limits) + if (!found) return empty + // The focused route path, innermost last, bounded at 16 levels of nesting. + var state = found.state + var active: string[] = [] + for (var depth = 0; state && state.routes && state.routes.length && depth < 16; depth++) { + var route = state.routes[state.index == null ? 0 : state.index] + if (!route) break + active.push(route.name) + state = route.state + } + var expected: Record = expectedParams || {} + var screen: any = null + var stack = agentCollectFiberRoots() + var seen = new Set() + var budget = limits.walkBudget + var deadline = Date.now() + limits.walkDeadlineMs + while (stack.length && budget-- > 0 && !screen) { + if ((budget & 255) === 0 && Date.now() > deadline) break + var fiber = stack.pop() + if (!fiber || seen.has(fiber)) continue + seen.add(fiber) + if (fiber.sibling) stack.push(fiber.sibling) + var props = fiber.memoizedProps + if (agentHidden(props)) continue + if (props && props.route && props.route.name === routeName) { + var focused = true + try { + if (props.navigation && typeof props.navigation.isFocused === 'function') { + focused = props.navigation.isFocused() + } + } catch (error) { + focused = true + } + if (focused) screen = fiber + } + if (fiber.child) stack.push(fiber.child) + } + var content = 0 + var loading = '' + var parts: string[] = [] + var missed: string[] = [] + if (screen) { + var inner = [screen.child] + var innerSeen = new Set() + var innerBudget = limits.innerWalkBudget + var loaderPattern = + /Skeleton|Shimmer|LoadingPlaceholder|LoadingIndicator|LoadingSpinner|LoadingView|LoadingScreen|ActivityIndicator|Spinner|^Loader$/ + while (inner.length && innerBudget-- > 0) { + var node = inner.pop() + if (!node || innerSeen.has(node)) continue + innerSeen.add(node) + // Same reason as the outer walk: pruning a hidden subtree must not take its siblings with it. + if (node.sibling) inner.push(node.sibling) + var nodeProps = node.memoizedProps + if (agentHidden(nodeProps)) continue + if (!loading) { + var typeName = agentComponentName(node.type) || agentComponentName(node.elementType) + if (typeName && loaderPattern.test(typeName)) loading = 'skeleton' + else if (node.tag === 13 && node.memoizedState !== null) loading = 'suspense' + else if (nodeProps) { + var busy = + (nodeProps.accessibilityState && nodeProps.accessibilityState.busy) || + nodeProps['aria-busy'] + if (busy === true) loading = 'busy' + else if ( + nodeProps.isLoading === true || + nodeProps.loading === true || + nodeProps.animating === true + ) { + loading = 'busy' + } else if ( + nodeProps.accessibilityRole === 'progressbar' || + nodeProps.role === 'progressbar' + ) { + loading = 'busy' + } + } + } + if (node.tag === 5 && nodeProps) { + var text = typeof nodeProps.children === 'string' ? nodeProps.children : null + var media = nodeProps.source || nodeProps.src + var label = + typeof nodeProps.accessibilityLabel === 'string' ? nodeProps.accessibilityLabel : null + if (text || media || label) { + content += 1 + if (parts.length < 250) { + parts.push(String(node.type) + ':' + String(text || label || 'src').slice(0, 60)) + } + } + } + if (node.child) inner.push(node.child) + } + // Only the keys the caller asked about, compared on the focused leaf's params. A screen + // carrying extra params is still the screen that was asked for. + var leaf = found.state + while (leaf && leaf.routes && leaf.routes.length) { + var next = leaf.routes[leaf.index == null ? 0 : leaf.index] + if (!next) break + leaf = next.state || next + } + for (var key in expected) { + var actual = leaf && leaf.params ? leaf.params[key] : undefined + if (String(actual) !== String(expected[key]) && missed.length < 4) missed.push(key) + } + if (missed.length) { + screen = null + content = 0 + } + } + // A 32-bit hash of the structure: only equality across polls is ever asked of it, and the + // structure itself runs to thousands of characters. + var joined = parts.join('|') + var hash = 0 + for (var at = 0; at < joined.length; at++) { + hash = (hash * 31 + joined.charCodeAt(at)) | 0 + } + return { + found: !!screen, + content: content, + loading: loading ? loading : null, + signature: (hash >>> 0).toString(36), + active: active, + paramMismatch: missed + } +} + +/** + * Replaces `target[name]` with a wrapper, safely, and registers the reversal. Returns whether the + * property is now patched; **never throws**, because a signal that cannot be installed must be + * reported as missing, not abort the install. + * + * The rules every patch in this agent follows, kept in one place so none of them can be forgotten: + * + * - **Refuse rather than force.** A missing property (when `requireOriginal`), a throwing getter, a + * frozen object, or an accessor with no setter all end in `false`. Writability is checked by + * *doing* the assignment and reading it back: a non-writable property silently ignores the write + * in sloppy mode and throws in strict mode, and the evaluator's mode is not ours to assume. + * - **No stacking.** If the current value is already one of our wrappers (it carries + * `__aiHarnessRelease`, the owning agent's `restore`), that agent is restored first, so a second + * install under a different key wraps the real original instead of wrapping the first wrapper. + * A same-key reinstall is already covered by `agentInstall` restoring its predecessor. + * - **Exact reversal.** The original value is kept and written back, or the own property is deleted + * when it was inherited, so the object ends up as it started. If something else wrapped the + * property after us, the reversal does *not* overwrite it (that would silently unhook a third + * party); it flips `control.live` so our wrapper becomes a pure pass-through and leaves it in the + * chain. + * + * `makeWrapper(original, control)` receives the original (null if it is not a function) and the + * control flag, and must return the replacement. + */ +export function agentPatchProperty( + target: any, + name: string, + requireOriginal: boolean, + makeWrapper: (original: any, control: { live: boolean }) => any, + undo: Array<() => void>, + release: () => void +): boolean { + if (!target || (typeof target !== 'object' && typeof target !== 'function')) return false + var current: any + try { + current = target[name] + } catch (error) { + return false + } + if (current && typeof current.__aiHarnessRelease === 'function') { + try { + current.__aiHarnessRelease() + } catch (error) { + current = null + } + try { + current = target[name] + } catch (error) { + return false + } + } + if (requireOriginal && typeof current !== 'function') return false + var hadOwn = Object.prototype.hasOwnProperty.call(target, name) + var control = { live: true } + var wrapper: any + try { + wrapper = makeWrapper(typeof current === 'function' ? current : null, control) + } catch (error) { + return false + } + if (typeof wrapper !== 'function') return false + try { + wrapper.__aiHarnessRelease = release + } catch (error) { + /* an unmarked wrapper still works; it just cannot be released by a later install */ + } + try { + target[name] = wrapper + } catch (error) { + return false + } + var applied = false + try { + applied = target[name] === wrapper + } catch (error) { + applied = false + } + if (!applied) return false + undo.push(function () { + control.live = false + if (target[name] !== wrapper) return + if (hadOwn) target[name] = current + else delete target[name] + }) + return true +} + +/** + * Installs the three observers and returns the state they write to. + * + * ## What each patch touches + * + * - **`__REACT_DEVTOOLS_GLOBAL_HOOK__.onCommitFiberRoot`**: replaced by a function that stamps + * `commitAt` and then calls whatever handler was there (DevTools' own, usually), with the same + * `this` and arguments, returning its result. React calls the property at commit time, so + * replacing it on the hook object is enough. The handler is optional: with none, only the stamp + * runs. Chained, never replaced. + * - **`fetch`**: replaced by a function that counts the call as in flight and returns the *same + * promise object* the original returned. Completion is observed by attaching a fulfilment and a + * rejection handler to it, so a rejection decrements the count and still reaches the app + * unchanged. One consequence, stated plainly: attaching a handler marks the promise as handled, so + * an app that never catches a failed request loses the engine's unhandled-rejection warning for + * it. The alternative (returning a derived promise) would preserve that warning but change the + * identity of the value the app holds, which is the worse trade for an apparatus that must not + * change behaviour. + * - **`XMLHttpRequest.prototype.send`**: replaced (the prototype, not the constructor, so + * `instanceof`, statics and subclasses are untouched). A `loadend` listener, plus a + * `readystatechange` at state 4 as a second route, decrements the count; the first to fire wins + * because each request holds a one-shot token. `send` itself is called with the original + * arguments and receiver and its result or exception propagates. + * + * ## Counting a request once + * + * React Native's `fetch` is a polyfill over `XMLHttpRequest` that calls `send` synchronously inside + * the promise executor, so without care one request counts twice. `fetchDepth` is raised for the + * duration of the synchronous original `fetch` call and `send` does not count while it is raised. + * This matters only for the count's honesty (`inflight`); quiet depends on it reaching zero. A + * native fetch that does not use `XMLHttpRequest` is unaffected. + * + * ## Blind spots (what quiet cannot see) + * + * Requests started before the install are invisible, so `commitAt` and `settledAt` start at the + * install time: the first `quietMs` after connecting is reported as not quiet rather than guessed. + * Native image loading, WebSocket traffic and anything that bypasses both globals is not counted. A + * long-lived request (long poll, server-sent events over XHR) holds `inflight` above zero for its + * whole life, so such a screen reports not quiet until the host's budget gives up. That is the safe + * direction to be wrong in. + * + * Bookkeeping lives in its own try block everywhere so an exception in it can never reach the app. + */ +export function agentCreateSignals(root: any, undo: Array<() => void>, release: () => void): any { + var started = Date.now() + var signals: any = { + installedAt: started, + commitAt: started, + settledAt: started, + pending: 0, + fetchDepth: 0, + available: { commits: false, fetch: false, xhr: false } + } + signals.begin = function () { + try { + signals.pending += 1 + return { done: false } + } catch (error) { + return null + } + } + signals.end = function (token: any) { + try { + if (!token || token.done) return + token.done = true + if (signals.pending > 0) signals.pending -= 1 + signals.settledAt = Date.now() + } catch (error) { + token = null + } + } + + var hook: any = null + try { + hook = root.__REACT_DEVTOOLS_GLOBAL_HOOK__ + } catch (error) { + hook = null + } + signals.available.commits = agentPatchProperty( + hook, + 'onCommitFiberRoot', + false, + function (original: any, control: any) { + return function (this: any) { + if (control.live) { + try { + signals.commitAt = Date.now() + } catch (error) { + control.live = true + } + } + if (typeof original === 'function') return original.apply(this, arguments) + return undefined + } + }, + undo, + release + ) + + signals.available.fetch = agentPatchProperty( + root, + 'fetch', + true, + function (original: any, control: any) { + return function (this: any) { + if (!control.live) return original.apply(this, arguments) + var token = signals.begin() + var result: any + signals.fetchDepth += 1 + try { + result = original.apply(this, arguments) + } catch (error) { + signals.end(token) + throw error + } finally { + signals.fetchDepth -= 1 + } + try { + if (result && typeof result.then === 'function') { + var done = function (): void { + signals.end(token) + } + result.then(done, done) + } else { + signals.end(token) + } + } catch (error) { + signals.end(token) + } + return result + } + }, + undo, + release + ) + + var xhrProto: any = null + try { + xhrProto = root.XMLHttpRequest ? root.XMLHttpRequest.prototype : null + } catch (error) { + xhrProto = null + } + if (xhrProto && typeof xhrProto.addEventListener === 'function') { + signals.available.xhr = agentPatchProperty( + xhrProto, + 'send', + true, + function (original: any, control: any) { + return function (this: any) { + if (!control.live || signals.fetchDepth > 0) return original.apply(this, arguments) + var token = signals.begin() + try { + var xhr = this + var done = function (): void { + signals.end(token) + } + xhr.addEventListener('loadend', done) + xhr.addEventListener('readystatechange', function () { + if (xhr.readyState === 4) done() + }) + } catch (error) { + signals.end(token) + } + try { + return original.apply(this, arguments) + } catch (error) { + signals.end(token) + throw error + } + } + }, + undo, + release + ) + } + return signals +} + +/** + * Folds a stateless snapshot and the in-app evidence into the verdict. Pure: `now` is passed in so + * the arithmetic is testable without a clock. + * + * - `commitQuietMs = now - lastCommit` (null if commits cannot be observed). + * - `networkQuietMs = 0` while anything is in flight, else `now - lastCompletion` (null if neither + * `fetch` nor `XMLHttpRequest` could be wrapped). + * - `quiet` needs each *available* clock to have reached its threshold and nothing in flight. An + * unavailable signal does not block quiet (it cannot be consulted) but it is named in `missing` + * and clears `confident`, so the host can tell a verdict made on fewer signals. + * - `settled` = found, no loading affordance, quiet, and not `painted === false`. `painted` null + * (no `requestAnimationFrame`) does not block; it is listed as missing like any other. + * + * ## What no longer has parity with the old `readinessExpr` path, and why that is correct + * + * The structure signature, `hidden()` pruning, focused-screen identification, `content`, `loading`, + * `active` (apart from path length) and `paramMismatch` are the same code in the same order and are + * still asserted equal. `settled`, `quiet`, `painted`, the timings and `missing` have no + * counterpart: the old path inferred quiet from two identical samples ~570ms apart, which is + * meaningless at 14ms. A snapshot that is identical twice in a row is no longer treated as evidence + * of anything; the app's own commit and request clocks are. + */ +export function agentVerdict( + snapshot: any, + signals: any, + painted: boolean | null, + now: number, + quietMs: number, + idleMs: number +): ReadinessReply { + var missing: string[] = [] + if (!signals.available.commits) missing.push('commits') + if (!signals.available.fetch) missing.push('fetch') + if (!signals.available.xhr) missing.push('xhr') + if (painted === null) missing.push('paint') + var commitQuietMs: number | null = signals.available.commits + ? Math.max(0, now - signals.commitAt) + : null + var networkTracked = signals.available.fetch || signals.available.xhr + var networkQuietMs: number | null = networkTracked + ? signals.pending > 0 + ? 0 + : Math.max(0, now - signals.settledAt) + : null + var quiet = + (commitQuietMs === null || commitQuietMs >= quietMs) && + (networkQuietMs === null || (signals.pending === 0 && networkQuietMs >= idleMs)) + var out: any = snapshot + out.quiet = quiet + out.painted = painted + out.commitQuietMs = commitQuietMs + out.networkQuietMs = networkQuietMs + out.inflight = networkTracked ? signals.pending : null + out.missing = missing + out.confident = missing.length === 0 + out.settled = !!out.found && !out.loading && quiet && painted !== false + return out +} + +/** + * The `readiness` command: waits for two frames, then answers once with the verdict. + * + * ## The paint check + * + * Two nested `requestAnimationFrame` callbacks: the first runs before the next paint, the second + * before the one after, so by the time the second fires a frame has been presented for whatever the + * tree looked like when the first was queued. This is what the 570ms CLI round-trip used to supply + * for free. `painted` is also false if a commit landed while the frames were pending, because then + * the frames painted a tree that is no longer the tree. + * + * A backgrounded app or a stalled UI thread never delivers the frames, so a timer (`paintTimeoutMs`) + * answers `painted: false` instead of leaving the host to time out with no information. With no + * `requestAnimationFrame` at all the check is reported unavailable (`painted: null`) and the reply is + * immediate. The snapshot and the timestamps are taken *after* the frames, so the verdict describes + * the tree as it is when the reply is sent. + */ +export function agentSettle( + signals: any, + routeName: string, + expectedParams: Record | null, + limits: AgentLimits, + quietMs: number, + idleMs: number, + reply: (value: unknown) => void +): void { + var root = globalThis as any + var finished = false + var timer: any = null + var commitAtStart = signals.commitAt + var finish = function (painted: boolean | null): void { + if (finished) return + finished = true + try { + if (timer !== null) root.clearTimeout(timer) + } catch (error) { + timer = null + } + try { + var now = Date.now() + var snapshot = agentReadiness(routeName, expectedParams, limits) + reply(agentVerdict(snapshot, signals, painted, now, quietMs, idleMs)) + } catch (error) { + reply({ + error: 'command-failed', + detail: String(error && (error as any).message).slice(0, 200) + }) + } + } + var raf = root.requestAnimationFrame + if (typeof raf !== 'function') { + finish(null) + return + } + try { + var timeout = + typeof limits.paintTimeoutMs === 'number' && limits.paintTimeoutMs > 0 + ? limits.paintTimeoutMs + : 500 + timer = root.setTimeout(function () { + finish(false) + }, timeout) + raf.call(root, function () { + raf.call(root, function () { + finish(signals.commitAt === commitAtStart) + }) + }) + } catch (error) { + finish(null) + } +} + +/** + * Installs the agent at `globalThis[key]` and arms the lease. See the file doc for the lease and + * the fail-closed rules; the order below matters: + * + * 1. Restore any previous agent under the same key (a reconnect re-installs under the key it used + * before, and two live agents would double-arm the watchdog). + * 2. Refuse outright if there is no timer to arm. An agent that cannot expire is the one outcome + * this design exists to prevent. + * 3. Arm the lease *before* publishing the global, and restore on any failure, so there is no window + * in which the agent is reachable but unguarded. + * + * `invoke` takes the command as a JSON string and replies at most once, through the callback the + * host wraps around its binding. It never throws: a malformed or failing command becomes an + * `AgentErrorReply`, because an exception inside `Runtime.evaluate` would surface on the host as an + * evaluation failure and be indistinguishable from the wrapper itself failing to compile. + */ +export function agentInstall(key: string, leaseMs: number, limits: AgentLimits): void { + var root = globalThis as any + var previous = root[key] + if (previous && typeof previous.restore === 'function') { + try { + previous.restore() + } catch (error) { + previous = null + } + } + if (typeof root.setTimeout !== 'function' || typeof root.clearTimeout !== 'function') { + throw new Error('no timers available: refusing to install without a watchdog') + } + var lease = typeof leaseMs === 'number' && isFinite(leaseMs) && leaseMs > 0 ? leaseMs : 10000 + var stopped = false + var watchdog: any = null + var undo: Array<() => void> = [] + var agent: InstalledAgent | null = null + var signals: any = null + + function restore(): void { + if (stopped) return + stopped = true + try { + root.clearTimeout(watchdog) + } catch (error) { + watchdog = null + } + watchdog = null + for (var i = undo.length - 1; i >= 0; i--) { + try { + undo[i]() + } catch (error) { + continue + } + } + undo.length = 0 + // Only remove the global if it is still ours, so a stale watchdog cannot delete a newer agent. + try { + if (root[key] === agent) delete root[key] + } catch (error) { + try { + root[key] = undefined + } catch (inner) { + stopped = true + } + } + } + + function renewLease(): void { + if (stopped) return + try { + root.clearTimeout(watchdog) + watchdog = root.setTimeout(function () { + restore() + }, lease) + } catch (error) { + // Fail closed: an agent that cannot re-arm its watchdog must not keep running. + restore() + } + } + + /* Runs a command and hands its reply to the callback; readiness calls it after the paint check. */ + function dispatch(command: any, done: (value: unknown) => void): void { + var type = command && command.type + if (type === 'heartbeat') { + done({ alive: true }) + } else if (type === 'nav-state') { + done(agentNavState(limits)) + } else if (type === 'readiness') { + var quiet = + typeof command.quietMs === 'number' && isFinite(command.quietMs) && command.quietMs >= 0 + ? command.quietMs + : limits.quietMs + var idle = + typeof command.idleMs === 'number' && isFinite(command.idleMs) && command.idleMs >= 0 + ? command.idleMs + : limits.idleMs + agentSettle( + signals, + String(command.route), + command.expectedParams || null, + limits, + quiet, + idle, + done + ) + } else { + done({ error: 'unknown-command' }) + } + } + + agent = { + invoke: function (commandJson: string, reply: (value: unknown) => void): void { + var replied = false + var once = function (value: unknown): void { + if (replied) return + replied = true + try { + reply(value) + } catch (error) { + replied = true + } + } + var command: any + try { + command = JSON.parse(commandJson) + } catch (error) { + once({ error: 'invalid-command' }) + return + } + if (command && command.type === 'restore') { + restore() + once({ restored: true }) + return + } + if (stopped) { + once({ stopped: true }) + return + } + renewLease() + if (stopped) { + once({ stopped: true }) + return + } + try { + dispatch(command, once) + } catch (error) { + once({ + error: 'command-failed', + detail: String(error && (error as any).message).slice(0, 200) + }) + } + }, + restore: restore + } + renewLease() + if (stopped) throw new Error('could not arm the lease watchdog') + // Patch only after the watchdog is armed, so the app is never instrumented without an expiry, and + // restore on any failure so a half-installed agent cannot leave patches behind. + try { + signals = agentCreateSignals(root, undo, restore) + root[key] = agent + } catch (error) { + restore() + throw error + } +} + +/** + * Everything that is injected, in the order it is declared in the app. The key is the name each + * function is declared and called by; `runtime-source.ts` verifies `fn.name` matches it. + */ +export const AGENT_FUNCTIONS = { + agentCollectFiberRoots, + agentFindNavigator, + agentFlattenRoutes, + agentNavState, + agentStyleFlags, + agentHidden, + agentComponentName, + agentReadiness, + agentPatchProperty, + agentCreateSignals, + agentVerdict, + agentSettle, + agentInstall +} as const diff --git a/src/app-flow/cdp/connection.ts b/src/app-flow/cdp/connection.ts new file mode 100644 index 00000000..e747cc3c --- /dev/null +++ b/src/app-flow/cdp/connection.ts @@ -0,0 +1,814 @@ +/** + * A persistent connection to the app under test's JS runtime, through Metro's debugger socket. + * + * This replaces one `agent-device cdp runtime eval` process per evaluation (~0.5s each, measured) + * whose returned string **truncates at exactly 200 characters**, and the chunking protocol + * `nav-state.ts` built to live with that. Here a command's reply never travels as an evaluation + * result at all: the app calls a per-connection binding installed with `Runtime.addBinding`, and the + * payload arrives as a `Runtime.bindingCalled` event with no length limit short of the socket's + * `maxPayload`. The design follows `codex-mobile-dev-plugin/src/server/app-flow/connection.ts`; this + * file is that module minus its presentation/portal orchestration, plus the reconnect it leaves to + * its callers. + * + * ## Two layers, and where reconnect lives + * + * - `CdpLink` is **one socket and one installed agent**. It never recovers: when anything goes wrong + * (socket closed, the app reloaded, the agent vanished, three heartbeats missed) it fails every + * pending call with a typed `CdpConnectionLostError` and reports itself lost, once. + * - `CdpConnection` owns a link and **replaces it** when it is lost. + * + * The reference fails the connection on `Runtime.executionContextsCleared` and leaves the retry to + * a loop three layers up (`runs.ts`, backoff `min(500 * 2^min(attempt-1, 4), 5000)`). That works for + * them because a reload ends a capture run. Our sweep **relaunches the app routinely** and has to + * carry on, and every consumer of this transport would otherwise need its own copy of that loop. So + * it lives here, in the layer that already knows the facts a retry needs: + * + * - **A relaunch changes the debugger URL.** Metro reassigns target ids across app restarts, so + * reconnecting to the old URL cannot work; each attempt calls `resolveTarget()` again, which is + * where `reidentifyTarget` (discovery.ts) matches the app by identity rather than by id. + * - **The agent is gone and must be re-installed**, which is the link's own handshake. + * - **The lease means the old agent expires by itself**, so there is nothing to clean up on the old + * socket: a reconnect is only ever "start again". + * + * What recovery deliberately does **not** do is retry the command that was in flight. After a + * reload we cannot know whether it ran, and today's three commands are reads but the next ones + * will not be. That command rejects with `CdpConnectionLostError`; the caller, which knows whether + * its command is idempotent, decides. A command issued *while* recovery is running waits for it + * (bounded by its own timeout) instead of failing, because that is the "relaunch mid-sweep" case. + * + * `reconnect()` is also public. After `maxReconnectAttempts` failed attempts the connection stops + * trying and reports `failed`; the caller, which knows Metro is back or the app was started again, + * re-arms it explicitly rather than having a background loop run forever against a dead Metro. + * + * ## Correlation: one counter, two kinds of id + * + * A single sequence feeds both kinds of request so ids never collide. **Positive ids** are plain CDP + * calls (`Runtime.enable`, `Runtime.addBinding`, the install evaluate) and their result is the + * message with that `id`. **Negative ids** are app commands. A command is itself a `Runtime.evaluate` + * sent under its negative id, but the evaluate's own result is empty (the wrapper returns nothing); + * the answer arrives later as `Runtime.bindingCalled` carrying the same id inside its JSON payload. + * So `message.id > 0` resolves a CDP call, a binding event resolves a command, and a CDP-level + * `error` or `result.exceptionDetails` on *either* kind rejects by `message.id`. That last rule is + * what lets a Hermes compile failure of the command wrapper land on the right promise instead of + * stranding it until its timeout. + * + * ## Surrogates (mandatory, not an optimisation) + * + * The reference documents a real bug: the debugger path re-encodes a raw non-BMP character (an + * emoji) into two separate surrogate code units that Hermes cannot compile. So every evaluated + * expression has every surrogate rewritten as a `\uXXXX` escape. That is only valid where the + * character sits inside a string, regex, template or comment, which is the only place source + * carries user text. Command values additionally go through `toWellFormed()` as they are + * serialised, so a lone surrogate (which JSON would carry as an escape the app would then rebuild + * into an unpaired unit) becomes U+FFFD at the boundary instead of travelling at all. + * + * ## Expiry in the app's own clock + * + * A command that waited in a socket buffer behind a busy JS thread can run long after the host gave + * up on it, which for a navigation is the worst outcome (it happens, late, with nobody waiting). The + * install handshake therefore returns the app's clock reading, the host computes + * `appClock - installStarted`, and every command carries an absolute deadline in the **app's** + * clock: the app drops it with `commandExpired` rather than running it. The offset is taken at the + * *start* of the install round trip, so transit time only ever makes the deadline later than the + * host's, never earlier: a fresh command cannot be expired by a slow handshake. Monotonic clocks on + * both sides (`performance.now`) mean wall-clock skew cannot matter; the app falls back to + * `Date.now` only when it has no `performance`, and the offset is then simply relative to that. + * + * ## Things that were changed from the reference, and why + * + * - **Teardown waits for its three trailing sends to be written** (bounded) before `terminate()`. + * The reference sends and terminates in the same tick; `terminate()` destroys the socket and can + * discard anything still buffered, which would leave the binding installed. + * - **A malformed binding payload is dropped**, not fatal. The reference treats any parse failure + * as a dead connection; a frame of the wrong shape is a protocol surprise, not evidence the + * socket is gone, and the command it belonged to still has its own timeout. A malformed *frame* + * (not JSON at all) is still fatal, since the stream itself can no longer be trusted. + * - **A `commandExpired` reply counts as a reply** for heartbeat purposes: the app answered, so it + * is alive and the link works. + * - **`restore` carries no deadline.** The teardown command must not expire in a slow app. + * + * ## Limits worth knowing + * + * `maxPayload` is 2MB; a larger reply makes `ws` close the socket (1009), which surfaces here as a + * lost link, not as a truncated result. The handshake and install timeouts below are not measured + * against a real Hermes compile of the agent; they are the reference's 2s, widened for the install. + */ +import { randomUUID } from 'node:crypto' +import type { ClientOptions } from 'ws' +import { WebSocket } from 'ws' +import { + AGENT_LEASE_MS, + DEFAULT_AGENT_LIMITS, + type AgentCommand, + type AgentLimits, + type HeartbeatReply +} from './agent-runtime.ts' +import { listMetroTargets, reidentifyTarget, type MetroCdpTarget } from './discovery.ts' +import { agentSource } from './runtime-source.ts' + +/** Origin of the failure, so a caller can branch without string-matching messages. */ +export type CdpErrorKind = + 'timeout' | 'command-expired' | 'runtime-unavailable' | 'evaluation-failed' | 'connection-lost' + +export class CdpError extends Error { + readonly kind: CdpErrorKind + constructor(kind: CdpErrorKind, message: string) { + super(message) + this.name = new.target.name + this.kind = kind + } +} + +/** The host gave up waiting. The command may still run in the app unless its deadline expires it. */ +export class CdpTimeoutError extends CdpError { + readonly operation: string + readonly timeoutMs: number + constructor(operation: string, timeoutMs: number) { + super('timeout', `${operation} did not complete within ${timeoutMs}ms`) + this.operation = operation + this.timeoutMs = timeoutMs + } +} + +/** The app dropped the command because its in-app deadline had already passed. It did not run. */ +export class CommandExpiredError extends CdpError { + readonly operation: string + constructor(operation: string) { + super('command-expired', `${operation} reached the app after its deadline and was not run`) + this.operation = operation + } +} + +/** No agent is installed at the key: the app reloaded, or the lease restored and removed it. */ +export class RuntimeUnavailableError extends CdpError { + readonly operation: string + constructor(operation: string) { + super('runtime-unavailable', `${operation}: the in-app agent is not installed`) + this.operation = operation + } +} + +/** The debugger rejected an evaluation (typically a compile error) or the app threw in it. */ +export class CdpEvaluationError extends CdpError { + readonly operation: string + readonly detail?: string + constructor(operation: string, detail?: string) { + super( + 'evaluation-failed', + `${operation} was rejected by the runtime${detail ? `: ${detail}` : ''}` + ) + this.operation = operation + this.detail = detail + } +} + +export type ConnectionLostReason = + | 'socket' + | 'reload' + | 'heartbeat' + | 'runtime-unavailable' + | 'closed' + | 'reconnecting' + | 'exhausted' + +/** The socket, the app's JS context, or the agent went away. Nothing about the call is known. */ +export class CdpConnectionLostError extends CdpError { + readonly reason: ConnectionLostReason + constructor(reason: ConnectionLostReason) { + super('connection-lost', `the debugger connection was lost (${reason})`) + this.reason = reason + } +} + +/** Handshake: the reference's 2s per step. The install also compiles the agent, so it gets more. */ +const HANDSHAKE_TIMEOUT_MS = 2_000 +const INSTALL_TIMEOUT_MS = 5_000 +const DEFAULT_COMMAND_TIMEOUT_MS = 1_500 +const HEARTBEAT_INTERVAL_MS = 2_000 +const HEARTBEAT_COMMAND_TIMEOUT_MS = 2_500 +/** A reply this recent already proves the app is alive; do not queue a heartbeat behind real work. */ +const HEARTBEAT_RECENT_REPLY_MS = 1_500 +const HEARTBEAT_FAILURE_LIMIT = 3 +const RESTORE_TIMEOUT_MS = 5_000 +/** How long teardown waits for its trailing sends to reach the socket. */ +const TEARDOWN_FLUSH_MS = 500 +const SOCKET_HANDSHAKE_TIMEOUT_MS = 3_000 +const SOCKET_MAX_PAYLOAD_BYTES = 2 * 1024 * 1024 +const KEY_PREFIX = '__aiHarnessCdp_' + +/** + * `min(500 * 2^min(attempt-1, 4), 5000)` -- the reference's reconnect schedule, kept so the two + * projects behave the same against the same Metro: 500, 1000, 2000, 4000, then 5000 forever. + * `attempt` is 1-based. + */ +export function reconnectBackoffMs(attempt: number): number { + return Math.min(500 * 2 ** Math.min(Math.max(attempt, 1) - 1, 4), 5_000) +} + +/** + * Rewrites every UTF-16 surrogate as a `\uXXXX` escape. See "Surrogates" in the file doc. Exported + * for the tests, which assert on the wire bytes rather than on this function in isolation. + */ +export function escapeSurrogates(source: string): string { + return source.replace( + /[\ud800-\udfff]/g, + (unit) => `\\u${unit.charCodeAt(0).toString(16).padStart(4, '0')}` + ) +} + +/** `JSON.stringify` replacer: every string leaves the host well formed. */ +const wellFormed = (_key: string, value: unknown): unknown => + typeof value === 'string' ? escapeSurrogates(value) : value + +/** + * The expression that reads the app's clock, spelled out in ES5 because the command wrapper and the + * install handshake are compiled by the same debugger evaluator as the agent (see the syntax + * constraint in `agent-runtime.ts`). Both sides of the offset must use the same expression. + */ +const APP_CLOCK_EXPR = + "(typeof performance!=='undefined'&&performance&&typeof performance.now==='function'?performance.now():Date.now())" + +/** The slice of the `ws` socket the link uses, so tests can supply a fake. */ +export interface CdpSocket { + readyState: number + send(data: string, callback?: (error?: Error) => void): void + terminate(): void + on(event: 'message', listener: (data: { toString(): string }) => void): unknown + on(event: 'error', listener: (error: Error) => void): unknown + on(event: 'close', listener: () => void): unknown + once(event: 'open', listener: () => void): unknown + once(event: 'error', listener: (error: Error) => void): unknown + once(event: 'close', listener: () => void): unknown +} + +export type CdpSocketFactory = (url: string, options: ClientOptions) => CdpSocket + +const defaultSocketFactory: CdpSocketFactory = (url, options) => + new WebSocket(url, options) as unknown as CdpSocket + +const OPEN = 1 + +interface Pending { + resolve(value: unknown): void + reject(error: Error): void + timer: NodeJS.Timeout + operation: string +} + +export interface CdpLinkOptions { + /** Shared by every link of one `CdpConnection`, so a re-install restores the agent it replaces. */ + key?: string + leaseMs?: number + limits?: AgentLimits + createSocket?: CdpSocketFactory + heartbeatIntervalMs?: number + heartbeatRecentReplyMs?: number + heartbeatTimeoutMs?: number + now?: () => number + /** Called once if the link dies of its own accord. Not called after `close()`. */ + onLost?: (error: CdpConnectionLostError) => void +} + +/** + * One socket and one installed agent. See "Two layers" in the file doc. + */ +export class CdpLink { + private readonly socket: CdpSocket + private readonly key: string + private readonly binding: string + private readonly leaseMs: number + private readonly limits: AgentLimits + private readonly now: () => number + private readonly heartbeatIntervalMs: number + private readonly heartbeatRecentReplyMs: number + private readonly heartbeatTimeoutMs: number + private readonly onLost?: (error: CdpConnectionLostError) => void + private readonly pending = new Map() + /** ONE counter for both id kinds: positive for CDP calls, negated for app commands. */ + private sequence = 0 + private dead = false + private closing?: Promise + private heartbeat?: NodeJS.Timeout + private heartbeatInFlight = false + private heartbeatFailures = 0 + private lastReply = Number.NEGATIVE_INFINITY + private appClockOffset?: number + /** Resolves once the handshake has installed the agent; rejects if any step fails. */ + readonly ready: Promise + + constructor(url: string, options: CdpLinkOptions = {}) { + this.key = options.key ?? `${KEY_PREFIX}${randomUUID().replaceAll('-', '')}` + // Unguessable per link: the page can enumerate its own globals, and a forged reply could + // otherwise resolve a pending command. Prefixed by the key so the next install can sweep it. + this.binding = `${this.key}_reply_${randomUUID().replaceAll('-', '')}` + this.leaseMs = options.leaseMs ?? AGENT_LEASE_MS + this.limits = options.limits ?? DEFAULT_AGENT_LIMITS + this.now = options.now ?? (() => performance.now()) + this.heartbeatIntervalMs = options.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS + this.heartbeatRecentReplyMs = options.heartbeatRecentReplyMs ?? HEARTBEAT_RECENT_REPLY_MS + this.heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? HEARTBEAT_COMMAND_TIMEOUT_MS + this.onLost = options.onLost + + // Metro's own origin, as its inspector proxy expects from a debugger front end. Node's global + // WebSocket cannot set it, nor `maxPayload`, which is why this uses `ws`. + const origin = new URL(url) + origin.protocol = 'http:' + this.socket = (options.createSocket ?? defaultSocketFactory)(url, { + origin: origin.origin, + handshakeTimeout: SOCKET_HANDSHAKE_TIMEOUT_MS, + maxPayload: SOCKET_MAX_PAYLOAD_BYTES, + followRedirects: false + }) + + const opened = new Promise((resolve, reject) => { + this.socket.once('open', () => resolve()) + this.socket.once('error', () => reject(new CdpConnectionLostError('socket'))) + this.socket.once('close', () => reject(new CdpConnectionLostError('socket'))) + }) + this.socket.on('message', (data) => this.onMessage(data.toString())) + this.socket.on('error', () => this.fail(new CdpConnectionLostError('socket'))) + this.socket.on('close', () => this.fail(new CdpConnectionLostError('socket'))) + this.ready = opened.then(() => this.install()) + // The caller observes `ready`; an unobserved rejection must not crash the process. + this.ready.catch(() => undefined) + } + + private async install(): Promise { + await this.send('Runtime.enable', {}, HANDSHAKE_TIMEOUT_MS, 'runtime-enable') + await this.send( + 'Runtime.addBinding', + { name: this.binding }, + HANDSHAKE_TIMEOUT_MS, + 'add-binding' + ) + // The agent's constructor path is the only thing here that can throw inside the app, and it + // does so on purpose (no timers, so no watchdog). That surfaces as `exceptionDetails`. + const keyPrefix = `${this.key}_reply_` + const expression = + `(function(){var __name=function(value){return value};` + + // A previous connection's reply global is dead weight and, worse, still a callable. + `var names=Object.keys(globalThis);for(var i=0;i this.beat(), this.heartbeatIntervalMs) + this.heartbeat.unref() + } + + /** + * Skipped when one is already in flight, or when a real reply landed recently: every command + * renews the app's lease, so a recent reply is proof of life and a heartbeat queued behind real + * work would only add a traversal's worth of latency to it. Three failures in a row with no reply + * in between closes the link; a reply that arrives while a heartbeat is outstanding clears the + * count, which is why failure is judged against when the heartbeat was *sent*. + */ + private beat(): void { + if (this.heartbeatInFlight || this.now() - this.lastReply < this.heartbeatRecentReplyMs) return + this.heartbeatInFlight = true + const sentAt = this.now() + this.invoke({ type: 'heartbeat' }, this.heartbeatTimeoutMs) + .catch(() => { + if (this.lastReply < sentAt && ++this.heartbeatFailures >= HEARTBEAT_FAILURE_LIMIT) { + this.fail(new CdpConnectionLostError('heartbeat')) + } + }) + .finally(() => { + this.heartbeatInFlight = false + }) + } + + private onMessage(text: string): void { + let message: { + id?: number + method?: string + params?: { name?: string; payload?: string } + result?: { exceptionDetails?: { text?: string; exception?: { description?: string } } } + error?: { message?: string } + } + try { + message = JSON.parse(text) + } catch { + this.fail(new CdpConnectionLostError('socket')) + return + } + if ( + message.method === 'Runtime.executionContextsCleared' || + message.method === 'Runtime.executionContextDestroyed' + ) { + // The app reloaded: the agent, the binding and every pending reply are gone with it. + this.fail(new CdpConnectionLostError('reload')) + return + } + if (message.method === 'Runtime.bindingCalled') { + if (message.params?.name !== this.binding) return + let value: { id?: unknown; result?: unknown } + try { + value = JSON.parse(message.params.payload ?? '') + } catch { + return + } + if (typeof value.id === 'number') this.settleReply(value.id, value.result) + return + } + if (typeof message.id !== 'number' || message.id === 0) return + const details = message.result?.exceptionDetails + if (message.error || details) { + const detail = details?.exception?.description ?? details?.text ?? message.error?.message + const operation = this.pending.get(message.id)?.operation ?? 'other' + this.finish(message.id, undefined, new CdpEvaluationError(operation, detail)) + return + } + // A negative id's own evaluate result is empty by design; the answer comes as a binding event. + if (message.id > 0) this.finish(message.id, message.result) + } + + private settleReply(id: number, result: unknown): void { + const pending = this.pending.get(id) + if (!pending) return + const reply = result as { commandExpired?: boolean; runtimeUnavailable?: boolean } | null + if (reply && reply.runtimeUnavailable) { + this.finish(id, undefined, new RuntimeUnavailableError(pending.operation)) + // No agent means nothing here can work again; hand the decision to whoever can reinstall. + this.fail(new CdpConnectionLostError('runtime-unavailable')) + return + } + this.lastReply = this.now() + this.heartbeatFailures = 0 + if (reply && reply.commandExpired) { + this.finish(id, undefined, new CommandExpiredError(pending.operation)) + return + } + this.finish(id, result) + } + + private finish(id: number, value?: unknown, error?: Error): void { + const pending = this.pending.get(id) + if (!pending) return + this.pending.delete(id) + clearTimeout(pending.timer) + if (error) pending.reject(error) + else pending.resolve(value) + } + + /** Marks the link dead, fails everything pending, and tells the owner, exactly once. */ + private fail(error: CdpConnectionLostError): void { + if (this.dead) return + this.dead = true + clearInterval(this.heartbeat) + for (const id of [...this.pending.keys()]) this.finish(id, undefined, error) + if (!this.closing) { + this.socket.terminate() + this.onLost?.(error) + } + } + + private send( + method: string, + params: unknown, + timeoutMs: number, + operation: string, + id: number = ++this.sequence + ): Promise { + if (this.dead || this.socket.readyState !== OPEN) { + return Promise.reject(new CdpConnectionLostError(this.dead ? 'closed' : 'socket')) + } + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => this.finish(id, undefined, new CdpTimeoutError(operation, timeoutMs)), + timeoutMs + ) + this.pending.set(id, { resolve, reject, timer, operation }) + const safe = + method === 'Runtime.evaluate' && + typeof (params as { expression?: unknown }).expression === 'string' + ? { + ...(params as object), + expression: escapeSurrogates((params as { expression: string }).expression) + } + : params + this.socket.send(JSON.stringify({ id, method, params: safe }), (error) => { + if (error) this.finish(id, undefined, new CdpConnectionLostError('socket')) + }) + }) + } + + /** + * Runs one agent command. Resolves with the agent's reply, or rejects with a typed `CdpError`. + * `timeoutMs` bounds both the host's wait and, through the embedded deadline, the app's willingness + * to start the command at all. + */ + async invoke( + command: AgentCommand, + timeoutMs: number = DEFAULT_COMMAND_TIMEOUT_MS + ): Promise { + await this.ready + const id = -++this.sequence + // `restore` must run even in a slow app, so it is the one command with no deadline. + const deadline = + this.appClockOffset === undefined || command.type === 'restore' + ? undefined + : this.now() + timeoutMs + this.appClockOffset + const reply = + `var reply=function(result){var send=globalThis[${JSON.stringify(this.binding)}];` + + `if(typeof send==='function')send(JSON.stringify({id:${id},result:result===undefined?null:result}))};` + const expression = + `(function(){var runtime=globalThis[${JSON.stringify(this.key)}];${reply}` + + (deadline === undefined + ? '' + : `if(${APP_CLOCK_EXPR}>${deadline}){reply({commandExpired:true});return}`) + + `if(!runtime||typeof runtime.invoke!=='function'){reply({runtimeUnavailable:true});return}` + + `runtime.invoke(${JSON.stringify(JSON.stringify(command, wellFormed))},reply)})()` + const result = await this.send( + 'Runtime.evaluate', + { expression, silent: true, returnByValue: true, objectGroup: this.key }, + timeoutMs, + command.type, + id + ) + if (command.type === 'heartbeat' && (result as HeartbeatReply | null)?.alive !== true) { + throw new CdpEvaluationError('heartbeat', 'invalid response') + } + return result as T + } + + /** + * Idempotent teardown, in this order: stop the heartbeat; ask the agent to restore itself + * (best effort, bounded; the lease restores it anyway if this fails); remove the binding, delete + * its global, release the object group; fail whatever is still pending; terminate the socket. + * Every call after the first returns the first's promise. + */ + close(options: { restore?: boolean } = {}): Promise { + return (this.closing ??= this.dispose(options.restore !== false)) + } + + private async dispose(restore: boolean): Promise { + clearInterval(this.heartbeat) + if (restore && !this.dead && this.socket.readyState === OPEN) { + let timer: NodeJS.Timeout | undefined + try { + await Promise.race([ + this.ready.then(() => this.invoke({ type: 'restore' }, RESTORE_TIMEOUT_MS)), + new Promise((resolve) => { + timer = setTimeout(resolve, RESTORE_TIMEOUT_MS + 500) + }) + ]) + } catch { + // The in-app watchdog restores after the host goes quiet; teardown must not hang on it. + } finally { + clearTimeout(timer) + } + } + if (this.socket.readyState === OPEN) { + const writes = [ + { method: 'Runtime.removeBinding', params: { name: this.binding } }, + { + method: 'Runtime.evaluate', + params: { + expression: `delete globalThis[${JSON.stringify(this.binding)}]`, + silent: true, + returnByValue: true + } + }, + { method: 'Runtime.releaseObjectGroup', params: { objectGroup: this.key } } + ].map( + (call) => + new Promise((resolve) => { + try { + this.socket.send(JSON.stringify({ id: ++this.sequence, ...call }), () => resolve()) + } catch { + resolve() + } + }) + ) + let flush: NodeJS.Timeout | undefined + await Promise.race([ + Promise.all(writes), + new Promise((resolve) => { + flush = setTimeout(resolve, TEARDOWN_FLUSH_MS) + }) + ]) + clearTimeout(flush) + } + this.fail(new CdpConnectionLostError('closed')) + this.socket.terminate() + } +} + +export type ConnectionState = 'idle' | 'connecting' | 'ready' | 'reconnecting' | 'failed' | 'closed' + +export interface CdpConnectionOptions { + /** + * Yields the debugger URL to connect to, **freshly each call**: a relaunched app is a new Metro + * target with a new URL. See `createTargetResolver` for the discovery-backed implementation. + */ + resolveTarget: () => Promise<{ webSocketDebuggerUrl: string }> + /** Stable across this connection's links; defaults to a random one. */ + sessionId?: string + leaseMs?: number + limits?: AgentLimits + createSocket?: CdpSocketFactory + heartbeatIntervalMs?: number + heartbeatRecentReplyMs?: number + heartbeatTimeoutMs?: number + /** Failed reconnect attempts tolerated before the state becomes `failed`. Default 10. */ + maxReconnectAttempts?: number + /** Injected so tests do not wait out the real backoff. */ + sleep?: (ms: number) => Promise + backoffMs?: (attempt: number) => number + log?: (message: string) => void + onStateChange?: (state: ConnectionState) => void +} + +/** + * Builds a `resolveTarget` that follows an app across Metro restarts and relaunches: it lists the + * targets on `metroPort` and re-identifies the one first attached, by app plus device rather than + * by id (`reidentifyTarget`). Throws when no single target matches, which the connection treats as + * a failed attempt and retries: right after a relaunch the new target is often not registered yet. + */ +export function createTargetResolver( + metroPort: number, + initial: MetroCdpTarget, + options: { list?: typeof listMetroTargets } = {} +): () => Promise { + const list = options.list ?? listMetroTargets + let current = initial + return async () => { + const targets = await list(metroPort) + const match = reidentifyTarget(targets, current.id, current) + if (!match) throw new Error(`no Metro target matches the app on port ${metroPort}`) + current = match + return match + } +} + +const defaultSleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)) + +/** + * The connection the rest of the app uses. See "Two layers" in the file doc for what it recovers + * from, what it does not retry, and why. + */ +export class CdpConnection { + private link?: CdpLink + private recovery?: Promise + private currentState: ConnectionState = 'idle' + private readonly key: string + private readonly sleep: (ms: number) => Promise + private readonly backoffMs: (attempt: number) => number + private readonly maxAttempts: number + private readonly options: CdpConnectionOptions + + constructor(options: CdpConnectionOptions) { + this.options = options + this.key = `${KEY_PREFIX}${(options.sessionId ?? randomUUID()).replaceAll('-', '')}` + this.sleep = options.sleep ?? defaultSleep + this.backoffMs = options.backoffMs ?? reconnectBackoffMs + this.maxAttempts = options.maxReconnectAttempts ?? 10 + } + + get state(): ConnectionState { + return this.currentState + } + + private setState(state: ConnectionState): void { + if (this.currentState === state) return + this.currentState = state + this.options.onStateChange?.(state) + } + + /** The first connection. One attempt, and it throws: whether to carry on without us is the caller's call. */ + async connect(): Promise { + if (this.currentState !== 'idle') + throw new Error(`cannot connect from state ${this.currentState}`) + this.setState('connecting') + try { + await this.openLink() + this.setState('ready') + } catch (error) { + this.setState('idle') + throw error + } + } + + private async openLink(): Promise { + const target = await this.options.resolveTarget() + const link = new CdpLink(target.webSocketDebuggerUrl, { + key: this.key, + leaseMs: this.options.leaseMs, + limits: this.options.limits, + createSocket: this.options.createSocket, + heartbeatIntervalMs: this.options.heartbeatIntervalMs, + heartbeatRecentReplyMs: this.options.heartbeatRecentReplyMs, + heartbeatTimeoutMs: this.options.heartbeatTimeoutMs, + onLost: (error) => { + // A link that dies before its handshake completes is reported by `ready`, not here. + if (this.link === link && this.currentState === 'ready') this.recover(error) + } + }) + try { + await link.ready + } catch (error) { + await link.close({ restore: false }).catch(() => undefined) + throw error + } + if (this.currentState === 'closed') { + await link.close() + throw new CdpConnectionLostError('closed') + } + this.link = link + } + + private recover(cause: CdpConnectionLostError): void { + if (this.recovery || this.currentState === 'closed') return + this.options.log?.(`link lost (${cause.reason}); reconnecting`) + this.recovery = this.runRecovery().finally(() => { + this.recovery = undefined + }) + // Callers observe recovery through `invoke`/`reconnect`; this keeps it from being unhandled. + this.recovery.catch(() => undefined) + } + + private async runRecovery(): Promise { + this.setState('reconnecting') + for (let attempt = 1; attempt <= this.maxAttempts; attempt++) { + await this.sleep(this.backoffMs(attempt)) + if (this.currentState === 'closed') throw new CdpConnectionLostError('closed') + try { + await this.openLink() + this.setState('ready') + return + } catch (error) { + this.options.log?.( + `reconnect attempt ${attempt}/${this.maxAttempts} failed: ${error instanceof Error ? error.message : String(error)}` + ) + } + } + this.setState('failed') + throw new CdpConnectionLostError('exhausted') + } + + /** + * Replaces the link now, from any state but `closed`. Used after `failed`, or by a caller that + * knows it just relaunched the app and does not want to wait for the heartbeat to notice. + * Resolves when a fresh link is installed; rejects with `CdpConnectionLostError('exhausted')`. + */ + async reconnect(): Promise { + if (this.currentState === 'closed') throw new CdpConnectionLostError('closed') + if (this.recovery) return this.recovery + const old = this.link + this.link = undefined + if (old) await old.close({ restore: false }).catch(() => undefined) + this.recover(new CdpConnectionLostError('reconnecting')) + return this.recovery + } + + /** + * Runs one agent command. While a reconnect is in progress it waits for it, up to `timeoutMs` + * from now, rather than failing; a command already in flight when the link died has been rejected + * and is never replayed. See the file doc. + */ + async invoke( + command: AgentCommand, + timeoutMs: number = DEFAULT_COMMAND_TIMEOUT_MS + ): Promise { + if (this.currentState === 'closed') throw new CdpConnectionLostError('closed') + if (this.currentState === 'failed') throw new CdpConnectionLostError('exhausted') + if (this.recovery) { + let timer: NodeJS.Timeout | undefined + try { + await Promise.race([ + this.recovery, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new CdpConnectionLostError('reconnecting')), timeoutMs) + }) + ]) + } finally { + clearTimeout(timer) + } + } + if (!this.link) throw new CdpConnectionLostError('socket') + return this.link.invoke(command, timeoutMs) + } + + /** Tears the link down (restoring the app) and stops any recovery. Idempotent. */ + async close(): Promise { + if (this.currentState === 'closed') return + this.setState('closed') + const link = this.link + this.link = undefined + await link?.close() + } +} diff --git a/src/app-flow/cdp/discovery.ts b/src/app-flow/cdp/discovery.ts new file mode 100644 index 00000000..3703b381 --- /dev/null +++ b/src/app-flow/cdp/discovery.ts @@ -0,0 +1,221 @@ +/** + * Finds the React Native debugger WebSocket target for a running Metro server. + * + * Metro serves its inspector targets at `/json/list`. This module does that one HTTP GET and turns + * the answer into typed, validated targets; it opens no socket (the connection lives elsewhere) + * and imports nothing from Electron or the device layer, so it stays loadable under `node --test` + * like the rest of `app-flow/` (see the note at the top of `reach/types.ts`). + * + * The Metro port is taken as a plain number, the way every other consumer already holds it + * (`ProfilingDevice.metroPort`, `selectNavStateTarget`, `selectMetroCdpTarget`), and the origin is + * built the same way they build it: `http://127.0.0.1:`. Nothing new discovers Metro. + * + * **A Metro response is untrusted input.** The body is capped in size and in target count, and every + * `webSocketDebuggerUrl` is checked before it is handed back: `ws:` only, a loopback host, the + * port Metro itself was reached on, and no credentials or fragment. A target failing any of those + * is dropped, not repaired, so a compromised or spoofed listener on that port cannot steer the + * caller's socket at an arbitrary host. The port check also matters in the benign case: an + * inspector may advertise `localhost` where discovery used `127.0.0.1`, which is accepted, but one + * advertising a different port is not the Metro we were asked about. + * + * ### `supportsMultipleDebuggers` + * + * Whether the target reports `reactNative.capabilities.supportsMultipleDebuggers` is returned as + * data and deliberately NOT enforced here. A target without it allows exactly one debugger + * attached at a time, and a second attach either fails or evicts the first. That matters to us + * because a sweep wants to hold one socket for its whole duration while the CPU profiler + * separately attaches `agent-device cdp` to the same inspector + * (`profiling/profilers/cpu-devtool.ts`, `selectMetroCdpTarget`). On a single-debugger target those + * two collide, and `shell.ts` already SIGKILLs a hung eval for exactly that contention. Refusing + * such targets outright would make the feature unusable on older runtimes that otherwise work fine + * for a sweep with no profiler attached, so the decision is the caller's: it knows whether a + * profiler is going to run. + */ + +/** How long `/json/list` may take. Metro answers locally in milliseconds; 5s is a hung server. */ +export const DISCOVERY_TIMEOUT_MS = 5_000 +/** Largest `/json/list` body accepted. Real lists are a few hundred bytes per target. */ +export const MAX_DISCOVERY_BODY_BYTES = 1024 * 1024 +/** Most targets parsed from one response. A dev machine has a handful; beyond this is noise. */ +export const MAX_DISCOVERY_TARGETS = 100 +/** Longest string kept from any descriptive field of a target. */ +const MAX_FIELD_LENGTH = 512 + +const LOOPBACK_HOSTS: ReadonlySet = new Set(['127.0.0.1', 'localhost', '[::1]']) + +/** One validated Metro inspector target. */ +export interface MetroCdpTarget { + /** Metro's id for the target. Volatile: Metro reassigns it across restarts. Do not persist it. */ + id: string + title: string + appId?: string + deviceName?: string + /** `reactNative.logicalDeviceId`: stable per device, unlike `id`. */ + deviceId?: string + /** Validated: `ws:`, loopback host, Metro's own port. Safe to hand to a WebSocket client. */ + webSocketDebuggerUrl: string + /** + * True only when the target explicitly reports `supportsMultipleDebuggers`. False means it was + * absent or not `true` (indistinguishable on purpose; both mean "assume one debugger"). See the + * module doc for why this is data rather than a refusal. + */ + supportsMultipleDebuggers: boolean +} + +export interface ListMetroTargetsOptions { + /** Cancels the request; the rejection is the abort reason, not a "cannot reach Metro" error. */ + signal?: AbortSignal + /** Injected so tests never touch the network. Defaults to the global `fetch`. */ + fetchImpl?: typeof fetch + timeoutMs?: number +} + +/** The parts of a target that survive a Metro restart. */ +export interface MetroTargetIdentity { + appId?: string + deviceId?: string + deviceName?: string +} + +const asString = (value: unknown): string | undefined => + typeof value === 'string' ? value.slice(0, MAX_FIELD_LENGTH) : undefined + +/** + * Validates one advertised debugger URL against the Metro port we reached. Returns the normalized + * href, or null when it must be rejected. Relative URLs resolve against the Metro origin, which + * then fails the scheme check, so they are rejected rather than guessed at. + */ +function validateDebuggerUrl(raw: string, metroPort: number): string | null { + let url: URL + try { + url = new URL(raw, `http://127.0.0.1:${metroPort}`) + } catch { + return null + } + if (url.protocol !== 'ws:') return null + if (!LOOPBACK_HOSTS.has(url.hostname)) return null + if (url.username || url.password || url.hash) return null + // `ws:` defaults to port 80, which `URL` reports as the empty string. + if (Number(url.port || 80) !== metroPort) return null + return url.href +} + +/** Reads a response body as text, giving up as soon as it exceeds the cap rather than after. */ +async function readCappedText(response: Response, maxBytes: number): Promise { + const tooLarge = (): Error => new Error('Metro returned too much data for its target list.') + const declared = Number(response.headers.get('content-length')) + if (Number.isFinite(declared) && declared > maxBytes) throw tooLarge() + if (!response.body) { + const text = await response.text() + if (Buffer.byteLength(text) > maxBytes) throw tooLarge() + return text + } + const reader = response.body.getReader() + const decoder = new TextDecoder() + let received = 0 + let text = '' + for (;;) { + const { done, value } = await reader.read() + if (done) break + received += value.byteLength + if (received > maxBytes) { + await reader.cancel().catch(() => undefined) + throw tooLarge() + } + text += decoder.decode(value, { stream: true }) + } + return text + decoder.decode() +} + +/** + * Lists the inspector targets of the Metro server on `metroPort`, keeping only those whose + * debugger URL passes `validateDebuggerUrl`. Throws when Metro is unreachable, answers non-2xx, + * sends an oversized or malformed body, or sends something that is not a list; an empty array + * means Metro is up with nothing attached. + */ +export async function listMetroTargets( + metroPort: number, + options: ListMetroTargetsOptions = {} +): Promise { + const { signal, fetchImpl = fetch, timeoutMs = DISCOVERY_TIMEOUT_MS } = options + const origin = `http://127.0.0.1:${metroPort}` + const timeout = AbortSignal.timeout(timeoutMs) + let response: Response + try { + response = await fetchImpl(`${origin}/json/list`, { + // A redirect is a different server answering; following it would defeat the port check. + redirect: 'error', + signal: signal ? AbortSignal.any([timeout, signal]) : timeout + }) + } catch (error) { + if (signal?.aborted) throw error + throw new Error(`Cannot reach Metro at ${origin}.`, { cause: error }) + } + if (!response.ok) throw new Error(`Metro returned HTTP ${response.status}.`) + + const body = await readCappedText(response, MAX_DISCOVERY_BODY_BYTES) + let payload: unknown + try { + payload = JSON.parse(body) + } catch (error) { + throw new Error('Metro returned a target list that is not valid JSON.', { cause: error }) + } + if (!Array.isArray(payload)) throw new Error('Metro returned an invalid target list.') + + const targets: MetroCdpTarget[] = [] + for (const item of payload.slice(0, MAX_DISCOVERY_TARGETS)) { + if (typeof item !== 'object' || item === null) continue + const record = item as Record + if (typeof record.id !== 'string' || typeof record.webSocketDebuggerUrl !== 'string') continue + const webSocketDebuggerUrl = validateDebuggerUrl(record.webSocketDebuggerUrl, metroPort) + if (!webSocketDebuggerUrl) continue + const reactNative = (record.reactNative ?? {}) as { + logicalDeviceId?: unknown + capabilities?: { supportsMultipleDebuggers?: unknown } + } + targets.push({ + id: record.id, + title: asString(record.title) ?? record.id, + appId: asString(record.appId), + deviceName: asString(record.deviceName), + deviceId: asString(reactNative.logicalDeviceId), + webSocketDebuggerUrl, + supportsMultipleDebuggers: reactNative.capabilities?.supportsMultipleDebuggers === true + }) + } + return targets +} + +const normalizeDeviceName = (name: string): string => name.toLowerCase().replace(/[^a-z\d]/g, '') + +/** + * Finds the target we were attached to before, after Metro may have restarted and handed out new + * ids. Matching is by app plus device, never by id alone: the same app on two simulators shares an + * `appId`, and a different app on the same simulator shares a device, so both must agree. + * + * Device identity is the logical device id when both sides have one, else the device name compared + * loosely (case and punctuation), since a name is all older runtimes report. When several targets + * still match, the one whose id is unchanged wins, otherwise the answer is `undefined`: picking one + * of two equally plausible targets would silently drive the wrong app. With no previous identity + * there is nothing to re-identify, so only the id is consulted. + */ +export function reidentifyTarget( + targets: readonly T[], + previousId: string, + previous?: MetroTargetIdentity +): T | undefined { + if (!previous) return targets.find((target) => target.id === previousId) + const matches = targets.filter((target) => { + const sameApp = previous.appId + ? target.appId === previous.appId + : target.id === previousId && !target.appId + const sameDevice = + (previous.deviceId !== undefined && target.deviceId === previous.deviceId) || + (previous.deviceName !== undefined && + target.deviceName !== undefined && + normalizeDeviceName(target.deviceName) === normalizeDeviceName(previous.deviceName)) + return sameApp && sameDevice + }) + const exact = matches.find((target) => target.id === previousId) + return exact ?? (matches.length === 1 ? matches[0] : undefined) +} diff --git a/src/app-flow/cdp/runtime-source.ts b/src/app-flow/cdp/runtime-source.ts new file mode 100644 index 00000000..25da162f --- /dev/null +++ b/src/app-flow/cdp/runtime-source.ts @@ -0,0 +1,132 @@ +/** + * Assembles the in-app agent's source text from the real functions in `agent-runtime.ts`. + * + * The reference implementation does the same (`fn.toString()` over its runtime functions) but runs + * its agent through an esbuild `target: 'es2015'` build first, and keeps a JSON-packaged copy for + * the shipped server with a hand-written validity check. We take neither: the agent is written in + * conservative syntax by hand (see the doc comment in `agent-runtime.ts`), so there is no build + * step to forget and no second artefact to go stale. What replaces the build step is the check in + * this file, which is deliberately cheap and deliberately textual. + * + * ## What the integrity check can and cannot promise + * + * It catches the failures that would otherwise show up only against a live device: + * + * - **A renamed function.** A bundler may rename a top-level declaration to avoid a collision. The + * functions call each other by their declared names, so injecting one under a new name would + * compile and then fail on first call. `fn.name` is compared with its registry key and a mismatch + * throws. + * - **A missing or extra function**, which means the registry and the check disagree about what the + * agent is. + * - **A runaway size.** The source rides in a single `Runtime.evaluate` frame that Hermes has to + * compile on every connect; a limit turns an accidental import of something large into a loud + * failure instead of a slow handshake. + * - **Syntax the Hermes debugger evaluator cannot compile**: `async`/`await`, optional chaining, + * nullish coalescing, spread/rest, `for...of`, `class`, and template literals. + * + * It cannot prove the source is *correct*, and being textual it cannot tell a comment from code, so + * an in-body comment spelling out a banned token is a false positive (and the fix is to reword the + * comment). Free references to module-scope names are not detected here at all; the test suite + * evaluates the assembled source in an empty `node:vm` context to flush those out. + */ +import { AGENT_FUNCTIONS } from './agent-runtime.ts' + +/** The number of functions the agent is made of. Changing the registry means changing this. */ +export const EXPECTED_AGENT_FUNCTION_COUNT = 13 +/** + * Upper bound on the assembled source. The current agent is well under a third of this; the limit + * exists to catch an accident, not to budget bytes, and the reference's own packaged check uses + * 500,000 per function for the same reason. + */ +export const MAX_AGENT_SOURCE_BYTES = 64 * 1024 + +/** Each pattern names a construct the debugger evaluator is not trusted to compile. */ +const BANNED_SYNTAX: ReadonlyArray = [ + ['async function', /\basync\b/], + ['await', /\bawait\b/], + ['optional chaining', /\?\.(?![0-9])/], + ['nullish coalescing', /\?\?/], + ['spread or rest', /\.\.\./], + ['for...of', /\bfor\s*\(\s*(?:var|let|const)\s+[^;)]*\bof\b/], + ['class', /\bclass\s+[A-Za-z_$]/], + ['template literal', /`/] +] + +export class AgentSourceError extends Error { + constructor(message: string) { + super(message) + this.name = 'AgentSourceError' + } +} + +type AgentFunctionMap = Readonly unknown>> + +/** + * Checks the registry and the text built from it. Exported so the tests can feed it a deliberately + * bad registry and confirm each rule fires, rather than trusting that the passing case passes for + * the right reason. + */ +export function assertAgentSourceIntegrity( + functions: AgentFunctionMap, + source: string, + expectedCount: number = EXPECTED_AGENT_FUNCTION_COUNT +): void { + const names = Object.keys(functions) + if (names.length !== expectedCount) { + throw new AgentSourceError( + `agent has ${names.length} functions, expected ${expectedCount}; update EXPECTED_AGENT_FUNCTION_COUNT with the registry` + ) + } + for (const name of names) { + if (functions[name]!.name !== name) { + throw new AgentSourceError( + `agent function "${name}" was renamed to "${functions[name]!.name}" by the build; injected functions call each other by name` + ) + } + } + if (Buffer.byteLength(source, 'utf8') > MAX_AGENT_SOURCE_BYTES) { + throw new AgentSourceError(`agent source exceeds ${MAX_AGENT_SOURCE_BYTES} bytes`) + } + for (const [label, pattern] of BANNED_SYNTAX) { + const hit = pattern.exec(source) + if (hit) { + throw new AgentSourceError( + `agent source contains ${label} near "${source.slice(Math.max(0, hit.index - 20), hit.index + 30).replace(/\s+/g, ' ')}"; see the syntax constraint in agent-runtime.ts` + ) + } + } +} + +/** + * The source of an expression that evaluates to the install function. Calling the result with + * `(key, leaseMs, limits)` installs the agent; every helper lives in the closure that expression + * creates, so nothing is added to the app's global scope but the one key. + * + * Each function is emitted as a declaration (`fn.toString()` already is one), so they are hoisted + * and may be listed in any order. + */ +export function buildAgentSource(functions: AgentFunctionMap = AGENT_FUNCTIONS): string { + const declarations = Object.values(functions) + .map((fn) => fn.toString()) + .join('\n') + const source = `(function () {\n${declarations}\nreturn agentInstall;\n})()` + assertAgentSourceIntegrity(functions, source, Object.keys(functions).length) + return source +} + +let cached: string | undefined + +/** + * The agent source for the real registry, built and checked once. Nothing about it varies per + * connection (the key, lease and limits are arguments at the call site), so rebuilding it per + * connect would only repeat the check. Also asserts the registry size against the constant, which + * `buildAgentSource` cannot do for a registry passed in. + */ +export function agentSource(): string { + if (cached === undefined) { + const source = buildAgentSource() + assertAgentSourceIntegrity(AGENT_FUNCTIONS, source) + cached = source + } + return cached +} diff --git a/src/app-flow/cli-args.ts b/src/app-flow/cli-args.ts new file mode 100644 index 00000000..103e07c8 --- /dev/null +++ b/src/app-flow/cli-args.ts @@ -0,0 +1,12 @@ +/** + * Shell quoting for the `agent-device …` lines persisted into `pathToReach`. + * + * Its own module because both `capture.ts` (which builds the boot/install/open/replay lines) and + * `manual/commands.ts` (which builds the press/fill/scroll ones) need it, and neither should import + * the other: the core capture module must not depend on a feature folder, and the manual recorder's + * command builders must stay free of Electron so they can be tested under `node --test`. + */ +export function quoteCliArg(value: string): string { + if (/^[A-Za-z0-9._:+/=@-]+$/.test(value)) return value + return `'${value.replace(/'/g, `'\\''`)}'` +} diff --git a/src/app-flow/decision-provider.ts b/src/app-flow/decision-provider.ts new file mode 100644 index 00000000..b421e615 --- /dev/null +++ b/src/app-flow/decision-provider.ts @@ -0,0 +1,85 @@ +import type { TokenUsage } from '../shared/app-flow.ts' + +export type SubAgentThinkingLevel = 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max' +export interface ModelCallDiagnostic { [key: string]: unknown } + +export interface SubAgentOptions { + cwd: string + systemPrompt: string + userPrompt: string + tools?: string[] + thinkingLevel?: SubAgentThinkingLevel + providerId?: string + modelId?: string + imagePath?: string + timeoutMs?: number + onProgress?: (message: string, level?: 'info' | 'warn') => void + onUsage?: (usage: TokenUsage) => void + onDiagnostic?: (diagnostic: ModelCallDiagnostic) => void + diagnosticContext?: { agent: string; fixerExecutionAttempt?: number } + returnPartialOnTimeout?: boolean +} + +export interface SubAgentResult { + text: string + toolCalls: Array<{ id: string; name: string; arguments: Record }> + timedOut: boolean + filesChanged: string[] + tokens: TokenUsage + diagnostics: ModelCallDiagnostic[] +} + +export interface CompactHistoryOptions { + keepRecentUserTurns?: number + placeholder?: string +} +export interface SubAgentSessionOptions { + cwd: string + systemPrompt: string + tools?: string[] + thinkingLevel?: SubAgentThinkingLevel + providerId?: string + modelId?: string + compactHistory?: CompactHistoryOptions +} +export type SubAgentCallOptions = Omit< + SubAgentOptions, + 'cwd' | 'systemPrompt' | 'tools' | 'thinkingLevel' | 'providerId' | 'modelId' +> + +export function parseSubAgentJson(text: string, validate?: (value: unknown) => value is T): T | null { + const candidates: string[] = [] + let start = -1, depth = 0, inString = false, escaped = false + for (let index = 0; index < text.length; index += 1) { + const char = text[index] + if (inString) { + if (escaped) escaped = false + else if (char === '\\') escaped = true + else if (char === '"') inString = false + continue + } + if (char === '"') inString = true + else if (char === '{') { if (depth === 0) start = index; depth += 1 } + else if (char === '}' && depth > 0) { depth -= 1; if (depth === 0 && start >= 0) candidates.push(text.slice(start, index + 1)) } + } + for (const candidate of candidates.reverse()) { + try { + const value = JSON.parse(candidate) as unknown + if (!validate || validate(value)) return value as T + } catch { /* Try the preceding complete JSON object. */ } + } + return null +} + +export interface AppFlowDecisionSession { + prompt(options: SubAgentCallOptions, label: string): Promise + historyLength(): number + dispose(): void +} +export interface AppFlowDecisionProvider { + run(options: SubAgentOptions, label: string): Promise + createSession(options: SubAgentSessionOptions): Promise + activeNow?(): number + /** Single-pending-request host providers set this false so the engine never hedges. */ + supportsConcurrentRequests?: boolean +} diff --git a/src/app-flow/exploration.ts b/src/app-flow/exploration.ts new file mode 100644 index 00000000..9893b06b --- /dev/null +++ b/src/app-flow/exploration.ts @@ -0,0 +1,1178 @@ +import type { + AppFlowDestinationLocator, + AppFlowNavigationAction, + AppFlowNavigationDecision +} from './capture.ts' +import type { + AppFlowCaptureStatus, + AppFlowGuard, + AppFlowPressable, + AppFlowTrigger +} from '../shared/app-flow.ts' +import { PARAM_VALUE_RE } from './param-store.ts' +import type { ParamResolution, ParamRouteNode, ParamSource } from './param-store.ts' +import { fingerprint, similarity, tokenize } from './screen-signals.ts' + +const LOCATOR_KINDS = new Set(['text', 'label', 'id']) + +/** + * `qualifiers` appends extra selector terms to narrow an otherwise ambiguous match — agent-device's + * selector grammar is a space-separated term list, so `label="Menu" hittable=true` is one selector + * matching only the tappable `Menu`. Used by `executeNavigationActions`'s ambiguity retry; see there + * for why a plain locator is tried first. + */ +export function serializeLocator( + locator: { by: 'text' | 'label' | 'id'; value: string }, + qualifiers?: Readonly> +): string { + const value = locator.value.trim() + if (!LOCATOR_KINDS.has(locator.by) || !value) throw new Error('Invalid navigation locator') + const base = `${locator.by}=${JSON.stringify(value)}` + const extra = Object.entries(qualifiers ?? {}).map( + ([key, term]) => `${key}=${typeof term === 'boolean' ? term : JSON.stringify(term)}` + ) + return [base, ...extra].join(' ') +} + +/** agent-device refuses to act on a selector matching more than one element, rather than guessing + * which one the caller meant. Recognised so a caller can narrow and retry instead of giving up. */ +export function isAmbiguousSelectorMessage(message: string): boolean { + return /matched \d+ distinct actionable elements/i.test(message) +} + +export function parseNavigationDecision(value: unknown): AppFlowNavigationDecision | null { + if (!value || typeof value !== 'object') return null + const decision = value as Record + if (decision.decision === 'arrived') { + if (typeof decision.reason !== 'string') return null + // A malformed `destinationLocator` is dropped rather than failing the decision. The locator + // only decides whether the walk's recording can be published; the arrival itself was judged by + // a separate screenshot verdict. Rejecting the whole decision over it would throw away a + // correct, already-walked journey because the model named a landmark the wrong way — the + // caller's landmark fallback handles the gap far better than a retry loop would. + const destinationLocator = asDestinationLocator(decision.destinationLocator) + return destinationLocator + ? { decision: 'arrived', reason: decision.reason, destinationLocator } + : { decision: 'arrived', reason: decision.reason } + } + if (decision.decision === 'blocked') { + return typeof decision.reason === 'string' + ? { decision: 'blocked', reason: decision.reason } + : null + } + if (decision.decision !== 'act' || !isNavigationAction(decision.action)) return null + return { + decision: 'act', + action: decision.action, + reason: typeof decision.reason === 'string' ? decision.reason : '' + } +} + +function isNavigationAction(value: unknown): value is AppFlowNavigationAction { + if (!value || typeof value !== 'object') return false + const action = value as Record + if (!['tap', 'swipe', 'input', 'wait'].includes(String(action.type))) return false + if (action.type === 'tap' || action.type === 'input') { + if (!isLocator(action.locator)) return false + } + if (action.type === 'input' && typeof action.value !== 'string') return false + if ( + action.type === 'swipe' && + action.direction !== undefined && + !['up', 'down', 'left', 'right'].includes(String(action.direction)) + ) { + return false + } + if (action.duration !== undefined && typeof action.duration !== 'number') return false + return true +} + +/** + * Narrows an arbitrary value to a locator usable as a saved recording's destination guard, or + * returns null. + * + * `by: 'text'` is deliberately not accepted, and this is the whole reason this is separate from + * `isLocator`. agent-device's publish check wants a `wait` whose matched element it could record a + * re-verifiable identity for, and it only records that identity when the element has an id or an + * accessibility label — a matched node with neither yields no evidence, and the guard is then + * rejected for having none. `label=` and `id=` selectors match *on* those attributes, so anything + * they resolve necessarily carries one; `text=` matches rendered text, which on a plain RN `Text` + * routinely exists with no `nativeID` and no `accessibilityLabel` at all. Accepting `text` would + * therefore produce a guard that works sometimes and silently loses the script the rest of the + * time, which is worse than never offering it. + */ +function asDestinationLocator(value: unknown): AppFlowDestinationLocator | null { + if (!isLocator(value) || value.by === 'text') return null + return { by: value.by, value: value.value.trim() } +} + +// `chromeLocatorValues`, `isChromeLocator` and `pickLandmarkLocator` lived here. All three existed +// to choose the *destination guard* a recorded `.ad` script needed before agent-device would +// publish it — a wait on a durable, non-chrome landmark of the screen the walk ended on. The sweep +// records no scripts now (see `walk-commands.ts`), so nothing needs a guard chosen for it, and the +// manual recorder picks its own from the live snapshot (`manual/locators.ts`'s +// `destinationLocator`). `chrome-inventory.ts` still refers to `pickLandmarkLocator` by name in a +// comment contrasting its own locator rules; that contrast is with the rules, not the function. + +function isLocator(value: unknown): value is { by: 'text' | 'label' | 'id'; value: string } { + if (!value || typeof value !== 'object') return false + const locator = value as Record + return ( + LOCATOR_KINDS.has(String(locator.by)) && + typeof locator.value === 'string' && + !!locator.value.trim() + ) +} + +/** Tracks active exploration time while allowing user-controlled waits to be excluded. */ +export class ActiveTimeBudget { + private excludedMs = 0 + private readonly limitMs: number + private readonly startedAt: number + private readonly now: () => number + + constructor(limitMs: number, startedAt = Date.now(), now: () => number = Date.now) { + this.limitMs = limitMs + this.startedAt = startedAt + this.now = now + } + + exclude(ms: number): void { + this.excludedMs += Math.max(0, ms) + } + + get remainingMs(): number { + return Math.max(0, this.limitMs - (this.now() - this.startedAt - this.excludedMs)) + } + + get expired(): boolean { + return this.remainingMs === 0 + } +} + +/** + * Remembers which deep-link URLs the arrival judge has already rejected this sweep, so a caller + * can skip reopening one it already knows fails instead of paying for the open, the + * `waitForRoute`, the screenshot and the judge call again to relearn the same fact. + * + * This exists because `paramFreeDeepLink` truncates every route with an unresolved param to a + * shared ancestor prefix (`/profile/:name…` becomes `/profile`), so every sibling route under + * that ancestor computes the identical URL — and a node that gets requeued after the blocked-run + * breaker fires recomputes the same URL for itself a second time too. A device run against + * social-app hit both: five distinct nodes each independently rediscovered that `/profile` 404s, + * and one of them (`ProfileFollows`) rediscovered it twice after a requeue. Keyed by the exact URL + * opened rather than by node, so it catches both cases without knowing anything about the route + * tree. + * + * Deliberately in-memory and scoped to one sweep, not persisted on the node — that is a later, + * schema-backed step of the larger plan. This is only the cheap, self-contained subset of it. + */ +export class DeepLinkRejectionMemo { + private readonly rejected = new Set() + + wasRejected(url: string): boolean { + return this.rejected.has(url) + } + + reject(url: string): void { + this.rejected.add(url) + } +} + +/** + * Remembers which in-component guard conditions this build has already been *observed* to hold + * closed, so sibling screens behind the same flag skip the device probe that would rediscover it. + * + * `explainUnavailable` can already name the gate from the source the moment a guarded route 404s — + * but it is only consulted *after* the deep link has been opened, settled, screenshotted and + * judged. That ordering is right for a single node (the parser knows the flag exists, not what the + * build set it to, and only the device can answer that) and wasteful across a family of them: a + * sweep against social-app paid 4.1s apiece to learn four separate times that + * `ax.features.ModerationInboxEnable` is off, each probe ending in the identical 'Page not found' + * frame and the identical static explanation. + * + * So the first node behind a condition still pays full price and settles the question empirically; + * every later node carrying the *same* condition text reads the answer off this memo for free. The + * verdict recorded is therefore always a measured one, never an assumption from static source — + * which is the distinction `explainUnavailable` exists to protect and this memo must not erode. + * + * Keyed by the condition's source text because that is what makes two guards the same gate: the + * fallback component is too coarse (every gated screen in an app tends to render the same + * `NotFoundScreen`) and the node is too fine (the whole point is to span siblings). Two screens + * whose conditions are written differently but evaluate alike simply each pay once, which is the + * cheap failure. + * + * In-memory and scoped to one sweep, exactly like `DeepLinkRejectionMemo` above and for the same + * reason: a feature flag is a fact about one build, and the per-node durable record + * (`deep-link-policy.ts`) is the thing that already rides across sweeps. + */ +export class GuardGateMemo { + private readonly closed = new Set() + + wasClosed(condition: string): boolean { + return this.closed.has(condition) + } + + close(condition: string): void { + this.closed.add(condition) + } +} + +// `discardRecordingCandidate` lived here: unlink a walk's `.candidate.ad` and soft-home the app. +// Exploration writes no candidate file now, so there is nothing to discard and no caller left. + +/** + * What one unresolved hop costs relative to a whole extra screen in the journey. A hop the parser + * resolved to a named control is something the planner can tap; one it could not is a claim that + * the two screens are connected somehow, and walking it means finding the link yourself — often on + * content that only exists once the app has loaded data. Any number of named taps is a better + * journey than one guess, so the penalty is larger than any realistic path length. + */ +const UNRESOLVED_HOP_PENALTY = 1_000 + +/** + * The most reliable path through the parsed route graph from a screen the app opens on to the + * target: fewest hops whose control the parser could not name, and among those, fewest hops. + * + * Hop count alone picks the wrong journey. The graph is built by scanning each screen for + * `navigate('X')`, so a screen that lists other records gets an edge to every screen those records + * open — and that edge is a shortcut on paper only, because the control that walks it is a row the + * app renders at runtime, not something the planner can find on a cold screen. The journey the + * source spells out control-by-control is longer and actually walkable. + * + * `entryIds` are the screens reachable straight from the app's entry chrome — its tabs. A path is + * only returned when it actually starts at one of them: walking back to "whatever has no incoming + * edge" invents journeys that begin on an arbitrary screen the user can't even see at launch. No + * honest path is better than a fabricated one — the caller then tells the planner to explore from + * the tab bar instead. + */ +export function findNavigationPath( + edges: readonly { from: string; to: string | null; trigger?: AppFlowTrigger | null }[], + targetId: string, + entryIds: Iterable +): string[] | null { + const entries = new Set(entryIds) + if (entries.has(targetId)) return [targetId] + + const incoming = new Map() + for (const edge of edges) { + if (!edge.to || edge.from === edge.to) continue + const hop = { from: edge.from, cost: edge.trigger ? 1 : 1 + UNRESOLVED_HOP_PENALTY } + const sources = incoming.get(edge.to) + if (sources) sources.push(hop) + else incoming.set(edge.to, [hop]) + } + + // Walked backwards from the target, cheapest partial journey first. `cheapest` is what stops a + // cyclic graph looping: a screen is only re-expanded when a strictly cheaper way to it turns up. + const cheapest = new Map([[targetId, 0]]) + const frontier: { path: string[]; cost: number }[] = [{ path: [targetId], cost: 0 }] + let best: { path: string[]; cost: number } | null = null + while (frontier.length) { + frontier.sort((a, b) => a.cost - b.cost) + const current = frontier.shift()! + // Everything still queued costs at least this much, so no cheaper journey can come out of it. + if (best && current.cost >= best.cost) break + for (const hop of incoming.get(current.path[0]) ?? []) { + const cost = current.cost + hop.cost + const known = cheapest.get(hop.from) + if (known !== undefined && known <= cost) continue + cheapest.set(hop.from, cost) + const path = [hop.from, ...current.path] + // An entry screen ends the journey rather than extending it: a path is what the user walks + // from launch, so it starts at the first entry it reaches and never passes through another. + if (entries.has(hop.from)) { + if (!best || cost < best.cost) best = { path, cost } + continue + } + frontier.push({ path, cost }) + } + } + return best?.path ?? null +} + +/** + * Renders a journey as the taps that walk it, naming the control for each hop the parser resolved. + * A hop with no known control is stated as such rather than left out, so the planner can see which + * single step it has to work out for itself instead of re-deriving the whole route. + */ +export function describeJourney( + edges: readonly { from: string; to: string | null; trigger?: AppFlowTrigger | null }[], + path: readonly string[] +): string { + if (path.length < 2) return path[0] ?? '' + const hops: string[] = [path[0]] + for (let index = 1; index < path.length; index++) { + const edge = edges.find( + (candidate) => candidate.from === path[index - 1] && candidate.to === path[index] + ) + const trigger = edge?.trigger + hops.push( + trigger + ? `--[tap ${trigger.by}=${JSON.stringify(trigger.value)}]--> ${path[index]}` + : `--[control unknown, find it on screen]--> ${path[index]}` + ) + } + return hops.join(' ') +} + +export type ReachMethod = 'saved-replay' | 'deep-link' | 'agent-navigation' + +export interface ExplorationObservation { + pressables: AppFlowPressable[] + screenshotPath: string +} + +export interface ExplorationResult { + arrived: boolean + actions: AppFlowNavigationAction[] + reason: string + /** + * Set only on an arrival, and only when the planner named a usable one. The caller needs it to + * guard the recording it is about to publish — see `AppFlowNavigationDecision` — and falls back + * to `pickLandmarkLocator` over the arrival observation's controls when it is absent. + */ + destinationLocator?: AppFlowDestinationLocator +} + +/** + * How many planner decisions one exploration attempt gets before it gives up on the target. + * + * The walk itself is short when it is going to work at all: a screen behind a tab bar is one tap, a + * screen behind a drawer is two, and a seeded walk starts a hop away by construction. What the + * higher cap bought was not deeper walks but longer flailing on screens that cannot be reached at + * all from where the explorer is standing — a route whose list is empty so no row exists to open, + * or a destination whose entry point sits behind a feature flag that is off in this build. Those + * look identical to the planner on every step: a plausible screen, no control leading onward. It + * cannot conclude "unreachable" from one frame, so it keeps proposing the next most promising tap + * until something else stops it, and every one of those taps is a screenshot, a control read and a + * vision call. + * + * Seven is chosen to clear a realistic walk with room to recover from a wrong turn, and to stop + * well before a doomed one has spent a whole budget. Two attempts per node (seeded, then from the + * entry screen) still apply, so a genuinely reachable screen that the first walk approached badly + * gets a second, independent seven. + */ +export const EXPLORATION_MAX_DECISIONS = 7 + +/** Screenmap-style observe → one action → observe loop, with injected device/model operations. */ +export async function runInteractiveExploration(options: { + budget: ActiveTimeBudget + maxDecisions?: number + maxNoProgress?: number + maxPlannerFailures?: number + /** + * How many times the planner may claim arrival and be overruled by `verifyArrival` before the + * walk gives up. Two by default. + * + * This is its own budget because a rejected arrival is the one unproductive step that no other + * counter sees. It executes no action, so `actionAttempted` stays false and the no-progress + * fingerprint comparison never runs; it parses fine, so `maxPlannerFailures` is not touched. On + * device that left `maxDecisions` as its only bound, and a `SearchTab/Search` walk spent all of + * it: the planner said "arrived" on steps 2, 3 and 4 of the same unchanged screen, the judge + * said mismatch twice and match on the third, and each round cost a full planner call plus a + * full judge call — about 12k tokens to re-roll a verdict on pixels that never changed. + */ + maxArrivalRejections?: number + observe(step: number): Promise + /** + * `history` carries only what went wrong along the way — a rejected arrival, a failed action, a + * planner call that errored — never a log of every successful step. That is deliberate: `decide` + * is expected to be backed by one long-lived model session per exploration attempt (a fresh + * sub-agent session per call would pay for resending the full history every time, which is the + * quadratic-cost problem this shape exists to avoid), so the model already has its own + * successful actions in its own conversation. Only the failures are new information the model + * would not otherwise have — a rejected `decide` call never becomes a turn in that conversation. + */ + decide( + observation: ExplorationObservation, + history: readonly string[] + ): Promise + act(action: AppFlowNavigationAction): Promise + verifyArrival(observation: ExplorationObservation): Promise + check?(): void + /** + * Fired once per observation, after it is taken but before it feeds the no-progress fingerprint + * comparison or the planner. Exists so the orchestrator can opportunistically recognise a + * *different* pending screen the walk toward this target happened to land on — Settings, Profile + * and the other hub screens this loop passes through on the way to almost anywhere — without this + * module knowing anything about capture maps, slugs, or checkpoints. A failure here must not derail + * the walk toward the real target, so it is swallowed rather than propagated. + */ + observed?(observation: ExplorationObservation): Promise +}): Promise { + const actions: AppFlowNavigationAction[] = [] + const history: string[] = [] + let previous: Set | null = null + let noProgress = 0 + let actionAttempted = false + let successfulActionPendingObservation = false + let plannerFailures = 0 + const maxDecisions = options.maxDecisions ?? 30 + const maxNoProgress = options.maxNoProgress ?? 3 + const maxPlannerFailures = options.maxPlannerFailures ?? 3 + const maxArrivalRejections = options.maxArrivalRejections ?? 2 + let arrivalRejections = 0 + + for (let step = 0; step < maxDecisions && !options.budget.expired; step++) { + options.check?.() + const observation = await options.observe(step) + options.check?.() + if (options.observed) { + try { + await options.observed(observation) + } catch (error) { + // Opportunistic capture is a side effect of observing, never a reason the real target's + // walk should stop — see the doc comment on `observed` above. + console.error('[app-flow] observed hook failed:', error) + } + options.check?.() + } + if (options.budget.expired) break + const current = fingerprint(observation.pressables) + if (actionAttempted) { + if (previous && similarity(previous, current) >= 0.98) noProgress += 1 + else noProgress = 0 + } + previous = current + actionAttempted = false + successfulActionPendingObservation = false + if (noProgress >= maxNoProgress) { + return { + arrived: false, + actions, + reason: 'Three consecutive actions made no visible progress.' + } + } + + let decision: AppFlowNavigationDecision | null + try { + decision = await options.decide(observation, history) + } catch (error) { + // A planner call that times out or errors says nothing about the screen — the app is still + // sitting where the last action left it. Losing the whole target to one slow call throws away + // every step already walked, so the failure costs this step and the loop observes again. + plannerFailures += 1 + const detail = error instanceof Error ? error.message : String(error) + history.push(`Planner call failed: ${detail}; decide again from the fresh observation.`) + options.check?.() + if (plannerFailures >= maxPlannerFailures) { + return { + arrived: false, + actions, + reason: `Planner failed ${plannerFailures} times in a row: ${detail}` + } + } + continue + } + options.check?.() + if (options.budget.expired) break + if (!decision) { + // Counted against the same budget as a thrown planner call, and for a sharper reason than + // symmetry. A call that returns unparseable JSON is the *most* expensive kind: on device, the + // two `/profile/:name` steps that produced no usable decision burned 3,076 and 5,890 reasoning + // tokens before emitting 44 and 203 characters of text. Leaving this path uncounted meant the + // costliest failure mode was the only one with no failure budget at all — it ran until + // `maxDecisions`, and every step after it also re-read the dead trace as input. + plannerFailures += 1 + history.push('Planner returned an invalid decision; inspect the fresh UI and correct it.') + if (plannerFailures >= maxPlannerFailures) { + return { + arrived: false, + actions, + reason: `Planner failed ${plannerFailures} times in a row: returned an invalid decision.` + } + } + continue + } + // Reset here rather than the moment `decide` returns: a call that came back with unparseable + // JSON has not succeeded, and resetting on return would have made the counter above unreachable. + plannerFailures = 0 + if (decision.decision === 'blocked') return { arrived: false, actions, reason: decision.reason } + if (decision.decision === 'arrived') { + if (await options.verifyArrival(observation)) { + return decision.destinationLocator + ? { + arrived: true, + actions, + reason: decision.reason, + destinationLocator: decision.destinationLocator + } + : { arrived: true, actions, reason: decision.reason } + } + arrivalRejections += 1 + history.push(`Arrival rejected: ${decision.reason}`) + // The planner and the judge disagree about the screen in front of them, and nothing the loop + // does next changes that screen — it re-observes the same frame and asks the same two + // questions again. Past a couple of rounds that is not exploration, it is paying twice per + // round to re-roll a non-deterministic verdict, so the walk stops and says so rather than + // spending the rest of `maxDecisions` on it. + if (arrivalRejections >= maxArrivalRejections) { + return { + arrived: false, + actions, + reason: `Planner claimed arrival ${arrivalRejections} times and the arrival check rejected each one: ${decision.reason}` + } + } + continue + } + + try { + actionAttempted = true + await options.act(decision.action) + actions.push(decision.action) + successfulActionPendingObservation = true + // No "action succeeded" note: a successful action needs no correction, and with a + // session-scoped planner (see `decide`'s doc comment) the model already knows what it just + // did — it is sitting right there in its own conversation. Recording it here too would only + // resurrect the per-step history dump this shape exists to avoid. + } catch (error) { + history.push( + `Action failed: ${JSON.stringify(decision.action)} — ${error instanceof Error ? error.message : String(error)}` + ) + } + } + // A decision cap limits model calls, not observations. If the final permitted decision was a + // successful action, inspect its result once before giving up. Previously an action that reached + // the target on decision seven was guaranteed to be reported missing because the loop ended + // before the corresponding observe/arrival check; Moderation did exactly that in the regression + // run. This adds no planner call and only pays the verifier when there is a new final frame. + if (successfulActionPendingObservation && !options.budget.expired) { + options.check?.() + const observation = await options.observe(maxDecisions) + options.check?.() + if (options.observed) { + try { + await options.observed(observation) + } catch (error) { + console.error('[app-flow] observed hook failed:', error) + } + options.check?.() + } + if (!options.budget.expired && (await options.verifyArrival(observation))) { + return { + arrived: true, + actions, + reason: 'The final navigation action reached the target.' + } + } + } + return { + arrived: false, + actions, + reason: options.budget.expired + ? 'Navigation budget exhausted after five minutes.' + : `Navigation stopped after ${maxDecisions} decisions.` + } +} + +/** + * Every ancestor of `urlPath` that is itself a route, deepest first — `/moderation/inbox` then + * `/moderation` for `/moderation/inbox/settings`. Deepest first because the shortest remaining + * journey is the one least likely to go wrong. + * + * Kept here, verbatim, as the ancestry primitive `reach/seed.ts`'s `findSeedSource` builds on — + * that module supersedes this file's old `findSeedAncestor`, which offered only one proof of + * reachability (a deep link opened this run). `findSeedSource` adds a second, stronger one (a + * verified replay) without touching how ancestry itself is computed. + */ +export function findRouteAncestors( + urlPath: string, + nodes: readonly T[] +): T[] { + const byPath = new Map() + for (const node of nodes) { + if (node.urlPath) byPath.set(node.urlPath.replace(/\/+$/, ''), node) + } + const segments = urlPath.replace(/\/+$/, '').split('/') + const found: T[] = [] + for (let cut = segments.length - 1; cut > 1; cut--) { + const ancestor = byPath.get(segments.slice(0, cut).join('/')) + if (ancestor) found.push(ancestor) + } + return found +} + +/** + * The nearest ancestor of `urlPath` that is itself a route, e.g. `/moderation/inbox` for + * `/moderation/inbox/settings`. Used to judge whether a 404 is about this one screen or about a + * whole branch of the app being absent. + */ +export function findRouteAncestor( + urlPath: string, + nodes: readonly T[] +): T | null { + return findRouteAncestors(urlPath, nodes)[0] ?? null +} + +/** + * The settled explanation for a screen its own build gates off, in one place because two callers + * now produce it: `explainUnavailable` below, for the node that discovers the gate by probing it, + * and the deep-link strategy's `GuardGateMemo` short-circuit, for every sibling that then skips + * the probe. The two must read identically — they are the same finding about the same gate, and a + * reader comparing two nodes' recorded reasons should not have to work out whether a wording + * difference means something. + */ +export function explainGuardGate(guard: AppFlowGuard): string { + return `Registered but gated off in this build: the screen renders <${guard.fallback}/> unless \`${guard.condition}\` is false.` +} + +/** + * Explains a deep link that resolved but landed on the app's own not-found screen, or returns null + * when the sweep still has to explore to find out. + * + * A registered route rendering 404 means one of two things: the build gates the screen off, or the + * URL the sweep built was not the one the route wanted. Telling them apart matters — the first is a + * final answer worth recording, the second is exactly the case exploration exists for. + */ +export function explainUnavailable( + node: { + urlPath: string | null + params: readonly string[] + guard: AppFlowGuard | null + }, + ancestorStatus: AppFlowCaptureStatus | null +): string | null { + // A sample param the sweep invented is at least as likely as a gate to be why the route 404s, + // and unlike a gate it is something exploration can get past. + if (node.params.length) return null + if (node.guard) return explainGuardGate(node.guard) + // No gate found in the source — but a 404 under a branch the sweep has already reached cannot be + // a wrong URL, because the surrounding routes resolved. Something in the build withholds it. + if (ancestorStatus === 'ok' || ancestorStatus === 'unavailable') { + return 'Registered but not available in this build: the route resolves under a reachable branch yet renders the not-found screen.' + } + return null +} + +/** The subset of a node `indexDeepLinkPath` reads. */ +export interface IndexDeepLinkNode { + id: string + group: string + file: string | null +} + +/** + * The route a root-path screen (`urlPath: '/'`) should be deep-linked at, or **null when no honest + * answer exists** and the screen has to be reached through the UI instead. + * + * Opening the bare scheme does not switch tabs, so a screen registered at `/` cannot be deep-linked + * directly and something has to stand in for it. Exactly one substitution is grounded: an Expo + * Router route group (`(tabs)/index` → `/(tabs)`) is a real route the app declares, read off the + * node's own id or group. + * + * Everything else used to fall back to the first segment of the source file path, and that is not a + * guess so much as an invention. On Bluesky — React Navigation, no route groups — + * `HomeTab/Home` (`src/view/screens/Home.tsx`) produced `bluesky://src`, a path the app has never + * heard of, and the only thing it could possibly do was render the build's own 404. The device log + * shows the full price paid per root-path node: a URL open, a `waitForRoute`, a screenshot and a + * 3.5s arrival call, all to be told `Page not found`, before the ladder fell through to the walk + * that was always going to be the answer. Worse, the mismatch is indistinguishable from a route the + * build genuinely withholds, so it also fed `registerDeepLinkFailure` — an invented URL voting on + * whether the app's URL scheme works. + * + * Returning null makes "there is no URL for this screen" a first-class answer, which is the truth + * for a React Navigation root screen, and lets `deepLinkStrategy` skip straight to the walk. + * `probe-route.ts` already refuses to *probe* a root path for this same reason; this is the + * per-node half of the same rule. + */ +export function indexDeepLinkPath(node: IndexDeepLinkNode): string | null { + const group = node.id.split('/').find((seg) => seg.startsWith('(') && seg.endsWith(')')) + if (group) return `/${group}` + if (node.group.startsWith('(') && node.group.endsWith(')')) return `/${node.group}` + return null +} + +/** Whether `node` is a root-path screen with no grounded stand-in route — the case where a deep + * link can only be invented, and so must not be attempted at all. */ +export function isUndeeplinkableRoot( + node: IndexDeepLinkNode & { urlPath: string | null } +): boolean { + return (node.urlPath === '/' || node.urlPath === '') && indexDeepLinkPath(node) === null +} + +/** + * The longest prefix of `urlPath` that carries no unresolved param, or null when the route needs one + * in its very first segment. + * + * A route the call sites never supply a value for is not automatically unreachable by URL: apps + * routinely resolve the parameterised route's own prefix on their own — `/profile/:name` opens the + * signed-in user's profile. Trying that costs one open and one arrival check, and the arrival + * verifier is what decides whether it landed on the target, so a prefix that means something else + * simply falls through to exploration. + */ +export function paramFreeDeepLink( + urlPath: string, + unresolvedParams: readonly string[] +): string | null { + if (!unresolvedParams.length) return urlPath + const wanted = new Set(unresolvedParams) + const segments = urlPath.replace(/\/+$/, '').split('/') + const at = segments.findIndex((segment) => wanted.has(paramKey(segment) ?? '')) + if (at === -1) return urlPath + const prefix = segments.slice(0, at).join('/') + return prefix && prefix !== '/' ? prefix : null +} + +/** The param a route segment declares — `:name` and `[name]` alike — or null for a literal one. */ +function paramKey(segment: string): string | null { + if (segment.startsWith(':')) return segment.slice(1) + if (segment.startsWith('[') && segment.endsWith(']')) { + return segment.slice(1, -1).replace(/^\.\.\./, '') + } + return null +} + +/** Longest control caption the planner is shown before it is cut. */ +const PRESSABLE_TEXT_LIMIT = 80 + +/** + * The visible controls trimmed to what a planner can act on. A content-heavy screen — a feed, a + * search result list — exposes hundreds of pressables whose captions are whole posts, and sending + * them verbatim has cost a single navigation step over twenty thousand input tokens and a hundred + * seconds. Captions are cut to a length that still identifies the control, entries that collapse + * onto each other once cut are dropped as the duplicate locators they are, and the count left out + * is reported so the planner knows it is looking at a window rather than the whole screen. + * + * `identifier` — when a control has one — is put first in the object and `text` is left out + * entirely, rather than the two of them sitting alongside `label` as equally-weighted fields. The + * ProfileFollowers walk in session_2026-09-22_22-48-44.log is why: the row's name was only ever + * exposed as `label` (`{"label":"Clockwork Creature","role":"StaticText"}`, no `identifier` at + * all), and the planner still spent its first step on `by:"text"`, which agent-device rejected + * outright ("Selector did not match: text=\"Clockwork Creature\""), then a second step on a + * same-shaped neighbour (`label:"Clockwork Creature's avatar"`) that matched but opened a post + * thread instead of the profile — two of five steps burned, versus the very next screen + * (ProfileFollows) landing in two taps because every control on it carried a testID + * (`bottomBarProfileBtn`, `profileHeaderFollowsButton`). A control list where `text` and `label` + * flank `identifier` as one option among three invites exactly that guessing; leading with + * `identifier` and dropping the field that duplicates `label` most often (`text` is `node.value` in + * capture.ts, which for a plain pressable is usually the same caption `label` already carries) + * makes the stable handle the obvious first read and shrinks the payload besides — this list is + * resent for the two most recent observations on every step, so every byte trimmed here is paid + * back on every subsequent step. + */ +export function summarizePressables( + pressables: readonly AppFlowPressable[], + limit = 60, + maxChars = PRESSABLE_TEXT_LIMIT +): { shown: AppFlowPressable[]; omitted: number } { + const seen = new Set() + const unique: AppFlowPressable[] = [] + for (const pressable of pressables) { + const identifier = pressable.identifier ? cut(pressable.identifier, maxChars) : undefined + const trimmed: AppFlowPressable = identifier + ? { + identifier, + ...(pressable.role ? { role: pressable.role } : {}), + // `text` is dropped only when `label` is there to carry the caption: capture.ts sets + // `text` from node.value and `label` from node.label (two independent fields), and on an + // identified pressable that has both, node.value is almost always the same caption + // node.label already carries — repeating it a third way just spends tokens nudging the + // planner back toward the selectors that keep failing. When there is no `label`, node.value + // is the control's only caption and dropping it would leave `{identifier, role}` saying + // nothing about what the control holds — a text field's current contents, a control whose + // state lives in its value. capture.ts keeps a pressable when ANY of text/label/identifier + // is set, so that shape is one the capture side explicitly expects to emit. + ...(pressable.label + ? { label: cut(pressable.label, maxChars) } + : pressable.text + ? { text: cut(pressable.text, maxChars) } + : {}) + } + : { + ...(pressable.text ? { text: cut(pressable.text, maxChars) } : {}), + ...(pressable.label ? { label: cut(pressable.label, maxChars) } : {}), + ...(pressable.role ? { role: pressable.role } : {}) + } + const identity = JSON.stringify(trimmed) + if (seen.has(identity)) continue + seen.add(identity) + unique.push(trimmed) + } + return { shown: unique.slice(0, limit), omitted: Math.max(0, unique.length - limit) } +} + +function cut(value: string, maxChars: number): string { + const collapsed = value.replace(/\s+/g, ' ').trim() + return collapsed.length > maxChars ? `${collapsed.slice(0, maxChars)}\u2026` : collapsed +} + +/** + * Whether a message describes the agent-device session or daemon being gone rather than the app + * doing anything in particular — the daemon was replaced mid-walk (agent-device does this + * transparently and logs it to stderr, never to a thrown error) and the session the replaced + * daemon owned dies with it, so the next call against that session fails with wording like + * "No active proxy device lease for this session; run open first." None of that says anything + * about the app; it says the harness's connection to the device needs to be re-established. + * + * `capture.ts`'s `isMissingSessionError` (which decides whether to recover by relaunching) and + * this file's `isToolingFailure` (which decides whether a reach failure counts against the + * no-progress breaker) both need to recognise the same set of messages, so this is the one place + * that wording lives — `isMissingSessionError` defers to it instead of keeping its own regex. + */ +export function isSessionLostMessage(message: string): boolean { + return /no active session|run open first|session_not_found/i.test(message) +} + +/** + * One log-ready clause saying what a recovery site saw when it decided the session was gone: which + * field carried the session-loss wording and what it said. + * + * "no active session, recovering" used to name only the step that caught it. The two causes that + * matter — the literal wording on the message (the daemon was replaced, the session died with it) + * and the wording only on `details.hint` (a replay's post-failure snapshot hit the dead session, + * and the message says "capture-failed") — are different failures with different fixes, and + * neither is recoverable from the bare line. Collapsed and cut so a replay failure that quotes a + * long locator cannot flood the log; the caller adds what it was doing, which only it knows. + */ +export function describeSessionLoss(message: string, hint?: string): string { + const cut = (value: string): string => { + const collapsed = value.replace(/\s+/g, ' ').trim() + return collapsed.length > 160 ? `${collapsed.slice(0, 160)}\u2026` : collapsed + } + if (isSessionLostMessage(message)) return `error message: ${cut(message)}` + if (hint !== undefined && isSessionLostMessage(hint)) { + return `error hint: ${cut(hint)} (message: ${cut(message)})` + } + return `error message: ${cut(message)}` +} + +/** + * Whether a failed reach is about the harness rather than the app — a planner that ran out of time, + * an exhausted budget, a device command that could not complete, or the agent-device session it was + * using having died (see `isSessionLostMessage`, most recently observed when a daemon replacement + * mid-walk killed the session `saveAndReplayRecording` needed). + * + * The no-progress counter exists to spot an app stuck behind one screen it will not leave, and it + * decides that by comparing the UI left behind by consecutive failures. Sibling screens look alike + * — a followers list and a follows list are near-identical — so counting timeouts alongside real + * navigation failures stops the sweep to ask the user to clear a blocker that was never there. + */ +export function isToolingFailure(reason: string | null): boolean { + if (!reason) return false + return ( + /sub-agent timed out|planner failed|budget exhaust|runner busy|screenshot failed/i.test( + reason + ) || isSessionLostMessage(reason) + ) +} + +/** + * Whether a walk seeded at a deep-linked ancestor is worth repeating from the app entry after it + * failed. + * + * A retry earns its cost only when the first attempt left something unlearned. Two things do: the + * walk never ran (the seed's recording could not launch, so the entry walk is the first real try), + * or it died of the harness rather than the app (`isToolingFailure` — a planner timeout or a lost + * session says nothing about whether the target is reachable). A walk that ran its decision loop + * to an ordinary end — decisions spent, no visible progress, the planner returning blocked — has + * already put the same planner, the same target and the same app in front of the question, from a + * start the sweep had *proved* it could reach. Repeating it from the entry screen buys a second + * seven decisions on the same evidence: measured, the retry cost ~55s and ended at the same + * unresolved-param path, and two never-captured screens in the baseline sweep took 129.3s between + * them. The replay-seeded branch already stops here for the same reason (`seedBaseReplayFailed`); + * this extends that gate to the deep-link-seeded one instead of adding a second counter beside the + * no-progress breaker, which counts *sweep-wide* failures and could not see a single node's two. + */ +export function shouldRetryFromEntry(seeded: { + /** The seeded attempt reached the decision loop and ended without arriving. */ + walkRan: boolean + reason: string | null +}): boolean { + return !seeded.walkRan || isToolingFailure(seeded.reason) +} + +/** + * Real values for a route's params, harvested from the call sites that navigate to it. Only params + * this finds evidence for appear in the result: an invented id is not a weaker deep link but a + * different one, pointing at a record the app does not have, and the 404 it lands on says nothing + * about the screen. + */ +export function resolveRouteParams( + node: { id: string; params: readonly string[] }, + edges: readonly { to: string | null; raw: string; target: string | null }[] +): Record { + const incoming = edges.filter((edge) => edge.to === node.id && edge.raw) + const resolved: Record = {} + for (const key of node.params) { + const escaped = escapeParamKey(key) + for (const edge of incoming) { + const candidate = + // `?name=alice` or `/name=alice` in a href-style target. + edge.raw.match(new RegExp(`[/?&]${escaped}=([^&/'"\`\\s)]+)`))?.[1] ?? + // `navigate('Profile', { name: 'alice' })` — the params object at the call site. + edge.raw.match(new RegExp(`\\b${escaped}\\s*:\\s*['"\`]([^'"\`]+)['"\`]`))?.[1] ?? + // A literal segment sitting where the route declares the param. + segmentForParam(edge, key) + if (candidate && PARAM_VALUE_RE.test(candidate)) { + resolved[key] = candidate + break + } + } + } + return resolved +} + +function escapeParamKey(key: string): string { + return key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +/** The subset of `ParamStore` this resolver needs, so callers and tests can pass a stub. */ +export interface ParamStoreLike { + noteStatic(node: ParamRouteNode, resolved: Record): void + best(node: ParamRouteNode): ParamResolution | null +} + +/** A route's params as far as anything can honestly fill them, and where the values came from. */ +export interface ResolvedRouteParams { + /** Every param with a real value. Complete when `unresolved` is empty, partial otherwise. */ + values: Record + /** Params still without one — what `paramFreeDeepLink` truncates the URL at. */ + unresolved: string[] + /** Where a *complete* set came from, for the sweep log. Null while the set is partial. */ + source: ParamSource | null + /** The screen a complete harvested set was observed on, for the same log line. */ + seenOn: string | null + /** + * The route that owns the record a complete harvested set came from — not always this node; see + * `sharesRouteParams`. Null for a static or partial set, which belong to no record. `demote` is + * keyed on this. + */ + ownedBy: string | null +} + +/** + * The values a route's params can be opened with, from the call sites *and* from what the app has + * been seen holding. + * + * `resolveRouteParams` above stays exactly as pure as it was: it reads literals out of source text + * and nothing else, and its tests are unchanged. This wraps it with the observed layer, and is the + * single place the two meet — the orchestrator previously computed the static half twice per node, + * once for the agent-device manifest's descriptive path and once for the gate that decides whether + * a deep link is attempted at all, which is also how the two could drift. + * + * A complete harvested record **replaces** the static partial outright rather than topping it up. + * That is the whole point of storing records: a `/profile/:name/lists/:rkey` built from a static + * `name` and some other record's `rkey` opens cleanly, captures the wrong list, and nothing + * downstream can tell. Precedence inside the store (`static` first, then navigation observations, + * then learned testID shapes, then guessed ones) decides which single record wins; see `best`. + */ +export function resolveNodeParams( + node: ParamRouteNode & { id: string; params: readonly string[] }, + edges: readonly { to: string | null; raw: string; target: string | null }[], + store?: ParamStoreLike | null +): ResolvedRouteParams { + const fromSource = resolveRouteParams(node, edges) + // Registering the static read is what gives the store its shape baseline: a param whose call + // sites spell out `alice.test` will not later accept a bare `42` scraped off a list row. It also + // promotes a *complete* static read into a record, so the precedence comparison below has + // something to compare against instead of being special-cased here. + store?.noteStatic(node, fromSource) + const harvested = store?.best(node) ?? null + if (harvested) { + return { + values: { ...harvested.values }, + unresolved: [], + source: harvested.source, + seenOn: harvested.seenOn, + ownedBy: harvested.ownedBy + } + } + const unresolved = node.params.filter((key) => !(key in fromSource)) + return { + values: fromSource, + unresolved, + source: !node.params.length || unresolved.length ? null : 'static', + seenOn: null, + ownedBy: null + } +} + +/** The segment of `edge.raw` that lines up with `:key` in the edge's target pattern. */ +function segmentForParam(edge: { raw: string; target: string | null }, key: string): string | null { + if (!edge.target) return null + const pattern = edge.target.split(/[?#]/)[0].split('/') + const actual = edge.raw.split(/[?#]/)[0].split('/') + if (pattern.length !== actual.length) return null + const at = pattern.findIndex((seg) => seg === `:${key}` || seg === `[${key}]`) + return at === -1 ? null : (actual[at] ?? null) +} + +/** What the candidate scorer below needs to know about a still-pending node. */ +export interface OpportunisticCandidateNode { + id: string + title: string + urlPath: string | null +} + +/** + * Splits `ProfileFollowers`-style identifiers into words before tokenizing. Route ids and + * component names are source identifiers, not prose — they carry no spaces — while a live + * observation's pressable text is real on-screen words (`fingerprint` tokenizes actual UI + * labels). Without this, an id's own token never matches anything a screen actually shows. + */ +function splitCamelCase(text: string): string { + return text.replace(/([a-z0-9])([A-Z])/g, '$1 $2') +} + +/** Whether a url segment is a param placeholder (`:name`, `[name]`) rather than static text. */ +function isParamSegment(segment: string): boolean { + return segment.startsWith(':') || (segment.startsWith('[') && segment.endsWith(']')) +} + +/** + * The vocabulary a node's own static description contributes — the same shape `fingerprint` + * builds from a live observation's pressables, so the two are comparable. `id`'s last `/`-segment + * is used rather than the whole id, because parser ids for a duplicate screen name are qualified + * as `layoutName/ScreenName` (see `parser.ts`); the layout half is the same for every screen it + * registers and would dilute every one of that layout's candidates by the same amount rather than + * help tell them apart. Param placeholders in `urlPath` (`:name`, `[id]`) are dropped for the same + * reason: a param name is shared by every route that happens to take one, not evidence this + * particular screen is on screen. + */ +export function expectedNodeTokens(node: OpportunisticCandidateNode): Set { + const parts = [node.title, node.id.split('/').pop() ?? node.id] + if (node.urlPath) { + for (const segment of node.urlPath.split('/')) { + if (segment && !isParamSegment(segment)) parts.push(segment) + } + } + const out = new Set() + for (const part of parts) for (const token of tokenize(splitCamelCase(part))) out.add(token) + return out +} + +/** + * How much of a node's own descriptive vocabulary must actually appear in the live observation + * before it is worth a vision call at all. This is coverage of `expected`, not a symmetric + * (Jaccard) overlap: a content-heavy screen's observation routinely carries dozens of pressable + * words having nothing to do with any one candidate's identity, and dividing by that huge union + * would keep every genuine match near zero. What matters here is simpler — does this node's own + * few words show up on screen — so the score is shared tokens over `expected.size` alone. + * + * Exported (plan Step 3d) so `reach/chrome-inventory.ts` can score a chrome item's pressable text + * against a node's or a layout's own vocabulary with the identical scorer `pickOpportunisticCandidate` + * uses below, rather than forking a second copy of this exact formula that could drift from it. + */ +export function expectedCoverage( + observed: ReadonlySet, + expected: ReadonlySet +): number { + if (expected.size === 0) return 0 + let shared = 0 + for (const token of expected) if (observed.has(token)) shared += 1 + return shared / expected.size +} + +/** + * Coverage an opportunistic candidate needs before it is worth a vision call at all. Below this, + * the shared app chrome most screens expose (a back button, a tab bar) produces enough incidental + * overlap with almost any node's title to nominate something on every single step — exactly the + * per-step-per-pending-node cost this pre-filter exists to avoid paying. + * + * This was 0.5, and a measured run showed that was far too lenient: 19 opportunistic checks were + * made and 17 were rejected by the vision call — a 15% hit rate — costing 33.6s and 15,043 tokens + * to confirm just 2 screens. A node's expected vocabulary is only a handful of words, so coverage + * is coarse (1/2, 2/3, 3/5 all landed at or above 0.5), and a node needing only half its own + * words was routinely nominated by a screen sharing a generic word or two. 0.7 sits just above + * 2/3 and 3/5, the partial matches that dominated the rejected checks, while still admitting 3/4 + * and anything stronger; it is deliberately not higher, because a genuine match can legitimately + * miss a word (a title the screen renders differently from its route name), and each missed + * confirmation only costs a screen its own real exploration, not a wrong capture. + * + * Raised again from 0.7 to 0.85. `session_2026-10-09_17-22-18.log` measured one such check at + * 3.2s with a wrong answer: it proposed `ProfileSearch` while the walk was heading to + * `ProfileFollowers`, two screens whose titles and URLs overlap heavily. 0.7 still admitted + * sibling screens, which are both the case this scorer cannot separate and the most common thing + * to walk past, so only a near-complete match is now worth a model call. + */ +const MIN_OPPORTUNISTIC_OVERLAP = 0.85 + +/** + * How much clearer the best candidate's coverage must be than the runner-up's before it is + * trusted without a vision call to break the tie. Sibling screens that share almost their entire + * vocabulary — a Followers list and a Follows list, see the comment on `isToolingFailure` above — + * sit close together in score; without a margin requirement the pre-filter would confidently + * nominate whichever of the two happened to sort first, and a mistaken opportunistic capture + * writes a result with no replay and no way to notice it was ever wrong. Below this margin, + * neither is proposed and both stay pending for their own real exploration. + * + * Raised from 0.25 to 0.35 alongside `MIN_OPPORTUNISTIC_OVERLAP` for the same measured reason + * (17 of 19 checks rejected, 33.6s and 15,043 tokens for 2 confirmed screens). Only candidates + * that already cleared the overlap floor reach this comparison, so they sit in [0.7, 1.0]; a + * 0.25 margin let a full match beat a 0.75 sibling, which is a near-tie on vocabularies this + * small, and the vision call then rejected it. 0.35 means a 1.0 winner needs the runner-up to + * be at or below 0.65 — in practice a runner-up must be clearly weaker, not merely one word short. + * + * Raised again from 0.35 to 0.5 for the same `session_2026-10-09_17-22-18.log` measurement + * (3.2s, wrong answer, `ProfileSearch` proposed over `ProfileFollowers`): siblings that close + * together still got through at 0.35, so a winner now needs a runner-up at half its coverage or + * less before the pre-filter nominates it. + */ +const MIN_OPPORTUNISTIC_MARGIN = 0.5 + +/** + * Cheap textual pre-filter for opportunistic capture: picks at most one still-pending node whose + * static description (title, id, url) overlaps the current live observation strongly and + * unambiguously enough to be worth a single confirming vision call. Returns null when nothing + * clears the bar or when the top two candidates are too close to call — either way, no candidate + * is worth spending a model call on, so the caller spends nothing. + */ +export function pickOpportunisticCandidate( + observedTokens: ReadonlySet, + pending: readonly T[] +): T | null { + const scored = pending + .map((node) => ({ node, overlap: expectedCoverage(observedTokens, expectedNodeTokens(node)) })) + .filter((entry) => entry.overlap >= MIN_OPPORTUNISTIC_OVERLAP) + .sort((a, b) => b.overlap - a.overlap) + if (scored.length === 0) return null + const [best, runnerUp] = scored + if (runnerUp && best.overlap - runnerUp.overlap < MIN_OPPORTUNISTIC_MARGIN) return null + return best.node +} + +/** + * Withdraw a harvested record once a rung that *used* it has failed on the device, whatever the + * node ended up as. + * + * Demotion used to run only when the whole ladder ended unreached. But the ladder ends with + * exploration, which routinely rescues a node whose deep link and render both just failed on a bad + * value — so the record stayed, and every sibling that borrowed it (`sharesRouteParams`) paid the + * same failed attempts again. Measured on one sweep: a value valid on its own route and rejected by + * two siblings, ~63s of wasted deep-link and render attempts each, ~126s in all, and the node was + * still captured at the end so the old gate never fired. + * + * Keyed on observed failure, not on the value: a rung that declined without touching the device + * says nothing about it, and neither does a tooling failure (timeout, dead session). Static + * literals are the source's own claim and are never in the store to withdraw. After demotion the + * store has nothing for the route, so the next borrower is declined for free and goes straight to + * exploration rather than being handed a less trustworthy substitute. + * + * Returns whether a record was withdrawn. + */ +export function demoteFailedHarvest( + store: { demote(nodeId: string, values: Record): void }, + nodeId: string, + resolution: Pick, + attempts: readonly { + strategy: string + ok: boolean + touchedDevice: boolean + reason: string + }[] +): boolean { + if (resolution.unresolved.length || !resolution.source || resolution.source === 'static') { + return false + } + const failedOnValue = attempts.some( + (attempt) => + (attempt.strategy === 'deep-link' || attempt.strategy === 'render') && + !attempt.ok && + attempt.touchedDevice && + !isToolingFailure(attempt.reason) + ) + if (!failedOnValue) return false + store.demote(resolution.ownedBy ?? nodeId, resolution.values) + return true +} diff --git a/src/app-flow/hedge.ts b/src/app-flow/hedge.ts new file mode 100644 index 00000000..b16f1694 --- /dev/null +++ b/src/app-flow/hedge.ts @@ -0,0 +1,98 @@ +/** + * Re-issues a stalled model call instead of waiting it out. + * + * Measured across two instrumented sweeps, this backend's latency for these calls is tail-dominated + * rather than slow on average: the arrival judge's p50 is 2.1s and its p90 is 8.7s, but single calls + * reached 16.6s and 24.9s. The giveaway is what those calls returned — 165 and 130 output tokens, + * against a p50 call's ~140. They were not deliberating and they were not generating; they were + * waiting. Four such calls across one 39-call sweep cost ~81s, about 16% of the run's wall clock, + * and produced nothing a fast call would not have produced. + * + * A retry cannot recover that, because a retry needs the first attempt to fail and these attempts + * succeed — eventually. The only thing that converts "eventually" into p50 is a second attempt + * racing the first, which is what this does: start one, and if it has not answered within + * `afterMs`, start another and take whichever finishes first. + * + * Safe only for a call that can be run twice with no consequence. `runAgent`'s one-shot, `tools: []` + * judges qualify: one system prompt, one image, no tools, no session history, and a JSON verdict as + * the entire result. Session-backed calls do not, and the caller is where that line is drawn — see + * the guard at this module's one call site. Nothing here can check it. + */ + +export interface HedgeOutcome { + result: T + /** Attempts started, so a caller can bill the ones it did not use. 1 when no hedge fired. */ + attempts: number + /** Which attempt produced `result`, zero-based. 1 means the hedge won. */ + winner: number +} + +/** + * Runs `attempt(0)`, and if it has not settled within `afterMs`, runs `attempt(1)` alongside it and + * returns whichever fulfils first. + * + * `afterMs` of `Infinity` (or anything non-positive) disables hedging, which is how a caller turns + * this off without branching around it. + * + * A rejection does not end the race while the other attempt is still running. The whole premise is + * that one of these calls can be having a bad time independently of the other, and a hedge that + * surfaced the first error would throw away a good answer that was seconds away. Only when both + * have rejected does this reject, and with attempt 0's error: that is the call the caller actually + * asked for, and its failure is the one that describes the real problem rather than the duplicate's. + */ +export async function hedge(options: { + attempt: (index: number) => Promise + afterMs: number + /** Called when the hedge is issued, for logging. Never called when the first attempt wins. */ + onHedge?: () => void +}): Promise> { + const first = options.attempt(0) + if (!(options.afterMs > 0) || !Number.isFinite(options.afterMs)) { + return { result: await first, attempts: 1, winner: 0 } + } + + // Tagged so the race can tell a fulfilment from the timer firing without a sentinel value that a + // generic `T` could legitimately be. + const tagged = (promise: Promise, index: number): Promise<{ index: number; result: T }> => + promise.then((result) => ({ index, result })) + + let timer: ReturnType | undefined + const elapsed = new Promise<'elapsed'>((resolve) => { + timer = setTimeout(() => resolve('elapsed'), options.afterMs) + }) + + // A rejection here is the first attempt failing before the timer fired, and it propagates: there + // is no race left to run and no reason to pay for a duplicate, because a call that fails this + // fast is failing for a reason a second identical call will hit too. Only an *elapsed* timer + // means the attempt is still in flight and worth racing. + let firstOrTimer: { index: number; result: T } | 'elapsed' + try { + firstOrTimer = await Promise.race([tagged(first, 0), elapsed]) + } finally { + clearTimeout(timer) + } + if (firstOrTimer !== 'elapsed') { + return { result: firstOrTimer.result, attempts: 1, winner: 0 } + } + + options.onHedge?.() + const second = options.attempt(1) + // Attached before the race so that whichever promise loses cannot surface as an unhandled + // rejection after this function has already returned the winner's answer. + const settled = [tagged(first, 0), tagged(second, 1)].map((promise) => + promise.then( + (value) => ({ ok: true as const, value }), + (error: unknown) => ({ ok: false as const, error }) + ) + ) + + const firstSettled = await Promise.race(settled) + if (firstSettled.ok) { + return { result: firstSettled.value.result, attempts: 2, winner: firstSettled.value.index } + } + // One side has failed; the other is still the caller's best chance of an answer. + const both = await Promise.all(settled) + const won = both.find((outcome) => outcome.ok) + if (won?.ok) return { result: won.value.result, attempts: 2, winner: won.value.index } + throw both[0].ok ? new Error('Hedged call failed') : both[0].error +} diff --git a/src/app-flow/host.ts b/src/app-flow/host.ts new file mode 100644 index 00000000..30182374 --- /dev/null +++ b/src/app-flow/host.ts @@ -0,0 +1,44 @@ +import { join } from 'node:path' +import type { AppFlowProgressEvent, ManualRecordingEvent } from '../shared/app-flow.ts' +import type { AppFlowDecisionProvider } from './decision-provider.ts' + +export type AppFlowHostEvent = AppFlowProgressEvent | ManualRecordingEvent + +export interface AppFlowAgentDeviceModule { + createAgentDeviceClient(config?: Record): any +} + +/** Runtime environment supplied by the MCP server. */ +export interface AppFlowHostPorts { + dataDir: string + decisions?: AppFlowDecisionProvider + emit(event: AppFlowHostEvent): void + loadAgentDevice(): Promise + /** Shell-safe executable prefix for the vendored CLI (for example `node '/.../agent-device.mjs'`). */ + agentDeviceCli?: string + screenshotUri?(projectId: string, slug: string): string +} + +let configuredHost: AppFlowHostPorts | null = null + +export function configureAppFlowHost(host: AppFlowHostPorts): void { + configuredHost = host +} + +export function getAppFlowHost(): AppFlowHostPorts { + if (!configuredHost) throw new Error('App Flow host has not been configured.') + return configuredHost +} + +export function appFlowDataDir(): string { + return getAppFlowHost().dataDir +} + +export function appFlowAgentDeviceStateDir(): string { + return join(appFlowDataDir(), 'agent-device') +} + +export function resolveAgentDeviceCommand(command: string): string { + if (!command.startsWith('agent-device')) return command + return `${getAppFlowHost().agentDeviceCli ?? 'agent-device'}${command.slice('agent-device'.length)}` +} diff --git a/src/app-flow/index.ts b/src/app-flow/index.ts new file mode 100644 index 00000000..dcb6d627 --- /dev/null +++ b/src/app-flow/index.ts @@ -0,0 +1,203 @@ +import type { + AppFlowProgressEvent, + AppFlowPromptResolution, + StartAppFlowInput +} from '../shared/app-flow.ts' +import { parseProject } from './parser.ts' +import { AppFlowBuilder } from './orchestrator.ts' +import { bootDevice, closeDevice, waitForAppReady, type CaptureDevice } from './capture.ts' +import { resolveAppFlowStart } from './start-request.ts' +import { getProjectId, readAppFlowForRenderer, readManifest } from './storage.ts' +import { configureAppFlowHost, type AppFlowHostPorts } from './host.ts' + +const builders = new Map() +/** Apps the user launched from the App Flow screen, kept running for a later sweep to adopt. */ +const launchedApps = new Map() +const launching = new Set() + +export function getAppFlowBuilders(): Map { + return builders +} + +export function createAppFlowService(host: AppFlowHostPorts): { + getStatus: ( + projectPath: string + ) => Promise<{ exists: boolean; projectId: string; manifest?: unknown; error?: string }> + getResult: (projectPath: string) => Promise + start: (options: StartAppFlowInput) => Promise<{ ok: true } | { ok: false; error: string }> + launchApp: (projectPath: string) => Promise<{ ok: true } | { ok: false; error: string }> + isAppLaunched: (projectPath: string) => Promise + pause: (projectPath: string) => Promise<{ paused: boolean }> + cancel: (projectPath: string) => Promise<{ cancelled: boolean }> + resolvePrompt: ( + projectPath: string, + resolution: AppFlowPromptResolution, + promptId?: string + ) => Promise<{ resolved: boolean }> +} { + configureAppFlowHost(host) + return { + getStatus: async (projectPath: string) => { + const projectId = getProjectId(projectPath) + const manifest = await readManifest(projectId) + return { exists: manifest !== null, projectId, manifest: manifest ?? undefined } + }, + getResult: async (projectPath: string) => { + const projectId = getProjectId(projectPath) + return readAppFlowForRenderer(projectId) + }, + launchApp: async (projectPath: string) => { + const projectId = getProjectId(projectPath) + if (builders.get(projectId)?.isRunning()) { + return { ok: false, error: 'App flow build already in progress.' } + } + if (launching.has(projectId)) return { ok: false, error: 'App is already launching.' } + launching.add(projectId) + try { + const parsed = await parseProject(projectPath) + if (!parsed) { + return { + ok: false, + error: `No supported Expo Router or React Navigation routes found in ${projectPath}.` + } + } + const previous = launchedApps.get(projectId) + if (previous) { + launchedApps.delete(projectId) + await closeDevice(previous) + } + const ctx = { + projectPath, + projectId, + scheme: parsed.scheme, + appName: parsed.appName, + bundleId: parsed.bundleId, + slug: parsed.slug, + platform: 'ios' as const, + sessionName: projectId + } + const device = await bootDevice(ctx) + try { + await waitForAppReady(ctx, device) + } catch (error) { + await closeDevice(device) + throw error + } + launchedApps.set(projectId, device) + return { ok: true } + } catch (error) { + console.error('[app-flow] launch failed:', error) + return { ok: false, error: error instanceof Error ? error.message : 'Unknown error' } + } finally { + launching.delete(projectId) + } + }, + isAppLaunched: async (projectPath: string) => launchedApps.has(getProjectId(projectPath)), + start: async (input) => { + // Defaults live in `start-request.ts` so they are testable without Electron. + const { + projectPath, + mode, + screenLimit, + explorationMode, + renderTransportMode, + deepLinkJudgeMode + } = resolveAppFlowStart(input) + if (mode === 'screen-ui-and-structure' && !host.decisions) { + return { ok: false, error: 'App Flow decision provider is not available.' } + } + + const projectId = getProjectId(projectPath) + let builder = builders.get(projectId) + if (builder?.isRunning()) { + return { ok: false, error: 'App flow build already in progress.' } + } + + if (launching.has(projectId)) return { ok: false, error: 'App is still launching.' } + const launchedDevice = launchedApps.get(projectId) + + builder = new AppFlowBuilder() + builders.set(projectId, builder) + + builder.on('event', (event: AppFlowProgressEvent) => { + host.emit(event) + }) + + builder.on('done', () => { + console.log(`[app-flow] build done (${projectId})`) + builders.delete(projectId) + }) + + builder.on('error', (result: { error: string }) => { + console.error(`[app-flow] build error (${projectId}): ${result.error}`) + // buildAppFlow already emits an `error` event for every failure it handles, but this is + // the last line of defence: the renderer must never be left spinning on a dead build. + host.emit({ + type: 'error', + phase: 'render', + message: result.error, + projectId + }) + builders.delete(projectId) + }) + + console.log(`[app-flow] start requested for ${projectPath} (${projectId})`) + // Run without awaiting so the IPC call returns immediately. + void builder + .start({ + projectPath, + mode, + decisionProvider: host.decisions, + screenLimit, + explorationMode, + renderTransportMode, + deepLinkJudgeMode, + launchedDevice + }) + .catch((error: unknown) => { + console.error('[app-flow] builder.start rejected:', error) + builders.delete(projectId) + host.emit({ + type: 'error', + phase: 'render', + message: error instanceof Error ? error.message : 'Unknown error', + projectId + }) + }) + return { ok: true } + }, + resolvePrompt: async ( + projectPath: string, + resolution: AppFlowPromptResolution, + promptId?: string + ) => { + const projectId = getProjectId(projectPath) + const builder = builders.get(projectId) + const resolved = builder?.resolvePrompt(resolution, promptId) ?? false + console.log( + `[app-flow] prompt resolution "${resolution}" for ${projectId} — ${resolved ? 'delivered' : 'nothing waiting'}` + ) + return { resolved } + }, + pause: async (projectPath: string) => { + const projectId = getProjectId(projectPath) + const builder = builders.get(projectId) + console.log( + `[app-flow] pause requested for ${projectId} — ${builder ? 'builder found' : 'no running builder'}` + ) + if (!builder) return { paused: false } + builder.pause() + return { paused: true } + }, + cancel: async (projectPath: string) => { + const projectId = getProjectId(projectPath) + const builder = builders.get(projectId) + console.log( + `[app-flow] cancel requested for ${projectId} — ${builder ? 'builder found' : 'no running builder'}` + ) + if (!builder) return { cancelled: false } + builder.cancel() + return { cancelled: true } + } + } +} diff --git a/src/app-flow/launcher-deferral.ts b/src/app-flow/launcher-deferral.ts new file mode 100644 index 00000000..83f07b65 --- /dev/null +++ b/src/app-flow/launcher-deferral.ts @@ -0,0 +1,35 @@ +/** + * How long to let the dev-client launcher clear on its own before recovering from it. + * + * `startReplayRecording` arms a recording with an `apps.open --saveScript`, and the launcher + * recovery in `waitForAppReady` clears the launcher with an `apps.open` of its own — which + * agent-device treats as "a second successful open" and refuses to publish the script after. The + * deferral exists so that a launcher which is merely slow (a cold bundle build) is waited out + * rather than recovered from, keeping the recording publishable. + * + * That bet is only worth making while it can still be won. On the measured sweep it was not: the + * deferral fired on six separate nodes, ran its full 20s every single time, and every one of those + * six still ended with `This script recording was aborted by a second successful open and cannot + * be published` — because the launcher in that setup is pointed at no Metro, so nothing but the + * reconnect was ever going to clear it. 131s of waiting bought six publication failures that + * happened anyway, on a sweep that was 75s slower than the one before it. + * + * So the deferral is kept, and spent once. The first time it elapses with the launcher still on + * screen, that is the evidence that this device's launcher does not self-clear, and every later + * call recovers immediately. A launcher that *does* clear within the window never records an + * expiry, so a genuinely slow bundle keeps being waited out for the whole sweep. + * + * Deliberately per-device and per-run rather than a constant: "does the launcher clear itself" + * is a property of how this dev client was started, not of the harness, and a device that is + * reconnected to Metro between sweeps should get the benefit of the doubt again. + */ +export type LauncherDeferralOutcome = + /** No deferral has elapsed on this device yet — either none ran, or one ran and the launcher cleared. */ + | 'untested' + /** A deferral ran its full length and the launcher was still on screen at the end of it. */ + | 'expired' + +export function launcherDeferralMs(configuredMs: number, outcome: LauncherDeferralOutcome): number { + if (!(configuredMs > 0)) return 0 + return outcome === 'expired' ? 0 : configuredMs +} diff --git a/src/app-flow/manual/commands.ts b/src/app-flow/manual/commands.ts new file mode 100644 index 00000000..f712d557 --- /dev/null +++ b/src/app-flow/manual/commands.ts @@ -0,0 +1,29 @@ +/** + * The literal `agent-device …` lines a manual recording falls back to when its recording cannot be + * published as a `.ad` script. + * + * Pure, and deliberately free of any Electron or device import, so the round trip that actually + * matters — that `reachStartScreen` can parse and execute every line this produces — is testable + * under `node --test`. A fallback path nothing can replay is worth nothing. + */ +import type { ManualLocator } from './locators.ts' + +export { quoteCliArg } from '../cli-args.ts' + +/** + * The command builders themselves now live in `../walk-commands.ts`, which the sweep's own + * `pathToReach` emission shares — the manual recorder and the explorer write the same lines for + * the same reasons, and keeping two spellings of `agent-device press` was how they would drift. + * Re-exported from here so every existing import of this module is unchanged. + */ +export { fillCommand, pressCommand, pressPointCommand, scrollCommand } from '../walk-commands.ts' + +/** + * Whether a step survives a rebuild of the app. + * + * A `text=` locator breaks on any copy change and a coordinate breaks on any layout change, so the + * panel marks both so the user can choose to re-record against something sturdier. + */ +export function isPortable(locator: ManualLocator | null): boolean { + return locator !== null && locator.by !== 'text' +} diff --git a/src/app-flow/manual/context.ts b/src/app-flow/manual/context.ts new file mode 100644 index 00000000..91da580a --- /dev/null +++ b/src/app-flow/manual/context.ts @@ -0,0 +1,27 @@ +/** + * The `CaptureContext` every manual-mode device operation runs under — recording and replay alike. + * + * Shared so the two cannot drift on the one field that matters: `sessionName`. It is deliberately + * not the bare `projectId` the sweep uses, because `closeDevice` ends with + * `sessions.close({ shutdown: true })` and on a shared name that would tear down a runner the sweep + * is holding. + */ +import type { CaptureContext } from '../capture.ts' +import { readAppConfig } from '../parser.ts' + +export async function manualCaptureContext( + projectPath: string, + projectId: string +): Promise { + const config = await readAppConfig(projectPath) + return { + projectPath, + projectId, + scheme: config.scheme, + appName: config.name, + bundleId: config.bundleId, + slug: config.slug, + platform: 'ios', + sessionName: `${projectId}-manual` + } +} diff --git a/src/app-flow/manual/index.ts b/src/app-flow/manual/index.ts new file mode 100644 index 00000000..854c7435 --- /dev/null +++ b/src/app-flow/manual/index.ts @@ -0,0 +1,181 @@ +/** + * The manual-recording service — the IPC-facing wrapper around `ManualSession`, built the same way + * `createAppFlowService` wraps `AppFlowBuilder`: a map keyed by project id, events broadcast to the + * renderer, and work started without awaiting so the IPC call returns immediately. + */ +import type { ManualRecordingAction, ManualRecordingEvent } from '../../shared/app-flow.ts' +import { bootDevice, closeDevice, type CaptureDevice } from '../capture.ts' +import { getAppFlowBuilders } from '../index.ts' +import { getProjectId, readAgentDeviceScreens } from '../storage.ts' +import { manualCaptureContext } from './context.ts' +import { runPathToReach } from '../path-run.ts' +import { ManualSession } from './session.ts' +import { configureAppFlowHost, type AppFlowHostPorts } from '../host.ts' + +const sessions = new Map() + +/** + * Devices left running by the Replay button, keyed by project id. + * + * Replay exists to put the app on a screen and leave it there, so it cannot close its own device — + * `closeDevice` shuts the agent-device session down, and the app under test goes with the runner. + * What that leaks is the Metro child process and its log file, which only `closeDevice` reaps, so + * the device is parked here and closed by whatever starts next on this project. The agent-device + * session itself needs no such care: `bootDevice` opens with a `sessions.close` that drops any + * leftover one. + */ +const lingering = new Map() + +async function closeLingeringDevice(projectId: string): Promise { + const device = lingering.get(projectId) + if (!device) return + lingering.delete(projectId) + await closeDevice(device).catch((error: unknown) => { + console.error('[app-flow:manual] could not close the parked replay device:', error) + }) +} + +export function createManualRecordingService(host: AppFlowHostPorts): { + start: ( + projectPath: string, + nodeId: string + ) => Promise<{ ok: true } | { ok: false; error: string }> + act: ( + projectPath: string, + action: ManualRecordingAction + ) => Promise<{ ok: true } | { ok: false; error: string }> + stop: (projectPath: string) => Promise<{ ok: true } | { ok: false; error: string }> + cancel: (projectPath: string) => Promise<{ cancelled: boolean }> + replay: ( + projectPath: string, + nodeId: string + ) => Promise<{ ok: true } | { ok: false; error: string }> +} { + configureAppFlowHost(host) + return { + start: async (projectPath: string, nodeId: string) => { + const projectId = getProjectId(projectPath) + // A sweep and a manual session would fight over the same simulator and the same `.ad` files. + if (getAppFlowBuilders().get(projectId)?.isRunning()) { + return { ok: false, error: 'An App Flow build is already running for this project.' } + } + if (sessions.has(projectId)) { + return { ok: false, error: 'A recording is already in progress for this project.' } + } + await closeLingeringDevice(projectId) + + const session = new ManualSession(projectPath, projectId, nodeId) + sessions.set(projectId, session) + session.on('event', (event: ManualRecordingEvent) => { + host.emit(event) + if (event.type === 'complete' || event.type === 'error') sessions.delete(projectId) + }) + + console.log(`[app-flow:manual] start ${nodeId} for ${projectPath} (${projectId})`) + // Not awaited: booting a simulator and launching the app takes minutes, and the renderer + // follows along on the event channel. + void session.start().catch((error: unknown) => { + console.error('[app-flow:manual] start rejected:', error) + sessions.delete(projectId) + void session.failed(error) + }) + return { ok: true } + }, + + act: async (projectPath: string, action: ManualRecordingAction) => { + const session = sessions.get(getProjectId(projectPath)) + if (!session) return { ok: false, error: 'No recording in progress for this project.' } + try { + await session.act(action) + return { ok: true } + } catch (error) { + // An action that fails is not a failed session — a mistyped field or a stale element should + // leave the user on the device, able to try again. + return { ok: false, error: error instanceof Error ? error.message : 'Interaction failed.' } + } + }, + + stop: async (projectPath: string) => { + const projectId = getProjectId(projectPath) + const session = sessions.get(projectId) + if (!session) return { ok: false, error: 'No recording in progress for this project.' } + try { + await session.stop() + return { ok: true } + } catch (error) { + console.error('[app-flow:manual] stop failed:', error) + await session.failed(error) + return { + ok: false, + error: error instanceof Error ? error.message : 'Could not save the recording.' + } + } finally { + sessions.delete(projectId) + } + }, + + cancel: async (projectPath: string) => { + const projectId = getProjectId(projectPath) + const session = sessions.get(projectId) + if (!session) return { cancelled: false } + sessions.delete(projectId) + await session.cancel().catch((error: unknown) => { + console.error('[app-flow:manual] cancel failed:', error) + }) + return { cancelled: true } + }, + + /** + * Walks a screen's stored `pathToReach` and leaves the app sitting on it. + * + * Awaited rather than fired like `start`: there is no panel to follow along on, the renderer + * just wants to know whether it worked, and the one line of feedback it needs is the error when + * it did not. The whole call is a couple of minutes on a cold device, which `ipcMain.handle` + * carries fine. + */ + replay: async (projectPath: string, nodeId: string) => { + const projectId = getProjectId(projectPath) + if (getAppFlowBuilders().get(projectId)?.isRunning()) { + return { ok: false, error: 'An App Flow build is already running for this project.' } + } + if (sessions.has(projectId)) { + return { ok: false, error: 'A recording is in progress for this project.' } + } + + const screens = await readAgentDeviceScreens(projectId) + const entry = Object.values(screens).find((screen) => screen.screenId === nodeId) + if (!entry?.pathToReach.length) { + return { ok: false, error: 'This screen has no recorded path to replay yet.' } + } + + await closeLingeringDevice(projectId) + console.log( + `[app-flow:manual] replay ${nodeId} for ${projectPath} (${projectId})` + + // Said out loud before the device is booted, because this is the one place that knows + // which path is about to run and a failure two minutes from now reads very differently + // depending on the answer. `ModerationInbox`'s stored path was reconstructed from a + // chrome reveal and was never travelled; `My-Profile`'s was walked. See + // `AppFlowCapture.pathVerified`. + (entry.pathVerified === false ? ' — inferred path, never travelled' : '') + + (entry.pathContentSelectors?.length + ? ` — matches on rendered content: ${entry.pathContentSelectors.join(', ')}` + : '') + ) + const ctx = await manualCaptureContext(projectPath, projectId) + const device = await bootDevice(ctx) + // Parked before the walk, not after: if a step throws, the device is still up and still owns + // a Metro process, and the next start is the only thing that will ever clean it up. + lingering.set(projectId, device) + try { + await runPathToReach(ctx, device, entry.pathToReach) + return { ok: true } + } catch (error) { + console.error('[app-flow:manual] replay failed:', error) + return { + ok: false, + error: error instanceof Error ? error.message : 'Could not replay this screen.' + } + } + } + } +} diff --git a/src/app-flow/manual/locators.ts b/src/app-flow/manual/locators.ts new file mode 100644 index 00000000..4692f2e4 --- /dev/null +++ b/src/app-flow/manual/locators.ts @@ -0,0 +1,56 @@ +/** + * Locator derivation for the manual recorder. + * + * Pure — no device, no Electron — so the two rules that decide whether a manual recording can be + * published at all are testable under `node --test` without a simulator. + */ +import type { ManualSnapshotNode, ManualSnapshotRect } from '../../shared/app-flow.ts' +import type { AppFlowDestinationLocator } from '../capture.ts' + +export type ManualLocator = { by: 'text' | 'label' | 'id'; value: string } + +/** + * How a clicked element becomes a selector, in descending order of durability. + * + * `identifier` (a `nativeID`/accessibility id) survives copy changes and localisation; `label` is + * next; `value` (the rendered text) is the last resort and is the only one that cannot guard a + * saved recording — see `destinationLocator`. Returns null for a node carrying none of the three, + * which the caller presses by coordinate instead. + */ +export function locatorFor(node: ManualSnapshotNode): ManualLocator | null { + const identifier = node.identifier?.trim() + if (identifier) return { by: 'id', value: identifier } + const label = node.label?.trim() + if (label) return { by: 'label', value: label } + const value = node.value?.trim() + if (value) return { by: 'text', value } + return null +} + +/** + * The landmark that guards the published recording, picked off the screen the session ended on. + * + * agent-device's `sessions.saveScript` refuses to publish an active-session script unless, after + * the last mutating action, there is a `wait` on a selector whose matched element carries a + * re-verifiable identity — the "portable destination guard". Only an id or an accessibility label + * qualifies, which is why `by: 'text'` is excluded here exactly as it is from + * `AppFlowDestinationLocator`. Null means the session has to fall back to literal commands. + */ +export function destinationLocator( + nodes: readonly ManualSnapshotNode[] +): AppFlowDestinationLocator | null { + for (const node of nodes) { + const identifier = node.identifier?.trim() + if (identifier) return { by: 'id', value: identifier } + } + for (const node of nodes) { + const label = node.label?.trim() + if (label) return { by: 'label', value: label } + } + return null +} + +/** The point to press when a node carries no selector at all. */ +export function rectCenter(rect: ManualSnapshotRect): { x: number; y: number } { + return { x: Math.round(rect.x + rect.width / 2), y: Math.round(rect.y + rect.height / 2) } +} diff --git a/src/app-flow/manual/path-commands.ts b/src/app-flow/manual/path-commands.ts new file mode 100644 index 00000000..88522d66 --- /dev/null +++ b/src/app-flow/manual/path-commands.ts @@ -0,0 +1,88 @@ +/** + * Reading a `pathToReach` back: the inverse of `commands.ts`, plus the two lines a sweep writes. + * + * `reachStartScreen` ([profiling/device.ts](../../profiling/device.ts)) already does this for + * profiling, but it reaches for a shell for anything that is not `replay` or `open`, and it needs a + * `ProfilingContext` to do it. The Replay button has neither, so it dispatches in process against + * the `CaptureDevice` the recorder already uses — which means the literal `press`/`fill`/`scroll` + * lines a fallback recording produces have to be understood here rather than handed to `/bin/zsh`. + * + * Pure, and free of Electron and device imports, so every form `commands.ts` can emit is round + * tripped in a unit test. Anything unrecognised comes back `null` rather than being guessed at: the + * caller reports the line it could not run, which is far better than silently skipping a step and + * leaving the user on the wrong screen wondering why. + */ +import { tokenizeAgentDeviceCommand } from '../../server/app-flow-deps/profiling/agent-device-command.ts' + +export type PathStep = + /** `boot` and `install` — the device is already up by the time a path runs. */ + | { kind: 'skip' } + | { kind: 'replay'; path: string } + | { kind: 'open'; app: string; url: string | null } + | { kind: 'press'; selector: string } + | { kind: 'pressPoint'; x: number; y: number } + | { kind: 'fill'; selector: string; value: string } + | { kind: 'scroll'; direction: 'up' | 'down' | 'left' | 'right' } + +const SCROLL_DIRECTIONS = new Set(['up', 'down', 'left', 'right']) + +/** + * The positional arguments of a command, with flags and their values removed. + * + * `--settle` takes no value while `--timeout 5000` does, and nothing in the token stream says + * which is which. The rule matches `parseAgentDeviceOpen`'s: a flag swallows the next token unless + * that token is itself a flag, which is right for every flag these commands carry. + */ +function positionals(tokens: readonly string[], from: number): string[] { + const values: string[] = [] + for (let i = from; i < tokens.length; i++) { + const token = tokens[i] + if (!token.startsWith('--')) { + values.push(token) + continue + } + if (!token.includes('=') && tokens[i + 1] && !tokens[i + 1].startsWith('--')) i++ + } + return values +} + +export function parsePathStep(command: string): PathStep | null { + const tokens = tokenizeAgentDeviceCommand(command.trim()) + if (tokens[0] !== 'agent-device') return null + const verb = tokens[1] + const args = positionals(tokens, 2) + + if (verb === 'boot' || verb === 'install') return { kind: 'skip' } + + if (verb === 'replay') { + // `replay run ` is the long form of `replay `. + const path = args[0] === 'run' ? args[1] : args[0] + return path ? { kind: 'replay', path } : null + } + + if (verb === 'open') { + return args[0] ? { kind: 'open', app: args[0], url: args[1] ?? null } : null + } + + if (verb === 'press') { + // `press ` — two numbers, as `pressPointCommand` writes it. + if (args.length >= 2 && args.every((arg) => /^-?\d+(\.\d+)?$/.test(arg))) { + return { kind: 'pressPoint', x: Number(args[0]), y: Number(args[1]) } + } + return args[0] ? { kind: 'press', selector: args[0] } : null + } + + if (verb === 'fill') { + // An empty string is a legitimate value — clearing a field — so only a missing one is a parse + // failure, which is why this tests the count rather than the truthiness of `args[1]`. + return args.length >= 2 && args[0] ? { kind: 'fill', selector: args[0], value: args[1] } : null + } + + if (verb === 'scroll') { + return args[0] && SCROLL_DIRECTIONS.has(args[0]) + ? { kind: 'scroll', direction: args[0] as 'up' | 'down' | 'left' | 'right' } + : null + } + + return null +} diff --git a/src/app-flow/manual/session.ts b/src/app-flow/manual/session.ts new file mode 100644 index 00000000..e75b56ac --- /dev/null +++ b/src/app-flow/manual/session.ts @@ -0,0 +1,491 @@ +/** + * A manual, agent-device-assisted recording session — the third way an App Flow screen can acquire + * a `pathToReach`, alongside the parser-only mode (which produces none) and the LLM sweep. + * + * The user drives their own app through a live wireframe of the device while this module issues + * every tap as a real agent-device interaction inside an *active recording*. That is the whole + * trick: the session is opened with `apps.open({ saveScript })`, so the app launch is the first + * recorded step and each interaction appends itself, and on stop the recording publishes to + * `/.ad` — exactly the path `savedReplayStrategy` probes first. A screen recorded here + * is therefore replayed ahead of any deep link or exploration by a later sweep, with no change to + * the sweep or the reach ladder. + * + * No model is consulted anywhere in this file. + */ +import { EventEmitter } from 'node:events' +import { rename, unlink } from 'node:fs/promises' +import { join } from 'node:path' +import type { + AppFlowNode, + AppFlowPressable, + ManualRecordingAction, + ManualRecordingEvent, + ManualRecordingStatus, + ManualSnapshotNode, + ManualSnapshotRect +} from '../../shared/app-flow.ts' +import { captureSnapshotFrame } from '../../server/app-flow-deps/profiling/snapshot.ts' +import { + agentDeviceOpenCommand, + bootDevice, + capturePressables, + captureScreenshot, + closeDevice, + pressWithLocator, + replayPathToReach, + saveAndReplayRecording, + startReplayRecording, + waitForRoute, + type CaptureContext, + type CaptureDevice +} from '../capture.ts' +import { serializeLocator } from '../exploration.ts' +import { manualCaptureContext } from './context.ts' +import { defaultScreenInfo, uniqueScreenKey } from '../screen-keys.ts' +import { + getFlowsDir, + getScreenshotPath, + readAgentDeviceScreens, + readAppFlow, + registerProject, + writeAgentDeviceScreens, + writeAppFlow +} from '../storage.ts' +import { fillCommand, isPortable, pressCommand, pressPointCommand, scrollCommand } from './commands.ts' +import { contentMatchedSelectors } from '../path-minimize.ts' +import { destinationLocator, locatorFor, rectCenter } from './locators.ts' + +/** How often the panel's wireframe is refreshed while the user is deciding what to tap. */ +const SNAPSHOT_INTERVAL_MS = 2_000 + +function log(step: string, detail?: string): void { + console.log(detail ? `[app-flow:manual] ${step} — ${detail}` : `[app-flow:manual] ${step}`) +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +export class ManualSession extends EventEmitter { + private readonly projectPath: string + private readonly projectId: string + private readonly nodeId: string + private device: CaptureDevice | null = null + private ctx: CaptureContext | null = null + private node: AppFlowNode | null = null + private candidatePath = '' + private publishedPath = '' + /** The literal `agent-device …` lines, in order, used when the recording cannot be published. */ + private readonly literalCommands: string[] = [] + private latestSnapshot: ManualSnapshotNode[] = [] + private revision = 0 + private pollTimer: NodeJS.Timeout | null = null + /** + * Serializes the poll against user actions. A snapshot taken while a press is mid-flight reports + * a half-transitioned screen, and two overlapping snapshots race the same agent-device session. + */ + private busy = false + private finished = false + + constructor( + projectPath: string, + projectId: string, + nodeId: string + ) { + super() + this.projectPath = projectPath + this.projectId = projectId + this.nodeId = nodeId + } + + private emitEvent(event: ManualRecordingEvent): void { + try { + this.emit('event', event) + } catch (error) { + console.error('Manual recording event listener error:', error) + } + } + + private status(status: ManualRecordingStatus, message?: string): void { + this.emitEvent({ + type: 'status', + projectId: this.projectId, + nodeId: this.nodeId, + status, + ...(message ? { message } : {}) + }) + } + + async start(): Promise { + this.status('starting', 'Resolving screen') + const result = await readAppFlow(this.projectId) + const node = result?.nodes.find((candidate) => candidate.id === this.nodeId) ?? null + if (!node) throw new Error('Screen is not in this project’s app flow. Generate the flow first.') + if (node.origin === 'chrome') { + throw new Error('A navigator surface is not a screen — there is nothing to record a path to.') + } + this.node = node + + this.ctx = await manualCaptureContext(this.projectPath, this.projectId) + + // Creates `flows/` and `screens/` if a previous build left them missing, and refreshes the + // registry's `lastOpenedAt` — the user is working on this project right now. + await registerProject(this.projectPath) + this.candidatePath = join(getFlowsDir(this.projectId), `.${node.slug}.manual.candidate.ad`) + this.publishedPath = join(getFlowsDir(this.projectId), `${node.slug}.ad`) + + this.status('starting', 'Booting simulator') + this.device = await bootDevice(this.ctx) + + this.status('starting', 'Launching app') + // Opens with `saveScript`, so the launch itself is the first line of the recording. This is + // what makes "the app launch is always part of pathToReach" true for both output shapes: the + // published script replays it, and the literal fallback reproduces it as `agent-device open`. + await startReplayRecording(this.ctx, this.device, this.candidatePath, this.device.launchUrl) + + this.status('recording') + await this.tick() + this.pollTimer = setInterval(() => void this.tick(), SNAPSHOT_INTERVAL_MS) + } + + /** + * Waits out an in-flight snapshot poll before taking the session. + * + * Bounded rather than an open loop: a snapshot has its own 15s timeout, so anything past that is + * a wedged call, and the user's tap is worth more than the stale tick blocking it. + */ + private async waitForIdle(timeoutMs = 20_000): Promise { + const deadline = Date.now() + timeoutMs + while (this.busy && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 50)) + } + } + + /** One wireframe refresh. Silently skipped while another device call holds the session. */ + private async tick(): Promise { + if (this.busy || this.finished || !this.device || !this.ctx) return + this.busy = true + try { + const frame = await captureSnapshotFrame(this.device, { + platform: this.ctx.platform, + timeoutMs: 15_000 + }) + this.latestSnapshot = frame.nodes as ManualSnapshotNode[] + this.revision += 1 + this.emitEvent({ + type: 'snapshot', + projectId: this.projectId, + nodeId: this.nodeId, + revision: this.revision, + nodes: this.latestSnapshot, + bounds: frame.bounds as ManualSnapshotRect + }) + } catch (error) { + // A dropped poll is not a failed session: the next tick two seconds later usually succeeds, + // and tearing the recording down over one busy runner would throw away the user's walk. + log('snapshot tick failed', errorMessage(error)) + } finally { + this.busy = false + } + } + + async act(action: ManualRecordingAction): Promise { + if (this.finished) throw new Error('This recording has already finished.') + const device = this.device + const ctx = this.ctx + if (!device || !ctx) throw new Error('Recording is still starting up.') + await this.waitForIdle() + + this.busy = true + this.status('acting') + try { + const step = await this.perform(ctx, device, action) + this.literalCommands.push(step.command) + this.emitEvent({ + type: 'step', + projectId: this.projectId, + nodeId: this.nodeId, + command: step.command, + portable: step.portable + }) + await waitForRoute() + this.status('recording') + } finally { + this.busy = false + } + // Refresh immediately rather than waiting out the interval: agent-device `@ref`s expire after + // any action, and the user is looking at a wireframe of the screen they just left. + await this.tick() + } + + private async perform( + ctx: CaptureContext, + device: CaptureDevice, + action: ManualRecordingAction + ): Promise<{ command: string; portable: boolean }> { + if (action.type === 'scroll') { + await device.client.interactions.scroll({ + platform: ctx.platform, + direction: action.direction + }) + return { command: scrollCommand(action.direction), portable: true } + } + + const locator = locatorFor(action.target) + if (!locator) { + const rect = action.target.rect + if (!rect) throw new Error('That element has neither a selector nor a position to press.') + const point = rectCenter(rect) + if (action.type === 'fill') { + throw new Error('That field has no id or label to type into reliably.') + } + await device.client.interactions.press({ + platform: ctx.platform, + x: point.x, + y: point.y, + settle: true, + timeoutMs: 10_000 + }) + // Coordinates are recorded faithfully but do not survive a layout or device-size change, so + // the panel marks this step as the weak one it is. + return { command: pressPointCommand(point.x, point.y), portable: false } + } + + const selector = serializeLocator(locator) + if (action.type === 'fill') { + await device.client.interactions.fill({ + platform: ctx.platform, + selector, + text: action.value, + settle: true, + timeoutMs: 10_000 + }) + return { + command: fillCommand(selector, action.value), + portable: isPortable(locator) + } + } + + await pressWithLocator(ctx, device, locator) + return { command: pressCommand(selector), portable: isPortable(locator) } + } + + /** + * Ends the walk and writes the artifacts. + * + * Tries hardest first: wait on a durable landmark, publish the recording, verify it by replaying + * it, and hand back a `pathToReach` that is one `agent-device replay`. Everything that can go + * wrong there falls back to the literal command list rather than failing the session — the user + * has already done the work, and a weaker path is worth far more than none. + */ + async stop(): Promise { + if (this.finished) return + const device = this.device + const ctx = this.ctx + const node = this.node + if (!device || !ctx || !node) throw new Error('Recording is still starting up.') + await this.waitForIdle() + this.finished = true + this.stopPolling() + this.status('saving') + + // The screen's own artifacts come first, while the app is still standing on the screen the user + // walked it to. Everything below relaunches: `saveAndReplayRecording` closes the session and + // re-runs the whole script to prove it replays, so by the time it returns the app has been + // through a cold launch and whatever the verification left on screen is what a screenshot would + // catch. Both calls are observation-only, which agent-device excludes from the recording by + // default, so taking them inside the still-open session does not add a step to the script. + const pressables = await this.collectPressables(ctx, device) + const screenshot = await this.collectScreenshot(ctx, device, node) + + let pathToReach: string[] = [] + let replayScript: string | null = null + let replayVerified = false + let fallbackReason: string | undefined + + const destination = destinationLocator(this.latestSnapshot) + if (!destination) { + fallbackReason = + 'This screen has no element with an id or accessibility label, so agent-device cannot guard a saved recording here.' + } else { + try { + await saveAndReplayRecording(ctx, device, this.candidatePath, serializeLocator(destination)) + await rename(this.candidatePath, this.publishedPath) + replayScript = this.publishedPath + replayVerified = true + pathToReach = replayPathToReach(device, this.publishedPath) + } catch (error) { + fallbackReason = `The recording could not be published or verified: ${errorMessage(error)}` + await unlink(this.candidatePath).catch(() => undefined) + } + } + + if (!replayVerified) { + await unlink(this.candidatePath).catch(() => undefined) + pathToReach = [ + ...device.setupCommands, + // `device.launchTarget` verbatim, exactly as `resolveDeepLink` passes it — the launch + // target is already the resolved bundle id by the time `bootDevice` returns. + agentDeviceOpenCommand(device.launchTarget, device.launchUrl ?? null), + ...this.literalCommands + ] + } + + await this.persist(node, { pathToReach, replayScript, replayVerified, pressables, screenshot }) + + this.emitEvent({ + type: 'complete', + projectId: this.projectId, + nodeId: this.nodeId, + pathToReach, + replayVerified, + ...(fallbackReason ? { fallbackReason } : {}) + }) + await this.teardown() + } + + private async collectPressables( + ctx: CaptureContext, + device: CaptureDevice + ): Promise { + try { + return await capturePressables(ctx, device) + } catch (error) { + log('pressables failed', errorMessage(error)) + return [] + } + } + + private async collectScreenshot( + ctx: CaptureContext, + device: CaptureDevice, + node: AppFlowNode + ): Promise { + const path = getScreenshotPath(this.projectId, node.slug) + try { + await captureScreenshot(ctx, device, path) + return join('screens', `${node.slug}.png`) + } catch (error) { + log('screenshot failed', errorMessage(error)) + return null + } + } + + /** + * Merges this one screen into both documents. Never rewrites either wholesale: every other + * screen's entry and every other node's capture is read back and preserved, because a manual + * recording is a single-screen edit to files a whole sweep also owns. + */ + private async persist( + node: AppFlowNode, + outcome: { + pathToReach: string[] + replayScript: string | null + replayVerified: boolean + pressables: AppFlowPressable[] + screenshot: string | null + } + ): Promise { + // Named here, once, for both documents. A manual path is the one kind nobody should be + // surprised by — the user watched every step go in — but it can still lean on a rendered + // string (a list, a post, a person's name), and a consumer deciding whether to re-record is + // entitled to know that before a replay on another account fails. + const contentSelectors = contentMatchedSelectors(outcome.pathToReach) + const result = await readAppFlow(this.projectId) + if (result) { + const target = result.nodes.find((candidate) => candidate.id === node.id) + if (target) { + target.capture = { + status: 'ok', + needsNavigation: false, + note: 'Recorded manually', + screenshot: outcome.screenshot, + pressables: outcome.pressables, + states: [], + captureMethod: 'manual', + replayScript: outcome.replayScript, + replayVerified: outcome.replayVerified, + // True on both branches above, and the one case where it is *more* than `replayVerified` + // claims: a recording that could not be published still produced a literal command list + // of steps the user themselves performed on the device. `pathVerified` asks whether + // anything travelled this path, not whether a script round-tripped, so the fallback + // qualifies exactly as the published script does — and either way this is nothing like + // the chrome-inferred guess the flag exists to mark (`AppFlowCapture.pathVerified`). + pathVerified: outcome.pathToReach.length > 0, + ...(contentSelectors.length ? { pathContentSelectors: contentSelectors } : {}), + // Always true in practice — a manual recording's path opens with the app launch — but + // written from the path itself rather than asserted, so the node and the screens entry + // below cannot disagree about whether there is something to replay. + replayable: outcome.pathToReach.length > 0, + ...(result.manifest.structureRevision + ? { structureRevision: result.manifest.structureRevision } + : {}) + } + } + await writeAppFlow(this.projectId, result) + } + + const screens = await readAgentDeviceScreens(this.projectId) + const existingKey = Object.keys(screens).find((key) => screens[key].screenId === node.id) + const used = new Set(Object.keys(screens).filter((key) => key !== existingKey)) + const key = existingKey ?? uniqueScreenKey(node.title || node.id, used) + screens[key] = { + key, + screenId: node.id, + info: screens[existingKey ?? '']?.info ?? defaultScreenInfo(node), + pressables: outcome.pressables, + pathToReach: outcome.pathToReach, + pathVerified: outcome.pathToReach.length > 0, + ...(contentSelectors.length ? { pathContentSelectors: contentSelectors } : {}), + replayScript: outcome.replayScript, + replayVerified: outcome.replayVerified, + replayStates: [] + } + await writeAgentDeviceScreens(this.projectId, screens) + } + + async cancel(): Promise { + if (this.finished) return + this.finished = true + this.stopPolling() + await unlink(this.candidatePath).catch(() => undefined) + this.status('stopped', 'Recording discarded') + await this.teardown() + } + + /** + * Always runs, on every exit path. A leaked agent-device session keeps the iOS runner lease, and + * the next sweep fails with "iOS runner is already owned by another agent-device daemon". + */ + async teardown(): Promise { + this.stopPolling() + const device = this.device + this.device = null + if (!device) return + try { + await closeDevice(device) + } catch (error) { + console.error('Failed to close manual recording device:', error) + } + } + + private stopPolling(): void { + if (!this.pollTimer) return + clearInterval(this.pollTimer) + this.pollTimer = null + } + + /** Reported to the renderer so a failed start still tears the device down. */ + async failed(error: unknown): Promise { + this.finished = true + this.stopPolling() + await unlink(this.candidatePath).catch(() => undefined) + this.emitEvent({ + type: 'error', + projectId: this.projectId, + nodeId: this.nodeId, + message: errorMessage(error) + }) + await this.teardown() + } +} diff --git a/src/app-flow/manual/validate.ts b/src/app-flow/manual/validate.ts new file mode 100644 index 00000000..6739104b --- /dev/null +++ b/src/app-flow/manual/validate.ts @@ -0,0 +1,65 @@ +/** + * Narrowing for the one manual-recording payload that is not a bare string. + * + * Lives beside the feature rather than in `ipc.ts` for the same reason `isDebuggingUserResponse` + * does: the shape belongs to the feature, and a guard this specific is worth unit-testing on its + * own. Pure — no Electron, no device. + */ +import type { ManualRecordingAction, ManualSnapshotNode } from '../../shared/app-flow.ts' + +const SCROLL_DIRECTIONS = new Set(['up', 'down', 'left', 'right']) +const MAX_TEXT = 4096 + +function optionalString(value: unknown, max = 1024): string | undefined { + if (value === undefined || value === null) return undefined + if (typeof value !== 'string' || value.length > max) return undefined + return value +} + +function parseTarget(value: unknown): ManualSnapshotNode | null { + if (!value || typeof value !== 'object') return null + const raw = value as Record + if (typeof raw.ref !== 'string' || raw.ref.length > 256) return null + + let rect: ManualSnapshotNode['rect'] + if (raw.rect !== undefined && raw.rect !== null) { + const candidate = raw.rect as Record + const numbers = ['x', 'y', 'width', 'height'].map((key) => candidate[key]) + if (!numbers.every((entry) => typeof entry === 'number' && Number.isFinite(entry))) return null + rect = { + x: candidate.x as number, + y: candidate.y as number, + width: candidate.width as number, + height: candidate.height as number + } + } + + return { + ref: raw.ref, + ...(optionalString(raw.role, 128) ? { role: raw.role as string } : {}), + ...(optionalString(raw.type, 128) ? { type: raw.type as string } : {}), + ...(optionalString(raw.label) ? { label: raw.label as string } : {}), + ...(optionalString(raw.value) ? { value: raw.value as string } : {}), + ...(optionalString(raw.identifier) ? { identifier: raw.identifier as string } : {}), + ...(rect ? { rect } : {}) + } +} + +export function parseManualRecordingAction(value: unknown): ManualRecordingAction | null { + if (!value || typeof value !== 'object') return null + const raw = value as Record + + if (raw.type === 'scroll') { + if (typeof raw.direction !== 'string' || !SCROLL_DIRECTIONS.has(raw.direction)) return null + return { type: 'scroll', direction: raw.direction as 'up' | 'down' | 'left' | 'right' } + } + + const target = parseTarget(raw.target) + if (!target) return null + if (raw.type === 'tap') return { type: 'tap', target } + if (raw.type === 'fill') { + if (typeof raw.value !== 'string' || raw.value.length > MAX_TEXT) return null + return { type: 'fill', target, value: raw.value } + } + return null +} diff --git a/src/app-flow/nav-route-target.ts b/src/app-flow/nav-route-target.ts new file mode 100644 index 00000000..ec9482c7 --- /dev/null +++ b/src/app-flow/nav-route-target.ts @@ -0,0 +1,39 @@ +interface RouteTargetNode { + id: string + file: string | null + params: readonly string[] +} + +const answersTo = (node: RouteTargetNode, routeName: string): boolean => + node.id === routeName || node.id.split('/').pop() === routeName + +/** + * The one param-declaring node a live navigator route's params belong to, or null. + * + * A navigator names a screen the way it registered it, and the parser may hold the same component + * under several nodes: one reached from a tab with no path (and so no params), another registered + * for its URL (and so declaring them). In a measured sweep the navigator exposed the route that + * carried the needed value under the first name, no param-declaring node answered to it, and 12 + * nodes stayed blocked behind ~400s of exploration. Two nodes backed by one component file are the + * same screen, so the values are bridged through the file. + * + * A direct name hit is tried first and is never overridden by the bridge. Ambiguity is fatal on + * every path, for the reason the name match always was: two candidates say nothing about which + * record these params belong to, and a wrong attachment is a confidently wrong capture. So the bridge + * is taken only when the name identifies exactly one node, that node has a file, and exactly one + * other node sharing the file declares params; a file shared by two param-declaring nodes yields + * none of them. A direct match that is itself ambiguous does not fall through to the bridge. + */ +export function resolveNavRouteTarget( + nodes: readonly T[], + routeName: string +): T | null { + const direct = nodes.filter((node) => node.params.length > 0 && answersTo(node, routeName)) + if (direct.length) return direct.length === 1 ? direct[0] : null + const named = nodes.filter((node) => answersTo(node, routeName)) + if (named.length !== 1 || !named[0].file) return null + const sharing = nodes.filter( + (node) => node !== named[0] && node.file === named[0].file && node.params.length > 0 + ) + return sharing.length === 1 ? sharing[0] : null +} diff --git a/src/app-flow/nav-state-gate.ts b/src/app-flow/nav-state-gate.ts new file mode 100644 index 00000000..0aabe9b1 --- /dev/null +++ b/src/app-flow/nav-state-gate.ts @@ -0,0 +1,37 @@ +import { findRouteAncestors } from './exploration.ts' + +/** + * Speculative nav-state reads one sweep may spend on screens that ancestry cannot vouch for. The + * worst case a sweep can add beyond the ancestry-gated reads is this many CLI round trips + * (~0.5-1s each); a sweep with nothing blocked adds none. + */ +export const NAV_STATE_SPECULATIVE_READS = 3 + +/** + * Which blocked nodes justify reading the navigator after capturing `from`. + * + * Ancestry is by urlPath prefix (`findRouteAncestors`), which is blind to a screen with no + * `urlPath`: the parser derives `params` from the path, so such a screen has none and is no + * node's prefix either, yet it is exactly the kind of screen (a detail view pushed with params in + * navigation state, never in a URL) that holds the values blocked nodes lack. In a measured sweep + * the values sat on one such screen at 169.7s, were never read, and 12 nodes stayed blocked behind + * ~400s of exploration. The gate is a cost heuristic, not a correctness constraint, so for those + * screens only, it spends from `speculativeBudget` instead of requiring ancestry. A screen with a + * urlPath that is nobody's ancestor still reads nothing: its position is known and says no. + */ +export function selectNavStateWaiters( + from: T, + blocked: readonly T[], + nodes: readonly T[], + speculativeBudget: number +): { waiting: T[]; speculative: boolean } { + if (!blocked.length) return { waiting: [], speculative: false } + const waiting = blocked.filter( + (other) => + other.urlPath && + findRouteAncestors(other.urlPath, nodes).some((ancestor) => ancestor.id === from.id) + ) + if (waiting.length) return { waiting, speculative: false } + if (!from.urlPath && speculativeBudget > 0) return { waiting: [...blocked], speculative: true } + return { waiting: [], speculative: false } +} diff --git a/src/app-flow/nav-state.ts b/src/app-flow/nav-state.ts new file mode 100644 index 00000000..acdc3eee --- /dev/null +++ b/src/app-flow/nav-state.ts @@ -0,0 +1,375 @@ +/** + * The routes the running app's navigator is currently holding, read out of the JS runtime. + * + * `param-store.ts` harvests param values from testIDs, which works only where the app happens to + * stamp a value into one *and* that testID survives into the accessibility tree. Both conditions + * fail often enough that this module is the difference between reaching a parameterised route and + * walking to it. Two ways they fail, measured on a real app rather than imagined: + * + * - **The value is in a testID that never reaches the tree.** An interpolated testID sitting under + * a wrapper that carries its own `accessibilityLabel` is replaced by that label in the read, so + * the row arrives as a sentence and the identifier is simply not there to harvest. + * - **The value is in no testID at all.** Opaque record keys — a post id, a list key — are + * routinely passed as navigation params without ever being stamped into a control. + * + * The navigator knows those values in both cases, because it is what the app passed them to. That + * is the whole reason to read it. + * + * This is **not** the CDP transport rewrite that was deliberately deferred. There is no new socket + * client, no second `DevicePort`, and nothing persistent: it is one `agent-device cdp runtime eval` + * per read, through the same CLI and the same target-selection helpers the CPU profiler already + * uses (`profiling/cpu-evidence.ts`). The whole module is read-only — the expression it evaluates + * must never call `navigate` or touch app state, and it is written so that it cannot. + * + * Everything here degrades to `null`. A release build has no DevTools hook, Metro may not be + * running, Hermes implements only a subset of CDP, and React Navigation's internals move between + * versions. None of those are errors the sweep should care about — they mean "no extra values + * today", and the sweep explores exactly as it does now. The caller latches the first failure (see + * `NavStateStatus`) so a sweep never pays for this twice. + */ + +import { FIND_NAVIGATOR_JS } from './runtime-navigator.ts' + +/** One route out of a flattened navigation state. */ +export interface NavRoute { + /** The navigator's own name for the screen — usually the route id the parser also knows. */ + name: string + /** The URL path the route was entered at, when the app recorded one. */ + path?: string + /** Only values that can honestly become a URL segment; see the expression below. */ + params?: Record +} + +/** What `readNavState` needs from the outside world, so tests never shell out. */ +export interface NavStateDeps { + /** Runs a shell command and returns its combined output. Rejects on a non-zero exit. */ + run(command: string, timeoutMs: number): Promise + /** `agent-device --session cdp `. Injected so the quoting stays in one place. */ + cdpCommand(args: string): string + /** Shell-quotes one argument. */ + quote(value: string): string + log?(message: string): void +} + +/** + * How long the one-off target list and select may take, matching `selectMetroCdpTarget`'s own + * budget. Generous because `agent-device cdp` runs a pinned helper through npm and the very first + * invocation on a machine may download it; a tighter bound here would latch `unavailable` for a + * sweep purely because a package was cold. + */ +const SETUP_TIMEOUT_MS = 30_000 +/** A read that has not answered in this long is not going to; the walk is waiting behind it. */ +const EVAL_TIMEOUT_MS = 5_000 + +/** + * The expression evaluated in the app's JS context. + * + * Written as one self-contained arrow IIFE that can only return a string or null, because anything + * it throws surfaces as a CLI failure that says nothing useful. Three constraints shaped it: + * + * - **No instrumentation to rely on.** The comparison project injects a Babel transform and can + * assume a global navigation ref. We have neither, so the ref has to be *found*: walk the fiber + * tree from React DevTools' own hook and look for anything the app has handed a navigator to. + * - **It runs on the app's JS thread.** An unbounded walk over a large tree would stall the UI, + * which is the one thing a read must never do, so the traversal is bounded and gives up rather + * than finishing at any cost. + * - **Only URL-able values are worth returning.** Params are filtered to strings and finite + * numbers in the page, so a route object holding a whole post or a callback neither crosses the + * wire nor arrives as `[object Object]` pretending to be an id. + * + * ## Why this returned nothing on every real run, and what actually fixed it + * + * The first version of this module logged `navigation state eval returned nothing usable` on every + * sweep against Bluesky, which latched `unavailable` and left the app's ten `:rkey` routes with no + * value source at all. The obvious suspect was the fiber walk, and it was the wrong one. Evaluated + * live against the running app, the original expression returns the full, correct payload: + * + * ``` + * [{"name":"HomeTab"},{"name":"Home"},{"name":"PostThread","params":{"name":"alice.test", + * "rkey":"3kabc7xyz"}},{"name":"PostQuotes","params":{…}},{"name":"SearchTab"},…] + * ``` + * + * **The payload was being cut off in transit.** `agent-device cdp runtime eval` truncates a + * returned string at exactly 200 characters — measured, not inferred: `'X'.repeat(400)` comes back + * as 200 `X`s. A real navigator's flattened state is 400-plus characters, so what reached + * `parseNavStateOutput` was a string containing `[` and no closing `]`. Both candidate forms failed + * to parse, the function returned null, and the module reported the runtime as unable to answer + * when in fact it had answered perfectly. A 200-character ceiling is also why no amount of + * re-running or re-scheduling would ever have helped. + * + * So the read is now **chunked**: the expression stashes its own payload on one namespaced global + * and returns a short header plus the first slice, and `readNavState` asks for the remaining slices + * until it has the whole string. A payload that fits in one slice still costs exactly one eval, + * which is the common case. See `NAV_STATE_CHUNK` for the slice size and the header format. + * + * Stashing is the one write this module performs, and it is deliberate: recomputing the state for + * each slice would be read-only but could interleave slices from two different navigation states + * and assemble a payload that never existed. A single harness-owned global under a reserved name + * is the lesser risk, and it touches nothing the app can observe. + * + * ## Why the carrier search still changed + * + * The truncation fix alone would have been enough on this app, so the discovery changes below are + * robustness rather than the bug fix, and they are kept because each covers a case this app simply + * does not exercise: + * + * - **`getRootState` is the container's method, not a screen's.** It lives on what + * `useNavigationContainerRef()` holds and on `NavigationContainerRefContext`, neither of which is + * guaranteed to sit in the fiber tree. What *is* in the tree, on every screen React Navigation + * renders, is the `navigation` prop, which carries `getState` and `getParent`. That shape is now + * accepted and climbed to the root, as the comparison project's runtime does (`runtime.js`'s + * `navigation(value)` helper). It discriminates on read-only methods rather than on `dispatch`, + * which is what that project uses, so this expression never so much as names a mutating method + * and the test asserting that stays a real guard instead of a thing to be exempted from. + * - **Renderer ids are not stable and 1 was a guess.** On this app id 1 is in fact the React + * Native renderer, but `hook.renderers` is keyed by registration order, so a bundle that + * registers anything first would have left `getFiberRoots(1)` empty. The renderers are now + * enumerated and filtered by `rendererPackageName`, with the fixed-id call kept as a fallback. + * - **A carrier that answers badly must not end the search.** Each `getRootState()` is called in + * its own try/catch and its result is required to have `routes`, so a detached or still-mounting + * container ref is skipped rather than accepted as a successful read of an empty sweep. + * + * The budget moved for a measured reason: the live tree is **17,822 fibers**, and the comparison + * project's own comment records a busy feed exceeding 16,000, which made the original 20,000-fiber + * cap a coin flip on exactly the screens that matter. The real protection for the JS thread is the + * wall-clock deadline, so the node count is now generous and the deadline is what stops the walk. + * A `seen` set guards cycles, which a `child`/`sibling` push cannot do on its own. + */ + +/** + * How much payload one eval may return, and the header that frames it. + * + * `agent-device cdp runtime eval` truncates a returned string at 200 characters. The header is + * `AIHNS1::` — a format marker, the full payload length, and a colon — so 160 characters of + * slice leaves room for a five-digit length and never approaches the ceiling. + */ +const NAV_STATE_CHUNK = 160 +/** The global the expression stashes its payload on, so slices come from one consistent read. */ +const NAV_STATE_STASH = '__aiHarnessNavState' + +export const NAV_STATE_EXPR = `(() => { + try { + ${FIND_NAVIGATOR_JS} + var found = findNavigator() + if (!found) return null + var state = found.state + if (!state) return null + var flat = [] + var walk = function (node, depth) { + if (!node || depth > 12 || !Array.isArray(node.routes)) return + for (var i = 0; i < node.routes.length; i++) { + var route = node.routes[i] + if (!route || typeof route.name !== "string") continue + var entry = { name: route.name } + if (typeof route.path === "string" && route.path) entry.path = route.path + var raw = route.params + if (raw && typeof raw === "object" && !Array.isArray(raw)) { + var kept = {} + var any = false + for (var key in raw) { + var value = raw[key] + if (typeof value === "string") { kept[key] = value; any = true } + else if (typeof value === "number" && isFinite(value)) { kept[key] = String(value); any = true } + } + if (any) entry.params = kept + } + flat.push(entry) + if (route.state) walk(route.state, depth + 1) + } + } + walk(state, 0) + var payload = JSON.stringify(flat) + // Stashed so the remaining slices come from this same read; see the doc comment above. + try { globalThis[${JSON.stringify(NAV_STATE_STASH)}] = payload } catch (error) {} + return "AIHNS1:" + payload.length + ":" + payload.slice(0, ${NAV_STATE_CHUNK}) + } catch (error) { + return null + } +})()` + +/** + * Asks for one more slice of the payload the last `NAV_STATE_EXPR` stashed. + * + * Returns the same `AIHNS1::` framing so the caller has one parser, and `null` when + * the stash is gone — an app reload between slices drops it, and a partial assembly must be + * abandoned rather than parsed, because half a JSON array is not a smaller answer. + */ +export function navStateChunkExpr(offset: number): string { + return `(() => { + try { + var payload = globalThis[${JSON.stringify(NAV_STATE_STASH)}] + if (typeof payload !== "string") return null + return "AIHNS1:" + payload.length + ":" + payload.slice(${Math.trunc(offset)}, ${Math.trunc(offset) + NAV_STATE_CHUNK}) + } catch (error) { + return null + } +})()` +} + +/** + * Pulls `AIHNS1::` back out of the CLI's own reporting. + * + * The slice is raw JSON, so it arrives escaped inside the CLI's `string: "…"` envelope and has to + * be unescaped before it can be concatenated — the same quote-and-parse trick + * `navStateCandidates` uses below. Returns null for anything that is not this framing, which + * covers both an expression that returned null and a CLI that reported an error instead. + */ +export function parseNavStateChunk(output: string): { total: number; slice: string } | null { + const envelope = /"((?:\\.|[^"\\])*)"/g + for (const match of output.matchAll(envelope)) { + const text = tryParse(`"${match[1]}"`) + if (typeof text !== 'string') continue + const framed = /^AIHNS1:(\d{1,9}):([\s\S]*)$/.exec(text) + if (!framed) continue + return { total: Number(framed[1]), slice: framed[2] ?? '' } + } + return null +} + +/** + * Finds the JSON array in whatever `agent-device cdp runtime eval` printed around it. + * + * The CLI wraps the evaluated value in its own reporting, and the exact shape belongs to a pinned + * helper package rather than to this repo, so matching it exactly would be a version dependency + * nobody would remember to update. Scanning for the payload is version-proof, and the cost of + * being wrong is a `null` that latches `unavailable` — the same outcome as not trying. + * + * The expression returns a JSON *string*, so the array usually arrives escaped inside the CLI's + * own JSON; both the escaped and the bare form are tried. + */ +export function parseNavStateOutput(output: string): NavRoute[] | null { + for (const candidate of navStateCandidates(output)) { + const routes = asRoutes(tryParse(candidate)) + if (routes) return routes + } + return null +} + +function* navStateCandidates(output: string): Generator { + // The bare array, as the expression returned it. + const open = output.indexOf('[') + const close = output.lastIndexOf(']') + if (open !== -1 && close > open) yield output.slice(open, close + 1) + // The same array escaped inside the CLI's own JSON envelope, e.g. "value": "[{\"name\":…}]". + for (const match of output.matchAll(/"(\[(?:\\.|[^"\\])*\])"/g)) { + const unescaped = tryParse(`"${match[1]}"`) + if (typeof unescaped === 'string') yield unescaped + } +} + +function tryParse(text: string): unknown { + try { + return JSON.parse(text) + } catch { + return null + } +} + +/** Accepts only the shape this module promises; anything else is treated as "not the payload". */ +function asRoutes(value: unknown): NavRoute[] | null { + if (!Array.isArray(value)) return null + const routes: NavRoute[] = [] + for (const item of value) { + if (!item || typeof item !== 'object' || Array.isArray(item)) return null + const entry = item as { name?: unknown; path?: unknown; params?: unknown } + if (typeof entry.name !== 'string' || !entry.name) return null + const route: NavRoute = { name: entry.name } + if (typeof entry.path === 'string' && entry.path) route.path = entry.path + if (entry.params && typeof entry.params === 'object' && !Array.isArray(entry.params)) { + const params: Record = {} + for (const [key, raw] of Object.entries(entry.params)) { + if (typeof raw === 'string') params[key] = raw + } + if (Object.keys(params).length) route.params = params + } + routes.push(route) + } + return routes +} + +/** + * Selects a Metro CDP target once, so later reads are a single eval. + * + * Separate from the read itself because selection is the expensive, once-per-sweep half and the + * read is the cheap repeatable one. Returns false rather than throwing: every caller's answer to + * "could not reach the runtime" is the same, and it is not an error. + */ +export async function selectNavStateTarget( + metroPort: number, + appLabel: string | null, + deps: NavStateDeps, + pickTarget: (output: string, preferred: Array) => string | null +): Promise { + const metroUrl = `http://127.0.0.1:${metroPort}` + try { + const listed = await deps.run( + deps.cdpCommand(`target list --url ${deps.quote(metroUrl)}`), + SETUP_TIMEOUT_MS + ) + const target = pickTarget(listed, [appLabel, 'React Native']) + if (!target) { + deps.log?.(`no Metro CDP target at ${metroUrl}`) + return false + } + await deps.run(deps.cdpCommand(`target select ${deps.quote(target)}`), SETUP_TIMEOUT_MS) + return true + } catch (error) { + deps.log?.(`could not select a Metro CDP target — ${errorText(error)}`) + return false + } +} + +/** + * The currently mounted navigator's routes, or null when the runtime cannot answer. + * + * **Honest limit, and it decides how this should be scheduled.** This sees the mounted navigator + * state and its stack history — not every record the app holds, only the ones it is showing or has + * shown. So it is worth reading right after capturing a screen that is a parent (or, through + * `findRouteAncestors`, an ancestor) of a still-unreached node with unresolved params, and worth + * nothing at all otherwise. Read on that schedule it is a handful of CLI round trips per sweep; + * read eagerly it is a tax on every screen for values that are not there yet. + */ +export async function readNavState(deps: NavStateDeps): Promise { + try { + const first = parseNavStateChunk( + await deps.run( + deps.cdpCommand(`runtime eval --expr ${deps.quote(NAV_STATE_EXPR)}`), + EVAL_TIMEOUT_MS + ) + ) + if (!first) { + deps.log?.('navigation state eval returned nothing usable') + return null + } + let payload = first.slice + // Bounded by the payload's own declared length, so a stash that keeps answering with the same + // slice cannot loop: every pass must extend `payload` or the read is abandoned. + while (payload.length < first.total) { + const next = parseNavStateChunk( + await deps.run( + deps.cdpCommand(`runtime eval --expr ${deps.quote(navStateChunkExpr(payload.length))}`), + EVAL_TIMEOUT_MS + ) + ) + if (!next?.slice.length || next.total !== first.total) { + // The app reloaded, or the stash went away mid-read. Half a JSON array is not a smaller + // answer, so this is reported the same way an unreachable runtime is. + deps.log?.( + `navigation state read truncated at ${payload.length} of ${first.total} characters` + ) + return null + } + payload += next.slice + } + const routes = parseNavStateOutput(payload) + if (!routes) deps.log?.('navigation state eval returned nothing usable') + return routes + } catch (error) { + deps.log?.(`navigation state eval failed — ${errorText(error)}`) + return null + } +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/src/app-flow/orchestrator.ts b/src/app-flow/orchestrator.ts new file mode 100644 index 00000000..ab8b098c --- /dev/null +++ b/src/app-flow/orchestrator.ts @@ -0,0 +1,4413 @@ +import { EventEmitter } from 'node:events' +import { createHash, randomUUID } from 'node:crypto' +import { copyFile, unlink } from 'node:fs/promises' +import { basename, join } from 'node:path' +import type { + AgentDeviceScreenEntry, + AgentDeviceScreensMap, + AppFlowCapture, + AppFlowCaptureStatus, + AppFlowBuildMode, + AppFlowChrome, + AppFlowDeepLinkRejection, + AppFlowEdge, + AppFlowManifest, + AppFlowNavigation, + AppFlowNode, + AppFlowPressable, + AppFlowProgressEvent, + AppFlowPrompt, + AppFlowPromptResolution, + AppFlowResult, + AppFlowStep, + AppFlowTrigger, + DeepLinkJudgeMode, + ExplorationMode, + RenderTransportMode, + TokenUsage +} from '../shared/app-flow.ts' +import { addTokenUsage } from '../shared/app-flow.ts' +import type { AppFlowDecisionProvider } from './decision-provider.ts' +import { + parseSubAgentJson, + type SubAgentCallOptions, + type SubAgentOptions, + type SubAgentResult, + type SubAgentSessionOptions +} from './decision-provider.ts' +import { emptyTokenUsage, snapshotUsage } from '../server/app-flow-deps/pi/usage.ts' +import { + APP_READY_SETTLE_SIMILARITY, + bootDevice, + buildAgentDevicePathToReach, + capturePressables, + captureScreenshot, + closeDevice, + executeNavigationActions, + goHome, + isOnDevClientLauncher, + openDeepLink, + replayPathToReach, + reopenBoundApp, + SUBSTANTIVE_FRAME_GRACE_MS, + waitForAppReady, + settleAfterRoute, + type CaptureContext, + type CaptureDevice +} from './capture.ts' +import { parseProject } from './parser.ts' +import { baselineLaunchCommands, walkCommands, walkStepCommand } from './walk-commands.ts' +import { contentMatchedSelectors, minimizeWalkCommands } from './path-minimize.ts' +import { isPathVerified } from './path-provenance.ts' +import { runPathToReach } from './path-run.ts' +import type { RenderTransport } from './reach/adapters.ts' +import { createReachPorts } from './reach/adapters.ts' +import { + AUTH_GATE_FRAME_WAIT_MS, + settlesAuthGateForRun, + type AuthGateResolution +} from './reach/auth-gate-policy.ts' +import { formatPhaseSummary, PhaseLedger } from './timing.ts' +import { hedge } from './hedge.ts' +import { + chromeEdges, + chromeVirtualNodes, + navigatorChrome, + rebuildChromeEdges, + detectChromeInventory, + findDrawerOpener, + findMatchingChrome, + inferChromeKind, + mergeChromeInventory, + type ChromeCandidate +} from './reach/chrome-inventory.ts' +import { buildChromePathToReach } from './reach/chrome-path.ts' +import { + carryForwardDeepLinkRejection, + initialDeepLinkBreakerState, + registerDeepLinkFailure, + registerDeepLinkSuccess, + type DeepLinkBreakerState +} from './reach/deep-link-policy.ts' +import { + chooseDeliverySources, + decideDelivery, + type DeliveryScreenshotOutcome +} from './reach/delivery.ts' +import { runReachLadder, strategiesFor, type ReachStrategy } from './reach/ladder.ts' +import { + NAVIGATION_SYSTEM_PROMPT, + RUNTIME_SCREEN_SYSTEM_PROMPT, + STATE_TRIGGER_SYSTEM_PROMPT +} from './reach/prompts.ts' +import { ReachQueue } from './reach/queue.ts' +import { defaultScreenInfo, fallbackScreenKey, uniqueScreenKey } from './screen-keys.ts' +import { + matchRuntimeNode, + mintRuntimeNode, + recordRuntimeSighting, + sanitizeScreenKey +} from './reach/runtime-nodes.ts' +import { + PROBE_FRAME_ELEMENT_FLOOR, + PROBE_LAUNCH_READY_BUDGET_MS, + isSubstantiveFrame, + judgeProbeFrame, + pickProbeRoute +} from './reach/probe-route.ts' +import { findSeedSource } from './reach/seed.ts' +import { + createWalkCursor, + mergeWalkEdge, + noteWalkAction, + noteWalkArrival, + type WalkCursor +} from './reach/walk-edges.ts' +import { computeStructureKey } from './reach/structure-key.ts' +import { renderStrategy } from './reach/strategies/render.ts' +import { deepLinkStrategy } from './reach/strategies/deep-link.ts' +import { savedReplayStrategy } from './reach/strategies/saved-replay.ts' +import { isFallbackRoute, orderSweepNodes } from './reach/sweep-order.ts' +import { ExplorationControl, type ExplorationPauseReason } from './reach/exploration-control.ts' +import type { + ArrivalVerdict, + DiscoveredFact, + ReachAttempt, + ReachEffect, + ReachEvidence +} from './reach/types.ts' +import { fingerprint, similarity, tokenize } from './screen-signals.ts' +import { + ActiveTimeBudget, + DeepLinkRejectionMemo, + GuardGateMemo, + EXPLORATION_MAX_DECISIONS, + demoteFailedHarvest, + describeJourney, + expectedNodeTokens, + findNavigationPath, + isToolingFailure, + shouldRetryFromEntry, + parseNavigationDecision, + pickOpportunisticCandidate, + resolveNodeParams, + summarizePressables, + type ParamStoreLike, + runInteractiveExploration, + type ExplorationResult +} from './exploration.ts' +import { + AUTH_CHECKPOINT_SLUG, + clearSweepProgress, + generateSlug, + generateStateSlug, + getFlowsDir, + getProjectId, + getScratchPath, + getScreenshotPath, + prepareScratchDir, + pruneTransientScreens, + registerProject, + readAppFlow, + readAgentDeviceScreens, + readParamStore, + readSweepProgress, + screenshotUrl, + writeAgentDeviceScreens, + writeAppFlow, + writeCaptureStatus, + writeManifest, + writeParamStore, + writeSweepProgress, + type AppFlowSweepProgress +} from './storage.ts' +import { ParamStore } from './param-store.ts' +import { readNavState, selectNavStateTarget, type NavStateDeps } from './nav-state.ts' +import { NAV_STATE_SPECULATIVE_READS, selectNavStateWaiters } from './nav-state-gate.ts' +import { resolveNavRouteTarget } from './nav-route-target.ts' +import { runShellCommand } from './shell.ts' +import { + agentDeviceCdp, + parseCdpTargetList, + pickMetroCdpTarget, + shellQuote +} from '../server/app-flow-deps/profiling/cdp.ts' + +export type AppFlowPhase = 'parse' | 'boot' | 'capture' | 'classify' | 'states' | 'render' + +/** Slug of the screenshot shown to the user when the build stops on a screen it cannot leave. */ +const BLOCKED_SLUG = '--blocked' + +// `computeStructureKey` moved to `reach/structure-key.ts` (plan Step 4b) — that module has no +// Electron import and is unit-testable under `node --test`, which `orchestrator.ts` itself is not. + +/** + * Above this fingerprint overlap, two failed destinations are treated as showing the same UI. + * Only a near-identical screen advances the no-progress counter because shared app chrome alone + * must not condemn a capture. + */ +const ARRIVAL_SIMILARITY = 0.9 + +/** + * How many routes in a row must fail to arrive before the build stops and asks the user. One + * failure is ordinary — a param that resolves to nothing. Three different routes rendering the + * same screen is the app refusing to navigate at all. + */ +const BLOCKED_RUN_LIMIT = 3 + +// The deep-link circuit breaker's limit is `reach/deep-link-policy.ts`'s `DEEPLINK_BREAKER_LIMIT` — +// moved there in landing-order item 4 along with the breaker's state and decisions, so its own +// module can be unit-tested with no device and no sweep-scoped closure. See that module's doc +// comment on `DeepLinkBreakerState`. + +/** + * How many opportunistic arrival checks a single exploration walk toward one target may spend on + * *other* pending screens it happens to pass through. Every one of them is a real vision call + * charged against that walk's own five-minute budget, and shared hub screens (Settings, Profile, + * Moderation) turn up on nearly every walk in a sweep like social-app's — an unbounded cap would + * let confirming screens seen many times over eat the budget the walk needs for its actual target. + * Two is enough to catch most of what a five-minute walk passes through without meaningfully + * competing with the target's own decisions for that budget. + */ +const OPPORTUNISTIC_VERIFY_LIMIT = 2 + +/** + * Below this much budget left, opportunistic verification is skipped outright rather than spent — + * a check that itself times out mid-call would burn the walk's last reserves confirming a screen + * it did not come here for, right as the real target most needs whatever time remains. + */ +const OPPORTUNISTIC_MIN_REMAINING_MS = 60_000 + +/** + * How many nodes the sweep will mint for screens the parser never registered a route for at all + * (plan Step 4a). Every mint costs a model call (`RUNTIME_SCREEN_SYSTEM_PROMPT`) on top of whatever + * opportunistic budget it shares with known-node capture, but the real reason for a hard ceiling is + * that a bad affirmative — a modal or a partially-loaded frame the judge misjudged as distinct — is + * only ever caught by a human looking at the graph afterward. Ten is generous for a build the parser + * genuinely missed a handful of screens in, without letting a systematically wrong judge answer + * flood the graph with junk nodes for the rest of an 80-screen sweep. + */ +const RUNTIME_NODE_SWEEP_CAP = 10 + +/** + * How long the adopt path re-confirms the pre-launched app before deciding it is not ready and + * soft re-rooting it. + * + * Shorter than the 90s `APP_READY_TIMEOUT_MS` the fresh-boot branch takes because this is not a + * cold start and must not be budgeted like one: `index.ts`'s Launch already ran `waitForAppReady` + * to completion and only stored the device once it returned, so the bundle is built, Metro is + * warm, and the app was on a settled screen before the Start button was even enabled. On the + * 2026-10-09 social-app run that settle landed at t=43.9s and Start came at t=64.2s — twenty + * seconds in which the only things that can go wrong are the user backgrounding the app, the + * simulator sleeping, or the agent-device session dropping, and a warm app re-settles in about + * two reads. + * + * Fifteen seconds therefore buys headroom for a slow accessibility read or a screen the user left + * mid-transition, and nothing else. Spending the full 90s here would be strictly worse than the + * bug being fixed in the dead-app case, since the recovery that follows is cheap and the wait + * that precedes it is not: the floored wait only returns early on failed reads, so a backgrounded + * app that answers slowly would otherwise hold the sweep for a minute and a half before anyone + * tried the one thing that fixes it. + */ +const ADOPTED_APP_READY_BUDGET_MS = 15_000 + +export interface AppFlowBuildOptions { + projectPath: string + mode?: AppFlowBuildMode + /** Maximum screens to process in this device sweep. Omitted means no limit. */ + screenLimit?: number + /** + * Maximum screen *attempts* — reached or not — this sweep may spend. Omitted derives it from + * `screenLimit`; see `ExplorationControl`'s `DEFAULT_ATTEMPTS_PER_SCREEN` for why a sweep bounded + * only by successful captures has no bound at all. + */ + attemptLimit?: number + /** Wall-clock (active time, excluding waits on the user) this sweep may spend. Omitted derives it + * from `screenLimit` — see `DEFAULT_MS_PER_SCREEN`. */ + sweepDeadlineMs?: number + /** Cooperative pause request, observed only between durable per-screen checkpoints. */ + pauseRequested?: () => boolean + /** Host-provided decisions. Required for screen sweeps; structure-only builds do not use it. */ + decisionProvider?: AppFlowDecisionProvider + /** + * Which reach mechanisms this sweep may use for a screen no URL can address. Omitted means + * `auto`. See `ExplorationMode`, and `strategiesFor` for the ladder each one produces. + */ + explorationMode?: ExplorationMode + /** + * How the render path reaches the app's JS runtime. Omitted means `cli`. Consumed at exactly one + * place, marked `RENDER TRANSPORT SELECTION` below. See `RenderTransportMode`. + */ + renderTransportMode?: RenderTransportMode + /** + * Whether a deep link's arrival is confirmed by the vision judge. Omitted means `on` — the + * behaviour every sweep had before this option existed. Consumed at exactly one place, the + * deep-link rung's `arrivalJudge`. See `DeepLinkJudgeMode`, and `reach/not-found-signal.ts` for + * what stands in for the judge's `status: 'not-found'` when it is `off`. + */ + deepLinkJudgeMode?: DeepLinkJudgeMode + /** An app the user already launched. The sweep adopts it instead of building and booting again. */ + launchedDevice?: CaptureDevice + onEvent(event: AppFlowProgressEvent): void + signal?: AbortSignal + /** + * Asks the user to unblock the build by acting on the device, and resolves when they answer. + * The device is left running throughout. Omitted (or resolving to `stop`) ends the sweep while + * preserving the structural graph and partial captures. + */ + requestUserAction?(prompt: AppFlowPrompt): Promise +} + +export interface AppFlowBuildResult { + ok: true + projectId: string + result: AppFlowResult +} + +export interface AppFlowBuildError { + ok: false + error: string +} + +function checkSignal(signal?: AbortSignal): void { + if (signal?.aborted) throw new Error('Cancelled') +} + +function emit( + projectId: string, + onEvent: (event: AppFlowProgressEvent) => void, + event: AppFlowProgressEvent +): void { + try { + onEvent({ ...event, projectId }) + } catch (error) { + console.error('App flow event listener error:', error) + } +} + +// `sanitizeScreenKey` moved to `reach/runtime-nodes.ts` (plan Step 4a) — a runtime node's id is +// derived from the judge's own `key` through this exact function, so it lives in one place rather +// than risking a second copy drifting from the one this file used to keep privately. + +function buildAgentDeviceScreens( + nodes: AppFlowNode[], + edges: AppFlowEdge[], + runtimeEdges: AppFlowEdge[], + chrome: AppFlowChrome[], + ctx: CaptureContext, + device: CaptureDevice, + openedByNode: Record, + inferred: Record, + captures: Record +): AgentDeviceScreensMap { + const used = new Set() + const screens: AgentDeviceScreensMap = {} + + for (const node of nodes) { + const params = pickSampleParams(node, edges) + const guessed = inferred[node.id] + const key = uniqueScreenKey((guessed?.key ?? '').trim() || fallbackScreenKey(node), used) + const info = guessed?.info?.trim() || defaultScreenInfo(node) + const capture = captures[node.id] + // Four sources, first match wins, and the new first one is what replaced the sweep's `.ad` + // scripts: `capture.pathCommands` is the walk's own `agent-device …` lines, already carrying + // their baseline launch prefix (`walk-commands.ts`). It comes before `replayScript` because a + // node can hold both — a manual recording from an earlier session plus a walk from this one — + // and the command list is the one this sweep just proved. + // + // `replayScript` stays for exactly one producer: the manual recorder, which still publishes a + // real `.ad` file. Nothing in the sweep writes one now. + const pathToReach = capture?.pathCommands?.length + ? [...capture.pathCommands] + : capture?.replayScript + ? replayPathToReach(device, capture.replayScript) + : openedByNode[node.id] + ? buildAgentDevicePathToReach(ctx, device, node, params, openedByNode[node.id]) + : (buildChromePathToReach(node, edges, runtimeEdges, chrome, device) ?? []) + if (pathToReach.length === 0) continue + // Whether anything ever ran this path, as opposed to inferring it — the same four tests in the + // same order as the ladder just above, which is why they live in one pure, tested place rather + // than being spelled out a second time here. See `path-provenance.ts` for why the question + // needs asking at all, and `AppFlowCapture.pathVerified` for what the answer is used for. + const pathVerified = isPathVerified({ + pathCommands: capture?.pathCommands, + replayScript: capture?.replayScript, + replayVerified: capture?.replayVerified, + deepLinkOpened: Boolean(openedByNode[node.id]) + }) + const entry: AgentDeviceScreenEntry = { + key, + screenId: node.id, + info, + pressables: capture?.pressables ?? [], + pathToReach, + pathVerified, + // Reported, not acted on — see `contentMatchedSelectors`. Computed from the finished path + // rather than threaded out of the walk so it covers all four sources uniformly: the stored + // `Profile-List` path leans on `label="Dummy"` (a list this account owns) and the stored + // `TermsOfService` path on `label="Terms of Service"`, and both are facts about the path, not + // about which producer wrote it. + pathContentSelectors: contentMatchedSelectors(pathToReach), + replayScript: capture?.replayScript ?? null, + replayVerified: capture?.replayVerified ?? false, + // A state variant carries its own full command list now — the base screen's path plus the + // lines for the actions that opened the variant — so the filter is on that rather than on a + // `.ad` path. `replayScript` is still passed through for a states entry carried forward from + // a document written before the change. + replayStates: + capture?.states + .filter((state) => state.pathCommands?.length || state.replayScript) + .map((state) => ({ + name: state.name, + pathToReach: [...(state.pathCommands ?? [])], + replayScript: state.replayScript ?? null, + replayVerified: true + })) ?? [] + } + screens[key] = entry + } + + return screens +} + +/** + * Stamp each node's capture with whether `buildAgentDeviceScreens` gave it an entry. + * + * The Replay button needs to know whether a screen has a stored `pathToReach`, and the renderer + * never reads `agent-device-screens.json` — see `AppFlowCapture.replayable` for the two ways + * inferring it from `capture.status` went wrong. Taking the answer from the screens map means the + * document and the map cannot disagree about it. + * + * Shared because the sweep writes the document twice from two separately assembled node arrays — + * `persistCaptureUpdate` after every screen, and the final write at the end — and the second one + * lands last. Stamping only the first is the same as not stamping at all: the first sweep after + * this field was added still wrote `replayable: undefined` for all twenty screens, because the + * end-of-sweep rebuild had its own copy of the node assembly and no knowledge of the field. + */ +function withReplayableCaptures( + nodes: readonly T[], + screens: AgentDeviceScreensMap +): T[] { + const entries = new Map( + Object.values(screens) + .filter((entry): entry is AgentDeviceScreenEntry & { screenId: string } => + Boolean(entry.screenId) + ) + .map((entry) => [entry.screenId, entry]) + ) + return nodes.map((node) => { + if (!node.capture) return node + const entry = entries.get(node.id) + return { + ...node, + capture: { + ...node.capture, + replayable: Boolean(entry), + // Carried across for the same reason `replayable` is: the renderer never reads + // `agent-device-screens.json`, so anything the Replay button has to say about the path it + // is about to run has to be on the capture. Stamped from the entry the map actually holds, + // never re-derived, so the document and the map cannot disagree about whether a path was + // walked. + ...(entry ? { pathVerified: entry.pathVerified ?? false } : {}), + ...(entry?.pathContentSelectors?.length + ? { pathContentSelectors: [...entry.pathContentSelectors] } + : {}) + } + } + }) +} + +/** Where exploration starts when a deep link can put it partway to the target. */ +interface DeepLinkSeed { + /** Node id of the ancestor route the app is opened at. */ + id: string + /** Its route pattern, for the planner's benefit. */ + urlPath: string + /** The URL the recorded launch uses. */ + url: string +} + +/** + * Where a bounded last-hop exploration starts when `reach/seed.ts` found a screen this sweep has + * already reached and stored a journey for — plan Step 3e. + * + * `commands` replaces the `script` this used to carry. The walk begins by *running the parent's + * stored `pathToReach`* line by line (`DevicePort.runPathCommands`) rather than by replaying a + * `.ad` file, and the target's own path becomes those same lines plus whatever the last hop takes + * — "replay the parent, then do the last hop" with nothing on disk and nothing to arm, repair or + * publish. `trigger` is the named control the parent's outgoing edge pointed at the target with, + * when `findSeedSource` found one — handed to the planner as a strong hint rather than a command, + * since the recorded control can drift from what is on screen right now. + */ +interface CommandSeed { + id: string + /** The parent's stored `pathToReach`, baseline launch line included, so it stands alone. */ + commands: readonly string[] + trigger: AppFlowTrigger | null +} + +type ExplorationSeed = DeepLinkSeed | CommandSeed + +/** What one role has cost the sweep so far. */ +interface AgentTokenTally { + calls: number + tokens: TokenUsage +} + +/** + * What one model call was actually handed, logged before it runs. + * + * A token line says a call cost 324,000 tokens; it cannot say whether that was one enormous prompt, + * a screenshot, or the agent quietly reading source files for forty seconds. Those need different + * fixes, and telling them apart after the fact was impossible from the log alone. Images are named, + * never dumped — a base64 screenshot would bury every other line in the file — and prompts are + * reported by size with a short head, which is enough to recognise a prompt that has grown without + * anyone intending it to. + */ +function logAgentInput( + label: string, + input: { + systemPrompt?: string + userPrompt: string + imagePath?: string + tools?: string[] + /** Turns already in a session's history when this call was made. Absent for one-shot roles, + * which have none — its whole purpose is to show whether a walk's history stops growing. */ + historyMessages?: number + } +): void { + const parts = [`prompt ${input.userPrompt.length} chars`] + if (input.systemPrompt) parts.push(`system ${input.systemPrompt.length} chars`) + if (input.imagePath) parts.push(`image ${basename(input.imagePath)}`) + if (input.historyMessages !== undefined) parts.push(`history ${input.historyMessages} msgs`) + parts.push(`tools [${(input.tools ?? []).join(', ')}]`) + const head = input.userPrompt.replace(/\s+/g, ' ').slice(0, 160) + console.log(`[app-flow] agent-in ${label} — ${parts.join(', ')} :: ${head}`) +} + +/** + * The other half: what the call did once it had that input. `turns` and `toolCalls` are the two + * numbers that explain a call whose cost bears no relation to the prompt it was given — a role + * granted a file-reading tool can spend an entire budget on turns nobody asked for, and that is + * invisible in a usage total. + */ +function logAgentOutput(label: string, result: SubAgentResult): void { + const tools = result.toolCalls.length + ? result.toolCalls.map((call) => call.name).join(', ') + : 'none' + console.log( + `[app-flow] agent-out ${label} — turns ${result.diagnostics.length}, tool calls ${result.toolCalls.length} (${tools}), ` + + `text ${result.text.length} chars${result.timedOut ? ', TIMED OUT' : ''}` + ) +} + +/** + * Every model call in a sweep is its own disposable session, so a per-call line is the only place the + * cost is legible: a running total alone cannot say whether a long sweep was many cheap arrival + * checks or one exploration that would not stop. + */ +function logAgentTokens( + label: string, + used: TokenUsage, + elapsedMs: number, + running: TokenUsage, + /** + * Recorded here rather than at the four call sites because this is the one function every model + * call reaches on both the success and the failure path. A timed-out judge is the most expensive + * call a sweep can make, and tallying only successes would hide exactly that. + */ + ledger?: PhaseLedger +): void { + ledger?.record('model', elapsedMs) + const cache = + used.cacheRead || used.cacheWrite ? ` cache r${used.cacheRead}/w${used.cacheWrite}` : '' + const reasoning = used.reasoning ? ` reasoning ${used.reasoning}` : '' + const cost = used.cost ? ` $${used.cost.toFixed(4)}` : '' + console.log( + `[app-flow] tokens ${label} — in ${used.input} out ${used.output}${cache}${reasoning} ` + + `total ${used.total}${cost} in ${(elapsedMs / 1000).toFixed(1)}s ` + + `(sweep ${running.total})` + ) +} + +/** + * How long a one-shot judge may go unanswered before a duplicate is raced against it. + * + * Sized from two instrumented sweeps rather than picked: the arrival judge's p50 is 2.1s and its + * p90 is 8.7s, so this sits just above the p90 and leaves the ordinary call untouched. What it + * catches is the shape that is not deliberation — 16.6s for 165 output tokens, 24.9s for 130, where + * a p50 call returns ~140 — and in the measured run four such calls cost ~81s, 16% of the sweep. + * + * Lowering it would catch more stalls and would also start duplicating the calls that are slow + * because they are genuinely reasoning: the same sweep has a 7.6s arrival that spent 858 of its 967 + * output tokens on reasoning, and a duplicate of that call has to do the same thinking, so it pays + * twice for nothing. That is the trade this constant sets, and the next run's p-values are what + * should move it. + */ +const HEDGE_AFTER_MS = 8_000 + +/** + * `HEDGE_AFTER_MS` for a call that is safe to run twice, and `Infinity` — never hedge — for one + * that is not. + * + * Tools are the whole test. A judge with `tools: []` can only read a screenshot and return text, so + * a second copy of it costs tokens and nothing else; anything holding a tool could run that tool + * twice, and a duplicated device action is a corrupted sweep rather than a slow one. + */ +function hedgeableAfterMs(agentOptions: SubAgentOptions): number { + return (agentOptions.tools?.length ?? 0) === 0 ? HEDGE_AFTER_MS : Infinity +} + +/** Companion to `logAgentTokenSummary`: the other half of the same bill. */ +function logPhaseSummary(ledger: PhaseLedger): void { + for (const line of formatPhaseSummary(ledger)) console.log(line) +} + +function tokenUsageSince(before: TokenUsage, after: TokenUsage): TokenUsage { + return { + input: Math.max(0, after.input - before.input), + output: Math.max(0, after.output - before.output), + cacheRead: Math.max(0, after.cacheRead - before.cacheRead), + cacheWrite: Math.max(0, after.cacheWrite - before.cacheWrite), + reasoning: Math.max(0, after.reasoning - before.reasoning), + total: Math.max(0, after.total - before.total), + cost: Math.max(0, after.cost - before.cost) + } +} + +function logAgentFailure(label: string, error: unknown, elapsedMs: number): void { + console.error( + `[app-flow] agent-failed ${label} after ${(elapsedMs / 1000).toFixed(1)}s — ${error instanceof Error ? error.message : String(error)}` + ) +} + +/** Closing breakdown by role, so a sweep's bill can be read without replaying the log. */ +function logAgentTokenSummary( + tallies: ReadonlyMap, + total: TokenUsage, + /** + * Duplicates issued by `hedge`, reported because they are the one thing that stops the rows + * below from summing to the total. + * + * A hedged call bills two requests and keeps one answer. Both land in `total` through `onUsage`, + * which is correct — the sweep really did pay for both — but only the winner's usage is knowable + * at the per-call boundary, so only the winner's reaches a role's row. The gap between the rows + * and the total is exactly these duplicates, and naming the count here is what makes it legible + * instead of looking like a tallying bug. + */ + hedges: number +): void { + const byCost = [...tallies.entries()].sort((a, b) => b[1].tokens.total - a[1].tokens.total) + for (const [role, tally] of byCost) { + const share = total.total ? Math.round((tally.tokens.total / total.total) * 100) : 0 + console.log( + `[app-flow] tokens by role — ${role}: ${tally.calls} calls, ${tally.tokens.total} tokens (${share}%)` + + `${tally.tokens.cost ? `, $${tally.tokens.cost.toFixed(4)}` : ''}` + ) + } + // `reasoning` is a slice of `output`, not an addition to it, so it is reported rather than summed. + // It belongs on this line because it is usually the larger half: a sweep that matched every screen + // on the first try still spent 70% of its output thinking, and that is the knob `thinkingLevel` + // turns — unreadable from a total that only says how many tokens came back. + const reasoning = total.reasoning ? ` reasoning ${total.reasoning}` : '' + if (hedges) + console.log( + `[app-flow] tokens — ${hedges} hedged duplicate${hedges === 1 ? '' : 's'} billed to the total but not to any role above` + ) + console.log( + `[app-flow] tokens total — in ${total.input} out ${total.output}` + + `${reasoning} cache r${total.cacheRead}/w${total.cacheWrite} total ${total.total}` + + `${total.cost ? ` $${total.cost.toFixed(4)}` : ''}` + ) +} + +function pickSampleParams( + node: AppFlowNode, + edges: AppFlowEdge[], + store?: ParamStoreLike | null +): Record { + const resolved = resolveNodeParams(node, edges, store).values + // Display only. The `1` stand-in exists because the descriptive path written into the + // agent-device manifest reads better than a bare `:id`, and nothing opens that string. Every + // caller that builds a URL to open resolves its own params and declines when they are unfilled — + // `buildUrlPath` throws rather than substituting, so a new caller that forgets finds out. + return Object.fromEntries(node.params.map((key) => [key, resolved[key] ?? '1'])) +} + +function captureContext( + projectPath: string, + projectId: string, + parsed: { + scheme: string | null + appName: string | null + bundleId: string | null + slug: string | null + }, + signal?: AbortSignal, + timing?: PhaseLedger +): CaptureContext { + return { + projectPath, + projectId, + scheme: parsed.scheme, + appName: parsed.appName, + bundleId: parsed.bundleId, + slug: parsed.slug, + platform: 'ios', + sessionName: projectId, + signal, + timing + } +} + +export async function buildAppFlow( + options: AppFlowBuildOptions +): Promise { + const { projectPath, signal } = options + const mode = options.mode ?? 'structure-only' + const activeNow = (): number => options.decisionProvider?.activeNow?.() ?? Date.now() + const projectId = getProjectId(projectPath) + let activeManifest: AppFlowManifest | null = null + const startedAt = Date.now() + /** + * Where this sweep's wall clock goes. Started on `startedAt` so its total is the same number the + * manifest records as `durationMs` — a summary whose shares are taken against a different clock + * than the one the UI reports would be unreconcilable with it. + */ + const timing = new PhaseLedger(startedAt) + + /** + * Writes the manifest that ends this build, stamping how long it ran. + * + * Only the terminal writes go through here. The manifest is rewritten several times mid-build to + * move `sweepStatus` between `running` and `awaiting-user`, and stamping those would record the + * time up to an arbitrary checkpoint — the number would keep changing while a sweep sat waiting + * on the user, which is the opposite of a duration. + */ + const finishManifest = async (final: AppFlowManifest): Promise => { + final.durationMs = timing.wallMs + await writeManifest(projectId, final) + } + + // LLM usage of every sub-agent call in this build. Each emitted event carries a copy, so the + // renderer can show token usage while the build is still running. + let tokens: TokenUsage = emptyTokenUsage() + const onEvent = (event: AppFlowProgressEvent): void => + options.onEvent({ ...event, tokens: snapshotUsage(tokens) }) + + // Keyed by role — the label's first word — so the closing summary says where a sweep's tokens went. + const tallies = new Map() + /** Duplicates `hedge` issued this sweep. See `logAgentTokenSummary`'s `hedges` parameter. */ + let hedges = 0 + + const runAgent = async ( + agentOptions: SubAgentOptions, + label: string + ): Promise => { + logAgentInput(label, agentOptions) + const startedAt = Date.now() + const tokensBefore = snapshotUsage(tokens) + const callOnce = (): Promise => { + if (!options.decisionProvider) throw new Error('A decision provider is required for a screen sweep.') + return options.decisionProvider.run(agentOptions, label) + } + let result: SubAgentResult + try { + const outcome = await hedge({ + attempt: callOnce, + // The guard `hedge` cannot make for itself. A call that can run twice harmlessly is one + // with no tools to invoke and no session history to append to, which is every judge that + // arrives here: one system prompt, one screenshot, a JSON verdict. A call carrying tools + // could act on the device twice, so it waits however long it waits. + afterMs: options.decisionProvider?.supportsConcurrentRequests === false + ? Infinity + : hedgeableAfterMs(agentOptions), + onHedge: () => + console.log( + `[app-flow] hedge ${label} — no answer in ${(HEDGE_AFTER_MS / 1000).toFixed(1)}s, racing a duplicate` + ) + }) + result = outcome.result + hedges += outcome.attempts - 1 + if (outcome.attempts > 1) + console.log( + `[app-flow] hedge ${label} — ${outcome.winner === 0 ? 'the original' : 'the duplicate'} answered first after ${((Date.now() - startedAt) / 1000).toFixed(1)}s` + ) + } catch (error) { + const used = tokenUsageSince(tokensBefore, tokens) + const role = label.split(' ')[0] + const tally = tallies.get(role) ?? { calls: 0, tokens: emptyTokenUsage() } + tallies.set(role, { calls: tally.calls + 1, tokens: addTokenUsage(tally.tokens, used) }) + logAgentTokens(label, used, Date.now() - startedAt, tokens, timing) + logAgentFailure(label, error, Date.now() - startedAt) + throw error + } + tokens = addTokenUsage(tokens, result.tokens) + const role = label.split(' ')[0] + const tally = tallies.get(role) ?? { calls: 0, tokens: emptyTokenUsage() } + tallies.set(role, { + calls: tally.calls + 1, + tokens: addTokenUsage(tally.tokens, result.tokens) + }) + logAgentTokens(label, result.tokens, Date.now() - startedAt, tokens, timing) + logAgentOutput(label, result) + return result + } + + // Step 0's fix for exploration's token cost: one sub-agent session per exploration attempt + // instead of a fresh `runSubAgent` call — and its cold session, resent history, and no + // prompt-cache reuse — per navigation step. `prompt` mirrors `runAgent`'s tallying (same role + // bucket, same running total, same log line) so `logAgentTokenSummary` reads identically whether + // a role's calls came from one-shot `runAgent` roles or a long-lived session. + const runAgentSession = async ( + sessionOptions: SubAgentSessionOptions + ): Promise<{ + prompt: (callOptions: SubAgentCallOptions, label: string) => Promise + dispose: () => void + }> => { + if (!options.decisionProvider) throw new Error('A decision provider is required for a screen sweep.') + const hostSession = await options.decisionProvider.createSession(sessionOptions) + const historyLength = (): number => hostSession.historyLength() + return { + prompt: async (callOptions, label) => { + logAgentInput(label, { + ...callOptions, + systemPrompt: sessionOptions.systemPrompt, + tools: sessionOptions.tools, + historyMessages: historyLength() + }) + const startedAt = Date.now() + const tokensBefore = snapshotUsage(tokens) + let result: SubAgentResult + try { + result = await hostSession.prompt(callOptions, label) + } catch (error) { + const used = tokenUsageSince(tokensBefore, tokens) + const role = label.split(' ')[0] + const tally = tallies.get(role) ?? { calls: 0, tokens: emptyTokenUsage() } + tallies.set(role, { calls: tally.calls + 1, tokens: addTokenUsage(tally.tokens, used) }) + logAgentTokens(label, used, Date.now() - startedAt, tokens, timing) + logAgentFailure(label, error, Date.now() - startedAt) + throw error + } + tokens = addTokenUsage(tokens, result.tokens) + const role = label.split(' ')[0] + const tally = tallies.get(role) ?? { calls: 0, tokens: emptyTokenUsage() } + tallies.set(role, { + calls: tally.calls + 1, + tokens: addTokenUsage(tally.tokens, result.tokens) + }) + logAgentTokens(label, result.tokens, Date.now() - startedAt, tokens, timing) + logAgentOutput(label, result) + return result + }, + dispose: () => hostSession.dispose() + } + } + + try { + checkSignal(signal) + emit(projectId, onEvent, { type: 'phase', phase: 'parse', message: 'Parsing routes...' }) + console.log('[app-flow] parse routes') + + const { name } = await registerProject(projectPath) + console.log(`[app-flow] registered project "${name}" (${projectId}) at ${projectPath}`) + const parseStartedAt = Date.now() + const parsed = await parseProject(projectPath) + if (!parsed) { + // Thrown (not returned) so the catch below emits an `error` event — a bare return would + // leave the renderer stuck on "Parsing routes..." forever. + throw new Error( + `No supported Expo Router or React Navigation routes found in ${projectPath}.` + ) + } + console.log( + `[app-flow] parse done in ${Date.now() - parseStartedAt}ms — ${parsed.nodes.length} routes, ` + + `${parsed.edges.length} edges, mode=${parsed.mode}, appDir=${parsed.appDir}` + ) + + const structureRevision = `${Date.now().toString(36)}-${randomUUID().slice(0, 8)}` + const structureKey = computeStructureKey(parsed.nodes, parsed.edges) + // Persisted verbatim as `AppFlowResult.navigation` (plan Step 4d) so a reader of `app-flow.json` + // can say where a journey starts without re-parsing the project. Built once here, alongside + // `structureKey`, from the exact same parse the sweep runs against. + const navigation: AppFlowNavigation = { + layouts: parsed.layouts, + rootNavigator: parsed.rootNavigator, + entryLayouts: parsed.entryLayouts, + entryScreenIds: parsed.entryScreenIds + } + const manifest: AppFlowManifest = { + // Bumped for Step 4's new persisted shape (`node.origin`, `edge.origin`/`evidence`, + // `runtimeEdges`, `chrome`, `navigation`). Nothing branches on this number today; it exists so + // a future reader can refuse a document it cannot represent. + formatVersion: 4, + generator: 'ai-harness/app-flow', + generatedAt: new Date().toISOString(), + project: { + name, + path: projectPath, + scheme: parsed.scheme, + platform: 'ios', + mode: parsed.mode + }, + buildMode: mode, + structureRevision, + sweepStatus: mode === 'structure-only' ? 'not-requested' : 'running' + } + activeManifest = manifest + await writeManifest(projectId, manifest) + + const previous = await readAppFlow(projectId) + const previousById = new Map(previous?.nodes.map((node) => [node.id, node]) ?? []) + const usedSlugs = new Set() + const nodesWithSlug = parsed.nodes.map((node) => { + const baseSlug = generateSlug(node.id) + const slug = usedSlugs.has(baseSlug) + ? `${baseSlug}_${createHash('sha256').update(node.id).digest('hex').slice(0, 8)}` + : baseSlug + usedSlugs.add(slug) + return { + ...node, + slug, + capture: + previousById.get(node.id)?.file === node.file && previousById.get(node.id)?.capture + ? { + ...previousById.get(node.id)!.capture!, + structureRevision: previous?.manifest.structureRevision, + stale: true + } + : null, + // Carried forward alongside `capture`, but independently validated: a rejection survives a + // rebuild only when this node's own structure (id/file/urlPath/params) still matches what it + // was recorded against, not when its capture happens to survive too. + deepLinkRejection: carryForwardDeepLinkRejection( + previousById.get(node.id)?.deepLinkRejection, + node + ) + } + }) + + // Rehydrated so a value observed in a previous sweep lets a re-run deep-link straight to a + // parameterised route instead of re-walking to it. `fromJSON` marks every carried record as + // coming from a prior run, so anything seen live this sweep outranks it — the record it names + // may have been deleted since, and a stale handle should only ever be the fallback. + const paramStore = ParamStore.fromJSON(await readParamStore(projectId)) + + const partialResult: AppFlowResult = { + manifest, + nodes: nodesWithSlug, + edges: parsed.edges, + // This is the structure-only write, before the device sweep that discovers any of this even + // starts — written as empty rather than left absent so a fresh v4 document never looks like a + // mid-migration v3 one. The sweep below fills these in as it runs (plan Steps 3d/4a). + runtimeEdges: [], + chrome: [], + navigation + } + await writeAppFlow(projectId, partialResult) + const activeNodeIds = new Set(nodesWithSlug.map((node) => node.id)) + const previousScreens = await readAgentDeviceScreens(projectId) + await writeAgentDeviceScreens( + projectId, + Object.fromEntries( + Object.entries(previousScreens).filter( + ([, entry]) => entry.screenId && activeNodeIds.has(entry.screenId) + ) + ) + ) + emit(projectId, onEvent, { + type: 'structure-ready', + phase: 'render', + message: `Structure ready: ${nodesWithSlug.length} screens, ${parsed.edges.length} edges.` + }) + + if (mode === 'structure-only') { + await finishManifest(manifest) + emit(projectId, onEvent, { + type: 'complete', + phase: 'render', + message: 'Structure complete.' + }) + return { ok: true, projectId, result: partialResult } + } + if (!options.decisionProvider) { + throw new Error('A decision provider is required for a screen sweep.') + } + + checkSignal(signal) + const bootStartedAt = Date.now() + // Boot is excluded from every time measurement, so its device work is deliberately not + // recorded in the ledger (the boot context carries no `timing`). + const bootCtx = captureContext(projectPath, projectId, parsed, signal) + const reusedDevice = options.launchedDevice + let device: CaptureDevice + if (reusedDevice) { + emit(projectId, onEvent, { + type: 'phase', + phase: 'boot', + message: 'Checking the app you launched...' + }) + device = reusedDevice + } else { + emit(projectId, onEvent, { + type: 'phase', + phase: 'boot', + message: 'Booting the simulator, installing the app, and starting Metro...' + }) + console.log('[app-flow] bootDevice start') + device = await bootDevice(bootCtx) + console.log('[app-flow] bootDevice done') + } + + try { + const ctx = captureContext(projectPath, projectId, parsed, signal, timing) + checkSignal(signal) + if (reusedDevice) { + // The adopt path's whole purpose is that the app is *already* up: the user pressed Launch, + // `index.ts` ran `waitForAppReady` to completion, and the app has been sitting on a + // settled screen ever since. This branch used to throw that away with + // `relaunchBoundApp(bootCtx, device)` regardless, and on a dev-client build — which has no + // `launchUrl` — a bare `--relaunch` lands on the Expo launcher. The 2026-10-09 social-app + // sweep paid the whole bill for it: warm and settled at t=43.9s, relaunched at t=65.1s, + // and not back on a real screen until t=93.6s. In between it lost the entry-chrome pass + // outright ("app 'xyz.blueskyweb.app' is not running"), spent two ~3.4s `arrival Lists` + // model calls judging the Expo launcher, and cached an auth verdict taken off a dead app. + // Twenty-eight seconds and three wrong answers, to undo the one thing pre-launching is for. + // + // So: ask first, act only if the answer is no. The probe is the same floored wait the + // fresh-boot branch runs below — see item 3's note there — which is what makes "ready" + // mean a frame that both settled *and* cleared `PROBE_FRAME_ELEMENT_FLOOR`. The floor is + // load-bearing here and was the second half of the bug: the Expo launcher settles at 3 + // interactive elements, and an unfloored wait called that ready twice in the log above. + // + // What this deliberately does not check is *which* screen the app is on. A pre-launched + // app the user poked at for twenty seconds (43.9s → 64.2s in that run) can be several + // screens deep, and nothing cheap can tell that from the entry screen. Three reasons that + // is the right trade: the entry-chrome pass reads *chrome*, which in a tabs or drawer app + // is present on every screen and is already wrapped in a `try/catch` that treats a miss as + // "continue without it"; the deep-link probe and every reach strategy open a URL or walk + // from wherever they are rather than assuming home; and the alternative — re-rooting + // unconditionally — is precisely the 28s mistake above. A drifted-but-live app costs at + // worst a slightly poorer chrome inventory. A relaunched one costs the sweep's first + // half-minute every time. + const adoptedAppReady = await waitForAppReady( + bootCtx, + device, + signal, + ADOPTED_APP_READY_BUDGET_MS, + { + minInteractiveElements: PROBE_FRAME_ELEMENT_FLOOR, + minInteractiveElementsGraceMs: SUBSTANTIVE_FRAME_GRACE_MS + } + ) + if (adoptedAppReady) { + console.log( + '[app-flow] adopted the launched app as-is — it is already settled on a substantive frame, so no relaunch' + ) + } else { + // Genuinely not ready: backgrounded, wedged, sitting on the dev-client launcher, or the + // session died while the user was reading the App Flow screen. Re-root with the *soft* + // `goHome` rather than `relaunchBoundApp`, so a dev client that is merely on the wrong + // screen keeps the bundle it has already evaluated; and allow the dev-client fallback, + // because for a `native-run` build the URL that re-roots is `devClientUrl` and + // `launchUrl` is null — opening the null one is the no-op `reconnectDevClient`'s comment + // records sixteen failures of. Safe to ask for it only here, after the wait above has + // established the app is not loaded. + console.log( + '[app-flow] adopted app is not on a substantive frame — soft re-rooting it instead of relaunching' + ) + await goHome(bootCtx, device, { reconnectDevClientWhenUnloaded: true }) + // `goHome`'s own wait only asks "did something settle", which the 3-element launcher + // frame answers yes to. Re-assert the floor so the auth gate below is never handed one. + await waitForAppReady(bootCtx, device, signal, undefined, { + minInteractiveElements: PROBE_FRAME_ELEMENT_FLOOR, + minInteractiveElementsGraceMs: SUBSTANTIVE_FRAME_GRACE_MS + }) + } + } else { + emit(projectId, onEvent, { + type: 'phase', + phase: 'boot', + message: 'Waiting for the app to finish loading...' + }) + console.log('[app-flow] waitForAppReady start') + // See the history of this floor: the wait the sweep always pays must outlast a dev-client + // loading frame, or the auth gate re-waits for the same event. Graced rather than held to + // `timeoutMs` because this wait cannot tell a splash from a login screen. + await waitForAppReady(bootCtx, device, signal, undefined, { + minInteractiveElements: PROBE_FRAME_ELEMENT_FLOOR, + minInteractiveElementsGraceMs: SUBSTANTIVE_FRAME_GRACE_MS + }) + console.log('[app-flow] waitForAppReady done') + } + timing.excludeWall(Date.now() - bootStartedAt) + + await pruneTransientScreens(projectId) + await prepareScratchDir(projectId) + + // The gate runs only after a launch or relaunch, so a launch that is already past auth is + // classified once. `stored` keeps the first launch's screenshot; later launches overwrite it + // only when they turn out to be gated, and the version counter busts the renderer's cache + // for the single reused URL. + let authCheckpointStored = false + let authCheckpointVersion = 0 + /** Consecutive gate checks whose every attempt came back empty — see `runAuthGate`. */ + let consecutiveSilentAuthChecks = 0 + /** Who cleared the gate for this sweep, or null while it still has to be asked — see + * `reach/auth-gate-policy.ts` and the `checkAuthGate` wrapper below. */ + let authSettledBy: 'classifier' | 'user' | null = null + + /** One full run of the gate. Reports *how* it concluded, not just what it returned, because + * that is what decides whether the sweep may stop asking — see `settlesAuthGateForRun`. */ + const runAuthGate = async ( + remaining: number, + budget?: ActiveTimeBudget + ): Promise => { + /** Sticky across the loop below: a Continue sends it round again for a fresh screenshot, + * and the clear that follows is still the user's doing. */ + let userResolved = false + for (;;) { + checkSignal(signal) + if (budget?.expired) + return { outcome: 'exhausted', verdict: null, substantiveFrame: false, userResolved } + // Don't hand the classifier a splash. `waitForAppReady` has already run by now and + // reports a *stable* frame, which a dev-client loading screen is — see + // `AUTH_GATE_FRAME_WAIT_MS` for the run where that cost three calls and a user prompt. + // Bounded, and never a precondition: a real login screen is legitimately sparse, so on + // expiry the gate classifies whatever is on screen exactly as it always did. + const launchElements = await capturePressables(ctx, device) + .then((pressables) => pressables.length) + .catch(() => null) + /** + * Whether the frame about to be classified ever cleared `PROBE_FRAME_ELEMENT_FLOOR`. + * Carried into the resolution because it, and not the classifier, is what decides + * whether the answer may be cached for the sweep — see `substantiveFrame` in + * `reach/auth-gate-policy.ts`. + */ + let substantiveFrame = isSubstantiveFrame(launchElements) + if (!substantiveFrame) { + console.log( + `[app-flow] auth-gate — app shows ${launchElements ?? 'an unreadable frame'} interactive element(s), below the ${PROBE_FRAME_ELEMENT_FLOOR} a settled screen reads; waiting up to ${AUTH_GATE_FRAME_WAIT_MS / 1000}s before classifying` + ) + const waitStartedAt = Date.now() + // One continuous wait rather than a poll loop, and the floor pushed into the wait + // itself — the lesson `probeDeepLinkRoute` records just below, where asking a settled + // loading screen thirteen separate times cost 13.7s to learn nothing. + // + // The return value is kept, which it was not before. `waitForAppReady` reports false + // for three very different endings — the window expired, a launcher would not clear, + // or the device stopped answering three reads in a row — and the last of those is the + // only one that can end a fifteen-second window in 1.9s, which is exactly what the + // 2026-10-09 social-app sweep logged before the classifier was handed a screenshot of + // a dead app and said `clear` about it. + substantiveFrame = await waitForAppReady(ctx, device, signal, AUTH_GATE_FRAME_WAIT_MS, { + minInteractiveElements: PROBE_FRAME_ELEMENT_FLOOR + }) + checkSignal(signal) + console.log( + `[app-flow] auth-gate — classifying after ${((Date.now() - waitStartedAt) / 1000).toFixed(1)}s${substantiveFrame ? '' : ' on a frame that never became readable; this verdict will not be cached for the sweep'}` + ) + } + const probe = getScratchPath(projectId, 'auth-probe') + await captureScreenshot(ctx, device, probe) + let verdict: { verdict: 'auth' | 'clear' | 'uncertain'; reason: string } | null = null + // Distinguishes "the classifier looked and was unsure" from "the classifier never + // answered". Both leave `verdict` null, and treating them the same produced an + // unbreakable loop on device: a provider change made every call return an empty body, the + // gate read the missing verdict as "cannot rule out a login wall", prompted the user, and + // then re-ran the identical failing call when they pressed Continue — forever, with a + // message blaming their launch screen for what was actually a model outage. + let emptyResponses = 0 + let attempts = 0 + for ( + let attempt = 0; + attempt < 2 && (!verdict || verdict.verdict === 'uncertain'); + attempt++ + ) { + attempts++ + const response = await runAgent( + { + cwd: projectPath, + systemPrompt: + 'Classify the mobile screenshot. Return strict JSON: {"verdict":"auth"|"clear"|"uncertain","reason":"short reason"}. Auth includes login/signup landing screens even with guest access. An incidental sign-in link on a normal content screen is clear. Do not infer credentials or suggest actions.', + userPrompt: 'Is the current app UI an authentication gate?', + tools: [], + // The cheapest question the sweep asks: one screenshot, two outcomes, no context to + // weigh. It was nonetheless spending 472 output tokens per call, 91% of them + // reasoning, across 14 calls in one run. + // + // `off`, not `minimal`. This is the call that asked for `minimal` — a level this + // backend answers with `400 Unexpected reasoning effort minimal`, which is what + // produced the empty auth-gate verdicts blamed on the endpoint in `provider.ts`'s + // old comment. `off` maps to `reasoning_effort: "none"`, measured at 0 reasoning + // tokens with the JSON verdict still returned. + thinkingLevel: 'off', + imagePath: probe, + timeoutMs: Math.max(1_000, Math.min(60_000, budget?.remainingMs ?? 60_000)) + }, + 'auth-gate' + ) + if (!response.text.trim()) emptyResponses++ + verdict = parseSubAgentJson(response.text) + } + const classifierSilent = attempts > 0 && emptyResponses === attempts + const stored = getScreenshotPath(projectId, AUTH_CHECKPOINT_SLUG) + if (!authCheckpointStored || verdict?.verdict !== 'clear') { + await copyFile(probe, stored) + authCheckpointStored = true + authCheckpointVersion += 1 + } + await unlink(probe).catch(() => undefined) + if (verdict?.verdict === 'clear') + return { outcome: 'clear', verdict: 'clear', substantiveFrame, userResolved } + // Nothing the user can do at the device fixes a classifier that is not responding, and + // `Continue` only buys another identical failure — so say so plainly and give up rather + // than prompt. One retry round first, because a single empty response can be a blip. + if (classifierSilent) { + consecutiveSilentAuthChecks++ + console.log( + `[app-flow] auth-gate — classifier returned an empty response ${emptyResponses}/${attempts} times (round ${consecutiveSilentAuthChecks}); this is a model or endpoint failure, not the app's launch screen` + ) + if (consecutiveSilentAuthChecks >= 2) { + throw new Error( + 'The screen classifier returned no response. App Flow cannot tell whether the app is showing a login screen, so the build has stopped. This is a model or endpoint problem rather than something to fix in the app.' + ) + } + } else { + consecutiveSilentAuthChecks = 0 + } + if (!options.requestUserAction) + return { + outcome: 'stop', + verdict: verdict?.verdict ?? null, + substantiveFrame, + userResolved + } + manifest.sweepStatus = 'awaiting-user' + await writeManifest(projectId, manifest) + const prompt: AppFlowPrompt = { + id: randomUUID(), + kind: verdict?.verdict === 'auth' ? 'auth' : 'review', + detail: + verdict?.verdict === 'auth' + ? 'Complete auth in the simulator. Once logged in, click Continue.' + : `Could not confidently classify this launch screen${verdict?.reason ? `: ${verdict.reason}` : '.'}`, + screenshot: screenshotUrl( + projectId, + `screens/${AUTH_CHECKPOINT_SLUG}.png?v=${authCheckpointVersion}` + ), + remaining + } + emit(projectId, onEvent, { + type: 'prompt', + phase: 'capture', + message: + verdict?.verdict === 'auth' + ? 'Authentication required.' + : 'Launch screen needs review.', + prompt + }) + const waitStartedAt = Date.now() + const resolution = await options.requestUserAction(prompt) + budget?.exclude(Date.now() - waitStartedAt) + if (resolution === 'stop') + return { + outcome: 'stop', + verdict: verdict?.verdict ?? null, + substantiveFrame, + userResolved + } + userResolved = true + manifest.sweepStatus = 'running' + await writeManifest(projectId, manifest) + if (resolution === 'override' && verdict?.verdict !== 'auth') + return { + outcome: 'clear', + verdict: verdict?.verdict ?? null, + substantiveFrame, + userResolved + } + // Continue intentionally rechecks a fresh screenshot without relaunching. + } + } + + /** + * The gate as the sweep calls it: asked until something decisive comes back, then answered + * from memory for the rest of the run. + * + * The short-circuit is why this wrapper exists rather than the caching living inside + * `runAuthGate`: every caller — sweep start, `establishBase`, the state-variant replay pass, + * and the `AuthPort` the reach strategies go through — gets the skip for free, and none of + * them has to know the rule. A skipped call costs no screenshot and no classifier call, which + * is the whole saving; `settlesAuthGateForRun` decides what counts as decisive. + */ + const checkAuthGate = async ( + remaining: number, + budget?: ActiveTimeBudget + ): Promise<'clear' | 'stop' | 'exhausted'> => { + if (authSettledBy) return 'clear' + const resolution = await runAuthGate(remaining, budget) + authSettledBy = settlesAuthGateForRun(resolution) + if (authSettledBy) { + console.log( + `[app-flow] auth-gate — cleared by the ${authSettledBy}; settled for this sweep, later launches will not re-check` + ) + } else if (resolution.outcome === 'clear' && !resolution.substantiveFrame) { + // Worth a line of its own: the sweep is continuing on a `clear` it is deliberately + // refusing to remember, and without saying so the next launch's extra gate call looks + // like the duplicate-call bug this memo was added to fix. + console.log( + '[app-flow] auth-gate — cleared, but the classified frame never became readable; not cached, the next launch will ask again' + ) + } + return resolution.outcome + } + + // The seam the reach-attempt closures below call through instead of `capture.ts`/ + // `device.client` directly — see `reach/ports.ts` and `reach/adapters.ts`. Deliberately NOT + // used by `checkAuthGate` above, `resolveBlocked` below, device boot, or the state-variant + // pass further down: those own a user prompt, `manifest.sweepStatus`, or are one-shot setup, + // and the plan keeps them outside this seam permanently rather than routing them through a + // ladder of reach strategies. + /** + * RENDER TRANSPORT SELECTION — the single place `options.renderTransportMode` is consumed. + * + * TODO(socket transport): the persistent-WebSocket `RenderTransport` is constructed here, + * chosen by this value. Today only the `agent-device cdp runtime eval` transport below + * exists, so `socket` is recorded and logged but still served by the CLI; nothing else in the + * orchestrator may branch on this option. Grep `RENDER TRANSPORT SELECTION` to find it. + */ + const renderTransportMode: RenderTransportMode = options.renderTransportMode ?? 'cli' + console.log(`[app-flow] render transport selected: ${renderTransportMode}`) + + /** + * DEEP-LINK ARRIVAL JUDGE — the single place `options.deepLinkJudgeMode` is consumed. + * + * Passed straight through to the deep-link rung's `arrivalJudge` and nowhere else: the + * exploration walk's own arrival checks are deliberately out of this switch's reach (see + * `DeepLinkJudgeMode`). False means the strategy decides a 404 from the controls on screen + * instead of a model call — `reach/not-found-signal.ts`. + * + * "Off" is narrower than it was, and the log line says so rather than overstating it: the + * strategy keeps the judge for any route that declares params, because seven of the eight + * wrong captures in `session_2026-10-13_14-33-40.log` were param routes. The decision lives + * in `routeDeclaresParams` (`reach/strategies/deep-link.ts`), next to the node whose params + * it is about, rather than here where only the sweep-wide setting is known. + */ + const deepLinkArrivalJudge = (options.deepLinkJudgeMode ?? 'on') === 'on' + console.log( + `[app-flow] deep-link arrival judge: ${ + deepLinkArrivalJudge + ? 'on' + : 'off for routes with no params (not-found detected from the screen); routes with params keep the judge' + }` + ) + + /** + * The render path's transport, over the same CDP target `nav-state` already selects. + * + * It shares `paramStore.navState` deliberately: that field is the sweep's one answer to + * "can the app's JS context be reached", and a second latch would mean the reader and the + * navigator could disagree about it — and would re-pay for a handshake the other already + * discovered was hopeless. `ready()` performs the selection at most once and caches the + * verdict for the rest of the sweep. + */ + const renderTransport: RenderTransport = { + async ready(): Promise { + if (paramStore.navState === 'ready') return true + if (paramStore.navState === 'unavailable') return false + const selected = await selectNavStateTarget( + device.metroPort, + parsed.appName, + navStateDeps, + (output, preferred) => + pickMetroCdpTarget(parseCdpTargetList(output), preferred)?.id ?? null + ) + paramStore.navState = selected ? 'ready' : 'unavailable' + return selected + }, + evaluate: (expression, timeoutMs) => + runShellCommand( + agentDeviceCdp(ctx.sessionName, `runtime eval --expr ${shellQuote(expression)}`), + projectPath, + timeoutMs, + signal + ) + } + + const ports = createReachPorts({ + ctx, + device, + projectId, + projectPath, + runAgent, + checkAuthGate, + signal, + renderTransport + }) + + if ((await checkAuthGate(nodesWithSlug.length)) === 'stop') { + const paused = options.pauseRequested?.() ?? false + manifest.sweepStatus = paused ? 'paused' : 'stopped' + await finishManifest(manifest) + await writeAppFlow(projectId, { ...partialResult, manifest }) + emit(projectId, onEvent, { + type: 'complete', + phase: 'render', + message: paused ? 'Screen sweep paused.' : 'Screen sweep stopped.' + }) + return { ok: true, projectId, result: partialResult } + } + + checkSignal(signal) + emit(projectId, onEvent, { + type: 'phase', + phase: 'capture', + message: 'Capturing screenshots...' + }) + + // A sweep that was stopped, cancelled or crashed leaves a per-screen checkpoint behind. It + // is only reused when the parsed structure still fingerprints the same, so routes that moved + // force a full sweep rather than resuming onto screens that no longer exist. + const resumed = await readSweepProgress(projectId) + const checkpoint: AppFlowSweepProgress = + resumed?.structureKey === structureKey + ? resumed + : { structureKey, updatedAt: '', captures: {}, opened: {}, inferred: {}, statesDone: [] } + + const captureMap: Record = { ...checkpoint.captures } + const openedByNode: Record = { ...checkpoint.opened } + const inferredScreens: Record = { + ...checkpoint.inferred + } + const statesDone = new Set(checkpoint.statesDone) + + // Runtime state accumulated across the WHOLE sweep (plan Steps 3d, 4a) — never per-node, since + // a chrome or a runtime screen discovered while exploring toward one node is exactly as + // reusable for every other node behind it. Seeded from the checkpoint so a resumed sweep + // reuses the same ids/edges rather than re-minting/re-detecting them under new ones — see + // `AppFlowSweepProgress.runtimeNodes`'s doc comment in `storage.ts`. + const runtimeNodes: AppFlowNode[] = [...(checkpoint.runtimeNodes ?? [])] + const runtimeEdgesList: AppFlowEdge[] = [...(checkpoint.runtimeEdges ?? [])] + const chromeInventory: AppFlowChrome[] = [...(checkpoint.chrome ?? [])] + /** + * Where the walk currently in progress is standing, and what it has tapped since — the state + * `noteWalkArrival` (`reach/walk-edges.ts`) turns into `origin: 'runtime-walk'` edges. Held at + * sweep scope rather than inside `exploreFrom` because the confirmations that feed it arrive + * through `applyDiscoveredFact` below, which is deliberately ignorant of any one walk; null + * whenever no exploration is in flight, so a confirmation that reaches it from anywhere else + * (a deep-link arrival, an opportunistic sighting during delivery) records nothing rather than + * inventing a hop out of whichever screen a previous walk happened to end on. + * + * Not checkpointed: a resumed sweep relaunches the app, so the position a cursor held before + * the interruption says nothing about where the next walk starts. + */ + let walkCursor: WalkCursor | null = null + /** Folds a confirmed arrival into `runtimeEdgesList` when the hop that produced it was a + * single tap from a screen this sweep had already confirmed — see `noteWalkArrival`. */ + const noteArrival = (nodeId: string): void => { + if (!walkCursor) return + const edge = noteWalkArrival(walkCursor, nodeId, { + structureRevision, + observedAt: new Date().toISOString() + }) + if (!edge) return + mergeWalkEdge(runtimeEdgesList, edge) + console.log(`[app-flow] walked edge — ${edge.from} -> ${edge.to} (${edge.raw})`) + } + + // Deliberately seeded from `nodesWithSlug` (the carry-forward already validated against each + // node's current structure signature), never from `checkpoint` — `sweep-progress.json` is + // wiped by `clearSweepProgress` on every completed sweep, exactly when this suppression most + // needs to outlive it. Updated in place as the sweep runs and flushed to `app-flow.json` + // through `persistCaptureUpdate`, the same durable path `capture` already rides. + const deepLinkRejections: Record = {} + for (const node of nodesWithSlug) { + if (node.deepLinkRejection) deepLinkRejections[node.id] = node.deepLinkRejection + } + + // What the app opens on DIRECTLY — each entry layout's own `initialRouteName` (see + // `entryScreenIds` on `ParsedAppGraph`). This intentionally does NOT filter by `group` + // membership: `group` is attribution (which navigator registers a screen — a screen behind a + // shared helper called from five tabs belongs to all five of them), not reachability. Filtering + // by group previously put 78 of social-app's 83 screens "one tap from launch" — most of them + // behind that shared helper — which both fabricated a one-hop journey for screens actually many + // pushes deep and suppressed the "explore from the entry chrome" instruction for genuine + // orphans. `entryScreenIds` is already resolved to real node ids, so it is used as-is. + const entryNodeIds = parsed.entryScreenIds + const appEntryBrief = parsed.rootNavigator + ? `App entry: a ${parsed.rootNavigator} navigator. Its entries are ${parsed.entryLayouts.join(', ')}. Every journey starts there, so when you have no better lead, go back to that chrome and open the entry the target belongs to.` + : 'App entry: unknown; treat the launch screen as the start of every journey.' + const orphans = nodesWithSlug.filter( + (candidate) => !findNavigationPath(parsed.edges, candidate.id, entryNodeIds) + ).length + // How each screen can be reached by URL is the number that predicts what a sweep costs: the + // reach ladder tries the deep-link rung on every node regardless, so a screen with a plain + // path is one cheap `openUrl`, while one whose path has a `[param]` segment cannot be opened + // until something supplies a value, and one with no path (or only the root `/`, which every + // launch already lands on and so proves nothing) has no deep link to try at all. The orphan + // count is a different animal: it is a BFS over the *parsed* edge graph, which only sees + // `router.push` call sites the static parser could resolve, so it counts screens the parser + // failed to connect and says nothing about whether they are reachable. It gates nothing and + // was once reported as "need live exploration" while all 76 of Bluesky's real screens were in + // the linking config; it is kept only as a statistic about the parsed graph. + const hasUrlPath = (node: (typeof nodesWithSlug)[number]): boolean => + Boolean(node.urlPath) && node.urlPath !== '/' + const deepLinkable = nodesWithSlug.filter( + (node) => hasUrlPath(node) && node.params.length === 0 + ).length + const paramBlocked = nodesWithSlug.filter( + (node) => hasUrlPath(node) && node.params.length > 0 + ).length + const noUrl = nodesWithSlug.length - deepLinkable - paramBlocked + console.log( + `[app-flow] entry — root=${parsed.rootNavigator ?? 'unknown'} layouts=[${parsed.entryLayouts.join(', ')}] ` + + `entryScreens=${entryNodeIds.length} — ${deepLinkable}/${nodesWithSlug.length} reachable by deep link, ` + + `${paramBlocked} param-blocked until a value is observed, ${noUrl} with no URL; ` + + `${orphans} have no static path from an entry in the parsed edge graph` + ) + const total = nodesWithSlug.length + const alreadyCaptured = nodesWithSlug.filter((node) => captureMap[node.id]).length + if (alreadyCaptured > 0) { + console.log( + `[app-flow] resuming sweep — ${alreadyCaptured}/${total} screens already checkpointed` + ) + emit(projectId, onEvent, { + type: 'phase', + phase: 'capture', + message: `Resuming: ${alreadyCaptured} of ${total} screens already captured.` + }) + } + + const navStateDeps: NavStateDeps = { + run: (command, timeoutMs) => runShellCommand(command, projectPath, timeoutMs, signal), + cdpCommand: (args) => agentDeviceCdp(ctx.sessionName, args), + quote: shellQuote, + log: (message) => console.log(`[app-flow] nav-state — ${message}`) + } + + /** + * Ask the running app's navigator for the param values nothing else can see — but only when + * this screen is plausibly the one holding them. + * + * The read sees the *mounted* navigator state and its stack history, not the app's whole + * database, so it is worth making right after capturing a screen that is an ancestor of some + * route still blocked on a param, and worth nothing at all otherwise. That matters because + * it is a CLI round trip: run eagerly it would be a tax on all 76 screens for values that + * are not there; run on this schedule it is a handful of calls a sweep. Ten of Bluesky's + * routes carry an `:rkey` that appears in no testID anywhere in the app, so this is the only + * thing that reaches them by URL at all. + * + * "Sits above" is judged by `selectNavStateWaiters`: urlPath ancestry, plus a small fixed + * budget of speculative reads for screens ancestry cannot speak for. + * + * Strictly additive and strictly read-only: every failure path ends in `unavailable`, + * latched for the rest of the sweep, and the sweep carries on exploring exactly as it does + * today. A thrown error here must never cost a capture, hence the outer catch. + */ + let speculativeNavReads = 0 + const harvestNavState = async (from: AppFlowNode): Promise => { + if (paramStore.navState === 'unavailable') return + try { + const blocked = nodesWithSlug.filter( + (other) => + other.id !== from.id && + !captureMap[other.id] && + other.params.length > 0 && + other.urlPath && + resolveNodeParams(other, parsed.edges, paramStore).unresolved.length > 0 + ) + const { waiting, speculative } = selectNavStateWaiters( + from, + blocked, + nodesWithSlug, + NAV_STATE_SPECULATIVE_READS - speculativeNavReads + ) + if (!waiting.length) return + if (speculative) speculativeNavReads += 1 + + if (paramStore.navState === 'unknown') { + const ready = await selectNavStateTarget( + device.metroPort, + parsed.appName, + navStateDeps, + (output, preferred) => + pickMetroCdpTarget(parseCdpTargetList(output), preferred)?.id ?? null + ) + paramStore.navState = ready ? 'ready' : 'unavailable' + if (!ready) return + } + + const routes = await readNavState(navStateDeps) + if (!routes) { + paramStore.navState = 'unavailable' + return + } + let usable = 0 + for (const route of routes) { + if (route.path && paramStore.ingestUrl(route.path, from.id, nodesWithSlug)) usable += 1 + if (!route.params) continue + // A navigator names a screen the way the navigator registered it, which may be the + // tail of a route id the parser spells out in full (`Profile` vs `HomeTab/Profile`), + // or a sibling node backed by the same component file. Only an unambiguous match is + // used: two routes answering to one name says nothing about which record these + // params belong to. + const target = resolveNavRouteTarget(nodesWithSlug, route.name) + if (!target) continue + if (paramStore.ingestNavRoute(target, route.params, from.id)) usable += 1 + } + if (usable) { + console.log( + `[app-flow] nav-state — ${routes.length} route(s) read on ${from.urlPath ?? from.id}, ${usable} usable` + ) + } else { + // A read that returns routes and yields nothing is the silent failure that made a + // whole sweep unexplainable: the values were on screen, the store stayed empty, and + // the log said nothing at all. The route names are what discriminates the cause — a + // navigator that registered a screen under a name no node answers to is, in the + // store, indistinguishable from one that exposed no params in the first place. The + // first dozen names are enough to recognise a stack without flooding the log. + const seen = routes + .slice(0, 12) + .map( + (route) => + `${route.name}${route.path ? ' +path' : ''}${route.params ? ' +params' : ''}` + ) + .join(', ') + console.log( + `[app-flow] nav-state — ${routes.length} route(s) read on ${from.urlPath ?? from.id}, none usable, ${waiting.length} node(s) still blocked — saw: ${seen}${routes.length > 12 ? ', …' : ''}` + ) + } + } catch (error) { + paramStore.navState = 'unavailable' + console.log( + `[app-flow] nav-state — read failed, not retrying — ${error instanceof Error ? error.message : String(error)}` + ) + } + } + + const persistCheckpoint = async (): Promise => { + await writeSweepProgress(projectId, { + structureKey, + updatedAt: new Date().toISOString(), + captures: captureMap, + opened: openedByNode, + inferred: inferredScreens, + statesDone: [...statesDone], + runtimeNodes, + runtimeEdges: runtimeEdgesList, + chrome: chromeInventory + }) + } + + /** + * The screens manifest as it stands right now — pure, so every writer may call it. + * + * One definition rather than the three identical nine-argument calls this file had, because + * the document's nodes are stamped from this map (`withReplayableCaptures`) and a writer + * working from a differently-built map would write a different answer. + */ + const currentAgentDeviceScreens = (): AgentDeviceScreensMap => + buildAgentDeviceScreens( + [...nodesWithSlug, ...runtimeNodes], + parsed.edges, + runtimeEdgesList, + chromeInventory, + ctx, + device, + openedByNode, + inferredScreens, + captureMap + ) + + /** + * The crash-resume checkpoint a captured (or settled-as-unreached) node leaves behind. + * + * A checkpoint, not a publication: `persistCheckpoint` is what makes a crashed or stopped + * sweep resume at the right screen, and it is the only thing a resumed sweep reads. The + * flat capture-status map, the app-flow document and the screens manifest are all derived + * from the same in-memory state and are rebuilt wholesale at finish and on the stopped path, + * so rewriting them after every screen was O(n²) bytes for an intermediate nobody reads, + * sitting on the critical path between screens. + * + * `deepLinkRejections` is the reason the finish and stopped node builds now stamp + * `deepLinkRejection` themselves: this function's document write used to be its only + * durable path, and without that move every rejection learned this sweep would be lost and + * the next sweep would pay again for deep links already proven to 404. + */ + const persistCaptureUpdate = async (): Promise => { + await persistCheckpoint() + } + + const recordUnreached = async ( + node: AppFlowNode, + status: AppFlowCaptureStatus, + note: string | null + ): Promise => { + console.log(`[app-flow] ${node.id} recorded ${status} — ${note ?? 'no reason given'}`) + captureMap[node.id] = { + status, + // `unavailable` is a settled answer about this build, not a screen still to be reached, + // so it must not come back as outstanding navigation work on the next run. + needsNavigation: status !== 'unavailable', + note, + screenshot: null, + pressables: [], + states: [], + captureMethod: 'unreached', + replayScript: null, + replayVerified: false + } + await persistCaptureUpdate() + } + + // The static vocabulary a chrome item's pressable text is matched against — every still-known + // node's own title/id/url tokens (`expectedNodeTokens`, `exploration.ts`), plus each entry + // layout's own name for the case where no single node stands for the whole layout. Rebuilt on + // demand rather than cached once: it is cheap pure token math, and it must reflect nodes + // learned about since the sweep started — a runtime node minted earlier this same sweep is + // itself a legitimate destination for a later chrome item. + const chromeCandidates = (): ChromeCandidate[] => [ + ...nodesWithSlug.map((candidate) => ({ + label: candidate.id, + toNodeId: candidate.id, + tokens: expectedNodeTokens(candidate) + })), + ...runtimeNodes.map((candidate) => ({ + label: candidate.id, + toNodeId: candidate.id, + tokens: new Set(candidate.runtimeSignature ?? []) + })), + ...parsed.entryLayouts.map((name) => ({ + label: `layout:${name}`, + toNodeId: null, + tokens: new Set(tokenize(name.replace(/([a-z0-9])([A-Z])/g, '$1 $2'))) + })) + ] + + // Landing-order item 5's real payoff without item 5 itself (`exploreFrom` stays inline this + // round — see the CHECKPOINT): `observed` reports what it noticed as a `DiscoveredFact` + // (`reach/types.ts`) instead of writing `captureMap`/`runtimeNodes`/`chromeInventory` directly, + // and this is the one place, right next to the state it owns, that folds a fact into the + // sweep. Three call sites feed it — the known-node opportunistic capture that used to write + // `captureMap` inline, the chrome detector, and the runtime-node matcher/minter — kept in one + // place so all three write through the same `persistCaptureUpdate` checkpoint. + const applyDiscoveredFact = async (fact: DiscoveredFact): Promise => { + switch (fact.kind) { + case 'opportunistic-capture': { + const match = nodesWithSlug.find((candidate) => candidate.id === fact.nodeId) + if (!match) return + await copyFile(fact.screenshotPath, getScreenshotPath(projectId, match.slug)) + captureMap[match.id] = { + status: fact.arrival.status ?? 'ok', + // There is no saved journey here, only an accidental sighting — a resumed sweep + // still needs to actually navigate to this screen to redo anything with it. + needsNavigation: true, + note: fact.arrival.reason, + screenshot: `screens/${match.slug}.png`, + pressables: [...fact.pressables], + states: [], + captureMethod: 'opportunistic', + replayScript: null, + replayVerified: false, + structureRevision, + stale: false + } + inferredScreens[match.id] = { key: fact.arrival.key, info: fact.arrival.info } + // The judge just confirmed which screen the walk is standing on, which is exactly the + // fact a walked edge needs — and the only reason this sighting is worth more than the + // capture it already wrote. Recorded before the persist below so the edge rides the same + // checkpoint write as the capture that proved it. + noteArrival(match.id) + await persistCaptureUpdate() + console.log( + `[app-flow] opportunistic capture — reached ${match.id} while exploring toward another screen` + ) + emit(projectId, onEvent, { + type: 'step', + phase: 'capture', + message: `Captured ${match.urlPath ?? match.title} opportunistically`, + completed: nodesWithSlug.filter((candidate) => captureMap[candidate.id]).length, + total + }) + return + } + case 'chrome-observed': { + const kind = inferChromeKind(fact.reveal) + const detected: AppFlowChrome = { + id: `chrome:${kind}:${fact.entryNodeId}`, + kind, + entryNodeId: fact.entryNodeId, + reveal: fact.reveal, + items: fact.items, + observedAt: new Date().toISOString(), + structureRevision + } + // `findMatchingChrome` runs the same kind-plus-item-overlap identity + // `mergeChromeInventory` uses internally, called separately here only so the edges below + // can be built from the record this sighting actually landed on — the chrome's one + // canonical `id`/`entryNodeId`, carried forward from whichever sighting was folded in + // first, not this sighting's own (possibly different) entry screen. Without this, a + // repeat sighting from a new screen would still mint its own parallel `entry -> item` + // edges instead of reinforcing the one edge set the canonical entry already has. + const canonical = findMatchingChrome(chromeInventory, detected) ?? detected + const merged = mergeChromeInventory(chromeInventory, detected) + chromeInventory.splice(0, chromeInventory.length, ...merged) + for (const edge of chromeEdges({ + ...detected, + id: canonical.id, + entryNodeId: canonical.entryNodeId + })) { + const at = runtimeEdgesList.findIndex( + (existing) => existing.from === edge.from && existing.to === edge.to + ) + if (at === -1) runtimeEdgesList.push(edge) + else runtimeEdgesList[at] = edge + } + await persistCaptureUpdate() + console.log( + `[app-flow] chrome inventory recorded from ${fact.entryNodeId} — ${fact.items.length} items (${kind})` + ) + return + } + case 'runtime-screen-sighted': { + if (fact.match) { + const at = runtimeNodes.findIndex((candidate) => candidate.id === fact.match!.id) + if (at !== -1) { + runtimeNodes[at] = recordRuntimeSighting( + runtimeNodes[at], + new Set(fact.observedTokens) + ) + noteArrival(runtimeNodes[at].id) + await persistCaptureUpdate() + console.log(`[app-flow] runtime screen re-sighted — ${fact.match.id}`) + } + return + } + if (runtimeNodes.length >= RUNTIME_NODE_SWEEP_CAP) { + console.log( + `[app-flow] runtime screen discovery capped at ${RUNTIME_NODE_SWEEP_CAP} for this sweep — not minting "${fact.key}"` + ) + return + } + const minted = mintRuntimeNode({ + key: fact.key, + title: fact.info || fact.key, + observedTokens: new Set(fact.observedTokens), + existingIds: new Set([ + ...nodesWithSlug.map((candidate) => candidate.id), + ...runtimeNodes.map((candidate) => candidate.id) + ]), + usedSlugs + }) + await copyFile(fact.screenshotPath, getScreenshotPath(projectId, minted.slug)) + runtimeNodes.push({ + ...minted, + capture: { + status: 'ok', + needsNavigation: true, + note: fact.info, + screenshot: `screens/${minted.slug}.png`, + pressables: [...fact.pressables], + states: [], + captureMethod: 'opportunistic', + replayScript: null, + replayVerified: false, + structureRevision, + stale: false + } + }) + // A screen the parser never knew about is the case a walked edge matters most for: it + // has no static edges at all, so without this the only thing the graph would ever say + // about it is that it exists. + noteArrival(minted.id) + await persistCaptureUpdate() + console.log(`[app-flow] runtime screen discovered — ${minted.id} ("${fact.key}")`) + return + } + } + } + + // Failed destinations still participate in no-progress detection, including empty plans. + // This compares observed UI directly and never launches a made-up unmatched URL. + let failedReference: Set | null = null + let blockedRun = 0 + let sweepStopped = false + + /** + * The escalate-to-the-user half of getting the app unblocked: take a `BLOCKED_SLUG` + * screenshot, emit an `AppFlowPrompt` with `kind: 'blocked'`, and await + * `options.requestUserAction`. Pulled out of `resolveBlocked` below so the states loop's + * stuck-launcher case (see the hinted-nodes loop further down) can reach the same prompt + * without also carrying `resolveBlocked`'s other half — the `failedReference` fingerprint + * comparison and the silent one-shot relaunch retry are specific to the capture loop's + * no-progress detection, and the states loop has neither a `failedReference` nor a reason to + * relaunch before asking, since a launcher is a launcher on first sight. + * + * `detail` is the only thing that varies between callers — each describes what it actually + * saw — everything else (the screenshot, the manifest `sweepStatus` flips, the emit, the + * resume wait) is exactly what `resolveBlocked` always did, unchanged. + * + * `kind: 'blocked'`'s doc comment in shared/app-flow.ts notes it is "named for what was + * observed, not for what the screen is", which is why a stuck dev-client launcher reuses it + * rather than needing a prompt kind of its own. + */ + const promptToUnblock = async ( + detail: string, + remaining: number + ): Promise => { + const stop = (): 'stop' => 'stop' + if (!options.requestUserAction) return stop() + + const shot = getScreenshotPath(projectId, BLOCKED_SLUG) + await captureScreenshot(ctx, device, shot).catch(() => undefined) + const prompt: AppFlowPrompt = { + id: randomUUID(), + kind: 'blocked', + detail, + screenshot: screenshotUrl(projectId, `screens/${BLOCKED_SLUG}.png`), + remaining + } + manifest.sweepStatus = 'awaiting-user' + await writeManifest(projectId, manifest) + emit(projectId, onEvent, { + type: 'prompt', + phase: 'capture', + message: 'Waiting for you to get the app past this screen...', + prompt + }) + console.log(`[app-flow] blocked — waiting for the user (${remaining} screens left)`) + const resolution = await options.requestUserAction(prompt) + console.log(`[app-flow] blocked — user chose "${resolution}"`) + if (resolution === 'stop') return stop() + + manifest.sweepStatus = 'running' + await writeManifest(projectId, manifest) + + emit(projectId, onEvent, { + type: 'phase', + phase: 'capture', + message: 'Checking the app again...' + }) + await waitForAppReady(ctx, device, signal) + return 'continue' + } + + const resolveBlocked = async (remaining: number): Promise => { + // A React Native error boundary, once tripped, keeps rendering its fallback across every + // navigation after it — the app looks permanently blocked even though nothing but its JS + // state is actually broken. A relaunch clears that in-memory state for free and often + // resolves the block without ever bothering the user, so it is worth trying exactly once + // before escalating. `blockedRun` is deliberately NOT reset here: if the relaunch does not + // clear the block, the next iteration should escalate rather than loop back through another + // three failures before trying to relaunch again. + console.log( + `[app-flow] blocked — relaunching the app before asking the user (${remaining} screens left)` + ) + try { + // `reopenBoundApp` now waits for the relaunched app to settle itself, so the separate + // wait this call site used to need is part of it. + await reopenBoundApp(ctx, device) + const recovered = fingerprint(await capturePressables(ctx, device).catch(() => [])) + const stillBlocked = + failedReference !== null && similarity(recovered, failedReference) >= ARRIVAL_SIMILARITY + if (!stillBlocked) { + console.log( + '[app-flow] blocked — relaunch cleared it, resuming without asking the user' + ) + failedReference = null + blockedRun = 0 + return 'continue' + } + console.log('[app-flow] blocked — relaunch did not clear it, escalating to the user') + } catch (error) { + console.log( + `[app-flow] blocked — relaunch attempt failed, escalating to the user — ${error instanceof Error ? error.message : String(error)}` + ) + } + + const resolution = await promptToUnblock( + 'The app keeps rendering this one screen whatever route is opened, so nothing behind ' + + 'it can be captured. Resolve the visible blocker in the simulator; it stays running, ' + + 'then continue.', + remaining + ) + if (resolution === 'continue') { + failedReference = null + blockedRun = 0 + } + return resolution + } + + // `ArrivalResult` used to be declared inline here; it is now `reach/types.ts`'s + // `ArrivalVerdict`, aliased so every existing use site in this function keeps compiling + // unchanged. + type ArrivalResult = ArrivalVerdict + + // The two-attempt retry loop this used to run inline now lives in `JudgePort.verifyArrival` + // (`reach/adapters.ts`), moved verbatim — this is a pass-through so every call site below + // keeps its existing name and shape. + const verifyCurrentTarget = ( + node: AppFlowNode, + screenshotPath: string, + budget: ActiveTimeBudget + ): Promise => ports.judge.verifyArrival(node, screenshotPath, budget) + + // Whether `tryDeepLink` is allowed to touch the device at all this sweep. Per-node deep-link + // overhead is a URL open, a `waitForRoute`, a screenshot and a vision call — for a build whose + // scheme does not route at all, paying that on every one of up to 81 routes only to fall + // through to `explore()` anyway (with its own fresh five-minute budget) is the single most + // wasteful thing the sweep can do. A one-time probe up front, plus a running breaker below, + // catch that early instead of rediscovering it node by node. + // The breaker's decisions (`registerDeepLinkFailure`/`applyDeepLinkReprobeResult`/ + // `registerDeepLinkSuccess`) are pure, moved to `reach/deep-link-policy.ts` in landing-order + // item 4; this is only the running state they fold into, replacing the old `deepLinkEnabled`/ + // `consecutiveDeepLinkFailures` locals. + // Deliberate entry-chrome inventory pass, run once before the per-node sweep below. Every + // other chrome detection in this file is opportunistic — `observed`'s chrome-observed case + // (further down) only fires when a walk toward some *other* target happens to wander through + // the chrome, so a real app's drawer can go completely unrecorded for an entire sweep if no + // walk ever happens to open it. Bluesky is exactly that case: it has a working drawer (the + // model tapped `label="Open drawer menu"` and then a named item while reaching a screen behind + // it, mid-sweep, proving the control exists and works) but no sweep ever recorded it, because + // recording only ever rode on some unrelated walk's own steps. Reading the tab bar and opening + // the drawer once here means `findNavigationPath` has a locator-bearing hop to every screen + // behind either chrome from the very first node the sweep visits, rather than only from + // whichever node's walk happened to stumble onto it — which is the whole point: a deep link + // that fails now has a recorded, tappable alternative to fall back to immediately, not a maybe. + // + // Placement is load-bearing: this runs *before* the deep-link probe below, not after. The + // probe opens a URL and leaves the app wherever that URL landed, with no navigation back — so + // running afterwards inventoried whatever screen the probe had just produced. In the device + // run that caught this, the probe's own URL was invalid (`bluesky://src`), the app was still + // mid-transition, and the pass read 5 controls and concluded there was no drawer opener; the + // very same screen read 8 controls including `label="Open drawer menu"` moments later. Here + // the app is freshly launched and already settled — `bootDevice`'s `waitForAppReady` and the + // auth gate have both run and seen a stable frame — which is the one moment in the whole + // sweep the app is guaranteed to be sitting on its real entry screen. + // + // Detection reuses `detectChromeInventory` (`reach/chrome-inventory.ts`) — the exact same pure, + // token-only match the opportunistic path already runs — so this costs device round trips only + // and NOT ONE model call: reading a control list and matching its text against known node + // vocabulary is not a vision judgment. Bounded to at most two pressable reads, one tap to open + // the drawer and one best-effort tap to close it again, and wrapped in a `try/catch` a total + // failure never escapes: no drawer control found, a device hiccup, anything at all still leaves + // the sweep exactly where it would have started if this pass did not exist. + try { + const launchEntryNodeId = entryNodeIds[0] ?? 'app-launch' + const tabBarPressables = await ports.device.capturePressables(15_000) + const tabBarChrome = detectChromeInventory(tabBarPressables, chromeCandidates()) + if (tabBarChrome) { + await applyDiscoveredFact({ + kind: 'chrome-observed', + entryNodeId: launchEntryNodeId, + reveal: [], + items: tabBarChrome.items + }) + } + + const drawerOpener = findDrawerOpener(tabBarPressables) + if (drawerOpener) { + const revealStep: AppFlowStep = { type: 'tap', locator: drawerOpener } + // Hoisted out of the try below because the close fallback needs it: the drawer's own + // items are the only controls the sweep has *proved* are reachable while the drawer is + // open, which is exactly the property the fallback turned out to need. + let drawerChrome: ReturnType = null + // Its own try: the tab-bar inventory above has already been recorded by this point, and a + // drawer that refuses to open must not read in the log as the whole pass having failed — + // which is exactly how an ambiguous-selector throw presented before, sending the last + // investigation looking for a broken pass rather than a broken locator. + try { + await ports.device.executeNavigationActions([revealStep], 15_000) + const drawerPressables = await ports.device.capturePressables(15_000) + drawerChrome = detectChromeInventory(drawerPressables, chromeCandidates()) + if (drawerChrome) { + await applyDiscoveredFact({ + kind: 'chrome-observed', + entryNodeId: launchEntryNodeId, + reveal: [revealStep], + items: drawerChrome.items + }) + } + console.log( + `[app-flow] entry-chrome pass — drawer opened via ${drawerOpener.by}="${drawerOpener.value}", ${drawerChrome?.items.length ?? 0} items recorded from ${drawerPressables.length} controls` + ) + } catch (error) { + console.log( + `[app-flow] entry-chrome pass — drawer opener ${drawerOpener.by}="${drawerOpener.value}" could not be used, keeping the tab inventory — ${error instanceof Error ? error.message : String(error)}` + ) + } + // Closing it again is not best-effort, and the comment that used to say so was wrong. + // The claim was that a drawer left open leaves the app "no worse off than a normal walk + // that opened it and moved on". It does not: the very next thing the sweep does is the + // deep-link probe, whose single verdict decides the transport for every route. On the + // social-app run that caught this, the close tap threw in 287ms against a control the + // open drawer was covering, `.catch(() => undefined)` swallowed it without a line in the + // log, and the probe fired its URL into a modal and read the drawer's own 17 controls + // back. One silent toggle failure cost 79 routes their deep links. + // + // So the toggle is now checked rather than assumed, and the check is the opener's own + // visibility: an open drawer covers the control that opened it, so finding that control + // again is the app telling us the drawer is gone. That is the same reading of the same + // control list the pass already does, and it costs one accessibility read. + let closed = false + try { + await ports.device.executeNavigationActions([revealStep], 15_000) + closed = Boolean(findDrawerOpener(await ports.device.capturePressables(15_000))) + } catch (error) { + console.log( + `[app-flow] entry-chrome pass — close tap on ${drawerOpener.by}="${drawerOpener.value}" failed: ${error instanceof Error ? error.message : String(error)}` + ) + } + if (!closed) { + // The fallback has to tap a control that is *present while the drawer is open*, and + // the tab bar is not one. An earlier version reached for a tab entry on the reasoning + // that it "does not have to be" under the drawer; the device disagreed. An open + // drawer replaces the accessibility tree rather than overlaying it — the read taken + // one line above returns the drawer's own 17 controls and nothing behind them — so a + // tab locator misses for the same reason the opener did, in the same ~165ms, and the + // run that proved it went straight from the failed opener tap to the warning in 178ms + // without the tap ever reaching the device. + // + // The drawer's own items are the one set of controls the pass has just *observed*, so + // they are what the fallback presses. They navigate rather than toggle, which closes + // the drawer as a side effect of going somewhere — and somewhere is fine here: the + // probe below needs a settled real screen, not specifically the entry screen. + // Preferring the item that leads back to the launch entry node keeps it on the screen + // the pass started from when the drawer offers that route, which on a tabbed app it + // usually does. + const drawerItems = (drawerChrome?.items ?? []).filter( + (item) => item.trigger.value !== drawerOpener.value + ) + const exitTarget = ( + drawerItems.find((item) => item.toNodeId === launchEntryNodeId) ?? + drawerItems.find((item) => item.toNodeId !== null) ?? + drawerItems[0] + )?.trigger + if (exitTarget) { + try { + await ports.device.executeNavigationActions( + [{ type: 'tap', locator: exitTarget }], + 15_000 + ) + closed = Boolean(findDrawerOpener(await ports.device.capturePressables(15_000))) + } catch (error) { + console.log( + `[app-flow] entry-chrome pass — exit tap on ${exitTarget.by}="${exitTarget.value}" failed: ${error instanceof Error ? error.message : String(error)}` + ) + } + } + console.log( + closed + ? `[app-flow] entry-chrome pass — drawer would not toggle shut; left it via ${exitTarget?.by}="${exitTarget?.value}"` + : // Loud on purpose. Everything downstream now starts from a screen the sweep did + // not intend, and the probe's `frameChangedAfterOpen` guard is the only thing + // standing between that and the transport being switched off for every route. + `[app-flow] entry-chrome pass — WARNING: the drawer is still open and could not be dismissed; the next screens are being read through it` + ) + } + } else { + console.log( + `[app-flow] entry-chrome pass — no drawer opener among ${tabBarPressables.length} entry controls` + ) + } + } catch (error) { + console.log( + `[app-flow] entry-chrome pass failed, continuing without it — ${error instanceof Error ? error.message : String(error)}` + ) + } + + let deepLinkBreaker: DeepLinkBreakerState = initialDeepLinkBreakerState() + // The exact URL a deep-link attempt opened, once the arrival judge has rejected it this + // sweep. This matters most for a param-free prefix (`paramFreeDeepLink` truncating + // `/profile/:name` to `/profile`, say): every sibling route sharing that ancestor computes + // the identical `target`, and a node that gets requeued after the blocked-run breaker fires + // recomputes the same `target` for itself too. Without this, a build whose `/profile` 404s + // paid a URL open, a `waitForRoute`, a screenshot and a judge call *per node* to relearn + // exactly the same fact — five times in the device run that motivated this fix. This is + // deliberately in-memory and sweep-scoped, not persisted on the node: that is the later, + // schema-backed version of this idea (plan step 3a), and this is only the cheap subset that + // stops the immediate, self-contained waste. + const rejectedDeepLinkUrls = new DeepLinkRejectionMemo() + const closedGuardGates = new GuardGateMemo() + const probeRoute = pickProbeRoute(nodesWithSlug) + + /** + * The reach the one-time deep-link probe below already performed and verified, held for the + * sweep to claim instead of performing it a second time. + * + * The probe does the full thing — opens the URL, settles, screenshots, asks the judge — and + * then the sweep walked up to the same node and did all four again, identically. A device + * run shows the duplicate costing ~14s: `openDeepLink (tabs)/explore` and a `match` verdict, + * immediately followed by `openDeepLink (tabs)/explore` and the same `match` verdict. + * + * Only ever valid for the *first* node the sweep reaches, and `claimProbeCredit` enforces + * exactly that by clearing it on any other node. The app is left sitting on the probe route + * with nothing between the probe and the loop that touches the device, so if the sweep opens + * with that node the screen in front of it is still the verified one; if it opens with any + * other node, that node's own reach moves the app and the credit is worthless. Nothing here + * tries to be cleverer than that — a credit that survived one unrelated navigation would be + * a claim about a screen nobody has looked at. + */ + let probeCredit: { + nodeId: string + arrival: ArrivalResult + /** The frame the probe's judge accepted, for delivery to reuse — see `ReachEvidence.frame`. */ + frame: string + /** The controls read during that same judge call — see `ReachEvidence.pressables`. */ + pressables?: readonly AppFlowPressable[] | null + opened?: { app: string; url: string } + } | null = null + const claimProbeCredit = (nodeId: string): typeof probeCredit => { + if (!probeCredit) return null + const credit = probeCredit + probeCredit = null + return credit.nodeId === nodeId ? credit : null + } + + /** + * Interactive elements the device currently reports, or null when the read failed. + * + * Only the probe uses this, and only to decide whether a frame is substantive enough to draw + * a sweep-wide conclusion from (`isSubstantiveFrame`). A failed read deliberately returns null + * rather than 0: "I could not look" and "there is nothing there" are different claims, and + * `judgeProbeFrame` treats only the second as something to act on. The extra accessibility + * read costs a second or two, once per probe, against the multi-minute cost of the mistake it + * prevents — every route in the sweep losing its deep link to a splash screen. + * + * The counts are comparable with the `settled with N interactive elements` line + * `waitForAppReady` logs: on the reference run the same screens read 9 and 9, 38 and 48. + */ + const countInteractiveElements = async (): Promise => { + try { + return (await ports.device.capturePressables()).length + } catch { + return null + } + } + + /** One open-and-judge pass. `probeDeepLinkRoute` is what callers use; this is the single + * pass it may run twice. `pass` only ever appears in logs, so a device log shows whether a + * mismatch was confirmed or merely first observed. */ + const probeOpenPass = async ( + route: (typeof nodesWithSlug)[number], + pass: 1 | 2 + ): Promise<{ + outcome: 'match' | 'mismatch' | 'inconclusive' + arrival: ArrivalResult | null + opened?: { app: string; url: string } + frame?: string + pressables?: readonly AppFlowPressable[] | null + }> => { + const probeBudget = new ActiveTimeBudget(60_000, activeNow(), activeNow) + // The budget bounds the *judgement*, not the device work ahead of it, so getting the app + // onto the screen is excluded the same way a wait on the user is (see the auth gate's + // `budget.exclude`). Without this the two are charged to one clock, and a device that needs + // recovering spends the judge's entire allowance before it is asked anything: a device log + // shows `openDeepLink` taking ~72s against this 60s budget while the runner was wedged, + // after which `verifyArrival`'s `!budget.expired` guard skipped both attempts, made no model + // call at all, and the probe reported "inconclusive" having never actually looked. Slow is + // not the same as wrong, and only the second is something this probe is entitled to conclude. + const deviceStartedAt = Date.now() + // Settle *before* opening the URL, not only after, and keep settling until the app is + // actually there. `waitForAppReady` reports a stable frame and a dev-client loading screen + // is perfectly stable, so "settled" alone does not mean "ready to route". On the cold-cache + // social-app run the first read returned 2 interactive elements, the URL was fired into + // that, Metro logged `iOS Bundled 16079ms index.ts (6130 modules)` 13 seconds later, and + // the swallowed link left the app on its default screen — judged a mismatch, which took + // deep links away from all 79 routes and tripled the sweep's token cost. That is why the + // wait below has an element floor at all, and why it is a wait rather than the single + // retry it replaced. + let launchElements = await countInteractiveElements() + if (!isSubstantiveFrame(launchElements)) { + console.log( + `[app-flow] deep-link probe — app shows ${launchElements ?? 'an unreadable frame'} interactive element(s), below the ${PROBE_FRAME_ELEMENT_FLOOR} needed to trust a probe verdict; waiting up to ${PROBE_LAUNCH_READY_BUDGET_MS / 1000}s for it to finish loading before opening the URL` + ) + const waitStartedAt = Date.now() + // One continuous wait, not a loop around a wait. `waitForRoute` returns the moment two + // reads agree, and a dev-client loading screen agrees with itself immediately, so asking + // it repeatedly re-paid the whole settle handshake for a frame that had not changed: on + // the measured social-app run this printed the same poll/stale/settled triplet thirteen + // times between t=45.9s and t=59.6s, ~1.1s each, and spent 13.7s discovering what one + // poll would have noticed as it happened. Pushing the element floor *into* the wait lets + // a single poll hold until the frame is both stable and substantial. + await ports.device.waitForRoute({ + minInteractiveElements: PROBE_FRAME_ELEMENT_FLOOR, + timeoutMs: PROBE_LAUNCH_READY_BUDGET_MS + }) + checkSignal(signal) + launchElements = await countInteractiveElements() + const waitedSeconds = ((Date.now() - waitStartedAt) / 1000).toFixed(1) + console.log( + isSubstantiveFrame(launchElements) + ? `[app-flow] deep-link probe — app rendered ${launchElements} interactive element(s) after ${waitedSeconds}s, opening the URL now` + : `[app-flow] deep-link probe — app still shows ${launchElements ?? 'an unreadable frame'} interactive element(s) after ${waitedSeconds}s; opening the URL anyway, but a mismatch cannot be trusted from here` + ) + } + // Whether the app could have received the URL at all. Carried into `judgeProbeFrame` + // because no post-open measurement can recover it — see that function's mismatch branch. + const preOpenSubstantive = isSubstantiveFrame(launchElements) + // The frame as it stood immediately before the open, so the judge can be told whether the + // URL moved the app at all. Read here rather than reused from the settle wait above + // because that wait may have returned seconds ago; this is the last frame before the open + // and nothing else is. Best-effort: a failed read makes the question unanswerable, which + // `judgeProbeFrame` treats as a reason to defer rather than a reason to act. + const preOpenFrame = await ports.device + .capturePressables() + .then((pressables) => fingerprint(pressables)) + .catch(() => null) + const opened = await ports.device.openDeepLink(route, {}) + checkSignal(signal) + // Whether the app reached a stable frame at all. A frame captured while the device was + // still refusing accessibility reads ("runner busy", "still loading … capturing anyway") + // shows a splash or the previous screen no matter where the URL actually routed, so a + // mismatch judged off it is a statement about the device, not about the URL scheme. + const { settled } = await ports.device.waitForRoute() + probeBudget.exclude(Date.now() - deviceStartedAt) + const probeShot = getScratchPath(projectId, 'deep-link-probe') + await ports.device.captureScreenshot(probeShot) + // The controls are read inside the judge's own latency, so a probe that goes on to be + // claimed by `probeCreditRung` can hand delivery a read it did not have to spend — see + // `probeCredit`. Dropped on any outcome but a match, which costs nothing: the read ran + // concurrently with a call that had to happen either way. + const [arrival, probePressables] = await Promise.all([ + verifyCurrentTarget(route, probeShot, probeBudget), + ports.device.capturePressables().catch(() => null) + ]) + // Read the frame the judge actually looked at, so "was this worth judging" is answered + // about that frame rather than about whatever the device shows by the time we ask. The + // concurrent read above is the best available answer to that: it was taken *during* the + // judge call rather than after it, so it is nearer the judged frame than a fresh read + // would be — on the reference run the judge took up to 24.9s to reply, which is a long + // time to let the app drift before asking what it was looking at. `judgeProbeFrame` only + // consults this on a mismatch; the match branch returns before reading it. + const frameElements = + arrival.verdict === 'match' + ? launchElements + : (probePressables?.length ?? (await countInteractiveElements())) + // Compared with the same overlap bar `waitForAppReady` settles on rather than exact + // equality, for the same reason it uses one: a feed mutates under a stationary app, so + // byte-identical frames are the exception even when nothing navigated. + const frameChangedAfterOpen = + preOpenFrame && probePressables + ? similarity(preOpenFrame, fingerprint(probePressables)) < APP_READY_SETTLE_SIMILARITY + : null + const outcome = judgeProbeFrame({ + verdict: arrival.verdict, + settled, + elementCount: frameElements, + preOpenSubstantive, + frameChangedAfterOpen + }) + if (outcome === 'match') + return { outcome, arrival, opened, frame: probeShot, pressables: probePressables } + if (outcome === 'inconclusive' && arrival.verdict === 'mismatch') { + const why = !preOpenSubstantive + ? `was opened into an app showing only ${launchElements ?? 'an unreadable number of'} interactive element(s), which could not have routed it` + : frameChangedAfterOpen === false + ? 'left the screen exactly as it was, so the URL was swallowed rather than routed — something already on screen took it' + : frameChangedAfterOpen === null + ? 'could not be checked against the frame before the open, so there is no evidence the URL reached the app' + : !settled + ? 'mismatched on a frame the app never settled on' + : `mismatched on a frame carrying only ${frameElements ?? 'an unreadable number of'} interactive element(s) — too thin to be evidence about the scheme` + console.log( + `[app-flow] deep-link probe ${route.id} (pass ${pass}) ${why} — inconclusive, not evidence about the scheme` + ) + } + return { outcome, arrival: null } + } + + // The one check both the initial probe and the breaker's re-confirmation need: does opening + // `route` by URL actually land the app on it. Kept in one place so the two call sites can't + // drift into subtly different definitions of "the transport works". + // + // A mismatch is confirmed by a second pass before it is believed. Everything the first pass + // can measure about app readiness it measures *before* opening, but a URL can still be + // swallowed by a dev client that answers accessibility reads while its bundle is not yet + // wired to a navigator — the cold-boot failure `judgeProbeFrame` documents. The second pass + // runs against an app that has demonstrably just rendered a real screen, so it is the one + // observation that cannot be confounded that way. It costs one URL open and one judge call, + // around six seconds, set against the measured cost of a wrong disable: every route in the + // sweep walked by hand, 3.5x the tokens. Given that asymmetry a confirmation is close to + // free, and it is skipped entirely on the overwhelmingly common path where the first pass + // matches. + const probeDeepLinkRoute = async ( + route: (typeof nodesWithSlug)[number] + ): Promise<{ + outcome: 'match' | 'mismatch' | 'inconclusive' + /** The judge's own verdict, so a caller can reuse the reach this probe already proved + * rather than paying for an identical one — see `probeCredit` below. Null whenever the + * probe concluded without a usable verdict. */ + arrival: ArrivalResult | null + /** What `openDeepLink` recorded, so a caller reusing this reach reports the same + * `deep-link-opened` effect the deep-link strategy would have. */ + opened?: { app: string; url: string } + /** The frame the judge accepted, so a caller reusing this reach can deliver it rather + * than screenshot the same screen again. */ + frame?: string + /** The controls read at that same moment, for the same reason — null when the read failed, + * which only costs the claimant a read of its own. */ + pressables?: readonly AppFlowPressable[] | null + }> => { + const first = await probeOpenPass(route, 1) + if (first.outcome !== 'mismatch') return first + console.log( + `[app-flow] deep-link probe ${route.id} looks like the wrong screen — re-opening the URL once against the now-loaded app before disabling deep links for the sweep` + ) + const second = await probeOpenPass(route, 2) + if (second.outcome !== 'mismatch') { + console.log( + `[app-flow] deep-link probe ${route.id} did not reproduce the mismatch on the second open (${second.outcome}) — the first was the app not being ready, deep links stay enabled` + ) + } + return second + } + + if (probeRoute?.urlPath) { + // The probe may only ever *confirm* that deep links work. It must never be what turns them + // off, unless it actually saw the app settle somewhere other than the target. + // + // The asymmetry is brutal and was measured twice on device. Wrongly disabling costs every + // one of up to 84 routes its transport, sending each into a fresh five-minute exploration + // walk — the single most expensive mistake this sweep can make. Wrongly *enabling* costs a + // few per-node attempts before `registerDeepLinkFailure`'s breaker trips on real evidence + // from real nodes, which is a strictly better-informed judge than one screenshot. So + // anything short of a settled, confident mismatch leaves deep links on and defers to it. + // + // What that rules out, both seen in one device log: a sub-agent timeout (an infrastructure + // failure that says nothing about the URL) and a verdict read off a frame captured while + // the runner was busy and `waitForAppReady` had already given up. Neither is evidence, and + // between them they disabled deep links for an entire 84-route sweep. + let outcome: 'match' | 'mismatch' | 'inconclusive' = 'inconclusive' + let lastError: unknown = null + for (let attempt = 1; attempt <= 2 && outcome !== 'match'; attempt++) { + try { + const probed = await probeDeepLinkRoute(probeRoute) + outcome = probed.outcome + if (probed.arrival && probed.frame) + probeCredit = { + nodeId: probeRoute.id, + arrival: probed.arrival, + frame: probed.frame, + pressables: probed.pressables, + opened: probed.opened + } + lastError = null + if (outcome === 'mismatch') break + } catch (error) { + lastError = error + outcome = 'inconclusive' + if (attempt === 1) { + console.log( + `[app-flow] deep-link probe errored, retrying once — ${error instanceof Error ? error.message : String(error)}` + ) + } + } + } + deepLinkBreaker = initialDeepLinkBreakerState(outcome !== 'mismatch') + if (lastError) { + console.log( + `[app-flow] deep-link probe errored twice — inconclusive, leaving deep links enabled for the per-node breaker to judge — ${lastError instanceof Error ? lastError.message : String(lastError)}` + ) + } else if (outcome === 'match') { + console.log( + `[app-flow] deep-link probe ${probeRoute.id} (${probeRoute.urlPath}) matched — deep links enabled for this sweep` + ) + } else if (outcome === 'mismatch') { + console.log( + `[app-flow] deep-link probe ${probeRoute.id} (${probeRoute.urlPath}) settled on the wrong screen — deep links disabled for this sweep` + ) + } else { + console.log( + `[app-flow] deep-link probe ${probeRoute.id} (${probeRoute.urlPath}) inconclusive — leaving deep links enabled for the per-node breaker to judge` + ) + } + } else { + console.log( + '[app-flow] deep-link probe — no eligible probe route (need a urlPath with no params), leaving deep links enabled' + ) + } + + // The breaker's re-probe device call, handed to `deepLinkStrategy` as its `reprobe` callback + // (`reach/strategies/deep-link.ts`) — that module owns *when* to reprobe (a pure decision, + // `reach/deep-link-policy.ts`'s `registerDeepLinkFailure`), this owns *how*, reusing the exact + // same check the initial one-time probe above already runs. Absent when there is no eligible + // probe route, matching the original immediate-disable fallback in that case. + const reprobeDeepLinkRoute = probeRoute + ? async (): Promise => + (await probeDeepLinkRoute(probeRoute)).outcome !== 'mismatch' + : undefined + + // Entry screens are the cheapest to reach (a deep link or a one-hop walk) and are what produce + // the chrome inventory and the parent replays every later screen behind that chrome reuses — + // see `sweep-order.ts`'s doc comment for the full tier breakdown. `nodesWithSlug` itself stays + // in parser order: everything above and below the loop that looks a node up by id, counts + // orphans, or writes the final result reads it as-is, and reordering it in place would only + // make those call sites' order accidentally depend on the sweep's own order too. + // `deepLinkBreaker.enabled` is read here, after the one-time probe above has folded its + // result in, because it decides whether tier 5 may demote a node at all — see + // `OrderSweepNodesOptions.deepLinksEnabled`. With the transport live a param-free URL is a + // way in that owes nothing to the control graph; with it off, the graph is all there is. + const sweptNodes = orderSweepNodes( + nodesWithSlug, + parsed.edges, + parsed.entryScreenIds, + parsed.entryLayouts, + { + deepLinksEnabled: deepLinkBreaker.enabled, + // Only when a credit is actually in hand. `probeCredit` is set by a probe the judge + // accepted and cleared by the first `claimProbeCredit` call, so this is precisely "there + // is verified work on this node waiting to be claimed" — not merely "a route was probed". + probeRouteId: probeCredit?.nodeId ?? null + } + ) + + // `ReachQueue` replaces the old `for (let i = 0; i < nodesWithSlug.length; i++)` — see its doc + // comment for why the raw index could not survive `sweptNodes` reordering `nodesWithSlug`. + // `queue.completed` reads exactly like the old `i` did at the top of the loop body (nodes + // strictly before the current one, not yet counting the one in flight), and `queue.remaining` + // exactly like the old `total - i` — both used at every call site below that used to spell out + // that arithmetic by hand. + const sweepQueue = new ReachQueue(sweptNodes) + let sweepPaused = false + /** Nodes the blocked-run breaker has already re-served once. See the `requeueCurrent` call + * site for why a second retry is always wasted work rather than a second chance. */ + const requeuedIds = new Set() + // `activeNow` rather than `Date.now` for the deadline, for the same reason every + // `ActiveTimeBudget` above uses it: a sweep sitting on a blocked prompt is waiting on the + // user, and charging that to the run's clock would time out a sweep that was never slow. + const explorationControl = new ExplorationControl( + options.screenLimit, + options.pauseRequested, + { + attemptLimit: options.attemptLimit, + sweepDeadlineMs: options.sweepDeadlineMs, + now: activeNow + } + ) + /** The one place the four pause reasons turn into a line a reader can act on. The three + * ceilings are not interchangeable and must not all print "screen limit reached": that + * message on a run that captured three of eight screens and ran out of attempts sends the + * reader looking for five missing captures instead of at the unreachable routes that ate + * the budget. */ + const pauseMessage = (reason: ExplorationPauseReason): string => { + if (reason === 'requested') return '[app-flow] pausing at the latest screen checkpoint' + if (reason === 'limit') + return `[app-flow] screen limit reached (${options.screenLimit}) — pausing` + if (reason === 'attempts') + return `[app-flow] attempt limit reached (${explorationControl.screensAttempted} attempts for ${explorationControl.screensExplored}/${options.screenLimit} screens) — pausing; the remaining routes are costing more to reach than they are returning` + return `[app-flow] time limit reached (${explorationControl.screensExplored}/${options.screenLimit} screens in ${explorationControl.screensAttempted} attempts) — pausing` + } + const pauseBeforeNextScreen = (): boolean => { + const reason = explorationControl.beforeNextScreen() + if (!reason) return false + sweepPaused = true + sweepStopped = true + console.log(pauseMessage(reason)) + return true + } + const finishExploredScreen = (captured: boolean): boolean => { + const reason = explorationControl.screenFinished(captured) + if (!reason) return false + sweepPaused = true + sweepStopped = true + console.log(pauseMessage(reason)) + return true + } + for (let node = sweepQueue.next(); node !== null; node = sweepQueue.next()) { + checkSignal(signal) + // Pause before touching the next screen. A request made while a screen is in flight reaches + // the matching check after that screen's durable checkpoint has been written below. + if (pauseBeforeNextScreen()) break + // Two distinct reasons to skip, and only these two. The checkpoint is a snapshot taken + // before this loop started, so it cannot see a node captured opportunistically while + // exploring toward an earlier one this same run — that write only ever lands in the live + // `captureMap`, and checking the checkpoint alone re-explored every opportunistic hit from + // scratch with a fresh five-minute budget, the very waste that feature exists to avoid. + // + // What must NOT skip is a node this run already failed: `recordUnreached` writes a + // `missing` entry into the same live `captureMap`, so a bare `captureMap[node.id]` test + // also swallows the `queue.requeueCurrent()` retry the blocked prompt schedules once the + // user (or the relaunch) clears the blocker — the node would be marked missing and never + // reattempted. + const settled = checkpoint.captures[node.id] ?? null + const opportunistic = captureMap[node.id]?.captureMethod === 'opportunistic' + if (settled || opportunistic) { + console.log( + opportunistic + ? `[app-flow] skip ${node.id} — captured opportunistically while exploring another screen` + : `[app-flow] skip ${node.id} — already captured in a previous run` + ) + emit(projectId, onEvent, { + type: 'step', + phase: 'capture', + message: opportunistic + ? `Captured ${node.urlPath ?? node.title} opportunistically` + : `Already captured ${node.urlPath ?? node.title}`, + completed: sweepQueue.completed + 1, + total + }) + continue + } + const budget = new ActiveTimeBudget(5 * 60_000, activeNow(), activeNow) + const label = node.urlPath ?? node.title + emit(projectId, onEvent, { + type: 'step', + phase: 'capture', + message: `Reaching ${label}`, + completed: sweepQueue.completed, + total + }) + + // One resolution feeds both the values and the unresolved list, so the two cannot drift: the + // static call sites are consulted first and anything the app itself showed us earlier in the + // sweep (or in a previous one) fills in what they could not. + const resolution = resolveNodeParams(node, parsed.edges, paramStore) + // Only the params that actually have an observed value. These go to rungs that open a URL + // or dispatch a navigation, and a stand-in would make either of those land cleanly on some + // other record — see `buildUrlPath`, which now refuses a hole outright rather than filling + // it. `unresolvedParams` below is what stops a rung from getting that far. + const params = Object.fromEntries( + node.params + .filter((key) => resolution.values[key] !== undefined) + .map((key) => [key, resolution.values[key]]) + ) + // Params neither the route graph nor anything harvested could supply. A deep link built on a + // guessed id opens a record the app does not have, and the error screen that follows is + // indistinguishable from a route this build withholds — so the in-app path is the only + // honest way in. + const unresolvedParams = resolution.unresolved + if (unresolvedParams.length === 0 && resolution.source && resolution.source !== 'static') { + console.log( + `[app-flow] params — ${node.urlPath} resolved from ${resolution.source} seen on ${resolution.seenOn}: ${JSON.stringify(resolution.values)}` + ) + } + let captureMethod: AppFlowCapture['captureMethod'] = 'unreached' + let replayScript: string | null = null + // True whenever `replayScript` was itself re-confirmed by a fresh screenshot rather than + // merely recorded. Only `savedReplayStrategy` can set it now, replaying a script the + // *manual* recorder published; exploration records no script to verify. + let replayScriptVerified = false + /** + * The journey a walk actually took, as `agent-device …` lines — this node's stored + * `pathToReach` and the seed for anything that explores from here. Set by `exploreFrom` on + * arrival, null when no walk arrived or when the walk contained a step with no faithful + * CLI form (see `walkCommands`, which refuses to store a truncated path). + */ + let pathCommands: readonly string[] | null = null + let reachError: string | null = null + let acceptedArrival: ArrivalResult | null = null + // What was in hand at the instant the vision judge accepted this node's screen, kept because + // that instant is the only one in the whole reach that is *proven* to be looking at the + // target. Everything after it — publishing the recording, replaying it to check it, the + // auth-gate probe — can and does move the device somewhere else, and a replay that fails + // mid-walk leaves it stranded with no session at all. Delivery falls back to these two when + // the live device has moved on; see `chooseDeliverySources` (reach/delivery.ts). + // + // The frame lives in its own scratch slot rather than being read from `scratch/explore.png` + // later: that file is overwritten by the very next observation, and by the time delivery + // runs it is whatever the exploration of some *other* node last looked at. + let arrivalFrame: string | null = null + let arrivalPressables: AppFlowPressable[] = [] + /** + * The frame the winning strategy's judge accepted, when that strategy left the app standing + * on it with nothing further touching the device — `ReachEvidence.frame`. Distinct from + * `arrivalFrame` above, which is a *fallback* for a delivery capture that came back + * compromised; this one says the delivery capture need not be taken at all, because the + * screen has not changed since the frame that proved it. + */ + let verifiedFrame: string | null = null + /** The controls read at that same verified moment — `ReachEvidence.pressables`. */ + let verifiedPressables: readonly AppFlowPressable[] | null = null + let authStopped = false + // Set once a deep link proves the route resolves but this build withholds the screen. + // Exploration cannot get past that, so it is skipped rather than left to time out. + let unavailableReason: string | null = null + // The deep link resolved onto the app's 404 without proving the screen is gated — usually a + // sample param. The explorer is told, so it treats a missing entry point as confirmation + // rather than a reason to keep hunting. + let deepLinkNotFound = false + // `deepLinkAttempted` (whether the deep-link rung actually opened a URL for this node, as + // opposed to short-circuiting on the breaker, an unresolved param, or a missing urlPath) is + // now derived from the ladder's own `outcome.attempts` after it runs — see the `const + // deepLinkAttempted = outcome.attempts.some(...)` below — since that is exactly the + // `touchedDevice` field `deepLinkStrategy` (`reach/strategies/deep-link.ts`) already reports. + // Set only when a `CommandSeed`'s own stored path fails to run — a fact about the parent's + // journey, not about whether the target is reachable at all. The `explore` step below uses + // this to decide whether a bounded last hop that could not even start is worth retrying as + // a full, unseeded exploration (yes) versus one that started fine and simply ran out of its + // three decisions (no — see plan Step 3e). + let seedBaseFailed = false + // Set when the most recent `exploreFrom` reached its decision loop and ended without + // arriving, as opposed to failing before it (launch, base replay, auth gate) — reset at + // the top of every call. `shouldRetryFromEntry` reads it to tell a walk that ran out of + // ideas from one that never got to have any. + let seededWalkRan = false + // The *manual* recorder's published path, and the only `.ad` file the sweep still touches: + // `savedReplayStrategy` replays it when one exists. Exploration no longer writes a + // candidate or publishes anything here. + const savedScript = join(getFlowsDir(projectId), `${node.slug}.ad`) + // Bounded mode — plan Step 3e/3f's "last hop": the seed already proved everything up to one + // tap away, so a walk that has not arrived within three decisions is not a promising walk + // approached badly, it is a target that was never reachable this way at all. Tighter than + // `EXPLORATION_MAX_DECISIONS`'s general-purpose seven on purpose — see that constant's own + // doc comment for why a higher cap only buys longer flailing on a hopeless target, which is + // exactly what a proven seed should never need. + const exploreFrom = async ( + seed: ExplorationSeed | null, + exploreOptions: { bounded?: boolean } = {} + ): Promise => { + checkSignal(signal) + seededWalkRan = false + if (authStopped || budget.expired || isFallbackRoute(node) || unavailableReason) { + if (isFallbackRoute(node)) { + reachError = 'Fallback route has no in-app entry point.' + } + return false + } + const commandSeed = seed && 'commands' in seed ? seed : null + // The lines this walk's own stored path will open with, so the path it eventually stores + // is literally what the walk did. A command seed contributes the parent's whole journey; + // anything else contributes one baseline launch line carrying the URL the reset below + // actually opened. Built before the device is touched so the two cannot disagree. + const seedUrl = seed && 'url' in seed ? seed.url : null + const prefix = commandSeed + ? [...commandSeed.commands] + : baselineLaunchCommands(device, seedUrl) + // Collected as the walk goes, by `act` below. Null the moment a step turns out to have no + // faithful CLI form — see `walkCommands` for why a truncated path is worse than none. + let walkLines: string[] | null = [] + try { + try { + if (commandSeed) { + // Running the parent's stored `pathToReach` rather than replaying its `.ad`: the + // lines open with their own launch and then press their way to the parent's screen, + // which is the same journey the script encoded and the same one `path-run.ts` runs + // for the Replay button. What it no longer involves is a candidate file, an armed + // `--save-script` transaction, a divergence report or an in-place repair — the four + // things `replaySeedBase` existed to manage, all of which were consequences of the + // seed being a recording rather than a list. + // + // Repair has no analogue here and needs none. A recorded `.ad` carried per-step + // element *identities* that a fresh accessibility tree could contradict + // (`IDENTITY_MISMATCH`, the failure the strip existed to rescue); these lines carry + // only selectors, which are re-resolved against the live tree on every run. + await ports.device.runPathCommands(commandSeed.commands) + checkSignal(signal) + } else { + // A soft re-root, not a relaunch. The cold start this used to pay was the price of + // the recorded script's `open --relaunch` first line, not of the walk — see + // `startReplayRecording`'s own doc comment, which says so — and the walk only needs + // to start somewhere known. `seedUrl` carries a deep-link seed's URL so the walk + // starts where the stored baseline line says it does. + await ports.device.resetToLaunch( + seedUrl, + Math.max(1_000, Math.min(90_000, budget.remainingMs)) + ) + checkSignal(signal) + } + } catch (error) { + // Only a `commandSeed` failing here means the *parent's own proof* did not hold up — + // the caller (`runVerifiedReach`'s `explore` step) uses this to decide whether it is + // still worth paying for a full, unseeded exploration afterwards. A plain launch + // failing here is the pre-existing case and always did fall to the outer catch before + // this branch existed; recorded the same way now, just sooner and with a clearer note. + if (commandSeed) seedBaseFailed = true + reachError = `${commandSeed ? "Seed's stored path" : 'Launch'} failed: ${error instanceof Error ? error.message : String(error)}` + console.log(`[app-flow] ${node.id} ${reachError}`) + return false + } + const launchAuth = await ports.auth.checkGate(sweepQueue.remaining, budget) + if (launchAuth === 'stop') { + authStopped = true + return false + } + if (launchAuth === 'exhausted') { + reachError = 'Navigation budget exhausted after five minutes.' + return false + } + const graphPath = findNavigationPath(parsed.edges, node.id, entryNodeIds) + // Which control opens this screen, and from where — the parser reads it off the call + // site, so the planner is told what to tap instead of hunting for it screen by screen. + const incoming = parsed.edges + .filter((edge) => edge.to === node.id) + .map((edge) => + edge.trigger + ? `from ${edge.from}: tap ${edge.trigger.by}=${JSON.stringify(edge.trigger.value)}` + : `from ${edge.from}: control unknown (${edge.raw})` + ) + // `node.group` names only the first navigator a shared registration helper happened to + // be called from — true for id/display purposes, but stating it alone as WHERE the + // screen lives is actively misleading when it is really registered in several tabs (see + // `groups` on `AppFlowNode`). Naming all of them here is what stops the planner reading + // e.g. "registered in layout: HomeTab" for a screen that is just as reachable from + // Search, Notifications, MyProfile or Messages. + const layoutLine = + node.groups.length > 1 + ? `Target is registered via a shared helper in every one of: ${node.groups.join(', ')} — it is reachable from any of them, not only the first.` + : `Target is registered in layout: ${node.group || 'unknown'}` + // Bounds this one exploration attempt's opportunistic spend — see + // `OPPORTUNISTIC_VERIFY_LIMIT`. A fresh budget per `exploreFrom` call (seeded attempt, + // then a from-entry retry) rather than per node, since each is its own walk through the + // app and each can pass its own hub screens. + let opportunisticAttempts = 0 + // The action that produced whatever the next `observed` call sees, if any — set at the + // end of a successful `act` below. A chrome revealed by a tap (a hamburger button + // opening a drawer) needs this to record a non-empty `reveal`; a chrome that needed no + // reveal at all (an always-visible tab bar) simply never has one, and that is correct + // too — see the `chrome-observed` fact this feeds in `observed` below. + let lastAction: AppFlowStep | null = null + // A seeded walk starts on a screen this sweep already proved it can reach, so the first + // hop out of it is recordable. An unseeded walk starts from a fresh relaunch, where the + // landing screen is whatever the app restores — so it starts at null and the first + // `observed` confirmation sets the position without claiming an edge into it. + const cursor = createWalkCursor(seed?.id ?? null) + walkCursor = cursor + // One session for this whole exploration attempt (Step 0): the model keeps its own + // turn-by-turn history, so only the first step needs the full target briefing — + // every step after that sends just the fresh screenshot, the fresh control list, and + // anything that went wrong since the last step (`history`, which by this point only + // ever holds failures — see the doc comment on `decide` in exploration.ts). + // No tools. Every one of this role's decisions is made from the screenshot and the + // control list it is handed — the prompt never asks it to consult a file, and the + // source path in the briefing is context for naming, not an invitation to open it. + // Granting `read` let a walk that could not find its target start reading the repo + // instead: one ProfileKnownFollowers step ran 42s and 324k tokens, and that single + // unreachable screen accounted for 73% of an entire sweep's bill. `maxDecisions` + // cannot bound this, because it counts decisions and the reading happens inside one. + // + // History compaction: the session keeps its turns so a step does not resend what came + // before, but the provider still re-reads all of them every step. Measured on device: + // within one walk a step-1 planner call cost ~1,900 input tokens and step 3 cost + // ~3,900, and the sweep logged 137,600 cache-read tokens across 20 planner calls — + // more than four times its entire reasoning spend and the largest single line in the + // run. Older turns keep the model's own decision and lose their screenshot and control + // list; the briefing and the two most recent observations stay whole. See + // `pi/history-compaction.ts` for the exact rules. + const navigationSession = await runAgentSession({ + cwd: projectPath, + systemPrompt: NAVIGATION_SYSTEM_PROMPT, + tools: [], + // `low`, and deliberately not `off` like the judges. This is the one app-flow role + // that makes a genuine multi-step decision — where am I, what did my last tap do, + // which control moves me toward a screen I have not seen yet — so it is the role most + // likely to pay for zero reasoning in failed walks, and a failed walk costs far more + // than the reasoning it saved. `low` is still a large cut from what this role was + // actually getting: nothing was reaching the wire before, so every call ran at the + // backend's xhigh default (see `pi/provider.ts`). Worth retrying at `off` once a + // sweep has shown what `low` does to the arrival rate. + thinkingLevel: 'low', + compactHistory: { keepRecentUserTurns: 2 } + }) + let stepsSent = 0 + let exploration: ExplorationResult + try { + exploration = await runInteractiveExploration({ + budget, + // The five-minute budget alone is not a bound on how much a hopeless target costs: + // it bounds wall clock, not model calls, and a fast app burns decisions far quicker + // than it burns minutes. This is the bound on calls — see the constant's comment for + // why seven, and why an unreachable screen is exactly the case that used to run long. + // A bounded attempt gets a much tighter version of the same reasoning: the seed + // already proved everything up to one tap away, so three unproductive decisions (or + // even one that makes no visible difference, or one failed planner call) says the + // target is not one more tap from here after all. + maxDecisions: exploreOptions.bounded ? 3 : EXPLORATION_MAX_DECISIONS, + maxNoProgress: exploreOptions.bounded ? 1 : undefined, + maxPlannerFailures: exploreOptions.bounded ? 1 : undefined, + observe: async () => { + // No auth gate here: the app was relaunched and classified immediately before + // this loop started, and mid-exploration taps cannot reach a launch gate. The + // frame is the decider's vision input only, so every step overwrites one scratch + // file and `screens/` keeps just the delivered capture. + const screenshotPath = getScratchPath(projectId, 'explore') + await ports.device.captureScreenshot(screenshotPath) + return { + screenshotPath, + pressables: await ports.device + .capturePressables(Math.max(1_000, Math.min(30_000, budget.remainingMs))) + .catch(() => []) + } + }, + check: () => checkSignal(signal), + decide: async (observation, history) => { + const visible = summarizePressables(observation.pressables) + const recentFailures = history.length + ? `\nSince the last step: ${JSON.stringify(history)}` + : '' + const controlsLine = `Visible controls now: ${JSON.stringify(visible.shown)}${visible.omitted ? ` (+${visible.omitted} more controls on this screen, not listed)` : ''}${recentFailures}\nChoose one next decision from this fresh observation.` + // A bounded attempt is seeded one hop from the target by construction — the + // planner is told so directly instead of being handed the same "find your way + // from an entry screen" briefing a full exploration gets, and the parent's own + // outgoing edge is handed over as a strong hint (never a command: a recorded + // control can drift from what the live screen shows). + const seedLine = commandSeed + ? `\nYou replayed a proven journey to ${commandSeed.id} and are one tap away from the target.${commandSeed.trigger ? ` The last known control for this hop: tap ${commandSeed.trigger.by}=${JSON.stringify(commandSeed.trigger.value)} — use it if it is still on screen, otherwise find the equivalent control.` : ''}` + : seed && 'url' in seed + ? `\nThe app was opened directly at ${seed.urlPath} (${seed.id}), so you start there rather than at the entry screen.` + : '' + const boundedLine = exploreOptions.bounded + ? '\nYou have very few actions left to find this screen from here — act only on a control you are confident leads to the target, and return blocked as soon as none does.' + : '' + const userPrompt = + stepsSent === 0 + ? `${appEntryBrief}${seedLine}${boundedLine}\nTarget screen: ${node.id} (${node.title})${deepLinkNotFound ? "\nOpening this route by URL landed on the app's not-found screen, so this build may not offer it at all. Confirm quickly via the named entry control and return blocked if it is absent." : ''}${unresolvedParams.length ? `\nThis route needs ${unresolvedParams.map((key) => `:${key}`).join(', ')}, so it shows one concrete item. If the entry chrome has a tab or menu item named after this screen, tap that first; it goes straight there. Only when no such entry exists, open any one item from a list that already leads here — which one does not matter.` : ''}\n${layoutLine}\nSource: ${node.file ?? 'unknown'}\nJourney from an entry screen: ${graphPath ? describeJourney(parsed.edges, graphPath) : 'none in the route graph — find it by exploring from the entry chrome'}\nWays into this screen: ${incoming.length ? incoming.join(' | ') : 'none found statically'}\n${controlsLine}` + : controlsLine + stepsSent += 1 + const response = await navigationSession.prompt( + { + userPrompt, + imagePath: observation.screenshotPath, + // One planning step is a look at a screenshot and a control list, so a call + // still running after a minute is stuck. Capping it here leaves the + // five-minute budget for the steps that follow instead of spending it all + // on one of them. + timeoutMs: Math.max(1_000, Math.min(60_000, budget.remainingMs)) + }, + `navigate ${node.id} step ${stepsSent}` + ) + return parseNavigationDecision(parseSubAgentJson(response.text)) + }, + act: async (action) => { + console.log(`[app-flow] explore ${node.id} action ${JSON.stringify(action)}`) + try { + await ports.device.executeNavigationActions( + [action], + Math.max(1_000, budget.remainingMs) + ) + lastAction = action + noteWalkAction(cursor, action) + // Recorded here, after the action succeeded, so the stored path contains + // exactly the steps the device actually performed — the same point the old + // recording transaction captured them at. A step with no faithful CLI form + // poisons the whole list rather than being skipped; see `walkCommands`. + if (walkLines) { + const line = walkStepCommand(action) + if (line) walkLines.push(line) + else if (action.type !== 'wait') walkLines = null + } + } catch (error) { + // The loop turns this into a history note for the planner. Without a log line the + // run reads as an action that silently did nothing to the screen. + console.log( + `[app-flow] explore ${node.id} action failed — ${error instanceof Error ? error.message : String(error)}` + ) + throw error + } + }, + verifyArrival: async (observation) => { + const arrival = await verifyCurrentTarget( + node, + observation.screenshotPath, + budget + ) + if (arrival.verdict !== 'match') return false + acceptedArrival = arrival + // Only when a single tap separates us from the last confirmed screen — the same + // conservatism `noteWalkArrival` applies before minting an edge, and for the same + // reason: with two or more actions in between, which one produced the arrival is a + // guess. Read before `noteArrival`, which resets the cursor's pending actions. + if ( + cursor.actionsSinceConfirmation.length <= 1 && + lastAction?.locator?.by === 'id' + ) { + paramStore.learnFromArrival(lastAction.locator.value, node, node.id) + } + // The walk's own target, confirmed — the strongest hop the sweep ever observes, + // since this is the judge the whole ladder exists to satisfy. + noteArrival(node.id) + // Kept for the destination-guard fallback below and for delivery: these are the + // controls of the screen the judge just accepted, and the only place they are + // still in hand once the recording has been published. Every later observation + // overwrites the scratch frame, and the loop returns no observation of its own. + arrivalPressables = observation.pressables + try { + const frame = getScratchPath(projectId, 'arrival-frame') + await copyFile(observation.screenshotPath, frame) + arrivalFrame = frame + } catch (error) { + // Copying the frame aside is a delivery convenience, never a condition of the + // arrival — the judge has already spoken on this screenshot. A failed copy costs + // the fallback frame and leaves the arrival exactly as accepted. + console.log( + `[app-flow] ${node.id} could not keep the verified arrival frame — ${error instanceof Error ? error.message : String(error)}` + ) + } + return true + }, + // Side observations: a tap made while walking to `node` can land on, or reveal, + // something worth remembering that has nothing to do with this walk's own target — + // another still-pending screen (the original opportunistic-capture case), a + // navigator chrome (plan Step 3d), or a screen the parser never registered a node + // for at all (plan Step 4a). This purely observes: it never influences what `decide` + // returns, never injects a step, and never lets `node` itself resolve via this path — + // `verifyArrival` above is the only way this walk's own target is accepted. Each + // check below reports what it found as a `DiscoveredFact` rather than writing + // `captureMap`/`runtimeNodes`/`chromeInventory` directly, so this hook stays ignorant + // of the graph it feeds — see `DiscoveredFact`'s doc comment (`reach/types.ts`) and + // `applyDiscoveredFact` above, which is what actually folds one in. + observed: async (observation) => { + if (budget.remainingMs < OPPORTUNISTIC_MIN_REMAINING_MS) return + const observedTokens = fingerprint(observation.pressables) + + // 1. A tap made while walking to `node` landed on some other screen the sweep + // already has a parsed node for. Never the node this walk is already trying to + // reach, never a node the sweep has already settled (anything with a + // `captureMap` entry), and never a route with no in-app entry point at all. + // + // Only when the observation matches no known runtime node: a match there is a + // better answer than a title-overlap guess, and the later `matchRuntimeNode` + // call in this block will use it, so spending a model call on a sibling-prone + // guess first would only pay for an answer the sweep already has. + if ( + opportunisticAttempts < OPPORTUNISTIC_VERIFY_LIMIT && + !matchRuntimeNode(runtimeNodes, observedTokens) + ) { + const pending = nodesWithSlug.filter( + (other) => + other.id !== node.id && !captureMap[other.id] && !isFallbackRoute(other) + ) + const known = pending.length + ? pickOpportunisticCandidate(observedTokens, pending) + : null + if (known) { + opportunisticAttempts += 1 + const arrival = await verifyCurrentTarget( + known, + observation.screenshotPath, + budget + ) + if (arrival.verdict === 'match') { + await applyDiscoveredFact({ + kind: 'opportunistic-capture', + nodeId: known.id, + arrival, + screenshotPath: observation.screenshotPath, + pressables: observation.pressables + }) + return + } + console.log( + `[app-flow] opportunistic ${known.id} rejected while exploring ${node.id} — ${arrival.reason}` + ) + } + } + + // 2. A navigator chrome — pure token matching, no vision call, so it never + // competes with the two model-backed checks above/below for the walk's own + // opportunistic budget. + const chromeDetected = detectChromeInventory( + observation.pressables, + chromeCandidates() + ) + if (chromeDetected) { + // `node` is where this walk is *going*, not where it is standing — it has not + // been reached yet, or the walk would be over. Attributing the reveal to it said + // the drawer was opened from a screen the explorer had never seen, which is how a + // device run ended up recording the Settings screen's rows as a chrome revealed + // from `HomeTab/Home`. The cursor holds the last position an arrival judge + // actually confirmed; the target is only the fallback when nothing has been + // confirmed yet. + const revealedFrom = walkCursor?.confirmedNodeId ?? node.id + // Only a tap that reads as a drawer opener is a reveal. `lastAction` is just + // whatever the walk happened to tap before this frame, so passing it through + // unconditionally recorded an always-visible tab bar as needing a reveal — and + // `buildChromePathToReach` then published that tap as the first step of the + // journey to every screen behind it. On device that produced paths beginning + // `press 'id="feedItem-by-gregpak.net"'` — tapping a post in the feed — as the + // documented way to reach 40 different screens. + const reveal = + lastAction && inferChromeKind([lastAction]) === 'drawer' ? [lastAction] : [] + await applyDiscoveredFact({ + kind: 'chrome-observed', + entryNodeId: revealedFrom, + reveal, + items: chromeDetected.items + }) + return + } + + // 3. A screen the parser never registered a node for at all. Only worth spending a + // model call on once neither cheap check above accounted for this frame, and + // only within the walk's own opportunistic budget and the sweep-wide mint cap. + if (opportunisticAttempts >= OPPORTUNISTIC_VERIFY_LIMIT) return + if (runtimeNodes.length >= RUNTIME_NODE_SWEEP_CAP) return + // Similarity-only pass first, with no id yet: a repeat sighting of an + // already-minted runtime node is recognised for free, without spending the model + // call the judge check below costs. + const knownRuntime = matchRuntimeNode(runtimeNodes, observedTokens) + if (knownRuntime) { + await applyDiscoveredFact({ + kind: 'runtime-screen-sighted', + match: knownRuntime, + key: knownRuntime.id, + info: '', + observedTokens: [...observedTokens], + screenshotPath: observation.screenshotPath, + pressables: observation.pressables + }) + return + } + opportunisticAttempts += 1 + const distinctResponse = await runAgent( + { + cwd: projectPath, + systemPrompt: RUNTIME_SCREEN_SYSTEM_PROMPT, + userPrompt: 'Is this a distinct app screen not already accounted for?', + tools: [], + // A classification of the frame in front of it, like the arrival judge — real + // screen or transient overlay — so `off` for the same reason. This one is also + // purely opportunistic: it runs on frames a walk happened to pass through, and + // declining to mint a node is always a safe answer. + thinkingLevel: 'off', + imagePath: observation.screenshotPath, + timeoutMs: Math.max(1_000, Math.min(30_000, budget.remainingMs)) + }, + 'runtime-screen' + ).catch((error) => { + console.log( + `[app-flow] runtime-screen check failed while exploring ${node.id} — ${error instanceof Error ? error.message : String(error)}` + ) + return null + }) + const verdict = distinctResponse + ? parseSubAgentJson<{ distinct: boolean; key?: string; info?: string }>( + distinctResponse.text + ) + : null + if (!verdict?.distinct || !verdict.key) return + const candidateId = `runtime:${sanitizeScreenKey(verdict.key) || 'screen'}` + const exactMatch = matchRuntimeNode(runtimeNodes, observedTokens, candidateId) + await applyDiscoveredFact({ + kind: 'runtime-screen-sighted', + match: exactMatch, + key: verdict.key, + info: verdict.info ?? '', + observedTokens: [...observedTokens], + screenshotPath: observation.screenshotPath, + pressables: observation.pressables + }) + } + }) + } finally { + navigationSession.dispose() + // The walk is over, however it ended. Leaving the cursor in place would let the next + // confirmation from anywhere else in the sweep read the screen this walk stopped on as + // its own starting point and mint an edge across a relaunch that never happened. + walkCursor = null + } + if (!exploration.arrived) { + seededWalkRan = true + reachError = exploration.reason + console.log(`[app-flow] ${node.id} exploration stopped — ${exploration.reason}`) + return false + } + + // The whole publish-and-verify tail used to live here: pick a portable destination + // guard, `saveAndReplayRecording` the candidate, survive its failure, rename the + // candidate over the published `.ad`. Every line of it was in service of the recording, + // and on the measured sweep every single exploration-based capture failed somewhere in + // it — four walks, two distinct causes — so the arrival was kept and the script was + // lost anyway. What is stored instead is the walk itself. + // + // Nothing here can fail in a way that costs the arrival: `walkLines` was assembled as + // the walk went, and a walk whose steps had no faithful CLI form simply stores no path + // (`walkCommands`'s all-or-nothing rule) while keeping the screen reached, exactly as + // the missing-landmark case used to. + if (!walkLines) { + console.log( + `[app-flow] ${node.id} arrival preserved without a stored path — a walked step had no faithful agent-device form` + ) + return true + } + // Minimized before it is stored, never while it is being walked: `act` records what + // the device did, and the walk's own history is what the planner reasons over. The + // trim applies to this walk's lines only — `prefix` is either a baseline launch or a + // parent's already-stored journey, neither of which this walk is entitled to rewrite + // (see `minimizeWalkCommands`). + const minimized = minimizeWalkCommands(walkLines) + pathCommands = [...prefix, ...minimized.commands] + const contentSelectors = contentMatchedSelectors(pathCommands) + console.log( + `[app-flow] ${node.id} stored a ${pathCommands.length}-command path${ + commandSeed ? ` seeded from ${commandSeed.id}` : '' + }${ + minimized.removed.length + ? ` (minimized: dropped ${minimized.removed.length} — ${minimized.removed + .map((entry) => `${entry.rule}: ${entry.command}`) + .join('; ')})` + : '' + }${ + contentSelectors.length + ? ` — matches on rendered content, may not replay elsewhere: ${contentSelectors.join(', ')}` + : '' + }` + ) + return true + } catch (error) { + checkSignal(signal) + reachError = `Interactive exploration failed: ${error instanceof Error ? error.message : String(error)}` + // Every other exit from `exploreFrom` logs a line naming the node and the reason + // (the `!exploration.arrived` branch above, the explicit `reachError` assignments), + // so a candidate discarded here for an uncaught error was the one silent path — the + // device run this fix responds to jumped straight from the arrival judge's verdict to + // the next node with nothing in between to explain why the screen was lost. + console.log(`[app-flow] ${node.id} exploration failed — ${reachError}`) + return false + } + } + + // Effect folding — plan landing-order item 4. `savedReplayStrategy`/`deepLinkStrategy` + // (`reach/strategies/`) are pure: they read `ports` and return a `ReachAttempt`, never + // reaching into this closure's state themselves. This is the one place their `effects` + // turn into the sweep-level state `trySavedReplay`/`tryDeepLink` used to mutate directly — + // see `reach/types.ts`'s doc comment on `ReachEffect` for why each kind exists. + const scratchPath = (key: string): string => getScratchPath(projectId, key) + const applyReachEffects = (effects: ReachAttempt['effects']): void => { + for (const effect of effects) { + switch (effect.kind) { + case 'deep-link-opened': + openedByNode[effect.nodeId] = effect.opened + // Every URL the app really opened is a complete, co-occurring param set for + // whichever route it matches — the cleanest record there is, and it may belong to a + // different route than the one being reached. + paramStore.ingestUrl(effect.opened.url, effect.nodeId, nodesWithSlug) + break + case 'deep-link-rejection-recorded': + deepLinkRejections[effect.nodeId] = effect.rejection + console.log( + `[app-flow] ${effect.nodeId} recording deep-link rejection so later sweeps skip ${effect.rejection.url}` + ) + break + case 'deep-link-rejection-cleared': + deepLinkRejections[effect.nodeId] = null + console.log( + `[app-flow] ${effect.nodeId} deep link succeeded — clearing stored rejection` + ) + break + case 'deep-link-memo-rejected': + rejectedDeepLinkUrls.reject(effect.url) + break + case 'guard-gate-closed': + closedGuardGates.close(effect.condition) + break + case 'deep-link-not-found': + deepLinkNotFound = true + break + case 'deep-link-breaker-success': + deepLinkBreaker = registerDeepLinkSuccess(deepLinkBreaker) + break + case 'deep-link-breaker-failure': + deepLinkBreaker = registerDeepLinkFailure( + deepLinkBreaker, + Boolean(probeRoute) + ).state + break + case 'deep-link-breaker-disabled': + deepLinkBreaker = { ...deepLinkBreaker, enabled: false } + break + case 'deep-link-partial-landed': + // Consumed synchronously by the deep-link rung below, in the same call that + // produced it — see that rung's own comment for why this cannot wait for a later + // strategy to read it back out of `effects`. + break + } + } + } + + /** + * Claims the reach the one-time deep-link probe already performed and verified, instead of + * performing the identical one again — see `probeCredit`. + * + * Reports itself as `'deep-link'` rather than inventing a strategy name, because that is + * honestly how the app got here: the probe opened this node's URL and a judge confirmed + * the frame. Every downstream consumer (`captureMethod`, `node.reach`, `decideDelivery`) + * is then looking at the same evidence it would have had, from the same transport, and + * none of them needs to learn a new case. The effects match the deep-link strategy's own + * success path for the same reason. + * + * `touchedDevice: false` is the one place it differs, and it is accurate: this rung makes + * no device call at all. That is the entire point — the device is already on the screen. + */ + const probeCreditRung: ReachStrategy = async () => { + const credit = claimProbeCredit(node.id) + if (!credit) { + return { + strategy: 'deep-link', + ok: false, + reason: 'No verified probe reach is available for this node.', + touchedDevice: false, + elapsedMs: 0, + effects: [] + } + } + const effects: ReachEffect[] = [{ kind: 'deep-link-breaker-success' }] + if (credit.opened) { + effects.push({ kind: 'deep-link-opened', nodeId: node.id, opened: credit.opened }) + } + applyReachEffects(effects) + console.log( + `[app-flow] reach ${node.id} — reusing the deep-link probe's verified arrival, no second navigation` + ) + return { + strategy: 'deep-link', + ok: true, + reason: credit.arrival.reason, + touchedDevice: false, + arrival: credit.arrival, + evidence: { + arrival: credit.arrival, + frame: credit.frame, + pressables: credit.pressables, + replayScript: null, + replayVerified: false + }, + elapsedMs: 0, + effects + } + } + + const savedReplayRung: ReachStrategy = (_input, p) => + savedReplayStrategy( + { + node, + scriptPath: savedScript, + budget, + remaining: sweepQueue.remaining, + scratchPath, + signal + }, + p + ) + + // `tryDeepLink`'s pure open-verify-record-rejection sequence, moved to + // `reach/strategies/deep-link.ts`. One entanglement stays here rather than in that module: + // a param-free prefix landing on a real screen (Step 3f) used to call `exploreFrom` for a + // bounded last hop *from inside* `tryDeepLink`, and a successful hop was recorded as method + // `'deep-link'` (so `node.reach` becomes `'deep-link'`, not `'navigation-only'`) — not as an + // exploration success. `exploreFrom` has not moved out of this file yet (landing-order item + // 5), so the pure strategy cannot call it without reaching back into this closure; instead + // it reports the landing as a `deep-link-partial-landed` effect and this wrapper — still + // "exploreFrom, wrapped as a strategy" per the plan — performs the same bounded hop the + // closure used to, preserving the exact method/reach classification. + /** The controls last read off this build's own not-found route, when anything captured it. */ + const capturedFallbackPressables = (): AppFlowPressable[] | null => { + for (const candidate of nodesWithSlug) { + if (!isFallbackRoute(candidate)) continue + const captured = captureMap[candidate.id]?.pressables + if (captured?.length) return captured + } + return null + } + + const deepLinkRung: ReachStrategy = async (_input, p) => { + const attempt = await deepLinkStrategy( + { + node, + unresolvedParams, + params, + budget, + scratchPath, + signal, + breaker: deepLinkBreaker, + storedRejection: deepLinkRejections[node.id], + memoRejected: (url) => rejectedDeepLinkUrls.wasRejected(url), + guardGateClosed: (condition) => closedGuardGates.wasClosed(condition), + nodesForAncestry: nodesWithSlug.map((candidate) => ({ + id: candidate.id, + urlPath: candidate.urlPath, + capturedStatus: captureMap[candidate.id]?.status ?? null + })), + reprobe: reprobeDeepLinkRoute, + arrivalJudge: deepLinkArrivalJudge, + // Only assembled when it will be read. The build's own fallback route is the one + // build-specific signature the judge-free check has; `isFallbackRoute` is the same + // predicate the sweep order and this loop's own fallback handling use, so the three + // cannot disagree about which route that is. Usually null on a first sweep (fallback + // routes sort last) and populated on a resumed one, where the capture is read back + // off the document — see `detectNotFoundScreen`'s rule 1. + notFoundEvidence: deepLinkArrivalJudge + ? undefined + : { fallbackPressables: capturedFallbackPressables() } + }, + p + ) + applyReachEffects(attempt.effects) + if (attempt.ok || attempt.failure) return attempt + + const partial = attempt.effects.find( + (effect) => effect.kind === 'deep-link-partial-landed' + ) + if (!partial || partial.kind !== 'deep-link-partial-landed') return attempt + console.log( + `[app-flow] ${node.id} prefix ${partial.seed.urlPath} landed on a real screen — attempting a bounded last hop for the remaining segment` + ) + if ( + await exploreFrom( + { id: partial.seed.id, urlPath: partial.seed.urlPath, url: partial.seed.url }, + { bounded: true } + ) + ) { + openedByNode[node.id] = { app: partial.seed.app, url: partial.seed.url } + paramStore.ingestUrl(partial.seed.url, node.id, nodesWithSlug) + deepLinkBreaker = registerDeepLinkSuccess(deepLinkBreaker) + if (deepLinkRejections[node.id]) { + deepLinkRejections[node.id] = null + console.log( + `[app-flow] ${node.id} bounded last hop succeeded — clearing stored rejection` + ) + } + return { + ...attempt, + ok: true, + reason: 'Bounded last hop from the partial prefix reached the target.', + arrival: acceptedArrival, + evidence: { + arrival: acceptedArrival, + pathCommands, + replayScript, + replayVerified: replayScriptVerified + } + } + } + if (authStopped) return { ...attempt, failure: 'auth-stopped' } + if (!budget.expired) { + console.log( + `[app-flow] ${node.id} bounded last hop from ${partial.seed.urlPath} did not reach — falling back to full exploration — ${reachError}` + ) + } + return attempt + } + + // `runVerifiedReach`'s old `explore` step, unchanged: `findSeedSource` (plan Step 3e) picks + // the best proof this sweep has that some other screen leads here, and `exploreFrom` itself + // has not moved out of this file yet (landing-order item 5) — this is exactly "exploreFrom, + // wrapped as a strategy" per the plan for this step. + const exploreRung: ReachStrategy = async () => { + const runExplore = async (): Promise => { + const seed = findSeedSource({ + urlPath: node.urlPath, + targetId: node.id, + nodes: nodesWithSlug, + edges: parsed.edges, + // Any journey this sweep has stored, not only one a *previous* sweep re-verified. + // The old test was `capture.replayScript`, and before that `replayVerified` — which + // could only ever be set by a previous sweep's saved replay, so seeding never fired + // within one run and every walk re-walked the hops its own parent had just walked. + // + // `pathCommands` is the honest test now, and strictly better than the script ever + // was: it is set by exactly the walks that arrived, it stands alone (its first line + // is its own launch), and running it needs no file on disk. A capture carried + // forward from a previous sweep brings its own along, so a resumed run seeds from it + // too. + storedPath: (id) => captureMap[id]?.pathCommands ?? null, + deepLinkedThisRun: (id) => openedByNode[id]?.url ?? null + }) + + if (seed?.kind === 'path') { + console.log( + `[app-flow] ${node.id} bounded last hop from ${seed.id}'s stored path${seed.trigger ? ` — tap ${seed.trigger.by}=${JSON.stringify(seed.trigger.value)}` : ''}` + ) + if ( + await exploreFrom( + { id: seed.id, commands: seed.commands, trigger: seed.trigger }, + { bounded: true } + ) + ) { + return true + } + if (authStopped || budget.expired) return false + if (!seedBaseFailed) { + // The parent's own proof held — its script really did keep an active session — and + // the bounded loop still ran out its three decisions without arriving. That is not + // "approached badly", it is the seed's own evidence that this target is not one tap + // from there, so a full unseeded walk is not worth its five-minute price here; the + // node records unreached with the bounded loop's own reason instead. + return false + } + console.log( + `[app-flow] ${node.id} seed's stored path failed — falling back to full exploration — ${reachError}` + ) + } else if (seed?.kind === 'deep-link') { + console.log(`[app-flow] ${node.id} explore seeded at ${seed.urlPath} — ${seed.url}`) + if (await exploreFrom({ id: seed.id, urlPath: seed.urlPath, url: seed.url })) + return true + if (authStopped || budget.expired || unavailableReason) return false + if (!shouldRetryFromEntry({ walkRan: seededWalkRan, reason: reachError })) { + console.log( + `[app-flow] ${node.id} seeded exploration ran its decisions without arriving — not retrying from the app entry, it would have nothing new to try — ${reachError}` + ) + return false + } + console.log( + `[app-flow] ${node.id} seeded exploration failed, retrying from the app entry — ${reachError}` + ) + } + return exploreFrom(null) + } + + const exploreStartedAt = Date.now() + const ok = await runExplore() + return { + strategy: 'entry-exploration', + ok, + failure: !ok && authStopped ? 'auth-stopped' : undefined, + reason: ok + ? (acceptedArrival?.reason ?? 'Exploration arrived.') + : (reachError ?? 'Exploration failed.'), + touchedDevice: true, + arrival: acceptedArrival, + evidence: ok + ? { + arrival: acceptedArrival, + // Always null since verification moved to first use: nothing on the exploration + // path takes a post-replay frame any more, so delivery takes its own frame and + // read — the fallback this field always had. + frame: null, + pressables: null, + pathCommands, + replayScript, + replayVerified: replayScriptVerified + } + : undefined, + // Measured rather than the 0 this used to report: the `reach + // ()` line reads this field, and the most expensive rung in the ladder logged as + // free. + elapsedMs: Date.now() - exploreStartedAt, + effects: [] + } + } + + /** + * Navigate through the app's own navigator, for screens a URL cannot address. + * + * It reuses `params`/`unresolvedParams` exactly as the deep-link rung does, so a route + * whose values the store could not fill is skipped here too rather than dispatched with a + * hole in it. Arrival stays the judge's call; see `reach/strategies/render.ts`. + */ + const renderRung: ReachStrategy = (_input, p) => + renderStrategy({ node, params, unresolvedParams, budget, scratchPath }, p) + + const outcome = await runReachLadder({ + input: null, + ports, + budget, + signal, + // `probeCreditRung` first: it is the only rung that can succeed without touching the + // device at all, and when it has nothing to claim it falls through for free. Ahead of + // the saved replay because a replay would move the app off the very screen the credit + // is a statement about. + // `renderRung` between the deep link and exploration: a deep link is the more faithful + // journey and produces a reusable replay, so it keeps priority, but a navigator + // dispatch is far cheaper and far more predictable than asking a model to find the + // screen. See `reach/strategies/render.ts`. + // + // Which of the last two rungs are present is the user's choice — see `strategiesFor`. + strategies: strategiesFor(options.explorationMode, { + probeCredit: probeCreditRung, + savedReplay: savedReplayRung, + deepLink: deepLinkRung, + render: renderRung, + explore: exploreRung + }) + }) + + // Every rung the ladder tried, in order — the decline reasons that used to be dropped on + // the floor here. `touchedDevice` is the distinction that matters when reading a sweep + // back: `declined` short-circuited before any device call (an unresolved param, a tripped + // breaker, a cached rejection) and cost nothing, while `failed` actually ran and did not + // land. A rung that never fires and a rung that fires and misses need opposite fixes, and + // without this line the two are indistinguishable in the log — which is how a sweep came + // back showing 2 renders out of 42 screens with no record of why the other 40 skipped it. + // + // The winner is logged too, in the same `reach ()` shape, because a + // ladder that only narrates its misses leaves the rung that *worked* to be inferred from a + // separate `verify … via ` line — and the suffix names a diagnostic frame, not a + // rung. Its reason is left off: for a winner that is the arrival judge's description of the + // frame, which can quote whatever the screen shows (a handle, a record name), and the + // `verify` line already carries it. + for (const attempt of outcome.attempts) { + if (attempt.ok) { + console.log( + `[app-flow] ${node.id} reach ${attempt.strategy} succeeded (${attempt.elapsedMs}ms)` + ) + continue + } + const how = attempt.touchedDevice ? 'failed' : 'declined' + console.log( + `[app-flow] ${node.id} reach ${attempt.strategy} ${how} (${attempt.elapsedMs}ms) — ${attempt.reason}` + ) + } + + // What the ten mutable variables (`openedByNode`, `consecutiveDeepLinkFailures`, + // `deepLinkEnabled`, `authStopped`, `deepLinkNotFound`, `unavailableReason`, + // `deepLinkAttempted`, `reachError`, `acceptedArrival`, `replayScript`) used to be set to, + // directly, by whichever closure happened to succeed — now derived in one place from the + // ladder's own `outcome`. + if (outcome.status === 'reached') { + const winner = outcome.attempts[outcome.attempts.length - 1] + captureMethod = + winner.strategy === 'deep-link' + ? 'deep-link' + : winner.strategy === 'saved-replay' + ? 'saved-replay' + : winner.strategy === 'render' + ? 'rendered' + : 'agent-navigation' + if (winner.evidence) { + acceptedArrival = winner.evidence.arrival + pathCommands = winner.evidence.pathCommands ?? pathCommands + replayScript = winner.evidence.replayScript + replayScriptVerified = winner.evidence.replayVerified + verifiedFrame = winner.evidence.frame ?? null + verifiedPressables = winner.evidence.pressables ?? null + } else if (winner.arrival) { + acceptedArrival = winner.arrival + } + } else if (outcome.status === 'unavailable') { + unavailableReason = outcome.reason + } else if (outcome.status === 'stopped') { + if (outcome.reason === 'auth') authStopped = true + // A 'cancelled' stop only happens when `signal` was already aborted between rungs; every + // other cancellation path still throws directly (`checkSignal` inside a strategy). Doing + // the same here — rather than treating this as an ordinary "give up on this node" outcome + // — keeps a cancelled build behaving exactly as it did before this extraction: it unwinds + // out of `buildAppFlow` entirely instead of moving on to record this node as unreached. + else checkSignal(signal) + } else { + reachError = outcome.reason + } + const deepLinkAttempted = outcome.attempts.some( + (attempt) => attempt.strategy === 'deep-link' && attempt.touchedDevice + ) + + if (authStopped) sweepStopped = true + + if (sweepStopped) break + + // A harvested value can name a record the app no longer has, or one that is valid for the + // route that owns it and not for a sibling that borrowed it. This runs whatever the node + // ended up as: exploration frequently rescues a node whose deep link or render just failed + // on a bad value, and withdrawing the value only on an unreached verdict left every other + // borrower to pay for the same failure. + if (!unavailableReason) + demoteFailedHarvest(paramStore, node.id, resolution, outcome.attempts) + + if (captureMethod === 'unreached') { + if (unavailableReason) { + await recordUnreached(node, 'unavailable', unavailableReason) + emit(projectId, onEvent, { + type: 'step', + phase: 'capture', + message: `${label} is not available in this build`, + completed: sweepQueue.completed + 1, + total + }) + if (finishExploredScreen(false)) break + // A screen this build withholds is a verdict, not a stuck app, so it must not feed the + // no-progress counter that decides the sweep is blocked and asks the user for help. + continue + } + await recordUnreached(node, 'missing', reachError) + if (finishExploredScreen(false)) break + // A planner that ran out of time left the app wherever the last action put it, which says + // nothing about the app being stuck — and sibling screens look so alike that counting + // these would stop the sweep to ask the user to clear a blocker that is not there. + if (isToolingFailure(reachError)) continue + const current = fingerprint(await capturePressables(ctx, device).catch(() => [])) + const overlap = failedReference ? similarity(current, failedReference) : 0 + if (failedReference && overlap >= ARRIVAL_SIMILARITY) blockedRun += 1 + else { + failedReference = current + blockedRun = 1 + } + if (blockedRun >= BLOCKED_RUN_LIMIT) { + const resolution = await resolveBlocked(sweepQueue.remaining - 1) + // The retry is for a *stuck app* — the blocker cleared, so the walk that failed against + // it deserves another run. It is not for a node the planner reasoned its way out of: + // `ModerationInboxReportDetails` concluded "no safe action reaches the target", the + // relaunch cleared the unrelated blocked-run counter, and the node was walked a second + // time to reach the same conclusion for a second full planner price. One retry per node + // keeps the recovery the breaker exists for and drops the repeat. + if (resolution === 'continue') { + if (requeuedIds.has(node.id)) { + console.log( + `[app-flow] ${node.id} — not retrying a second time; this run already reached the same verdict once` + ) + continue + } + requeuedIds.add(node.id) + sweepQueue.requeueCurrent() + continue + } + sweepStopped = true + break + } + continue + } + + // The static parse only ever guesses `reach` — this is where the device settles it. A + // deep link that actually matched proves the route routes; landing here instead via + // exploration or a saved replay, after a deep link was attempted and failed, proves the + // opposite. Anything not covered by either (no urlPath, deep link never attempted this + // node) keeps the parser's guess rather than being overwritten on no evidence. + if (captureMethod === 'deep-link') { + node.reach = 'deep-link' + } else if ( + (captureMethod === 'agent-navigation' || captureMethod === 'saved-replay') && + deepLinkAttempted + ) { + node.reach = 'navigation-only' + } + + emit(projectId, onEvent, { + type: 'step', + phase: 'capture', + message: `Capturing ${label}`, + completed: sweepQueue.completed + 1, + total + }) + + // The screen is reached and verified by this point, so a failing device command here costs + // this one screen's detail — never the sweep. Losing the remaining screens to a transient + // runner fault (a watchdogged accessibility capture is the common one) would throw away + // every capture still ahead of it. See `decideDelivery` (reach/delivery.ts): a verified + // arrival or a published replay must survive a screenshot failure here, not be discarded. + const screenshotPath = getScreenshotPath(projectId, node.slug) + console.log( + `[app-flow] capture ${sweepQueue.completed + 1}/${total} screenshot ${node.urlPath}` + ) + // Captured to scratch first, never straight to the node's slug: on the missing-session + // recovery branch the frame is the launch screen (`relaunched: true`), and writing that + // under this node's filename would silently corrupt an existing good screenshot even + // though the delivery record below correctly declines to treat it as reached evidence. + const deliveryShot = getScratchPath(projectId, 'delivery') + const relativePath = `screens/${node.slug}.png` + // Either the frame the judge already accepted, or the one taken just below. Both are this + // reach's own look at the screen, which is what `chooseDeliverySources` means by 'live'; + // the difference is only whether a second screenshot had to be spent to get one. + const liveFrame = verifiedFrame ?? deliveryShot + let liveScreenshot: DeliveryScreenshotOutcome + if (verifiedFrame) { + // A strategy that reports `evidence.frame` reached this screen and then stopped touching + // the device, so the frame the judge accepted is still what the screen shows — only the + // judge call itself sits between the two moments. Taking a second screenshot of it cost a + // measured ~0.6s per screen to produce a near-identical image, and the one delivered was + // the one *nothing* had verified. A `relaunched` frame cannot arrive here: it is turned + // into an automatic mismatch inside `verifyWithDiagnostic`, so no reach that reports a + // frame ever succeeded on one. + console.log(`[app-flow] ${node.id} delivering the frame the arrival judge verified`) + liveScreenshot = { ok: true, relativePath } + } else { + try { + const { relaunched } = await ports.device.captureScreenshot(deliveryShot) + liveScreenshot = relaunched + ? { ok: false, reason: 'session was lost; the app was relaunched at its launch URL' } + : { ok: true, relativePath } + } catch (error) { + checkSignal(signal) + const detail = error instanceof Error ? error.message : String(error) + console.log(`[app-flow] ${node.id} screenshot failed — ${detail}`) + liveScreenshot = { ok: false, reason: `Screenshot failed: ${detail}` } + } + } + + console.log( + `[app-flow] capture ${sweepQueue.completed + 1}/${total} pressables ${node.urlPath}` + ) + let livePressables: AppFlowCapture['pressables'] = [] + let livePressablesError: string | null = null + if (verifiedPressables) { + // Read during the arrival judge's own latency rather than after it — see + // `withPressables` in `verify-with-diagnostic.ts`. Still this reach's own read of this + // screen, and taken at the same instant as the frame delivered above, so the two halves + // of the record now describe one moment instead of two. + livePressables = [...verifiedPressables] + } else { + try { + livePressables = await ports.device.capturePressables() + } catch (error) { + checkSignal(signal) + livePressablesError = error instanceof Error ? error.message : String(error) + console.log(`[app-flow] ${node.id} pressables failed — ${livePressablesError}`) + } + } + console.log( + `[app-flow] capture ${sweepQueue.completed + 1}/${total} pressables found ${livePressables.length} ${node.urlPath}` + ) + + // Both halves of the record come from the same judgement about whether the live device is + // still on this screen — see `chooseDeliverySources`. Note that the control read above runs + // either way: it is the fresher source whenever it can be trusted, and finding that out + // costs one device call that the screenshot has already established is safe to spend. + const sources = chooseDeliverySources({ + liveScreenshot, + livePressables, + livePressablesError, + arrivalFramePath: arrivalFrame, + arrivalPressables + }) + // This is where a captured screen's testIDs become openable URLs for *other* routes: a + // `profileCard-alice.test-link` on this screen is the value `/profile/:name` could not get + // from source text. + paramStore.ingestPressables(node.id, sources.pressables.pressables, nodesWithSlug) + await harvestNavState(node) + let screenshotOutcome: DeliveryScreenshotOutcome + if (sources.screenshot.source === 'none') { + screenshotOutcome = { ok: false, reason: sources.screenshot.reason } + } else { + const frame = + sources.screenshot.source === 'live' ? liveFrame : sources.screenshot.framePath + try { + await copyFile(frame, screenshotPath) + screenshotOutcome = { ok: true, relativePath } + if (sources.screenshot.source === 'arrival') { + console.log( + `[app-flow] ${node.id} delivered the verified arrival frame; the live device had moved on` + ) + } + } catch (error) { + const detail = error instanceof Error ? error.message : String(error) + console.log(`[app-flow] ${node.id} could not store the delivery frame — ${detail}`) + screenshotOutcome = { ok: false, reason: `Screenshot unavailable: ${detail}` } + } + } + if (sources.pressables.source === 'arrival') { + console.log( + `[app-flow] ${node.id} delivered the verified arrival's controls; the live read was taken after the device had moved on` + ) + } + + const finalArrival = acceptedArrival as ArrivalResult | null + if (finalArrival) { + inferredScreens[node.id] = { + key: finalArrival.key, + info: finalArrival.info + } + } + failedReference = null + blockedRun = 0 + + const evidence: ReachEvidence = { + arrival: finalArrival, + pathCommands, + replayScript, + replayVerified: replayScriptVerified + } + const delivery = decideDelivery({ + kind: 'attempt', + evidence, + screenshot: screenshotOutcome, + pressables: sources.pressables.pressables, + pressablesError: sources.pressables.error, + pressablesSource: sources.pressables.source, + captureMethod: captureMethod as NonNullable, + structureRevision + }) + captureMap[node.id] = delivery.capture + // A chrome edge is minted the moment the drawer or tab bar is *seen*, so it starts life + // unverified and scriptless — `chromeEdges` has nothing better to record at that point. + // Once this same destination is actually reached and a replay script published for it, the + // edge's own claim has been proven by a walk, so upgrade it. This is what makes + // `buildChromePathToReach`'s preferred `agent-device replay` form reachable at all: nothing + // else in the sweep ever sets `evidence.replayScript`, and without this the chrome path + // would forever fall back to replaying the reveal and item taps by selector. + if (delivery.capture.replayVerified && delivery.capture.replayScript) { + for (let at = 0; at < runtimeEdgesList.length; at++) { + const edge = runtimeEdgesList[at] + if (edge.to !== node.id || !edge.evidence) continue + runtimeEdgesList[at] = { + ...edge, + evidence: { + ...edge.evidence, + verified: true, + replayScript: delivery.capture.replayScript + } + } + } + } + await persistCaptureUpdate() + if ( + finishExploredScreen(delivery.kind === 'reached' && Boolean(delivery.capture.screenshot)) + ) + break + } + + if (options.pauseRequested?.()) sweepPaused = true + if (sweepStopped) { + manifest.sweepStatus = sweepPaused ? 'paused' : 'stopped' + // This path used to lean on `persistCaptureUpdate` having already written the capture map + // and the screens manifest after the last screen. That write is now only a checkpoint, so + // the stopped path publishes both itself, from the same state the document is built from. + await writeCaptureStatus(projectId, captureMap) + const stoppedScreens = currentAgentDeviceScreens() + await writeAgentDeviceScreens(projectId, stoppedScreens) + const screenNodes = withReplayableCaptures( + [ + ...nodesWithSlug.map((node) => ({ + ...node, + capture: captureMap[node.id] ?? node.capture, + deepLinkRejection: deepLinkRejections[node.id] ?? null + })), + ...runtimeNodes + ], + stoppedScreens + ) + const stoppedNavigators = navigatorChrome(chromeInventory, screenNodes) + const stoppedNodes = [...screenNodes, ...chromeVirtualNodes(stoppedNavigators)] + const stoppedResult: AppFlowResult = { + manifest, + nodes: stoppedNodes, + edges: parsed.edges, + runtimeEdges: rebuildChromeEdges(runtimeEdgesList, stoppedNavigators, stoppedNodes), + chrome: chromeInventory, + navigation + } + await finishManifest(manifest) + await writeAppFlow(projectId, stoppedResult) + // Persisted once per sweep rather than per screen: the store only matters to the *next* + // run, and `persistCheckpoint` already writes on every screen. + await writeParamStore(projectId, paramStore.toJSON()).catch(() => undefined) + emit(projectId, onEvent, { + type: 'complete', + phase: 'render', + message: sweepPaused + ? 'Screen sweep paused at a checkpoint; partial results were preserved.' + : 'Screen sweep stopped; partial results were preserved.' + }) + // A paused sweep is a *completed* run of a bounded sweep, not a degenerate one: hitting the + // screen limit is the ordinary outcome of `screenLimit`, and this is the exit it takes. It + // was nonetheless the one exit with no closing summary, so the runs whose cost you most + // want to read — the ones that stopped because they ran out of budget — were the runs that + // reported nothing, and a seven-minute sweep's breakdown had to be reconstructed by hand + // from the per-call lines. + logAgentTokenSummary(tallies, tokens, hedges) + logPhaseSummary(timing) + return { ok: true, projectId, result: stoppedResult } + } + + await writeCaptureStatus(projectId, captureMap) + + const agentDeviceScreens = buildAgentDeviceScreens( + [...nodesWithSlug, ...runtimeNodes], + parsed.edges, + runtimeEdgesList, + chromeInventory, + ctx, + device, + openedByNode, + inferredScreens, + captureMap + ) + await writeAgentDeviceScreens(projectId, agentDeviceScreens) + console.log( + `[app-flow] wrote agent-device screens — ${Object.keys(agentDeviceScreens).length} entries` + ) + + checkSignal(signal) + emit(projectId, onEvent, { + type: 'phase', + phase: 'states', + message: 'Exploring state variants...' + }) + + const hintedNodes = nodesWithSlug.filter((n) => n.stateHints.length > 0) + console.log(`[app-flow] states start — ${hintedNodes.length} hinted routes`) + for (let i = 0; i < hintedNodes.length; i++) { + checkSignal(signal) + if (options.pauseRequested?.()) { + sweepPaused = true + sweepStopped = true + break + } + const node = hintedNodes[i] + emit(projectId, onEvent, { + type: 'step', + phase: 'states', + message: `Exploring states for ${node.urlPath}`, + completed: i + 1, + total: hintedNodes.length + }) + + if (statesDone.has(node.id)) continue + const baseCapture = captureMap[node.id] + if (!baseCapture?.screenshot) continue + const stateBudget = new ActiveTimeBudget(5 * 60_000, activeNow(), activeNow) + const verifiedDeepLink = openedByNode[node.id] + + /** + * Put the device back on `node` so a state variant can be triggered from it. Returns + * whether the frame that follows is worth judging at all. + */ + const establishBase = async (): Promise => { + // The base screen's own stored journey, run line by line, in place of the + // `replay.run(baseCapture.replayScript)` this used to do. `runPathToReach` carries the + // readiness wait that call site had to add by hand — a session stays "active" while + // sitting on the dev-client launcher just as readily as while sitting on `node`, which + // is exactly how this pass once ended up screenshotting the launcher and calling it + // every hinted route — and it works for a screen reached by exploration, which a + // `.ad`-only base never did. + if (baseCapture.pathCommands?.length) { + try { + await runPathToReach(ctx, device, baseCapture.pathCommands) + return true + } catch (error) { + console.warn(`[app-flow] state base path for ${node.id} failed:`, error) + return false + } + } + if (!verifiedDeepLink) return false + // `pickSampleParams`' `1` stand-in is for the descriptive path in the agent-device + // manifest and has no business building a URL that will be opened and photographed. The + // node reached its base capture by deep link, so a value exists; bailing rather than + // inventing one is what keeps that true if it ever stops being. + const stateParams = resolveNodeParams(node, parsed.edges, paramStore) + if (stateParams.unresolved.length) return false + const reopened = await openDeepLink(ctx, device, node, stateParams.values) + // `settleAfterRoute`, not the bare `waitForRoute` that used to be here. That was a flat + // 2.5s sleep, which is only ever right when the app is already on screen and settled — + // and by the time this phase starts, nothing guarantees it is. The main capture loop + // switched to `settleAfterRoute` for exactly this reason; this pass was left behind, so + // it fired every deep link into whatever the sweep happened to leave behind and judged + // the frame 2.5 seconds later regardless. + // `settleAfterRoute` returns `waitForAppReady`'s own verdict, not just a delay — false + // when launcher recoveries are exhausted or the app never settled at all. The deep-link + // probe above (`probeDeepLinkRoute`) already treats a mismatch on an unsettled frame as + // "inconclusive, not evidence about the scheme" rather than a real mismatch; discarding + // the verdict here made the same mistake in the opposite direction, calling the URL + // delivered and the base established just because `reopened.url` matched, with nothing + // checking that the app was actually there to receive it and land on that URL. + const settled = await settleAfterRoute(ctx, device, signal) + return settled && reopened.url === verifiedDeepLink.url + } + + // Read by the arrival verifier and the trigger planner, then discarded: the delivered + // artefact of this phase is the state variant capture written further down. + const screenshotPath = getScratchPath(projectId, 'state-start') + + /** + * Two attempts, with a relaunch between them, because "the deep link opened the wrong + * screen" and "the app is not running" are indistinguishable from here and only the first + * is a reason to give up on this route. + * + * A dev client that is not running shows its launcher — a settled, fully interactive + * screen — so neither `settleAfterRoute` nor `isLoadingScreen` can tell it apart from a + * real one, and a deep link into it does nothing. The old code read that as a plain + * mismatch and skipped, which meant one stalled app silently cost *every* remaining + * hinted route its deep link, its auth-gate call, its screenshot and its arrival call, + * all to reach the same `continue`. Relaunching is the same recovery the capture loop + * already performs on a lost session, and it is what a human does by hand when they open + * the app from that launcher screen and watch the rest of the pass start working. + */ + let baseReady = false + let baseAuth: 'clear' | 'stop' | 'exhausted' = 'clear' + + // Pulled out of the retry loop below so the launcher-escalation branch that follows it can + // ask for exactly one more attempt after the user clears a stuck launcher, instead of + // re-running the whole two-attempt relaunch cycle (which would relaunch an app the user + // just finished opening by hand). + const attemptEstablishAndVerify = async (): Promise => { + if (!(await establishBase())) return false + baseAuth = await checkAuthGate(hintedNodes.length - i, stateBudget) + if (baseAuth !== 'clear') return false + await captureScreenshot(ctx, device, screenshotPath) + const baseArrival = await verifyCurrentTarget(node, screenshotPath, stateBudget) + if (baseArrival.verdict === 'match') return true + console.log(`[app-flow] states ${node.id} base mismatch — ${baseArrival.reason}`) + return false + } + + for (let attempt = 0; attempt < 2 && !stateBudget.expired; attempt++) { + if (attempt > 0) { + console.log(`[app-flow] states ${node.id} — base not reached, relaunching and retrying`) + await reopenBoundApp(ctx, device) + } + if (await attemptEstablishAndVerify()) { + baseReady = true + break + } + if (baseAuth !== 'clear') break + } + + // The two attempts above cover a lost session and a wrong-screen deep link, but neither + // recovers a dev-client launcher: relaunching the bound app *while it is showing the + // launcher* just puts the launcher back, and `establishBase`/`waitForAppReady` already + // tried the reconnect-then-relaunch sequence internally before giving up. The main capture + // loop survives exactly this condition because it has `explore()`'s LLM rung and + // `resolveBlocked`'s user prompt; this pass had neither, which is why it silently walked + // the launcher and reported `base mismatch` for every hinted route behind it instead of + // telling anyone. `promptToUnblock` (the capture loop's own escalation, factored out above) + // is reused rather than duplicated, since `kind: 'blocked'` is documented as being named for + // what was observed rather than for what the screen is. + if ( + !baseReady && + baseAuth === 'clear' && + !stateBudget.expired && + (await isOnDevClientLauncher(ctx, device)) + ) { + const resolution = await promptToUnblock( + "The app is showing the dev client's launcher instead of running, so nothing behind " + + 'it can be captured. Open the app from the launcher in the simulator, then continue.', + hintedNodes.length - i + ) + if (resolution === 'stop') { + baseAuth = 'stop' + } else if (resolution === 'continue') { + console.log( + `[app-flow] states ${node.id} — launcher cleared by the user, retrying base` + ) + if (await attemptEstablishAndVerify()) baseReady = true + } + } + + if (baseAuth === 'stop') { + sweepStopped = true + break + } + if (!baseReady) continue + const fileContent = node.file ? `\nSource file:\n${node.file}` : '' + const { text } = await runAgent( + { + cwd: projectPath, + systemPrompt: STATE_TRIGGER_SYSTEM_PROMPT, + userPrompt: `Route: ${node.id}\nURL path: ${node.urlPath}\nState hints: ${JSON.stringify(node.stateHints)}${fileContent}\n\nSuggest how to reveal the state variant.`, + // No tools, for the same reason as the navigator above: this decides from a screenshot + // and the hints it is given, and the source path in the prompt is a label, not a task. + tools: [], + // `low` rather than the judges' `off`: this one plans an action sequence to reveal a + // state variant instead of classifying what is already on screen, which is the kind of + // question the navigator gets and is held to the same level. + thinkingLevel: 'low', + imagePath: screenshotPath, + timeoutMs: 30_000 + }, + `state-trigger ${node.id}` + ) + + const plan = parseSubAgentJson<{ + actions: Array<{ + type: 'tap' | 'swipe' | 'input' | 'wait' + locator?: { by: 'text' | 'label' | 'id'; value: string } + value?: string + direction?: 'up' | 'down' | 'left' | 'right' + duration?: number + }> + stateName: string + destinationLocator: { by: 'text' | 'label' | 'id'; value: string } + note: string | null + }>(text) + + if (plan?.actions.length) { + const stateSlug = generateStateSlug(node.slug, plan.stateName) + const stateScreenshot = getScreenshotPath(projectId, stateSlug) + // The variant's journey is the base screen's, plus the lines for the actions that open + // it — the same composition `exploreFrom` performs for a seeded walk, and the reason + // this pass no longer needs `recordStateFromReplay`/`verifyAndSaveReplay` or the + // `.ad` candidate those two wrote. Null when a planned action has no faithful CLI form, + // in which case the variant is still captured, just without a stored path. + // + // Built before the actions run so the list describes what is about to be executed; it is + // only stored once they all succeeded and the screenshot came back. + // + // Minimized on the same terms as a walk's own lines, and for the same reason: a state + // plan that opens a menu and dismisses it again stores two commands whose only effect on + // a replay is a race against the popover's animation. The base screen's path is left + // alone — it was minimized when it was stored. + const actionLines = walkCommands(plan.actions) + const statePath = + baseCapture.pathCommands?.length && actionLines?.length + ? [...baseCapture.pathCommands, ...minimizeWalkCommands(actionLines).commands] + : null + try { + await executeNavigationActions(ctx, device, plan.actions, stateBudget.remainingMs) + await captureScreenshot(ctx, device, stateScreenshot) + } catch (error) { + console.warn(`[app-flow] state ${stateSlug} failed:`, error) + if (sweepStopped) break + continue + } + captureMap[node.id].states.push({ + name: plan.stateName, + screenshot: `screens/${stateSlug}.png`, + ...(statePath ? { pathCommands: statePath } : {}) + }) + } + statesDone.add(node.id) + await persistCheckpoint() + } + if (options.pauseRequested?.()) { + sweepPaused = true + sweepStopped = true + } + + await writeCaptureStatus(projectId, captureMap) + const finalScreens = currentAgentDeviceScreens() + await writeAgentDeviceScreens(projectId, finalScreens) + + // Stamped here too, not only in `persistCaptureUpdate`: this write is the last one a sweep + // makes, so whatever it omits is what the graph ends up rendering. + const finalScreenNodes: AppFlowNode[] = withReplayableCaptures( + [ + ...nodesWithSlug.map((node) => ({ + ...node, + capture: captureMap[node.id] ?? node.capture, + deepLinkRejection: deepLinkRejections[node.id] ?? null + })), + ...runtimeNodes + ], + finalScreens + ) + const finalNavigators = navigatorChrome(chromeInventory, finalScreenNodes) + const finalNodes: AppFlowNode[] = [ + ...finalScreenNodes, + ...chromeVirtualNodes(finalNavigators) + ] + + manifest.tokens = snapshotUsage(tokens) + manifest.sweepStatus = sweepPaused ? 'paused' : sweepStopped ? 'stopped' : 'completed' + await finishManifest(manifest) + // A finished sweep has nothing left to resume: the next build starts clean. A stopped one + // keeps its checkpoint so the user can pick up where they left off. + if (!sweepStopped) await clearSweepProgress(projectId) + + const finalResult: AppFlowResult = { + manifest, + nodes: finalNodes, + edges: parsed.edges, + runtimeEdges: rebuildChromeEdges(runtimeEdgesList, finalNavigators, finalNodes), + chrome: chromeInventory, + navigation + } + await writeAppFlow(projectId, finalResult) + await writeParamStore(projectId, paramStore.toJSON()).catch(() => undefined) + const harvest = paramStore.summary() + console.log( + `[app-flow] params — harvested ${harvest.records} record(s) from ${harvest.templates} testID shape(s), ${harvest.learned} demonstrated` + ) + + checkSignal(signal) + emit(projectId, onEvent, { + type: 'complete', + phase: 'render', + message: sweepPaused + ? 'Screen sweep paused at a checkpoint; partial results were preserved.' + : sweepStopped + ? 'Screen sweep stopped; partial results were preserved.' + : 'Done.' + }) + logAgentTokenSummary(tallies, tokens, hedges) + logPhaseSummary(timing) + console.log('[app-flow] done') + + return { ok: true, projectId, result: finalResult } + } finally { + // A device the user launched stays up so the next sweep can adopt it again. + if (!reusedDevice) await closeDevice(device) + } + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + let preserved: AppFlowResult | null = null + if (activeManifest) { + activeManifest.sweepStatus = message === 'Cancelled' ? 'stopped' : 'failed' + // The tokens a cancelled sweep spent are just as real as a finished one's, and this is the + // only place they are recorded: `manifest.tokens` is written on the success path alone, so a + // cancel used to hand the renderer a manifest still carrying the *previous* run's total — or + // none at all on a first build, which is why the usage chip vanished the moment you stopped a + // sweep. The by-role breakdown is logged here for the same reason: a cancelled run is exactly + // the one whose bill you want to read. + activeManifest.tokens = snapshotUsage(tokens) + logAgentTokenSummary(tallies, tokens, hedges) + logPhaseSummary(timing) + await finishManifest(activeManifest).catch(() => undefined) + const existing = await readAppFlow(projectId).catch(() => null) + if (existing) { + preserved = { ...existing, manifest: activeManifest } + await writeAppFlow(projectId, preserved).catch(() => undefined) + } + } + if (message === 'Cancelled' && preserved) { + emit(projectId, onEvent, { + type: 'complete', + phase: 'render', + message: 'Screen sweep cancelled; partial results were preserved.' + }) + return { ok: true, projectId, result: preserved } + } + console.error(`[app-flow] build failed: ${message}`) + emit(projectId, onEvent, { type: 'error', phase: 'render', message }) + return { ok: false, error: message } + } +} + +/** + * How long a prompt waits for the user before the build gives up on them and carries on marking + * what it could not reach. Long enough to sign in by hand; short enough that a build nobody is + * watching does not hold a simulator overnight. + */ +const PROMPT_TIMEOUT_MS = 15 * 60_000 + +export class AppFlowBuilder extends EventEmitter { + private controller = new AbortController() + private running = false + private pauseRequested = false + private pendingPrompt: { + id: string + settle: (resolution: AppFlowPromptResolution) => void + } | null = null + + /** Answers an open prompt. Returns false when nothing is waiting for one. */ + resolvePrompt(resolution: AppFlowPromptResolution, promptId?: string): boolean { + const pending = this.pendingPrompt + if (!pending || promptId !== pending.id) return false + pending.settle(resolution) + return true + } + + private requestUserAction = (prompt: AppFlowPrompt): Promise => { + // Supersede an older prompt if the orchestrator reaches a newer checkpoint. + this.pendingPrompt?.settle('stop') + return new Promise((resolve) => { + const signal = this.controller.signal + const settle = (resolution: AppFlowPromptResolution): void => { + if (this.pendingPrompt?.settle !== settle) return + this.pendingPrompt = null + clearTimeout(timer) + signal.removeEventListener('abort', onAbort) + resolve(resolution) + } + // Cancelling or walking away must not leave the sweep waiting on a click forever. + const onAbort = (): void => settle('stop') + const timer = setTimeout(() => { + console.log(`[app-flow] prompt (${prompt.kind}) timed out — stopping`) + settle('stop') + }, PROMPT_TIMEOUT_MS) + this.pendingPrompt = { id: prompt.id, settle } + if (signal.aborted) settle('stop') + else signal.addEventListener('abort', onAbort, { once: true }) + }) + } + + /** + * An options object rather than positional parameters: the list had reached seven, two of its + * callers passed `undefined` through the middle of it to reach the ones they cared about, and + * adding `explorationMode` would have made an eighth. + */ + async start(options: { + projectPath: string + mode: AppFlowBuildMode + decisionProvider?: AppFlowDecisionProvider + screenLimit?: number + explorationMode?: ExplorationMode + renderTransportMode?: RenderTransportMode + deepLinkJudgeMode?: DeepLinkJudgeMode + launchedDevice?: CaptureDevice + }): Promise { + if (this.running) throw new Error('Build already in progress') + this.running = true + this.controller = new AbortController() + this.pauseRequested = false + + let result: AppFlowBuildResult | AppFlowBuildError + try { + result = await buildAppFlow({ + ...options, + pauseRequested: () => this.pauseRequested, + signal: this.controller.signal, + onEvent: (event) => this.emit('event', event), + requestUserAction: this.requestUserAction + }) + } catch (error) { + // buildAppFlow catches its own failures, but a listener throwing (or an unexpected bug) + // must not leave `running` stuck true with no terminal event. + result = { ok: false, error: error instanceof Error ? error.message : 'Unknown error' } + } finally { + this.running = false + this.pendingPrompt = null + } + + if (result.ok) { + this.emit('done', result) + } else { + this.emit('error', result) + } + } + + cancel(): void { + console.log('[app-flow] cancel requested') + this.controller.abort() + } + + pause(): void { + console.log('[app-flow] pause requested; waiting for the next screen checkpoint') + this.pauseRequested = true + // A user-action prompt is itself a durable boundary, so it need not remain open before the + // orchestrator can observe the cooperative pause request. + this.pendingPrompt?.settle('stop') + } + + isRunning(): boolean { + return this.running + } +} diff --git a/src/app-flow/param-store.ts b/src/app-flow/param-store.ts new file mode 100644 index 00000000..acfa69a7 --- /dev/null +++ b/src/app-flow/param-store.ts @@ -0,0 +1,964 @@ +/** + * Real values for route params, harvested from what the app itself showed us. + * + * `resolveRouteParams` (exploration.ts) reads values out of *source text* — the literals a + * `navigate('Profile', {name: 'alice'})` call site spells out. On an app that passes variables, + * which is nearly all of them, that finds nothing, and a parameterised route is then reachable only + * by walking to it: `/profile/:name` cost 29.2s of exploration in the reference sweep for a screen a + * resolved param opens in ~5s. The values were on the screen the whole time — Bluesky stamps handles + * into testIDs (`profileCard-alice.test-link`) and every navigation the harness performs reports the + * URL it landed on. + * + * Everything here obeys one rule: **a value is only ever observed, never invented.** A wrong id does + * not produce a weaker deep link, it produces a confidently wrong capture of some other record, and + * the 404 it lands on says nothing about the screen we were asked about. So where there is no + * evidence the answer is "no value", and the caller explores instead — exactly what it does today. + * + * The unit of storage is a **record**, not a param. A record is a complete param set taken from one + * single place: one pressable, one navigation observation, one real URL. Params are never spliced + * across sources, because `/profile/:name/lists/:rkey` built from a right `name` and some other + * record's `rkey` is precisely the wrong-capture failure above, and it is indistinguishable from + * success until a human looks at the screenshot. + */ + +/** Where a record's values came from, loosely ordered by how directly the app vouched for them. */ +export type ParamSource = 'static' | 'nav-state' | 'a11y-learned' | 'a11y-affinity' + +/** Highest first. Applied wholesale to whole records — see `best`. */ +const SOURCE_RANK: Record = { + static: 0, + 'nav-state': 1, + 'a11y-learned': 2, + 'a11y-affinity': 3 +} + +/** + * A complete param set for one route, from one observation. + * + * `values` covers **every** param the route declares or the record is not stored for it: a partial + * fill is the one thing that lets a right `name` carry a wrong `rkey` into a URL that opens cleanly. + */ +export interface ParamRecord { + /** The route these values are valid for. */ + nodeId: string + /** + * The route pattern `nodeId` carried when the observation was made, so the record can also answer + * for a route that shares these parameters — see `sharesRouteParams`. Absent on records carried + * over from a sweep that predates this field, which are then only ever used for their own node. + */ + urlPath?: string | null + /** Complete param set; every value has passed `PARAM_VALUE_RE` and any known shape check. */ + values: Record + source: ParamSource + /** Screen id the observation was made on, for provenance in the log. */ + seenOn: string + /** Monotonic within a run; higher is more recent. */ + seq: number + /** Carried over from a previous sweep, so the record may point at a record the app has deleted. */ + fromPriorRun?: boolean +} + +/** + * A testID shape that is known to carry a value for one param of one route — `profileCard-`…`-link` + * for `/profile/:name`'s `name`. + * + * Extraction strips the prefix and suffix rather than splitting on `-`, so a value that itself + * contains a dash (`alice-bob.bsky.social`) survives intact. + */ +export interface PressableTemplate { + prefix: string + suffix: string + nodeId: string + paramKey: string + /** + * `learned` — the app demonstrated this mapping: a tap on a control with this identifier landed on + * a URL that contained the token, so the binding is observed fact. + * + * `affinity` — nothing has been demonstrated yet and the identifier merely *reads* like it belongs + * to the route (`profileCard` ↔ `/profile/:name`). This is a bootstrap guess about which route a + * value belongs to; the value itself is still real. Worst case is one wasted open, and then the + * same exploration that would have happened anyway. + */ + evidence: 'learned' | 'affinity' + /** 1 for learned; the token-overlap fraction for affinity. */ + score: number +} + +/** + * A value that can stand in for a route param — the one rule every source here is held to. + * + * The static layer reads values out of *source text*, so what sits where a value should be is often + * the expression around it: a `navigate('ProfileList')` call site once yielded the deep link + * `/profile/navigate('ProfileList')/lists/navigate('ProfileList')`. The harvesting layers have the + * mirror-image problem — a testID segment or a navigation param can be anything the app felt like + * putting there. Anything that is not a plain URL segment is an artefact rather than a value, and + * this lives here rather than in `exploration.ts` so both layers are measured against one rule. + */ +export const PARAM_VALUE_RE = /^[\w.~@%-]{1,128}$/ + +/** + * Param values are persisted to disk and replayed across sweeps, so anything that looks like a + * credential, or that could poison an object when it is used as a key, must never enter the store. + * This is additive to `PARAM_VALUE_RE`, which constrains the *shape* of a value but says nothing + * about its key and lets a short opaque token or a dotted JWT-like string through. + */ +const SENSITIVE_KEY_RE = /token|password|secret|authorization|cookie/i +const PROTOTYPE_KEYS: ReadonlySet = new Set(['__proto__', 'constructor', 'prototype']) +const MAX_PARAM_KEY_LENGTH = 64 +const MAX_PARAM_VALUE_LENGTH = 128 +const CREDENTIAL_VALUE_RE = /^(?:Bearer\s|eyJ[A-Za-z0-9_-]+\.)/ + +export type ParamRejectionReason = + 'sensitive-key' | 'prototype-key' | 'key-too-long' | 'value-too-long' | 'credential-shaped-value' + +/** Why one param may not be stored, or null when it may. Never includes the value itself. */ +export function unsafeParamReason(key: string, value: string): ParamRejectionReason | null { + if (PROTOTYPE_KEYS.has(key)) return 'prototype-key' + if (key.length > MAX_PARAM_KEY_LENGTH) return 'key-too-long' + if (SENSITIVE_KEY_RE.test(key)) return 'sensitive-key' + if (value.length > MAX_PARAM_VALUE_LENGTH) return 'value-too-long' + if (CREDENTIAL_VALUE_RE.test(value)) return 'credential-shaped-value' + return null +} + +/** One refused param set, for the sweep log. Carries the key and the reason, never the value. */ +export interface ParamRejection { + nodeId: string + key: string + reason: ParamRejectionReason + origin: ParamSource | 'disk' +} + +/** What this module needs to know about a route. Structural, so tests need no fixtures. */ +export interface ParamRouteNode { + id: string + title?: string + urlPath: string | null + params: readonly string[] +} + +/** What this module needs from a captured control. Matches `AppFlowPressable` structurally. */ +export interface ParamPressable { + identifier?: string + label?: string + text?: string + role?: string +} + +/** + * Whether two route patterns describe the same journey far enough in that one's observed values are + * literally the other's. + * + * A value is only ever observed, never invented — but an observation of `/profile/:name/post/:rkey` + * is an observation of a real `name` and a real `rkey` *in those positions*, and + * `/profile/:name/post/:rkey/liked-by` addresses the same post with the same two values. Holding + * records strictly per-route meant a sweep explored three descendants of a post whose ids it had + * just harvested, at around fifty seconds each, with the answer already in the store. + * + * The test is positional rather than by param name on purpose: two routes may both call a param + * `name` and mean different things, and nothing short of identical leading segments — literals and + * `:param` spellings alike — establishes that the shorter path is a prefix of the longer one's + * journey. Where they diverge, no value crosses. This is not splicing: a record still travels + * whole, from one observation, which is the rule that stops a right `name` carrying some other + * record's `rkey`. + */ +export function sharesRouteParams( + owner: string | null | undefined, + target: string | null +): boolean { + if (!owner || !target) return false + const ownerSegments = owner.split('/').filter(Boolean) + const targetSegments = target.split('/').filter(Boolean) + const shared = Math.min(ownerSegments.length, targetSegments.length) + if (!shared) return false + for (let at = 0; at < shared; at += 1) { + if (ownerSegments[at] !== targetSegments[at]) return false + } + return true +} + +/* ------------------------------------------------------------------ shapes */ + +/** + * The coarse family a value belongs to. Used only as a *negative* check, and only when the static + * layer already produced a literal for the same param: a handle-shaped route whose call sites say + * `alice.test` should not accept a bare `42` scraped off a list row. With no literal there is no + * baseline, so no check is applied — guessing a shape would be as inventive as guessing a value. + */ +export type ShapeClass = 'digits' | 'dotted' | 'token' | 'mixed' + +export function shapeClass(value: string): ShapeClass { + if (/^\d+$/.test(value)) return 'digits' + if (value.includes('.')) return 'dotted' + if (/^[a-z0-9]+$/.test(value)) return 'token' + return 'mixed' +} + +/* -------------------------------------------------------------- tokenizing */ + +/** + * `profileHeaderFollowersButton` / `profile-header-followers` / `ProfileFollowers` → the words a + * human reads in them. Identifiers are source text and carry no spaces, so without this an id's own + * words never line up with a route's. + */ +export function tokenizeIdentifier(text: string): string[] { + return text + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2') + .split(/[^A-Za-z0-9]+/) + .map((word) => word.toLowerCase()) + .filter(Boolean) +} + +/** Words so generic that sharing them says nothing about which route a control belongs to. */ +const STOP_TOKENS = new Set([ + 'btn', + 'button', + 'link', + 'item', + 'card', + 'row', + 'cell', + 'view', + 'screen', + 'tab', + 'by', + 'to', + 'the', + 'a', + 'id' +]) + +/** The words a route is identified by: its literal path segments plus its component name. */ +export function nodeTokens(node: ParamRouteNode): Set { + const fromPath = (node.urlPath ?? '') + .split('/') + .filter((segment) => segment && !segment.startsWith(':') && !segment.startsWith('[')) + .flatMap((segment) => tokenizeIdentifier(segment)) + const fromTitle = tokenizeIdentifier(node.title ?? node.id.split('/').pop() ?? '') + return new Set([...fromPath, ...fromTitle].filter((token) => !STOP_TOKENS.has(token))) +} + +/** + * How much of a testID's wording is accounted for by a route's own wording, in [0, 1]. Scored + * against the prefix rather than the whole identifier so the value itself cannot contribute. + */ +export function affinityScore(prefix: string, node: ParamRouteNode): number { + const prefixTokens = tokenizeIdentifier(prefix).filter((token) => !STOP_TOKENS.has(token)) + if (!prefixTokens.length) return 0 + const routeTokens = nodeTokens(node) + const shared = prefixTokens.filter((token) => routeTokens.has(token)) + // A single short coincidence ("to", "id" are already dropped; "list" is not) is not evidence. + if (!shared.some((token) => token.length >= 4)) return 0 + return shared.length / prefixTokens.length +} + +/** Below this, the wording is a coincidence rather than a hint. */ +const AFFINITY_FLOOR = 0.5 + +/* --------------------------------------------------------------- templates */ + +/** + * The template a demonstrated mapping implies, found by locating the real value *inside* the + * identifier. Substring search rather than segment splitting, so values containing `-` or `.` need + * no special case and the recovered prefix/suffix are exactly what extraction will strip later. + */ +export function learnTemplate( + identifier: string, + value: string, + nodeId: string, + paramKey: string +): PressableTemplate | null { + if (!value || value.length < 2) return null + const at = identifier.indexOf(value) + if (at === -1) return null + // A value that is the entire identifier leaves nothing to key on and would match every control. + if (at === 0 && value.length === identifier.length) return null + return { + prefix: identifier.slice(0, at), + suffix: identifier.slice(at + value.length), + nodeId, + paramKey, + evidence: 'learned', + score: 1 + } +} + +/** + * Candidate value positions inside an identifier, for the bootstrap case where nothing has been + * demonstrated yet. A segment only counts as a possible value when it does not read like a word a + * developer typed: it carries a dot, a digit, an `@`, or is long enough to be an opaque key. That + * keeps `tab-home-link` from offering `home` as somebody's handle while still finding `alice.test`, + * `3kabc7xyz2p1q` and `did:plc:…`-style tokens. + */ +export function splitTestId( + identifier: string +): { prefix: string; token: string; suffix: string }[] { + const segments = identifier.split('-') + if (segments.length < 2) return [] + const out: { prefix: string; token: string; suffix: string }[] = [] + for (let at = 0; at < segments.length; at += 1) { + const token = segments[at] + if (!token || !PARAM_VALUE_RE.test(token) || !looksLikeValue(token)) continue + const prefix = segments.slice(0, at).join('-') + if (!prefix) continue + const suffix = segments.slice(at + 1).join('-') + out.push({ + prefix: `${prefix}-`, + token, + suffix: suffix ? `-${suffix}` : '' + }) + } + return out +} + +function looksLikeValue(token: string): boolean { + if (/[.@%~]/.test(token)) return true + if (/\d/.test(token) && !/^\d{1,2}$/.test(token)) return true + return token.length >= 13 +} + +/** The value `identifier` carries for `template`, or null when the shape does not apply. */ +export function applyTemplate(template: PressableTemplate, identifier: string): string | null { + if (!identifier.startsWith(template.prefix)) return null + if (template.suffix && !identifier.endsWith(template.suffix)) return null + const token = identifier.slice( + template.prefix.length, + template.suffix ? identifier.length - template.suffix.length : undefined + ) + if (!token || !PARAM_VALUE_RE.test(token)) return null + return token +} + +/* ------------------------------------------------------------- url parsing */ + +/** + * The route a real URL belongs to, and the whole param set it fills. + * + * One observed URL is the cleanest record obtainable: every param comes from the same place by + * construction, which is what makes nested routes like `/profile/:name/lists/:rkey` correct rather + * than lucky. Ranked by literal segment count so a concrete route wins over a more parameterised one + * that happens to have the same arity. + */ +export function matchFlowLink( + url: string, + nodes: readonly ParamRouteNode[] +): { nodeId: string; values: Record } | null { + const path = normalizePath(url) + if (path === null) return null + const actual = routeSegments(path) + let best: { nodeId: string; values: Record; literals: number } | null = null + for (const node of nodes) { + if (!node.urlPath || node.urlPath === '/') continue + const pattern = routeSegments(node.urlPath) + if (pattern.length !== actual.length) continue + // A catch-all swallows the rest of the path, so the value it would yield is a path rather than + // a segment and cannot be put back into a URL through `PARAM_VALUE_RE`. Nothing honest to do. + if (pattern.some((segment) => segment.startsWith('[...'))) continue + let literals = 0 + const values: Record = {} + let ok = true + for (let at = 0; at < pattern.length; at += 1) { + const key = patternParamKey(pattern[at]) + if (key === null) { + if (pattern[at].toLowerCase() !== actual[at].toLowerCase()) { + ok = false + break + } + literals += 1 + continue + } + const value = decodeSegment(actual[at]) + if (!value || !PARAM_VALUE_RE.test(value)) { + ok = false + break + } + values[key] = value + } + if (!ok) continue + if (!best || literals > best.literals) best = { nodeId: node.id, values, literals } + } + return best ? { nodeId: best.nodeId, values: best.values } : null +} + +/** The path part of a URL or a bare path, or null when there is nothing routable in it. */ +function normalizePath(url: string): string | null { + const scheme = url.match(/^([a-zA-Z][\w+.-]*):\/\//)?.[1]?.toLowerCase() + let path = (scheme ? url.slice(url.indexOf('://') + 3) : url).split(/[?#]/)[0] + // Only `http(s)` puts an authority where a path segment would otherwise sit. An app's own scheme + // has no host — `bluesky://profile/alice.test` means `/profile/alice.test` — so dropping a + // "hostname" there eats the first real segment and the route never matches. + if (scheme === 'http' || scheme === 'https') path = path.split('/').slice(1).join('/') + const trimmed = `/${path}`.replace(/\/{2,}/g, '/').replace(/\/+$/, '') + return trimmed || null +} + +/** + * The routable segments of a path or template. + * + * `index` and Expo Router group segments (`(tabs)`) are spelled in a route's *file* path but never + * appear in the URL the app navigates to, so a template carrying them would never match a real + * link on length alone. Dropped from both sides, as `codex-mobile-dev-plugin`'s `matchFlowLink` + * does, so `(tabs)/profile/[name]/index` and `/profile/alice.test` line up. + */ +function routeSegments(path: string): string[] { + return path + .split(/[?#]/)[0] + .split('/') + .filter((segment) => segment && segment !== 'index' && !/^\(.+\)$/.test(segment)) +} + +/** The param a template segment declares — `:name`, `:name?` and `[name]` alike. */ +function patternParamKey(segment: string): string | null { + if (segment.startsWith(':')) return segment.slice(1).replace(/\?$/, '') + if (segment.startsWith('[') && segment.endsWith(']')) { + return segment.slice(1, -1).replace(/^\.\.\./, '') + } + return null +} + +function decodeSegment(segment: string): string | null { + try { + return decodeURIComponent(segment) + } catch { + return segment + } +} + +/* ------------------------------------------------------------------- store */ + +/** How a nav-state read has turned out this sweep. Latched on first failure; see `nav-state.ts`. */ +export type NavStateStatus = 'unknown' | 'ready' | 'unavailable' + +export interface ParamStoreJson { + records: ParamRecord[] + templates: PressableTemplate[] + /** `${nodeId}\u0000${paramKey}` → the shape a static literal established for that param. */ + shapes: Record +} + +/** One accepted record, with the provenance the sweep log prints. */ +export interface ParamResolution { + values: Record + source: ParamSource + seenOn: string + /** + * The route whose record this is, which is not always the route being resolved — see + * `sharesRouteParams`. `demote` is keyed on it, so a value that turns out to name something the + * app no longer has is forgotten where it actually lives rather than left for the next + * descendant to borrow. + */ + ownedBy: string +} + +export class ParamStore { + /** + * Indexed by route and by (route, param) rather than held as flat lists. + * + * `ingestPressables` runs once per captured screen over every control on it against every + * parameterised route — on Bluesky that is ~45 x 22 per screen, 76 times a sweep — and the + * bootstrap layer can mint a template per (testID shape, route) pair. Flat arrays turn each of + * those into a scan, which is a lot of work to spend inside a change whose entire purpose is to + * make the sweep faster. The buckets are small and kept sorted on insert, so selection is a map + * lookup and a short walk. + */ + private recordsByNode = new Map() + private templatesByParam = new Map() + private shapes = new Map() + private seq = 0 + /** Every param set refused as unsafe, in order. The same entries are logged as they happen. */ + readonly rejections: ParamRejection[] = [] + /** Set by the orchestrator so one failed nav-state read is not retried all sweep. */ + navState: NavStateStatus = 'unknown' + + /* ---------------------------------------------------------- static layer */ + + /** + * Register what `resolveRouteParams` found in source text. A complete set becomes the + * highest-precedence record; every literal, complete or not, establishes the shape later + * harvested values for that param have to match. + */ + noteStatic(node: ParamRouteNode, resolved: Record): void { + // Static values never pass through `addObserved`, so the guard is applied here as well. One + // unsafe literal poisons the whole set, and no shape is learned from it either. + const unsafe = this.firstUnsafe(node, resolved, 'static') + if (unsafe) return + for (const [key, value] of Object.entries(resolved)) { + if (PARAM_VALUE_RE.test(value)) this.shapes.set(shapeKey(node.id, key), shapeClass(value)) + } + if (node.params.length && node.params.every((key) => resolved[key])) { + this.add({ + nodeId: node.id, + values: Object.fromEntries(node.params.map((key) => [key, resolved[key]])), + source: 'static', + seenOn: 'call-site', + seq: (this.seq += 1) + }) + } + } + + /* ------------------------------------------------------- url observation */ + + /** + * A URL the app actually navigated to. Classified as a navigation observation because that is what + * it is: the app produced the string, so every param in it is simultaneously real and + * co-occurring. + */ + ingestUrl(url: string, seenOn: string, nodes: readonly ParamRouteNode[]): ParamRecord | null { + const match = matchFlowLink(url, nodes) + if (!match) return null + const node = nodes.find((candidate) => candidate.id === match.nodeId) + if (!node) return null + return this.addObserved(node, match.values, 'nav-state', seenOn) + } + + /** + * One route out of a flattened navigation state. Non-string params are dropped rather than + * coerced past numbers: an object or array in a param slot cannot be put in a URL, and stringifying + * it produces `[object Object]`, which is an invented value. + */ + ingestNavRoute( + node: ParamRouteNode, + params: Record, + seenOn: string + ): ParamRecord | null { + const values: Record = {} + for (const key of node.params) { + const raw = params[key] + if (typeof raw === 'string') values[key] = raw + else if (typeof raw === 'number' && Number.isFinite(raw)) values[key] = String(raw) + else return null + } + return this.addObserved(node, values, 'nav-state', seenOn) + } + + /* ------------------------------------------------------- a11y harvesting */ + + /** + * Teach the store a testID shape from a navigation the harness just performed: the control it + * tapped, and the URL the app reported afterwards. This is the only evidence class that ties a + * value to a *route* rather than merely reading like it might belong to one, so it also + * back-fills a record for the node it just demonstrated. + */ + learnFromNavigation( + identifier: string | undefined, + landedUrl: string, + seenOn: string, + nodes: readonly ParamRouteNode[] + ): void { + const match = matchFlowLink(landedUrl, nodes) + if (!match) return + const node = nodes.find((candidate) => candidate.id === match.nodeId) + if (node) this.addObserved(node, match.values, 'nav-state', seenOn) + if (!identifier) return + for (const [key, value] of Object.entries(match.values)) { + const template = learnTemplate(identifier, value, match.nodeId, key) + if (template) this.addTemplate(template) + } + } + + /** + * A walked arrival: the harness tapped a control and the arrival judge confirmed the app landed on + * `node`. No URL is involved — an in-app tap produces none — so the value has to come out of the + * identifier itself. The binding is still *demonstrated* rather than guessed, because the app is + * what performed the navigation; this is the only way a walk pays for itself on later routes. + * + * Restricted to a single-param route whose identifier offers exactly one candidate position. Two + * candidates are two readings of the same control with nothing to choose between them, and a + * multi-param route would need every value out of this one identifier with no way to tell which + * is which. Both stay unresolved, and the next route explores exactly as it does today. + */ + learnFromArrival( + identifier: string | undefined, + node: ParamRouteNode, + seenOn: string + ): ParamRecord | null { + if (!identifier || node.params.length !== 1) return null + const candidates = splitTestId(identifier) + if (candidates.length !== 1) return null + const [candidate] = candidates + const paramKey = node.params[0] + this.addTemplate({ + prefix: candidate.prefix, + suffix: candidate.suffix, + nodeId: node.id, + paramKey, + evidence: 'learned', + score: 1 + }) + return this.addObserved(node, { [paramKey]: candidate.token }, 'a11y-learned', seenOn) + } + + /** + * Read every value the controls on a captured screen carry. + * + * A record is only built from **one** pressable: all of a node's params must come out of the same + * identifier, via one template each. That is what stops a `name` from one row pairing with an + * `rkey` from another. Nodes whose params cannot all be filled from a single control stay + * unresolved, and the caller explores — the intended outcome, not a failure. + */ + ingestPressables( + seenOn: string, + pressables: readonly ParamPressable[], + nodes: readonly ParamRouteNode[] + ): ParamRecord[] { + const wanted = nodes.filter((node) => node.params.length) + if (!wanted.length) return [] + this.proposeAffinityTemplates(pressables, wanted) + const added: ParamRecord[] = [] + for (const pressable of pressables) { + const identifier = pressable.identifier + if (!identifier) continue + for (const node of wanted) { + const values: Record = {} + let evidence: 'learned' | 'affinity' = 'learned' + for (const key of node.params) { + const hit = this.templatesFor(node.id, key) + .map((template) => ({ template, value: applyTemplate(template, identifier) })) + .find((candidate) => candidate.value !== null) + if (!hit?.value) break + values[key] = hit.value + if (hit.template.evidence === 'affinity') evidence = 'affinity' + } + if (Object.keys(values).length !== node.params.length) continue + const record = this.addObserved( + node, + values, + evidence === 'learned' ? 'a11y-learned' : 'a11y-affinity', + seenOn + ) + if (record) added.push(record) + } + } + return added + } + + /** + * Bootstrap templates from wording alone, for routes nothing has demonstrated yet. Only ever + * produces `affinity` evidence, and only for a (node, param) pair that has no learned template — + * once the app has shown us the real shape there is no reason to keep guessing at it. + */ + private proposeAffinityTemplates( + pressables: readonly ParamPressable[], + nodes: readonly ParamRouteNode[] + ): void { + // One identifier carrying one value can only speak for a single-param route; a multi-param + // route needs every value from the same control, which affinity scoring has no way to + // establish. And once the app has demonstrated the real shape for a param there is no reason + // to keep guessing at it. + const bootstrap = nodes.filter( + (node) => node.params.length === 1 && !this.hasLearnedTemplate(node.id, node.params[0]) + ) + if (!bootstrap.length) return + // A feed shows the same row shape forty times over. Scoring is per distinct shape, not per + // control, so a long list costs what a short one does. + const shapes = new Map() + for (const pressable of pressables) { + if (!pressable.identifier) continue + for (const candidate of splitTestId(pressable.identifier)) { + shapes.set(`${candidate.prefix}\u0000${candidate.suffix}`, candidate) + } + } + for (const shape of shapes.values()) { + for (const node of bootstrap) { + const score = affinityScore(shape.prefix, node) + if (score < AFFINITY_FLOOR) continue + this.addTemplate({ + prefix: shape.prefix, + suffix: shape.suffix, + nodeId: node.id, + paramKey: node.params[0], + evidence: 'affinity', + score + }) + } + } + } + + /** Candidate shapes for one param, demonstrated ones first. Kept sorted by `addTemplate`. */ + private templatesFor(nodeId: string, paramKey: string): readonly PressableTemplate[] { + return this.templatesByParam.get(shapeKey(nodeId, paramKey)) ?? [] + } + + private hasLearnedTemplate(nodeId: string, paramKey: string): boolean { + return this.templatesFor(nodeId, paramKey).some((template) => template.evidence === 'learned') + } + + /* ------------------------------------------------------------- selection */ + + /** + * The best complete param set for `node`, or null when nothing observed covers it. + * + * Precedence is applied to whole records and never merged: if the static layer filled `name` but + * not `rkey`, a harvested record supplying both wins outright rather than contributing its `rkey` + * to the static `name`. + */ + best(node: ParamRouteNode): ParamResolution | null { + if (!node.params.length) return null + const covering = (records: readonly ParamRecord[]): ParamRecord[] => + records + .filter((record) => node.params.every((key) => record.values[key] !== undefined)) + .sort(recordOrder) + // The node's own observations first, whatever their source: a record made for this route is a + // statement about this route, and one offered by a neighbour is an inference from it — sound, + // but second in line. Precedence inside each group is unchanged. + const own = covering(this.recordsByNode.get(node.id) ?? []) + const chosen = + own[0] ?? + covering( + this.allRecords.filter( + (record) => record.nodeId !== node.id && sharesRouteParams(record.urlPath, node.urlPath) + ) + )[0] + if (!chosen) return null + return { + values: Object.fromEntries(node.params.map((key) => [key, chosen.values[key]])), + source: chosen.source, + seenOn: chosen.seenOn, + ownedBy: chosen.nodeId + } + } + + /** + * Forget a record that turned out to point at something the app does not have — a handle deleted + * since the previous sweep, an affinity template that bound the wrong route. Called when a deep + * link built from harvested values is rejected on arrival, so the next attempt falls through to + * the next-best record and ultimately to exploration. + * + * `nodeId` is the route that *owns* the record (`ParamResolution.ownedBy`), which is not always + * the route that just failed on it: a record borrowed by a descendant has to be forgotten where + * it lives, or every other descendant goes on borrowing the same dead value. + */ + demote(nodeId: string, values: Record): void { + const stamp = valueStamp(values) + const bucket = this.recordsByNode.get(nodeId) + if (!bucket) return + this.recordsByNode.set( + nodeId, + bucket.filter((record) => valueStamp(record.values) !== stamp) + ) + } + + /* ------------------------------------------------------------ bookkeeping */ + + private addObserved( + node: ParamRouteNode, + values: Record, + source: ParamSource, + seenOn: string + ): ParamRecord | null { + if (!node.params.length) return null + // The whole record is refused, not just the offending param: a record with a hole in it would + // be "resolved" and then replayed incomplete. + if (this.firstUnsafe(node, values, source)) return null + const complete: Record = {} + for (const key of node.params) { + const value = values[key] + if (!value || !PARAM_VALUE_RE.test(value)) return null + const expected = this.shapes.get(shapeKey(node.id, key)) + if (expected && shapeClass(value) !== expected) return null + complete[key] = value + } + return this.add({ + nodeId: node.id, + urlPath: node.urlPath, + values: complete, + source, + seenOn, + seq: (this.seq += 1) + }) + } + + /** + * The first unsafe param of a set, recorded and logged (key and reason only), or null when all are + * safe. Checks the keys the route declares as well as any extra keys supplied. + */ + private firstUnsafe( + node: ParamRouteNode, + values: Record, + origin: ParamRejection['origin'] + ): ParamRejection | null { + const keys = new Set([...node.params, ...Object.keys(values)]) + for (const key of keys) { + const raw = Object.hasOwn(values, key) ? values[key] : undefined + const reason = unsafeParamReason(key, typeof raw === 'string' ? raw : '') + if (!reason) continue + return this.reject({ nodeId: node.id, key, reason, origin }) + } + return null + } + + private reject(rejection: ParamRejection): ParamRejection { + this.rejections.push(rejection) + console.warn( + `[app-flow] param-store refused record for ${rejection.nodeId}: param "${rejection.key.slice(0, MAX_PARAM_KEY_LENGTH)}" ${rejection.reason} (${rejection.origin})` + ) + return rejection + } + + private add(record: ParamRecord): ParamRecord { + const stamp = valueStamp(record.values) + const bucket = this.recordsByNode.get(record.nodeId) ?? [] + const existing = bucket.find((candidate) => valueStamp(candidate.values) === stamp) + if (existing) { + // Re-seeing the same values from a better source is a promotion, not a duplicate. + if (SOURCE_RANK[record.source] < SOURCE_RANK[existing.source]) existing.source = record.source + // A record read back from a sweep older than `urlPath` learns its pattern the first time the + // same values are seen again, rather than staying unusable by its own descendants forever. + if (!existing.urlPath && record.urlPath) existing.urlPath = record.urlPath + existing.seq = record.seq + existing.fromPriorRun = false + return existing + } + bucket.push(record) + this.recordsByNode.set(record.nodeId, bucket) + return record + } + + private addTemplate(template: PressableTemplate): void { + const key = shapeKey(template.nodeId, template.paramKey) + const bucket = this.templatesByParam.get(key) ?? [] + const existing = bucket.find( + (candidate) => candidate.prefix === template.prefix && candidate.suffix === template.suffix + ) + if (!existing) bucket.push(template) + else if (existing.evidence === 'affinity' && template.evidence === 'learned') { + existing.evidence = 'learned' + existing.score = 1 + } else if (template.score > existing.score) existing.score = template.score + bucket.sort(templateOrder) + this.templatesByParam.set(key, bucket) + } + + private get allRecords(): ParamRecord[] { + return [...this.recordsByNode.values()].flat() + } + + private get allTemplates(): PressableTemplate[] { + return [...this.templatesByParam.values()].flat() + } + + /** Provenance for the sweep log: how much was learned and from where. */ + summary(): { records: number; templates: number; learned: number } { + const templates = this.allTemplates + return { + records: this.allRecords.length, + templates: templates.length, + learned: templates.filter((template) => template.evidence === 'learned').length + } + } + + toJSON(): ParamStoreJson { + return { + records: this.allRecords.map((record) => ({ ...record })), + templates: this.allTemplates.map((template) => ({ ...template })), + shapes: Object.fromEntries(this.shapes) + } + } + + /** + * Rehydrate from a previous sweep. Carried records are marked `fromPriorRun` and sort below + * anything seen live, because the record they name may have been deleted in the meantime; a 404 on + * one demotes it through `demote` exactly as a bad live record would be. + * + * Hostile or merely stale input is dropped rather than trusted: this file is on disk between runs. + */ + static fromJSON(raw: unknown, { fromPriorRun = true } = {}): ParamStore { + const store = new ParamStore() + if (!raw || typeof raw !== 'object') return store + const data = raw as Partial + for (const record of Array.isArray(data.records) ? data.records : []) { + if (!isPlainRecord(record)) continue + // A file written before the guard existed may hold an unsafe record; it is dropped whole. + const unsafe = Object.entries(record.values).find(([key, value]) => + unsafeParamReason(key, value) + ) + if (unsafe) { + store.reject({ + nodeId: record.nodeId, + key: unsafe[0], + reason: unsafeParamReason(unsafe[0], unsafe[1])!, + origin: 'disk' + }) + continue + } + store.add({ ...record, fromPriorRun: fromPriorRun || record.fromPriorRun }) + store.seq = Math.max(store.seq, record.seq ?? 0) + } + for (const template of Array.isArray(data.templates) ? data.templates : []) { + if (isPlainTemplate(template)) store.addTemplate(template) + } + for (const [key, value] of Object.entries(data.shapes ?? {})) { + if (value === 'digits' || value === 'dotted' || value === 'token' || value === 'mixed') { + store.shapes.set(key, value) + } + } + return store + } +} + +function shapeKey(nodeId: string, paramKey: string): string { + return `${nodeId}\u0000${paramKey}` +} + +/** Identity of a param set, so the same values seen twice do not become two records. */ +function valueStamp(values: Record): string { + return Object.keys(values) + .sort() + .map((key) => `${key}=${values[key]}`) + .join('&') +} + +/** + * Seen-this-run first, then precedence, then most recent. + * + * Freshness outranks source, which looks backwards until you notice that only the *observed* + * sources can be carried over at all: `static` is re-derived from the project's source text on + * every run and so is never `fromPriorRun`. Among observations, a handle read off the screen a + * moment ago is far more likely to still exist than one a sweep from last week recorded, and the + * record that no longer exists is the one that costs a wasted open and a 404. Within a single run + * this comparison is inert and precedence decides everything, which is the common case. + */ +function recordOrder(left: ParamRecord, right: ParamRecord): number { + const byRun = Number(left.fromPriorRun ?? false) - Number(right.fromPriorRun ?? false) + if (byRun !== 0) return byRun + const bySource = SOURCE_RANK[left.source] - SOURCE_RANK[right.source] + if (bySource !== 0) return bySource + return right.seq - left.seq +} + +/** Demonstrated shapes before guessed ones, then the better-scoring guess. */ +function templateOrder(left: PressableTemplate, right: PressableTemplate): number { + if (left.evidence !== right.evidence) return left.evidence === 'learned' ? -1 : 1 + return right.score - left.score +} + +function isPlainRecord(value: unknown): value is ParamRecord { + if (!value || typeof value !== 'object') return false + const record = value as Partial + if (typeof record.nodeId !== 'string' || !record.nodeId) return false + if (!record.values || typeof record.values !== 'object') return false + if (record.urlPath !== undefined && record.urlPath !== null && typeof record.urlPath !== 'string') + return false + const entries = Object.entries(record.values) + if (!entries.length) return false + if ( + !entries.every(([key, item]) => key && typeof item === 'string' && PARAM_VALUE_RE.test(item)) + ) { + return false + } + return record.source !== undefined && record.source in SOURCE_RANK +} + +function isPlainTemplate(value: unknown): value is PressableTemplate { + if (!value || typeof value !== 'object') return false + const template = value as Partial + return ( + typeof template.prefix === 'string' && + typeof template.suffix === 'string' && + typeof template.nodeId === 'string' && + typeof template.paramKey === 'string' && + (template.evidence === 'learned' || template.evidence === 'affinity') && + typeof template.score === 'number' + ) +} diff --git a/src/app-flow/parser.ts b/src/app-flow/parser.ts new file mode 100644 index 00000000..31b41ad5 --- /dev/null +++ b/src/app-flow/parser.ts @@ -0,0 +1,1667 @@ +import { readdir, readFile, stat } from 'node:fs/promises' +import { dirname, extname, join, posix, relative, resolve } from 'node:path' +import ts from 'typescript' +import type { + AppFlowEdge, + AppFlowGuard, + AppFlowLayout, + AppFlowNode, + AppFlowStateHint, + AppFlowTrigger +} from '../shared/app-flow.ts' + +// Re-exported so existing imports of `AppFlowLayout` from this module keep compiling — the type +// itself moved to `shared/app-flow.ts` because `AppFlowResult.navigation` persists it verbatim into +// `app-flow.json`, which the renderer also reads, and `parser.ts` is main-process-only. +export type { AppFlowLayout } + +function generateSlug(routeId: string): string { + return routeId + .replace(/[^a-zA-Z0-9]/g, '_') + .replace(/_+/g, '_') + .toLowerCase() +} + +/** + * How much to trust `scheme` for building a working deep link: + * - `verified-prefix`: the linking config's own `prefixes` array names this scheme, so React + * Navigation will actually route a URL opened with it. + * - `scheme-mismatch`: a `prefixes` array was found, but this scheme is not in it. iOS will still + * open the app (the scheme is registered in Info.plist independently of `prefixes`), but React + * Navigation ignores a URL whose prefix it does not recognise, so the app just sits on whatever + * it already had open — indistinguishable at runtime from landing on the wrong screen. + * - `assumed`: no `prefixes` array was found at all (or it held only universal `https://` links), + * so there is nothing to check `expo.scheme` against; it is used on faith, as before this fix. + */ +export type DeepLinkConfidence = 'verified-prefix' | 'scheme-mismatch' | 'assumed' + +export interface ParsedAppGraph { + mode: 'expo-router' | 'react-navigation' + appDir: string | null + routesFile: string | null + scheme: string | null + /** Confidence that `scheme` is one React Navigation's `linking.prefixes` will actually route. */ + deepLinkConfidence: DeepLinkConfidence + /** Raw `prefixes` entries read off the linking config, custom-scheme and universal alike. */ + linkingPrefixes: string[] + appName: string | null + bundleId: string | null + slug: string | null + nodes: AppFlowNode[] + edges: AppFlowEdge[] + /** + * The navigator tree. Without it nothing in the graph says the app opens on a tab bar, so an + * explorer has no idea where a journey starts and picks whichever screen an over-approximated + * edge chain happens to dead-end at. + */ + layouts: AppFlowLayout[] + /** Kind of the root navigator, e.g. `Tabs`. */ + rootNavigator: string | null + /** Layouts mounted directly on the root — the tabs a journey can start from. */ + entryLayouts: string[] + /** + * Node ids the app's entry chrome opens DIRECTLY — each entry layout's own declared + * `initialRouteName` (or, absent that, its first registered screen, matching React Navigation's + * own default). This is deliberately a different concept from `entryLayouts`/`group`: `group` is + * ATTRIBUTION (which navigator registers a screen — a helper called from five tabs belongs to + * all five), while this is REACHABILITY (which single screen in each of those tabs a person + * lands on without navigating anywhere first). Conflating the two previously meant that every + * screen sharing a `group` with an entry layout — 78 of 83 in social-app, most of them behind a + * shared registration helper — was treated as one tap from launch, which both fabricated a + * one-hop journey for screens nested many pushes deep and suppressed the "explore from the entry + * chrome" instruction the planner needs for a genuine orphan. + */ + entryScreenIds: string[] +} + +/** Navigator kind for a `create*Navigator` factory, including project-specific wrappers. */ +function classifyNavigatorFactory(factory: string): string { + if (/MaterialTopTab|TopTab/.test(factory)) return 'TopTabs' + if (/BottomTab|\bTabs?Navigator/.test(factory)) return 'Tabs' + if (/Drawer/.test(factory)) return 'Drawer' + return 'Stack' +} + +interface RawRouteEntry { + abs: string + noExt: string + base: string + dir: string +} + +interface LayoutEntry { + file: string + dir: string + navigator: string | null + src: string +} + +interface PathAlias { + prefix: string + target: string +} + +interface ScreenTitleHit { + name: string | null + href: string | null + title: string +} + +function escapeRe(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +// Both parsers build their edge list by scanning every screen's source for call sites, which is +// inherently noisy in two ways this cleans up once at the end rather than guarding every producer +// above: (1) a screen that calls `navigate()` on its own route name — observed from a NotFound +// fallback resolving `navigate('HomeTab')` back to the tab it is already inside — draws a self-loop +// that says nothing; (2) the same call site can be matched by more than one heuristic (e.g. a +// `navigate()` call inside an anchor's `onPress`, also caught by the `href` scan), producing an +// exact duplicate edge. Both are noise, never signal, so they are dropped after building rather +// than reasoned about while building. +function finalizeEdges(edges: AppFlowEdge[]): AppFlowEdge[] { + const seenKeys = new Set() + const out: AppFlowEdge[] = [] + for (const edge of edges) { + if (edge.from === edge.to) continue + const key = `${edge.from}\0${edge.to}\0${edge.kind ?? ''}` + if (seenKeys.has(key)) continue + seenKeys.add(key) + out.push(edge) + } + return out +} + +function routeMatcher(urlPath: string): RegExp { + const re = urlPath + .split('/') + .map((seg) => + seg.startsWith('[...') + ? '.*' + : seg.startsWith('[') || seg.startsWith(':') + ? '[^/]+' + : escapeRe(seg) + ) + .join('/') + return new RegExp('^' + re + '/?$') +} + +const SHEET_LIBS: Array<[string, string]> = [ + ['@gorhom/bottom-sheet', 'gorhom-bottom-sheet'], + ['react-native-actions-sheet', 'actions-sheet'], + ['@lodev09/react-native-true-sheet', 'true-sheet'], + ['react-native-raw-bottom-sheet', 'raw-bottom-sheet'] +] + +/** + * Components a screen renders *instead of itself* when a gate is closed. Anything outside this list + * is ordinary conditional content, not a whole-screen swap, so it must not read as a guard. + */ +const GUARD_FALLBACK_RE = /^(?:NotFound|Unavailable|Forbidden|NoAccess|AccessDenied)/ + +/** Reads a parenthesised span starting at `open`, honouring nesting. */ +function readBalancedParens(src: string, open: number): { text: string; end: number } | null { + let depth = 0 + for (let index = open; index < src.length; index++) { + const char = src[index] + if (char === '(') depth += 1 + else if (char === ')') { + depth -= 1 + if (depth === 0) return { text: src.slice(open + 1, index), end: index } + } + } + return null +} + +/** + * Finds a gate that replaces the whole screen with a not-found-style fallback. + * + * The route is registered either way, so a deep link to a gated screen resolves and then renders a + * 404 — indistinguishable at runtime from a URL that was simply wrong. Reading the gate off the + * component tells the sweep which of the two it is, and saves it exploring for an entry point that + * this build does not render. + */ +/** + * Inlines a condition that is just a local flag (`!isEnabled`) with the expression it was assigned, + * so the recorded reason names the gate itself rather than the variable holding it. + */ +function resolveGuardCondition(src: string, condition: string): string { + const bare = /^(!?)([A-Za-z_$][\w$]*)$/.exec(condition) + if (!bare) return condition + const assigned = new RegExp( + `\\b(?:const|let|var)\\s+${escapeRe(bare[2])}\\s*(?::[^=\n]*)?=\\s*([^\n;]+)` + ).exec(src)?.[1] + return assigned ? `${bare[1]}${assigned.trim().replace(/\s+/g, ' ')}` : condition +} + +export function extractGuard(src: string): AppFlowGuard | null { + for (const match of src.matchAll(/\bif\s*\(/g)) { + const open = match.index + match[0].length - 1 + const condition = readBalancedParens(src, open) + if (!condition) continue + // Only an immediate return counts: a fallback further down the block is reached by other paths + // too, so it says nothing about how this screen renders when the gate is closed. + const returned = /^\s*\{?\s*return\s*\(?\s*<\s*([A-Za-z_$][\w$.]*)/.exec( + src.slice(condition.end + 1, condition.end + 200) + ) + if (!returned || !GUARD_FALLBACK_RE.test(returned[1])) continue + const text = condition.text.replace(/\s+/g, ' ').trim() + if (text) return { condition: resolveGuardCondition(src, text), fallback: returned[1] } + } + return null +} + +function extractHints(src: string): AppFlowStateHint[] { + const hints: AppFlowStateHint[] = [] + for (const [lib, label] of SHEET_LIBS) { + if (src.includes(lib)) { + const snap = src.match(/snapPoints\s*[:=][^[\n]*\[([^\]]+)\]/) + const snapPoints = snap + ? snap[1] + .split(',') + .map((s) => s.trim().replace(/["'`]/g, '')) + .filter(Boolean) + .map((s) => { + const n = parseFloat(s) + return Number.isNaN(n) ? null : n + }) + .filter((n): n is number => n !== null) + : undefined + hints.push({ type: 'bottom-sheet', lib: label, snapPoints }) + } + } + if (/useDialogControl|Dialog\.Outer/.test(src)) { + hints.push({ type: 'bottom-sheet', lib: 'app-dialog' }) + } + if (/]/.test(src)) { + hints.push({ type: 'modal', lib: 'rn-modal' }) + } + return hints +} + +async function walk(dir: string): Promise { + const out: string[] = [] + const entries = await readdir(dir, { withFileTypes: true }) + for (const entry of entries) { + const p = join(dir, entry.name) + if (entry.isDirectory()) { + out.push(...(await walk(p))) + } else { + out.push(p) + } + } + return out +} + +export interface AppConfigInfo { + scheme: string | null + name: string | null + slug: string | null + bundleId: string | null +} + +function firstString(value: unknown): string | null { + if (typeof value === 'string' && value.trim()) return value.trim() + if (Array.isArray(value) && typeof value[0] === 'string' && value[0].trim()) { + return value[0].trim() + } + return null +} + +export async function readAppConfig(projectRoot: string): Promise { + const empty: AppConfigInfo = { scheme: null, name: null, slug: null, bundleId: null } + try { + const cfg = JSON.parse(await readFile(join(projectRoot, 'app.json'), 'utf-8')) as { + expo?: { + scheme?: string | string[] + name?: string + slug?: string + ios?: { bundleIdentifier?: string } + } + scheme?: string | string[] + name?: string + } + const expo = cfg.expo ?? {} + return { + scheme: firstString(expo.scheme ?? cfg.scheme), + name: firstString(expo.name ?? cfg.name), + slug: firstString(expo.slug), + bundleId: firstString(expo.ios?.bundleIdentifier) + } + } catch { + // fall through to app.config.* + } + for (const f of ['app.config.js', 'app.config.ts']) { + try { + const src = await readFile(join(projectRoot, f), 'utf-8') + return { + scheme: src.match(/\bscheme\s*:\s*["']([A-Za-z0-9._+-]+)["']/)?.[1] ?? null, + name: src.match(/\bname\s*:\s*["']([^"']+)["']/)?.[1] ?? null, + slug: src.match(/\bslug\s*:\s*["']([A-Za-z0-9._+-]+)["']/)?.[1] ?? null, + bundleId: src.match(/\bbundleIdentifier\s*:\s*["']([A-Za-z0-9.]+)["']/)?.[1] ?? null + } + } catch { + // ignore + } + } + return empty +} + +function toUrlPath(noExt: string): string { + const segments = noExt.split('/') + if (segments[segments.length - 1] === 'index') segments.pop() + const urlSegments = segments.filter((s) => !(s.startsWith('(') && s.endsWith(')'))) + return '/' + urlSegments.join('/') +} + +function extractParams(segments: string[]): string[] { + return segments + .filter((s) => s.startsWith('[') && s.endsWith(']')) + .map((s) => s.slice(1, -1).replace(/^\.\.\./, '')) +} + +function toSlug(noExt: string): string { + let s = noExt === 'index' ? 'index' : noExt.replace(/\/index$/, '') + s = s + .replace(/[()[\]]/g, '') + .replace(/\.\.\./g, '') + .replace(/\//g, '_') + return generateSlug(s || 'index') +} + +function fallbackTitle(noExt: string): string { + const last = posix.basename(noExt) + if (last === 'index') return 'index' + const cleaned = last + .replace(/^\[(?:\.\.\.)?/, '') + .replace(/\]$/, '') + .replace(/[-_]+/g, ' ') + return cleaned.replace(/\b[a-z]/g, (c) => c.toUpperCase()) || 'Screen' +} + +function isUpwardHref(raw: string): boolean { + return raw.split(/[?#]/)[0].split('/').includes('..') +} + +const SOURCE_EXTS = [ + '.ios.tsx', + '.native.tsx', + '.ios.ts', + '.native.ts', + '.tsx', + '.ts', + '.ios.jsx', + '.native.jsx', + '.jsx', + '.js' +] +const IMPORT_RE = /(?:from|import)\s*\(?\s*['"]([^'"]+)['"]/g + +async function loadPathAliases(projectRoot: string): Promise { + const defaults: PathAlias[] = [ + { prefix: '@/', target: 'src/' }, + { prefix: '@/', target: '' } + ] + try { + const configPath = ts.findConfigFile(projectRoot, ts.sys.fileExists, 'tsconfig.json') + if (!configPath) return defaults + const parsed = ts.getParsedCommandLineOfConfigFile( + configPath, + {}, + { + ...ts.sys, + onUnRecoverableConfigFileDiagnostic: () => undefined + } + ) + const paths = parsed?.options.paths + if (!paths) return defaults + const baseUrl = parsed.options.baseUrl ?? projectRoot + const aliases = Object.entries(paths).map(([key, targets]) => ({ + prefix: key.replace(/\*$/, ''), + target: resolve(baseUrl, (targets[0] ?? '').replace(/\*$/, '')) + })) + aliases.sort((a, b) => b.prefix.length - a.prefix.length) + return aliases.length > 0 ? aliases : defaults + } catch { + return defaults + } +} + +async function resolveWithExt(base: string): Promise { + for (const ext of SOURCE_EXTS) { + try { + const candidate = base + ext + if ((await stat(candidate)).isFile()) return candidate + } catch { + // keep looking + } + } + for (const ext of SOURCE_EXTS) { + try { + const candidate = join(base, 'index' + ext) + if ((await stat(candidate)).isFile()) return candidate + } catch { + // keep looking + } + } + try { + if ((await stat(base)).isFile()) return base + } catch { + // not a file + } + return null +} + +async function resolveImport( + fromFile: string, + spec: string, + aliases: PathAlias[] +): Promise { + if (spec.startsWith('.')) { + return resolveWithExt(resolve(dirname(fromFile), spec)) + } + for (const alias of aliases) { + if (spec === alias.prefix.replace(/\/$/, '') || spec.startsWith(alias.prefix)) { + const rest = spec.startsWith(alias.prefix) ? spec.slice(alias.prefix.length) : '' + const resolved = await resolveWithExt(resolve(alias.target, rest)) + if (resolved) return resolved + } + } + return null +} + +async function loadLayoutSource(layoutFile: string, aliases: PathAlias[]): Promise { + const src = await readFile(layoutFile, 'utf-8') + const chunks = [src] + for (const match of src.matchAll(IMPORT_RE)) { + const resolved = await resolveImport(layoutFile, match[1], aliases) + if (!resolved) continue + chunks.push(await readFile(resolved, 'utf-8')) + } + return chunks.join('\n') +} + +function extractScreenTitles(src: string): ScreenTitleHit[] { + const hits: ScreenTitleHit[] = [] + + for (const match of src.matchAll( + /]*)>([\s\S]*?)<\/NativeTabs\.Trigger>/g + )) { + const name = match[1].match(/\bname\s*=\s*["']([^"']+)["']/)?.[1] ?? null + const href = match[1].match(/\bhref\s*=\s*["']([^"']+)["']/)?.[1] ?? null + const title = + match[2] + .match(/([^<]+)<\/NativeTabs\.Trigger\.Label>/)?.[1] + ?.trim() ?? + match[2].match(/