diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index 5814855ead..d6bfb51342 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -1,35 +1,65 @@ name: Perf Nightly -# Parked to workflow_dispatch by #1781 A3: it only writes a report and compares nothing, so it -# structurally cannot catch a regression, and iOS wall-clock medians swing up to +122% -# night-to-night at n=5 with no consumer reading the report. `pnpm perf` / scripts/perf are -# unaffected — still runnable on demand. Un-park once a comparison step with multi-night -# smoothing (someone reading it) exists, or a perf arc needs nightly data. +# Nightly iOS command performance against the fixture app, compared night over night (#1781 A3). +# - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts +# failing. PR lanes check behavior, not time. +# - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians +# swing up to +122% between single nights, so a step is judged against the median of up to seven +# earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. A night +# runs 3 rounds, the minimum for a timing verdict, so one failed sample withholds that step's +# timing verdict. +# - Cost: one macos-26 job of about 45 minutes a night; summaries are kept 90 days, raw logs 30. +# - Delete or park when: no flagged change has been acted on for 60 nights. +# Android stays report-only and runs only on request. on: + schedule: + - cron: '40 2 * * *' workflow_dispatch: inputs: rounds: - description: 'Measured rounds per command (samples)' + description: 'Measured rounds per step (samples, at least 3)' required: false - default: '5' + default: '3' + warmup: + description: 'Warmup rounds dropped before measuring' + required: false + default: '1' + scenarios: + description: 'Comma-separated iOS scenario ids (empty runs all)' + required: false + default: '' + baseline-nights: + description: 'Earlier scheduled nights in the rolling baseline' + required: false + default: '7' + baseline-branch: + description: 'Branch whose runs form the baseline' + required: false + default: 'main' + android: + description: 'Also run the report-only Android Settings tour' + type: boolean + required: false + default: true permissions: contents: read + actions: read concurrency: - group: ci-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: perf-nightly-${{ github.ref }} + cancel-in-progress: false env: - AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts' - PERF_ROUNDS: ${{ github.event.inputs.rounds || '5' }} + PERF_ROUNDS: ${{ github.event.inputs.rounds || '3' }} + PERF_WARMUP: ${{ github.event.inputs.warmup || '1' }} jobs: perf-ios: name: iOS Command Perf runs-on: macos-26 - timeout-minutes: 80 + timeout-minutes: 100 env: IOS_RUNTIME_VERSION: '26.2' AGENT_DEVICE_XCUITEST_ARCHS: 'arm64' @@ -42,6 +72,22 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml + + - name: Build the shared CLI + uses: ./.github/actions/run-gate + with: + gate: build + + - name: Boot iOS test simulator + id: ios-simulator + uses: ./.github/actions/boot-ios-test-simulator + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + preferred-device-name: iPhone 17 Pro - name: Restore and build iOS XCTest runner uses: ./.github/actions/setup-apple-runner-build @@ -53,37 +99,76 @@ jobs: xcuitest-platform: ios xcuitest-destination: generic/platform=iOS Simulator - - name: Boot iOS test simulator - uses: ./.github/actions/boot-ios-test-simulator - with: - runtime-version: ${{ env.IOS_RUNTIME_VERSION }} - preferred-device-name: iPhone 17 Pro + - name: Install the fixture app + id: fixture-app + uses: ./.github/actions/setup-fixture-app - name: Preflight iOS runner through public CLI + env: + UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} run: | + # The keyboard must stay expanded for keyboard dismiss to find its Done key. + xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "$UDID" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" pnpm clean:daemon - - name: Run iOS command perf benchmark + - name: Run iOS command perf suite + env: + UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + APP_ID: ${{ steps.fixture-app.outputs.app-id }} + APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + SCENARIOS: ${{ github.event.inputs.scenarios }} run: | - pnpm perf --platform ios \ - --device "iPhone 17 Pro" \ - --n "$PERF_ROUNDS" --warmup 1 \ + pnpm perf --platform ios --udid "$UDID" \ + --app-id "$APP_ID" --app-path "$APP_PATH" \ + --rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP" \ + ${SCENARIOS:+--scenarios "$SCENARIOS"} \ --out-dir "$GITHUB_WORKSPACE/perf-results" - - name: Upload iOS perf report - if: always() + - name: Compare with earlier nights + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} + env: + GH_TOKEN: ${{ github.token }} + BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }} + BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }} + run: | + pnpm perf:compare --to perf-results \ + --branch "$BASELINE_BRANCH" \ + --nights "$BASELINE_NIGHTS" \ + --exclude-run "$GITHUB_RUN_ID" \ + --out perf-results --github + echo "Full report: the perf-ios-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY" + + - name: Upload iOS perf summary + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: perf-ios - path: perf-results/ + name: perf-ios-summary + path: | + perf-results/summary.json + perf-results/report.md + perf-results/comparison.json + perf-results/comparison.md + retention-days: 90 + if-no-files-found: error + + - name: Upload iOS perf raw logs + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: perf-ios-raw + path: perf-results/raw/ + retention-days: 30 if-no-files-found: warn perf-android: name: Android Command Perf + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.android == 'true' }} runs-on: ubuntu-latest timeout-minutes: 80 + env: + AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts' steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -110,7 +195,7 @@ jobs: adb -s emulator-5554 shell settings put global window_animation_scale 0 || true; adb -s emulator-5554 shell settings put global transition_animation_scale 0 || true; adb -s emulator-5554 shell settings put global animator_duration_scale 0 || true; - pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup 1 --out-dir "$GITHUB_WORKSPACE/perf-results" + pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup "$PERF_WARMUP" --out-dir "$GITHUB_WORKSPACE/perf-results" - name: Upload Android perf report if: always() diff --git a/package.json b/package.json index ab0968810a..52b5601250 100644 --- a/package.json +++ b/package.json @@ -131,6 +131,7 @@ "test:ios-snapshot-differential": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift test --package-path apple/snapshot-presentation && node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/ios-snapshot-differential.test.ts", "size": "node scripts/size-report.mjs", "perf": "node --experimental-strip-types scripts/perf/run.ts", + "perf:compare": "node --experimental-strip-types scripts/perf/compare-cli.ts", "bench:ios-snapshot": "node --experimental-strip-types scripts/ios-snapshot-benchmark/run.ts", "bench:ios-snapshot:deep-button": "node --experimental-strip-types scripts/ios-snapshot-benchmark/deep-button.ts", "bench:ios-snapshot:evidence": "node --experimental-strip-types scripts/ios-snapshot-benchmark/evidence.ts", diff --git a/packages/host-kit/src/command.ts b/packages/host-kit/src/command.ts index 73b8aeb4f3..c57e0157bd 100644 --- a/packages/host-kit/src/command.ts +++ b/packages/host-kit/src/command.ts @@ -26,3 +26,4 @@ export { withCommandHost, withoutCommandExecutorOverride, } from './internal/exec.ts'; +export { EXEC_COMMAND_PHASE } from './internal/exec-trace.ts'; diff --git a/packages/host-kit/src/diagnostics.ts b/packages/host-kit/src/diagnostics.ts index def5222700..3f69c72b4d 100644 --- a/packages/host-kit/src/diagnostics.ts +++ b/packages/host-kit/src/diagnostics.ts @@ -1,6 +1,7 @@ export { countDiagnosticEventsByPhase, createRequestId, + DIAGNOSTIC_STDERR_PREFIX, type DiagnosticEventInput, emitDiagnostic, flushDiagnosticsToSessionFile, diff --git a/packages/host-kit/src/internal/diagnostics.ts b/packages/host-kit/src/internal/diagnostics.ts index 8d26f95e5f..9855bec828 100644 --- a/packages/host-kit/src/internal/diagnostics.ts +++ b/packages/host-kit/src/internal/diagnostics.ts @@ -53,6 +53,9 @@ type DiagnosticsScope = DiagnosticsScopeOptions & { sortedSensitiveValues?: string[]; }; +/** Prefixes each diagnostic event that a `--debug` client without a log file writes to stderr. */ +export const DIAGNOSTIC_STDERR_PREFIX = '[agent-device][diag] '; + const diagnosticsStorage = new AsyncLocalStorage(); export function createRequestId(): string { @@ -164,7 +167,7 @@ export function emitDiagnostic(event: DiagnosticEventInput): void { appendDiagnosticLine(scope, scope.traceLogPath, fileLine); } if (scope.debug && !scope.logPath && !scope.traceLogPath) { - process.stderr.write(`[agent-device][diag] ${fileLine}`); + process.stderr.write(`${DIAGNOSTIC_STDERR_PREFIX}${fileLine}`); } } catch { // Best-effort diagnostics should not break request flow. diff --git a/packages/host-kit/src/internal/exec-trace.ts b/packages/host-kit/src/internal/exec-trace.ts index 3a0ca05526..9b1f516476 100644 --- a/packages/host-kit/src/internal/exec-trace.ts +++ b/packages/host-kit/src/internal/exec-trace.ts @@ -9,6 +9,9 @@ import { updateDiagnosticsScope, } from './diagnostics.ts'; +/** The diagnostic phase of each host process lifecycle event. */ +export const EXEC_COMMAND_PHASE = 'exec_command'; + export type ExecSettlementReason = | 'success' | 'exit_failure' @@ -52,7 +55,7 @@ export function createExecTraceContext( pid = data.pid ?? pid; emitDiagnostic({ level: 'debug', - phase: 'exec_command', + phase: EXEC_COMMAND_PHASE, durationMs: performance.now() - started, data: { execId, command, event, pid, timeoutMs, ...data }, }); diff --git a/packages/host-kit/src/internal/retry.ts b/packages/host-kit/src/internal/retry.ts index c636d985b8..82f50606c3 100644 --- a/packages/host-kit/src/internal/retry.ts +++ b/packages/host-kit/src/internal/retry.ts @@ -9,6 +9,9 @@ type RetryPolicy = { shouldRetry?: (error: unknown, attempt: number) => boolean; }; +/** The diagnostic phase of each retry attempt and outcome. */ +export const RETRY_PHASE = 'retry'; + /** * The read side of a deadline. Holders are typed against this rather than the class so a caller * can hand one a substitute clock; {@link Deadline} is the production implementation. @@ -281,7 +284,7 @@ export function sleep(ms: number, signal?: AbortSignal): Promise { function publishRetryEvent(event: RetryTelemetryEvent): void { emitDiagnostic({ level: event.event === 'attempt_failed' || event.event === 'exhausted' ? 'warn' : 'debug', - phase: 'retry', + phase: RETRY_PHASE, data: { ...event, }, diff --git a/packages/host-kit/src/retry.ts b/packages/host-kit/src/retry.ts index bfc36cfe26..83e9eb6946 100644 --- a/packages/host-kit/src/retry.ts +++ b/packages/host-kit/src/retry.ts @@ -5,4 +5,5 @@ export { pollUntilSettled, retryWithPolicy, sleep, + RETRY_PHASE, } from './internal/retry.ts'; diff --git a/scripts/ios-snapshot-benchmark/benchmark-config.ts b/scripts/ios-snapshot-benchmark/benchmark-config.ts index 777beb2c35..4293114545 100644 --- a/scripts/ios-snapshot-benchmark/benchmark-config.ts +++ b/scripts/ios-snapshot-benchmark/benchmark-config.ts @@ -1,6 +1,6 @@ import path from 'node:path'; import { parseLocalStates, parseRtt, parseSampleCount, parseScreenIds } from './definitions.ts'; -import { resolveRepoRoot } from './host.ts'; +import { resolveRepoRoot } from '../lib/bench-host.ts'; import { assertBenchmarkOwner, assertOwnedDerivedPath, diff --git a/scripts/ios-snapshot-benchmark/run.test.ts b/scripts/ios-snapshot-benchmark/run.test.ts index 4639d11b35..d17a60e820 100644 --- a/scripts/ios-snapshot-benchmark/run.test.ts +++ b/scripts/ios-snapshot-benchmark/run.test.ts @@ -4,7 +4,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; import { mkdtempForTest } from '@agent-device/test-kit'; -import { resolveRepoRoot } from './host.ts'; +import { resolveRepoRoot } from '../lib/bench-host.ts'; test('CLI refuses an unmarked existing root before any descendant can be cleared', async () => { const repoRoot = resolveRepoRoot(); diff --git a/scripts/ios-snapshot-benchmark/run.ts b/scripts/ios-snapshot-benchmark/run.ts index e312da6980..89eff16655 100644 --- a/scripts/ios-snapshot-benchmark/run.ts +++ b/scripts/ios-snapshot-benchmark/run.ts @@ -10,7 +10,7 @@ import { classifyFailure, formatCliFailure, openFixture, type CliContext } from import { BenchmarkControlError, runDeepButtonControls } from './deep-control.ts'; import { CONTRACT, screenFixture } from './definitions.ts'; import { deepButtonFixtureEvidence } from './deep-button.ts'; -import { readGitRevision, readHostIdentity, readTarget, readToolchain } from './host.ts'; +import { readGitRevision, readHostIdentity, readTarget, readToolchain } from '../lib/bench-host.ts'; import { BenchmarkCellAdmissionError, BenchmarkContentionError, diff --git a/scripts/ios-snapshot-benchmark/types.ts b/scripts/ios-snapshot-benchmark/types.ts index e807e819fe..0ebd2d6504 100644 --- a/scripts/ios-snapshot-benchmark/types.ts +++ b/scripts/ios-snapshot-benchmark/types.ts @@ -1,3 +1,5 @@ +import type { GitRevision, HostIdentity, Target, Toolchain } from '../lib/bench-host.ts'; + export const BENCHMARK_SCHEMA_VERSION = 'ios-snapshot-convergence.v1' as const; export const ISSUE_ID = '#2189'; export const PARENT_ISSUE_ID = '#2188'; @@ -95,38 +97,6 @@ export type Measurement = { failureCategories: Partial>; }; -export type Toolchain = { - node: string; - pnpm: string; - xcode: string; - simctl: string; - os: string; - arch: string; -}; - -export type HostIdentity = { - model: string; - modelIdentifier: string; - cpu: string; - cpuCores: number; -}; - -export type GitRevision = { - commit: string; - branch: string; - dirty: boolean; -}; - -export type Target = { - platform: 'ios'; - kind: 'simulator'; - udid: string; - name: string; - runtime: string; - appId: string; - appPath?: string; -}; - export type PackageSize = { status: 'measured' | 'not-run'; revision: string; diff --git a/scripts/ios-snapshot-benchmark/host.test.ts b/scripts/lib/bench-host.test.ts similarity index 93% rename from scripts/ios-snapshot-benchmark/host.test.ts rename to scripts/lib/bench-host.test.ts index b2d1124d16..1a9cdc75b4 100644 --- a/scripts/ios-snapshot-benchmark/host.test.ts +++ b/scripts/lib/bench-host.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { parseHardwareProfile } from './host.ts'; +import { parseHardwareProfile } from './bench-host.ts'; test('parses the Mac model and CPU identity from system_profiler output', () => { assert.deepEqual( diff --git a/scripts/ios-snapshot-benchmark/host.ts b/scripts/lib/bench-host.ts similarity index 88% rename from scripts/ios-snapshot-benchmark/host.ts rename to scripts/lib/bench-host.ts index 7df8217649..03a7b29c14 100644 --- a/scripts/ios-snapshot-benchmark/host.ts +++ b/scripts/lib/bench-host.ts @@ -1,7 +1,38 @@ import { execFileSync } from 'node:child_process'; import os from 'node:os'; import path from 'node:path'; -import type { GitRevision, HostIdentity, Target, Toolchain } from './types.ts'; + +export type Toolchain = { + node: string; + pnpm: string; + xcode: string; + simctl: string; + os: string; + arch: string; +}; + +export type HostIdentity = { + model: string; + modelIdentifier: string; + cpu: string; + cpuCores: number; +}; + +export type GitRevision = { + commit: string; + branch: string; + dirty: boolean; +}; + +export type Target = { + platform: 'ios'; + kind: 'simulator'; + udid: string; + name: string; + runtime: string; + appId: string; + appPath?: string; +}; export function readToolchain(): Toolchain { return { diff --git a/scripts/perf/README.md b/scripts/perf/README.md new file mode 100644 index 0000000000..b53c2e1725 --- /dev/null +++ b/scripts/perf/README.md @@ -0,0 +1,103 @@ +# Command performance suite + +`pnpm perf --platform ios` times every interaction command against the fixture app (`examples/test-app`). +The steps are in `ios-suite.ts`. Each step belongs to one scenario. +Each scenario starts from an untimed setup in every round. +The heavy-feed scenario uses a screen whose accessibility tree is deeper than 65 levels. +The suite records a reason for each catalog command that it does not time. `ios-suite.test.ts` keeps that list complete. +`pnpm perf --platform android` runs the older report-only Settings tour. + +## Run locally + +1. Run `pnpm build`. The suite drives the built CLI and the built SDK. +2. Use your own booted simulator. Install a current Release fixture on it. For the cached CI build and the JavaScript repack, see `examples/test-app/README.md`. +3. Set `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH` to a path in your worktree. +4. Run the suite: + +```sh +pnpm perf --platform ios --udid --rounds 3 --warmup 1 --out-dir .tmp/perf/local +``` + +`--rounds` is at least 3. `--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id. +The suite uses its own temporary state directory and daemon. It stops them at the end. +At the end, or on SIGINT or SIGTERM, the suite saves the logs to `raw/state/`, stops the session and daemon within a deadline, and saves the logs again. + +## Output + +- `summary.json`: the machine-readable run (`agent-device-perf-summary/v1`). It holds the metadata, the coverage, and every sample. +- `report.md`: one table per scenario, the backends and fallbacks, the failures, and the coverage. +- `raw/state/`: the daemon log, and for each session `runner.log`, `events.ndjson`, and the `--debug` request log of each request. `requestId` in a sample names its request log. +- `raw/failures/`: the JSON response and stderr of each failed sample. + +## Metrics + +All times are milliseconds. Statistics use only the successful samples of a step. + +| Metric | Meaning | +| --- | --- | +| `wall` | The CLI process time, from spawn to exit. | +| `daemon` | The daemon request round trip that the client measures (`daemon_request`). For the pinned-backend capture (`home-reads/snapshot-forced-tree`, marked SDK in the report), the span of its request log. Comparisons use this metric. | +| `mean ± sd`, `cv` | The mean, the sample standard deviation, and the standard deviation divided by the mean. | +| `median`, `p95`, `min`, `max` | The median (the mean of the two middle values for an even count), the nearest-rank 95th percentile, and the extremes. | +| `runner` | The median sum of XCTest runner command times in one request. | +| `spawns` | The median count and time of host processes that one request started. | +| `snapshot source` | The median time of host accessibility-bridge acquisitions in one request. | +| `settle wait` | The median `settle.waitedMs` that a `--settle` response reports. | +| `stabilization` | The median time of post-gesture stabilization phases in one request. | +| `retries` | The failed attempts of retried operations in all successful samples. | +| `top phases` | The five phases with the largest median time per request. | +| `producer` | The number of samples that each snapshot producer answered (`simulator-ax-bridge`, `apple-runner`). | +| backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected`. | +| fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. | +| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth, or that a backend other than the pinned one served the pinned capture. | + +## Nightly results + +`.github/workflows/perf-nightly.yml` runs the iOS suite every night on `main`. +Each run uploads `perf-ios-summary` (summary, report, comparison; 90 days) and `perf-ios-raw` (logs; 30 days). +The job compares the run with up to seven earlier nights and writes the result to the job summary. +A night is the newest scheduled run of one UTC day that ran every scenario of the judged run. +Manual dispatches are never part of the rolling baseline. + +To fetch and compare runs on your machine, use `gh` with access to the repository: + +```sh +pnpm perf:compare # latest night against the 7 nights before it +pnpm perf:compare --to 2026-10-28 # that night against the 7 nights before it +pnpm perf:compare --to latest --from 2026-10-21 # latest against the newest run of that day +pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare # every run in the range +pnpm perf:compare --to .tmp/perf/local # a local run against recent nights +``` + +A reference is `latest`, a run id, a `YYYY-MM-DD` date, or a local output path. +An explicit `--to` or `--from` reference accepts runs of any trigger, so it can compare dispatches and local runs. +Downloads stay in `.tmp/perf-nightly//`. +Do not judge a local run against CI nights. The comparison warns when the device, Xcode, or CPU differs. + +## Comparison rule + +The baseline of a step is the median of its nightly medians. +The noise of a step is the scaled median absolute deviation of those medians. +A step is a `regression` or an `improvement` only when its change is more than all of these: + +- 20% of the baseline. +- Three times the night-to-night noise. +- Two times the coefficient of variation of the run divided by √n. +- 50 ms. + +Each step gets a timing verdict and a failure verdict. +A timing verdict uses only successful samples: + +- `insufficient-samples`: the judged run has fewer than three successful samples of the step. +- `insufficient-baseline`: fewer than three baseline nights have three successful samples of the step. Nights with fewer do not count. + +The failure verdict compares failed samples with the worst baseline night: + +- `new-failures`: the step failed and no baseline night failed. +- `more-failures`: the step failed more often than on every baseline night, both in count and in share of its samples. +- `fixed`: the step did not fail and every baseline night failed. + +When failures are new or more, a faster median is `insufficient-samples`, not `improvement`. +The samples that still pass do not show that the step got faster. +Regressions and worse failures are workflow warnings. The job stays green. +The thresholds are in `compare.ts`. diff --git a/scripts/perf/cli.ts b/scripts/perf/cli.ts index 57ee22c56a..cada28867d 100644 --- a/scripts/perf/cli.ts +++ b/scripts/perf/cli.ts @@ -33,7 +33,11 @@ function jsonOk(json: unknown): boolean { // Invoke the built CLI once. `args` includes the command + positionals + dash-flags; // `baseFlags` carries the isolation + device flags shared by every call. -export function invokeCli(args: string[], baseFlags: string[]): CliResult { +export function invokeCli( + args: string[], + baseFlags: string[], + options: { timeoutMs?: number } = {}, +): CliResult { const full = [...CLI_ARGV, ...args, ...baseFlags, '--json']; const t0 = performance.now(); let stdout = ''; @@ -46,6 +50,7 @@ export function invokeCli(args: string[], baseFlags: string[]): CliResult { cwd: REPO_ROOT, maxBuffer: MAX_BUFFER, allowFailure: true, + timeoutMs: options.timeoutMs, }); stdout = r.stdout; stderr = r.stderr; diff --git a/scripts/perf/compare-cli.ts b/scripts/perf/compare-cli.ts new file mode 100644 index 0000000000..3b3593acfe --- /dev/null +++ b/scripts/perf/compare-cli.ts @@ -0,0 +1,196 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { compareRuns, renderComparison, type Comparison } from './compare.ts'; +import { REPO_ROOT } from './config.ts'; +import { + coversTarget, + fetchSummary, + listRuns, + readLocalSummary, + resolveRunRef, + selectNightlyBaseline, + selectRange, + SUMMARY_ARTIFACT, + type NightlySource, + type WorkflowRun, +} from './nightly-artifacts.ts'; +import type { PerfSummary } from './types.ts'; + +const USAGE = `Usage: pnpm perf:compare [--to ] [--from |..] [--nights ] [options] + +Compares one perf run with earlier runs and prints a Markdown verdict per step. + +A is latest, a workflow run id, a YYYY-MM-DD date (the newest run that UTC day), or a local +path to a summary.json or a \`pnpm perf\` output directory. + + --to Run to judge (default: latest). + --from Baseline run, or an inclusive range .. of runs of any trigger. + Without it the baseline is the --nights scheduled nights before --to. + --nights Rolling baseline size in UTC nights, one scheduled run each (default: 7). + --branch Branch whose nightly runs are listed (default: main). + --repo Repository (default: $GITHUB_REPOSITORY or callstack/agent-device). + --workflow Workflow file (default: perf-nightly.yml). + --cache-dir Download cache (default: .tmp/perf-nightly). + --exclude-run Never use this run as baseline (repeatable). + --out Also write comparison.json and comparison.md there. + --github Append the Markdown to $GITHUB_STEP_SUMMARY and annotate flagged steps.`; + +type Options = { + to: string; + from?: string; + nights: number; + out?: string; + github: boolean; + exclude: Set; + source: NightlySource; +}; + +function parse(argv: string[]): Options { + const options: Options = { + to: 'latest', + nights: 7, + github: false, + exclude: new Set(), + source: { + repo: process.env.GITHUB_REPOSITORY ?? 'callstack/agent-device', + workflow: 'perf-nightly.yml', + branch: 'main', + artifact: SUMMARY_ARTIFACT, + cacheDir: path.join(REPO_ROOT, '.tmp', 'perf-nightly'), + }, + }; + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]!; + const value = (): string => { + const next = argv[++i]; + if (next === undefined) throw new Error(`Missing value for ${flag}`); + return next; + }; + switch (flag) { + case '--to': + options.to = value(); + break; + case '--from': + options.from = value(); + break; + case '--nights': { + const n = Number(value()); + if (!Number.isInteger(n) || n < 1) throw new Error('--nights must be a positive integer'); + options.nights = n; + break; + } + case '--branch': + options.source.branch = value(); + break; + case '--repo': + options.source.repo = value(); + break; + case '--workflow': + options.source.workflow = value(); + break; + case '--cache-dir': + options.source.cacheDir = path.resolve(value()); + break; + case '--exclude-run': + options.exclude.add(value()); + break; + case '--out': + options.out = path.resolve(value()); + break; + case '--github': + options.github = true; + break; + case '--help': + case '-h': + process.stdout.write(`${USAGE}\n`); + process.exit(0); + break; + default: + throw new Error(`Unknown flag: ${flag}\n\n${USAGE}`); + } + } + return options; +} + +type Loaded = { summary: PerfSummary; label: string; run?: WorkflowRun }; + +function main(): void { + const options = parse(process.argv.slice(2)); + let runs: WorkflowRun[] | undefined; + const allRuns = (): WorkflowRun[] => (runs ??= [...listRuns(options.source)]); + const runLabel = (run: WorkflowRun) => `run ${run.id} (${run.createdAt.slice(0, 10)})`; + const load = (ref: string): Loaded => { + if (fs.existsSync(ref)) + return { summary: readLocalSummary(ref), label: path.basename(path.resolve(ref)) }; + const run = resolveRunRef(ref, allRuns()); + if (!run) + throw new Error( + `No completed ${options.source.workflow} run on ${options.source.branch} matches ${ref}.`, + ); + const summary = fetchSummary(options.source, run); + if (!summary) throw new Error(`Run ${run.id} has no ${options.source.artifact} artifact.`); + return { summary, label: runLabel(run), run }; + }; + const loadRuns = (candidates: WorkflowRun[]): Loaded[] => + candidates.flatMap((run) => { + const summary = fetchSummary(options.source, run); + return summary ? [{ summary, label: runLabel(run), run }] : []; + }); + + const target = load(options.to); + const exclude = new Set([...options.exclude, ...(target.run ? [target.run.id] : [])]); + let baseline: Loaded[]; + if (options.from?.includes('..')) { + const [fromRef, toRef] = options.from.split('..'); + const range = selectRange(allRuns(), fromRef!, toRef!).filter((run) => !exclude.has(run.id)); + baseline = loadRuns(range); + } else if (options.from) { + baseline = [load(options.from)]; + } else { + const before = target.run?.createdAt ?? target.summary.startedAt; + baseline = selectNightlyBaseline( + runs ?? listRuns(options.source), + { before, exclude, nights: options.nights }, + (run) => { + const summary = fetchSummary(options.source, run); + return summary && coversTarget(summary, target.summary) ? summary : undefined; + }, + ).map(({ run, value }) => ({ summary: value, label: runLabel(run), run })); + } + + const comparison = compareRuns(target, baseline); + const markdown = renderComparison(comparison); + process.stdout.write(`${markdown}\n`); + if (options.out) writeComparison(comparison, markdown, options.out); + if (options.github) publishToGitHub(comparison, markdown); +} + +function writeComparison(comparison: Comparison, markdown: string, dir: string): void { + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'comparison.json'), `${JSON.stringify(comparison, null, 2)}\n`); + fs.writeFileSync(path.join(dir, 'comparison.md'), markdown); +} + +function publishToGitHub(comparison: Comparison, markdown: string): void { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (summaryPath) fs.appendFileSync(summaryPath, `${markdown}\n`); + for (const row of comparison.rows) { + if (row.timing === 'regression') { + process.stdout.write( + `::warning title=perf regression::${row.id}: median ${row.target?.median.toFixed(0)} ms vs ${row.baseline?.median.toFixed(0)} ms baseline\n`, + ); + } + if (row.failures === 'new-failures' || row.failures === 'more-failures') { + process.stdout.write( + `::warning title=perf ${row.failures}::${row.id}: ${row.target?.failures}/${row.target?.samples} samples failed\n`, + ); + } + } +} + +try { + main(); +} catch (error) { + process.stderr.write(`[perf:compare] ${(error as Error).message}\n`); + process.exit(1); +} diff --git a/scripts/perf/compare.test.ts b/scripts/perf/compare.test.ts new file mode 100644 index 0000000000..79aec9562d --- /dev/null +++ b/scripts/perf/compare.test.ts @@ -0,0 +1,130 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { compareRuns, type ComparisonRow } from './compare.ts'; +import { summaryFixture, type StepFixture } from './summary.fixtures.ts'; + +const STEP = 'heavy-feed/snapshot'; + +function nights(...steps: StepFixture[][]) { + return steps.map((night, index) => ({ summary: summaryFixture(night), label: `night ${index}` })); +} + +function row(target: StepFixture[], baseline: StepFixture[][], id = STEP): ComparisonRow { + const comparison = compareRuns( + { summary: summaryFixture(target), label: 'tonight' }, + nights(...baseline), + ); + return comparison.rows.find((r) => r.id === id)!; +} + +const flat = (ms: number, extra: Partial = {}): StepFixture[] => [ + { id: STEP, daemonMs: [ms, ms, ms], ...extra }, +]; + +test('flags a slowdown that exceeds a quiet baseline and the relative floor', () => { + const result = row( + [{ id: STEP, daemonMs: [1450, 1500, 1550] }], + [flat(1000), flat(1010), flat(990)], + ); + assert.equal(result.timing, 'regression'); + assert.equal(result.metric, 'daemon'); + assert.equal(result.baseline?.median, 1000); + assert.equal(result.deltaMs, 500); + assert.equal(result.threshold, 0.2); +}); + +test('does not flag the same slowdown when nightly medians already swing that much', () => { + const result = row( + [{ id: STEP, daemonMs: [1450, 1500, 1550] }], + [flat(500), flat(1000), flat(1500), flat(700), flat(1300)], + ); + assert.equal(result.timing, 'unchanged'); + // 3 × scaled MAD: deviations from 1000 are 0, 300, 300, 500, 500. + assert.ok(Math.abs(result.threshold! - (3 * 1.4826 * 300) / 1000) < 1e-9); +}); + +test('does not flag a large relative change below the absolute floor', () => { + assert.equal(row(flat(60), [flat(40), flat(40), flat(40)]).timing, 'unchanged'); +}); + +test('flags a speedup beyond noise as an improvement', () => { + assert.equal(row(flat(500), [flat(1000), flat(1010), flat(990)]).timing, 'improvement'); +}); + +test('withholds a timing verdict when tonight or a baseline night has fewer than three successes', () => { + const one = (ms: number): StepFixture[] => [{ id: STEP, daemonMs: [ms] }]; + const fromOneSample = row(one(1500), [one(1000), one(1000), one(1000)]); + assert.equal(fromOneSample.timing, 'insufficient-samples'); + assert.equal(fromOneSample.baseline?.timedNights, 0); + // Three successes tonight still need three nights that each had three successes. + const thinNights = row(flat(1500), [flat(1000), flat(1000), one(1000)]); + assert.equal(thinNights.timing, 'insufficient-baseline'); + assert.equal(thinNights.baseline?.timedNights, 2); + assert.equal(thinNights.deltaMs, 500); +}); + +test('withholds a timing verdict until enough nights exist', () => { + const result = row(flat(1500), [flat(1000), flat(1000)]); + assert.equal(result.timing, 'insufficient-baseline'); + assert.equal(result.deltaRelative, 0.5); +}); + +test('reports failures that no baseline night had, and failures that went away', () => { + const clean = [flat(1000), flat(1000), flat(1000)]; + assert.equal(row(flat(1000, { failures: 1 }), clean).failures, 'new-failures'); + const failing = [ + flat(1000, { failures: 1 }), + flat(1000, { failures: 2 }), + flat(1000, { failures: 1 }), + ]; + assert.equal(row(flat(1000), failing).failures, 'fixed'); + // As many failures as the worst baseline night is not a change; timing is judged as usual. + const flaky = [flat(1000, { failures: 1 }), flat(1000), flat(1000)]; + const steady = row(flat(1000, { failures: 1 }), flaky); + assert.equal(steady.failures, 'unchanged'); + assert.equal(steady.timing, 'unchanged'); +}); + +test('reports worse failures over a flaky baseline and never as an improvement', () => { + const flaky = [flat(1000, { failures: 1 }), flat(1000), flat(1000)]; + const mostlyFailing = row([{ id: STEP, daemonMs: [500], failures: 2 }], flaky); + assert.equal(mostlyFailing.failures, 'more-failures'); + assert.equal(mostlyFailing.timing, 'insufficient-samples'); + // Enough successes, but they are the subset that still passes, so the speedup is withheld. + const faster = row(flat(500, { failures: 2 }), flaky); + assert.equal(faster.failures, 'more-failures'); + assert.equal(faster.timing, 'insufficient-samples'); + assert.equal(faster.deltaMs, -500); +}); + +test('falls back to wall time when any run lacks the daemon timer', () => { + const result = row(flat(1000), [ + flat(1000), + [{ id: STEP, wallMs: [1300, 1300, 1300] }], + flat(1000), + ]); + assert.equal(result.metric, 'wall'); +}); + +test('marks steps that only one side measured', () => { + const comparison = compareRuns( + { summary: summaryFixture([{ id: 'a/new', daemonMs: [1] }]), label: 't' }, + [{ summary: summaryFixture([{ id: 'a/old', daemonMs: [1] }]), label: 'b' }], + ); + assert.deepEqual( + comparison.rows.map((r) => [r.id, r.timing]), + [ + ['a/new', 'new'], + ['a/old', 'missing'], + ], + ); +}); + +test('warns when the baseline ran on another toolchain', () => { + const comparison = compareRuns( + { summary: summaryFixture(flat(1000), { xcode: 'Xcode 26.3' }), label: 't' }, + [{ summary: summaryFixture(flat(1000)), label: 'b' }], + ); + assert.equal(comparison.warnings.length, 1); + assert.match(comparison.warnings[0]!, /xcode/); +}); diff --git a/scripts/perf/compare.ts b/scripts/perf/compare.ts new file mode 100644 index 0000000000..22b1f28a61 --- /dev/null +++ b/scripts/perf/compare.ts @@ -0,0 +1,325 @@ +import { MIN_MEASURED_ROUNDS } from './config.ts'; +import { median, scaledMad } from './stats.ts'; +import type { Measurement, PerfSummary, Stat } from './types.ts'; + +export type ComparisonThresholds = { + // A change must exceed every one of these to be flagged. + minRelative: number; // fraction of the baseline median + minAbsoluteMs: number; + spreadFactor: number; // × night-to-night relative spread (scaled MAD of nightly medians) + withinFactor: number; // × tonight's cv / √n (standard error of tonight's median) + minBaselineNights: number; // fewer timed nights give `insufficient-baseline` + minSamples: number; // successful samples that tonight and each timed night need +}; + +const DEFAULT_THRESHOLDS: ComparisonThresholds = { + minRelative: 0.2, + minAbsoluteMs: 50, + spreadFactor: 3, + withinFactor: 2, + minBaselineNights: 3, + minSamples: MIN_MEASURED_ROUNDS, +}; + +export type TimingVerdict = + | 'regression' + | 'improvement' + | 'unchanged' + | 'insufficient-samples' + | 'insufficient-baseline' + | 'new' + | 'missing'; + +export type FailureVerdict = 'new-failures' | 'more-failures' | 'fixed' | 'unchanged'; + +type Flagged = Exclude; + +/** Verdicts that need a reader's attention, in report order. */ +const FLAGGED_VERDICTS: readonly Flagged[] = [ + 'new-failures', + 'more-failures', + 'regression', + 'fixed', + 'improvement', +]; + +export type ComparisonRow = { + id: string; + scenario: string; + command: string; + timing: TimingVerdict; + // Null when only one side measured the step. + failures: FailureVerdict | null; + metric: 'daemon' | 'wall'; + target: { median: number; n: number; cv: number; failures: number; samples: number } | null; + baseline: { + median: number; + nights: number; + timedNights: number; + spread: number; + failingNights: number; + } | null; + deltaMs: number | null; + deltaRelative: number | null; + threshold: number | null; +}; + +export type RunRef = { + label: string; + runId?: string; + commit: string; + startedAt: string; + device: string; + xcode?: string; + rounds: number; +}; + +const COMPARISON_SCHEMA = 'agent-device-perf-comparison/v1'; + +export type Comparison = { + schema: typeof COMPARISON_SCHEMA; + target: RunRef; + baseline: RunRef[]; + thresholds: ComparisonThresholds; + counts: { + timing: Partial>; + failures: Partial>; + }; + warnings: string[]; + rows: ComparisonRow[]; +}; + +function describeRun(summary: PerfSummary, label: string): RunRef { + return { + label, + runId: summary.metadata.runId, + commit: summary.metadata.commit, + startedAt: summary.startedAt, + device: `${summary.metadata.device.name}${summary.metadata.device.runtime ? ` ${summary.metadata.device.runtime}` : ''}`, + xcode: summary.metadata.xcode, + rounds: summary.config.rounds, + }; +} + +function metricStat(m: Measurement, metric: 'daemon' | 'wall'): Stat | null { + return metric === 'daemon' ? m.daemon : m.wallClock; +} + +function failureRate(m: Measurement): number { + return m.samples.length === 0 ? 0 : m.failures / m.samples.length; +} + +/** + * Failures worsen when tonight has more failed samples than every baseline night, both as a count + * and as a share of samples, so a different round count cannot fake a change. + */ +function judgeFailures(target: Measurement, history: Measurement[]): FailureVerdict { + const failingNights = history.filter((m) => m.failures > 0).length; + const worstCount = Math.max(...history.map((m) => m.failures)); + const worstRate = Math.max(...history.map(failureRate)); + if (target.failures > worstCount && failureRate(target) > worstRate) + return failingNights === 0 ? 'new-failures' : 'more-failures'; + if (target.failures === 0 && failingNights === history.length) return 'fixed'; + return 'unchanged'; +} + +function compareStep( + id: string, + target: Measurement | undefined, + history: Measurement[], + thresholds: ComparisonThresholds, +): ComparisonRow { + const any = target ?? history[0]!; + const base = { id, scenario: any.scenario, command: any.command }; + const metric: 'daemon' | 'wall' = [target, ...history].every( + (m) => !m || !m.wallClock || m.daemon, + ) + ? 'daemon' + : 'wall'; + const empty = { deltaMs: null, deltaRelative: null, threshold: null }; + const timed = (s: Stat | null): s is Stat => s !== null && s.n >= thresholds.minSamples; + const timedNights = history + .map((m) => metricStat(m, metric)) + .filter(timed) + .map((s) => s.median); + const baselineMedian = timedNights.length > 0 ? median(timedNights) : null; + const baseline = + history.length === 0 + ? null + : { + median: baselineMedian ?? Number.NaN, + nights: history.length, + timedNights: timedNights.length, + spread: baselineMedian ? scaledMad(timedNights) / baselineMedian : 0, + failingNights: history.filter((m) => m.failures > 0).length, + }; + if (!target) + return { ...base, timing: 'missing', failures: null, metric, target: null, baseline, ...empty }; + const stat = metricStat(target, metric); + const targetFacts = { + median: stat?.median ?? Number.NaN, + n: stat?.n ?? 0, + cv: stat?.cv ?? 0, + failures: target.failures, + samples: target.samples.length, + }; + const row = { ...base, metric, target: targetFacts, baseline }; + if (!baseline) return { ...row, timing: 'new', failures: null, ...empty }; + const failures = judgeFailures(target, history); + if (!stat || baselineMedian === null) { + const timing = timed(stat) ? 'insufficient-baseline' : 'insufficient-samples'; + return { ...row, timing, failures, ...empty }; + } + + const deltaMs = stat.median - baselineMedian; + const deltaRelative = baselineMedian === 0 ? 0 : deltaMs / baselineMedian; + const threshold = Math.max( + thresholds.minRelative, + thresholds.spreadFactor * baseline.spread, + (thresholds.withinFactor * stat.cv) / Math.sqrt(stat.n), + ); + const measured = { ...row, failures, deltaMs, deltaRelative, threshold }; + if (!timed(stat)) return { ...measured, timing: 'insufficient-samples' }; + if (timedNights.length < thresholds.minBaselineNights) { + return { ...measured, timing: 'insufficient-baseline' }; + } + const beyondNoise = + Math.abs(deltaRelative) > threshold && Math.abs(deltaMs) > thresholds.minAbsoluteMs; + if (!beyondNoise) return { ...measured, timing: 'unchanged' }; + if (deltaMs > 0) return { ...measured, timing: 'regression' }; + // The samples that still succeed while failures worsen do not show that the step got faster. + const worsened = failures === 'new-failures' || failures === 'more-failures'; + return { ...measured, timing: worsened ? 'insufficient-samples' : 'improvement' }; +} + +function environmentWarnings(target: PerfSummary, baseline: PerfSummary[]): string[] { + const facts = (s: PerfSummary) => ({ + device: `${s.metadata.device.name} ${s.metadata.device.runtime ?? ''}`.trim(), + xcode: s.metadata.xcode ?? 'none', + cpu: s.metadata.host.cpu, + }); + const mine = facts(target); + const warnings: string[] = []; + for (const key of ['device', 'xcode', 'cpu'] as const) { + const differing = baseline.filter((s) => facts(s)[key] !== mine[key]).length; + if (differing > 0) { + warnings.push( + `${differing}/${baseline.length} baseline runs used a different ${key} than tonight (${mine[key]}); their timings are not like for like.`, + ); + } + } + return warnings; +} + +/** + * Compares one run against a rolling baseline of earlier runs. The baseline value of a step is the + * median of its nightly medians, and its noise is the scaled MAD of those medians. A step is flagged + * only when its change exceeds the relative floor, the night-to-night spread, tonight's own + * sampling error, and an absolute floor together. Tonight and every night that counts need + * `minSamples` successful samples. Failures are judged apart from timing. + */ +export function compareRuns( + target: { summary: PerfSummary; label: string }, + baseline: { summary: PerfSummary; label: string }[], + thresholds: ComparisonThresholds = DEFAULT_THRESHOLDS, +): Comparison { + const targetById = new Map(target.summary.measurements.map((m) => [m.id, m])); + const ids = [ + ...target.summary.measurements.map((m) => m.id), + ...baseline.flatMap((b) => b.summary.measurements.map((m) => m.id)), + ]; + const rows = [...new Set(ids)].map((id) => + compareStep( + id, + targetById.get(id), + baseline.flatMap((b) => b.summary.measurements.filter((m) => m.id === id)), + thresholds, + ), + ); + const counts: Comparison['counts'] = { timing: {}, failures: {} }; + for (const row of rows) { + counts.timing[row.timing] = (counts.timing[row.timing] ?? 0) + 1; + if (row.failures) counts.failures[row.failures] = (counts.failures[row.failures] ?? 0) + 1; + } + return { + schema: COMPARISON_SCHEMA, + target: describeRun(target.summary, target.label), + baseline: baseline.map((b) => describeRun(b.summary, b.label)), + thresholds, + counts, + warnings: environmentWarnings( + target.summary, + baseline.map((b) => b.summary), + ), + rows, + }; +} + +function ms(n: number | null | undefined): string { + return typeof n === 'number' && Number.isFinite(n) ? n.toFixed(0) : '–'; +} + +function pct(n: number | null | undefined, signed = false): string { + if (typeof n !== 'number' || !Number.isFinite(n)) return '–'; + const value = `${(n * 100).toFixed(0)}%`; + return signed && n > 0 ? `+${value}` : value; +} + +function flagRank(row: ComparisonRow): number { + const ranks = [row.failures, row.timing].map((v) => FLAGGED_VERDICTS.indexOf(v as Flagged)); + const flagged = ranks.filter((rank) => rank >= 0); + return flagged.length === 0 ? -1 : Math.min(...flagged); +} + +function tableRow(row: ComparisonRow): string { + const failures = row.target ? `${row.target.failures}/${row.target.samples}` : '–'; + const nights = row.baseline ? `${row.baseline.timedNights}/${row.baseline.nights}` : '0'; + return `| ${row.timing} | ${row.failures ?? '–'} | \`${row.id}\` | ${row.metric} | ${ms(row.target?.median)} | ${ms(row.baseline?.median)} | ${ms(row.deltaMs)} | ${pct(row.deltaRelative, true)} | ${pct(row.threshold)} | ${failures} | ${nights} |`; +} + +const TABLE_HEAD = [ + '| timing | failures | step | metric | tonight median | baseline median | Δ ms | Δ | threshold | failed tonight | timed nights |', + '|---|---|---|---|---|---|---|---|---|---|---|', +]; + +function describeCounts(counts: Partial>): string { + return ( + Object.entries(counts) + .map(([verdict, n]) => `${n} ${verdict}`) + .join(', ') || 'none' + ); +} + +/** Renders the comparison for the job summary and the artifact. */ +export function renderComparison(comparison: Comparison): string { + const t = comparison.thresholds; + const flagged = comparison.rows + .filter((row) => flagRank(row) >= 0) + .sort((a, b) => flagRank(a) - flagRank(b)); + const lines = [ + `## agent-device perf comparison`, + '', + `- **Tonight**: ${comparison.target.label} · ${comparison.target.commit.slice(0, 12)} · ${comparison.target.startedAt} · ${comparison.target.device}`, + `- **Baseline**: ${comparison.baseline.length === 0 ? 'none' : comparison.baseline.map((b) => `${b.label} (${b.commit.slice(0, 7)})`).join(', ')}`, + `- **Rule**: flag a step when its median moves by more than max(${pct(t.minRelative)}, ${t.spreadFactor}× night-to-night spread, ${t.withinFactor}× tonight cv/√n) and more than ${t.minAbsoluteMs} ms against the median of the baseline nights; a timing verdict needs ${t.minBaselineNights} nights, and tonight and each of those nights need ${t.minSamples} successful samples. Failures are flagged when tonight has more failed samples than every baseline night, in count and in share.`, + `- **Timing**: ${describeCounts(comparison.counts.timing)}`, + `- **Failures**: ${describeCounts(comparison.counts.failures)}`, + '', + ]; + for (const warning of comparison.warnings) lines.push(`> ⚠️ ${warning}`, ''); + if (flagged.length > 0) { + lines.push(...TABLE_HEAD, ...flagged.map(tableRow), ''); + } else { + lines.push('No step moved beyond its noise threshold.', ''); + } + lines.push( + '
All steps', + '', + ...TABLE_HEAD, + ...comparison.rows.map(tableRow), + '', + '
', + '', + ); + return lines.join('\n'); +} diff --git a/scripts/perf/config.test.ts b/scripts/perf/config.test.ts new file mode 100644 index 0000000000..d8bb3b363a --- /dev/null +++ b/scripts/perf/config.test.ts @@ -0,0 +1,9 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { parseConfig } from './config.ts'; + +test('a run measures at least three rounds, so every step can get a timing verdict', () => { + assert.equal(parseConfig(['--udid', 'sim', '--rounds', '3']).rounds, 3); + assert.throws(() => parseConfig(['--udid', 'sim', '--rounds', '2']), /--rounds must be .* >= 3/); + assert.throws(() => parseConfig(['--platform', 'android', '--n', '1']), /--n must be .* >= 3/); +}); diff --git a/scripts/perf/config.ts b/scripts/perf/config.ts index 10c3e84630..32623a8ebf 100644 --- a/scripts/perf/config.ts +++ b/scripts/perf/config.ts @@ -13,11 +13,19 @@ export type PerfConfig = { warmup: number; // leading rounds dropped from stats keepArtifacts: boolean; // keep temp state dir + leave device booted outDir: string; - udid?: string; // iOS device override (UDID) - device?: string; // device override by name (e.g. "iPhone 17 Pro"); preferred over udid + udid?: string; // iOS simulator UDID (required for iOS) + device?: string; // Android AVD name for boot serial?: string; // Android device override + appId: string; // iOS fixture bundle id + appPath?: string; // iOS fixture bundle path, recorded in the run metadata + scenarios?: string[]; // iOS scenario ids; all when omitted }; +export const DEFAULT_IOS_APP_ID = 'com.callstack.agentdevicelab'; + +/** The fewest measured rounds that a run accepts; a comparison needs as many successful samples. */ +export const MIN_MEASURED_ROUNDS = 3; + // How to invoke the CLI. Defaults to the built dist binary (bin/agent-device.mjs). // Set AGENT_DEVICE_PERF_CLI to run from source instead, e.g. on CI: // AGENT_DEVICE_PERF_CLI="--experimental-strip-types src/bin.ts" @@ -50,10 +58,11 @@ function readIntValue(argv: string[], i: number, flag: string, min: number): num export function parseConfig(argv: string[]): PerfConfig { const cfg: PerfConfig = { platform: 'ios', - rounds: 5, + rounds: MIN_MEASURED_ROUNDS, warmup: 1, keepArtifacts: false, - outDir: DEFAULT_OUT_DIR, + outDir: path.join(DEFAULT_OUT_DIR, new Date().toISOString().replaceAll(/[:.]/g, '-')), + appId: DEFAULT_IOS_APP_ID, }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; @@ -66,7 +75,7 @@ export function parseConfig(argv: string[]): PerfConfig { } case '--n': case '--rounds': - cfg.rounds = readIntValue(argv, i++, a, 1); + cfg.rounds = readIntValue(argv, i++, a, MIN_MEASURED_ROUNDS); break; case '--warmup': cfg.warmup = readIntValue(argv, i++, a, 0); @@ -86,9 +95,22 @@ export function parseConfig(argv: string[]): PerfConfig { case '--serial': cfg.serial = readValue(argv, i++, a); break; + case '--app-id': + cfg.appId = readValue(argv, i++, a); + break; + case '--app-path': + cfg.appPath = path.resolve(readValue(argv, i++, a)); + break; + case '--scenarios': + cfg.scenarios = readValue(argv, i++, a) + .split(',') + .map((id) => id.trim()) + .filter(Boolean); + break; default: throw new Error(`Unknown flag: ${a}`); } } + if (cfg.platform === 'ios' && !cfg.udid) throw new Error('--udid is required for --platform ios'); return cfg; } diff --git a/scripts/perf/forced-backend.ts b/scripts/perf/forced-backend.ts new file mode 100644 index 0000000000..b72619b3e3 --- /dev/null +++ b/scripts/perf/forced-backend.ts @@ -0,0 +1,89 @@ +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { pathToFileURL } from 'node:url'; +import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { createRequestId } from '@agent-device/host-kit/diagnostics'; +import { captureHostEnvironment } from '@agent-device/host-kit/process'; +import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot'; +import { readDaemonInfo } from '../../src/daemon-client/daemon-client-metadata.ts'; +import { sendRequest } from '../../src/daemon-client/daemon-client-transport.ts'; +import { REPO_ROOT } from './config.ts'; +import type { CliResult } from './types.ts'; + +type AgentDeviceSdk = typeof import('../../src/sdk/index.ts'); + +const FORCED_CAPTURE_TIMEOUT_MS = 90_000; + +/** + * Captures one snapshot with the internal backend pin that the CLI does not expose. The + * transport talks to the daemon that the built CLI already started and never starts or replaces + * it, so the measured daemon stays the daemon under test. + */ +export async function captureWithForcedBackend(options: { + stateDir: string; + session: string; + udid: string; + backend: SnapshotPreferredBackend; +}): Promise<{ result: CliResult; requestId: string }> { + const requestId = createRequestId(); + const sdk = (await import( + pathToFileURL(path.join(REPO_ROOT, 'dist/src/index.js')).href + )) as AgentDeviceSdk; + const client = sdk.createAgentDeviceClient( + { session: options.session, stateDir: options.stateDir }, + { transport: forcedBackendTransport(options.backend, requestId) }, + ); + const t0 = performance.now(); + try { + const snapshot = await client.capture.snapshot({ + platform: 'ios', + udid: options.udid, + timeoutMs: FORCED_CAPTURE_TIMEOUT_MS, + }); + const json = { success: true, data: snapshot }; + const wallClockMs = performance.now() - t0; + return { + result: { exitCode: 0, wallClockMs, stdout: '', stderr: '', json, ok: true }, + requestId, + }; + } catch (error) { + const normalized = sdk.normalizeAgentDeviceError(error); + const json = { success: false, error: normalized }; + const wallClockMs = performance.now() - t0; + return { + result: { exitCode: 1, wallClockMs, stdout: '', stderr: normalized.message, json, ok: false }, + requestId, + }; + } +} + +function forcedBackendTransport( + backend: SnapshotPreferredBackend, + requestId: string, +): AgentDeviceDaemonTransport { + const host = captureHostEnvironment(process); + return async (request, context) => { + const stateDir = request.flags?.stateDir; + if (request.session === undefined || typeof stateDir !== 'string') { + throw new Error('A forced-backend capture requires an explicit session and state dir.'); + } + const paths = resolveDaemonPaths(stateDir, { env: process.env }); + const info = readDaemonInfo(paths.infoPath); + if (!info) throw new Error(`No daemon is running for state dir ${stateDir}.`); + return await sendRequest( + info, + { + ...request, + session: request.session, + token: info.token, + flags: { ...request.flags, snapshotPreferredBackend: backend }, + meta: { ...request.meta, debug: true, requestId }, + }, + 'auto', + paths, + FORCED_CAPTURE_TIMEOUT_MS, + { host, signal: context?.signal, timeoutRecovery: 'none' }, + ); + }; +} diff --git a/scripts/perf/harness.ts b/scripts/perf/harness.ts index 49f0bb6f93..09063440a7 100644 --- a/scripts/perf/harness.ts +++ b/scripts/perf/harness.ts @@ -10,8 +10,8 @@ import { } from './cli.ts'; import type { PerfConfig } from './config.ts'; import { resolveProfile, type ResolvedProfile } from './platform-profiles.ts'; +import { buildMeasurement } from './measurement.ts'; import { buildSettingsTour, type ScenarioStep } from './scenario.ts'; -import { summarize } from './stats.ts'; import type { CliResult, Measurement, Sample } from './types.ts'; export type IsolationContext = { @@ -32,7 +32,7 @@ export function setupIsolation(cfg: PerfConfig): IsolationContext { fs.mkdirSync(artifactsDir, { recursive: true }); const baseFlags = ['--state-dir', stateDir, '--session', 'perf', ...profile.platformFlags]; log(`state-dir: ${stateDir}`); - log(`device: ${profile.deviceName} (${profile.udid ?? profile.serial})`); + log(`device: ${profile.deviceName} (${profile.serial ?? 'default'})`); return { stateDir, artifactsDir, baseFlags, profile }; } @@ -72,8 +72,8 @@ function toSample(r: CliResult, round: number): Sample { return sample; } -// The first interaction after open/relaunch pays the iOS XCUITest runner startup (~10s+ cold) -// and a per-relaunch first-AX-query settle cost. Run an untimed throwaway interaction so that +// The first interaction after open/relaunch pays the Android helper startup and a per-relaunch +// first-query settle cost. Run an untimed throwaway interaction so that // cost is never attributed to a measured command. function warmRunner(ctx: IsolationContext): void { invokeCli(['snapshot', '-i'], ctx.baseFlags); @@ -91,39 +91,31 @@ function runStep(step: ScenarioStep, ctx: IsolationContext, round: number): Samp : invokeBatchStep(step.step, ctx.baseFlags); const sample = toSample(r, round); if (step.execMode === 'batch') { - sample.daemonDurationMs = readBatchStepDurationMs(r); + sample.daemonMs = readBatchStepDurationMs(r); if (step.isSnapshot) sample.elementCount = countElements(r); } return sample; } -function buildMeasurement( +const TOUR_SCENARIO = 'settings-tour'; + +function tourMeasurement( step: Pick, platform: ResolvedProfile['platform'], samples: Sample[], warmupDropped: number, ): Measurement { - const ok = samples.filter((s) => s.ok); - const failures = samples.length - ok.length; - const notes: string[] = []; - if (failures > 0) { - const codes = [...new Set(samples.filter((s) => !s.ok).map((s) => s.errorCode))].join(', '); - notes.push(`${failures}/${samples.length} samples failed: ${codes}`); - } - const num = (xs: (number | undefined)[]) => xs.filter((n): n is number => typeof n === 'number'); - return { - command: step.command, + return buildMeasurement({ + id: `${TOUR_SCENARIO}/${step.label}`, + scenario: TOUR_SCENARIO, label: step.label, + command: step.command, + args: [], platform, execMode: step.execMode, samples, warmupDropped, - wallClock: summarize(ok.map((s) => s.wallClockMs)), - daemonDuration: summarize(num(ok.map((s) => s.daemonDurationMs))), - elementCount: summarize(num(ok.map((s) => s.elementCount))), - failures, - notes, - }; + }); } function setupMeasurementFailed(measurement: Measurement): boolean { @@ -137,7 +129,7 @@ function bootOnce(ctx: IsolationContext): Measurement { const bootFlags = ['--state-dir', ctx.stateDir, ...ctx.profile.platformFlags]; const r = invokeCli(['boot', ...ctx.profile.selectorFlags], bootFlags); const sample = toSample(r, 0); - return buildMeasurement( + return tourMeasurement( { command: 'boot', label: 'boot device', execMode: 'standalone' }, ctx.profile.platform, [sample], @@ -152,7 +144,7 @@ function establishSession(ctx: IsolationContext): Measurement { log('establishing session (open with device selectors)'); const r = invokeCli(['open', ctx.profile.appTarget, ...ctx.profile.selectorFlags], ctx.baseFlags); const sample = toSample(r, 0); - return buildMeasurement( + return tourMeasurement( { command: 'open', label: 'open (establish + cold)', execMode: 'standalone' }, ctx.profile.platform, [sample], @@ -176,10 +168,8 @@ export function runScenario(ctx: IsolationContext, cfg: PerfConfig): Measurement // Android accessibility dumps time out while the UI is animating; disable animations // up front (untimed) so snapshot/get/is/fill can read an idle hierarchy. - if (ctx.profile.platform === 'android') { - log('disabling animations (android)'); - invokeCli(['settings', 'animations', 'off'], ctx.baseFlags); - } + log('disabling animations'); + invokeCli(['settings', 'animations', 'off'], ctx.baseFlags); const totalRounds = cfg.warmup + cfg.rounds; for (let round = 0; round < totalRounds; round++) { @@ -197,7 +187,7 @@ export function runScenario(ctx: IsolationContext, cfg: PerfConfig): Measurement } const tourMeasurements = steps.map((step) => - buildMeasurement(step, ctx.profile.platform, acc.get(step.label)!, cfg.warmup), + tourMeasurement(step, ctx.profile.platform, acc.get(step.label)!, cfg.warmup), ); return [boot, establish, ...tourMeasurements]; } diff --git a/scripts/perf/ios-harness.test.ts b/scripts/perf/ios-harness.test.ts new file mode 100644 index 0000000000..e8e88f5129 --- /dev/null +++ b/scripts/perf/ios-harness.test.ts @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { test, vi } from 'vitest'; + +// Stands in for the CLI: records each teardown call, and whether the logs were already saved, +// then appends to the daemon log and fails like a teardown that went wrong. +const FAKE_CLI = ` +import fs from 'node:fs'; +const args = process.argv.slice(2); +const stateDir = args[args.indexOf('--state-dir') + 1]; +const command = args[0] === 'daemon' ? 'daemon stop' : args[0]; +const saved = fs.existsSync(process.env.PERF_FAKE_SAVED_LOG); +fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' saved=' + saved + '\\n'); +fs.appendFileSync(stateDir + '/daemon.log', command + '\\n'); +process.exit(1); +`; + +test('finalizing a run saves the logs before and after a failing teardown, once', async () => { + const root = mkdtempForTestSync('perf-finalize-'); + const fakeCli = path.join(root, 'fake-cli.mjs'); + fs.writeFileSync(fakeCli, FAKE_CLI); + const stateDir = path.join(root, 'state'); + const rawDir = path.join(root, 'out', 'raw'); + fs.mkdirSync(path.join(stateDir, 'sessions', 'perf'), { recursive: true }); + fs.writeFileSync(path.join(stateDir, 'daemon.log'), 'suite\n'); + fs.writeFileSync(path.join(stateDir, 'sessions', 'perf', 'runner.log'), 'runner\n'); + fs.writeFileSync(path.join(stateDir, 'daemon.json'), '{"token":"secret"}'); + const calls = path.join(root, 'calls.log'); + vi.stubEnv('AGENT_DEVICE_PERF_CLI', fakeCli); + vi.stubEnv('PERF_FAKE_CALLS', calls); + vi.stubEnv('PERF_FAKE_SAVED_LOG', path.join(rawDir, 'state', 'daemon.log')); + // The CLI command is read when the harness loads, so load it after the stub. + const { iosRunFinalizer } = await import('./ios-harness.ts'); + + const finalize = iosRunFinalizer( + { stateDir, udid: 'sim', appId: 'app', artifactsDir: path.join(rawDir, 'artifacts'), rawDir }, + false, + ); + finalize(); + // The interrupt handler and the normal finish both call it; the second call does nothing. + finalize(); + + assert.equal(fs.readFileSync(calls, 'utf8'), 'close saved=true\ndaemon stop saved=true\n'); + const saved = path.join(rawDir, 'state'); + assert.equal( + fs.readFileSync(path.join(saved, 'daemon.log'), 'utf8'), + 'suite\nclose\ndaemon stop\n', + ); + assert.equal( + fs.readFileSync(path.join(saved, 'sessions', 'perf', 'runner.log'), 'utf8'), + 'runner\n', + ); + assert.equal(fs.existsSync(path.join(saved, 'daemon.json')), false); + assert.equal(fs.existsSync(stateDir), false); +}); diff --git a/scripts/perf/ios-harness.ts b/scripts/perf/ios-harness.ts new file mode 100644 index 0000000000..16579f7506 --- /dev/null +++ b/scripts/perf/ios-harness.ts @@ -0,0 +1,326 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + resolveSessionDir, + resolveSessionRequestLogPath, +} from '@agent-device/daemon-contracts/session-artifact-paths'; +import { invokeCli } from './cli.ts'; +import type { PerfConfig } from './config.ts'; +import { captureWithForcedBackend } from './forced-backend.ts'; +import { expectationViolation, type SuiteScenario, type SuiteStep } from './ios-suite.ts'; +import { buildMeasurement } from './measurement.ts'; +import { + profileRequestLog, + readClientDiagnostics, + readSettleFacts, + readSnapshotFacts, +} from './profile.ts'; +import type { CliResult, Measurement, Sample } from './types.ts'; + +const SESSION = 'perf'; + +export type IosSuiteContext = { + stateDir: string; + udid: string; + appId: string; + artifactsDir: string; + rawDir: string; + // Set when a request outlived the client timeout. That request still holds the session, so + // the suite restarts the daemon before the next scenario. + daemonStuck?: boolean; +}; + +function log(msg: string): void { + process.stderr.write(`[perf] ${msg}\n`); +} + +function baseFlags(ctx: IosSuiteContext): string[] { + return ['--state-dir', ctx.stateDir, '--session', SESSION, '--platform', 'ios']; +} + +function run(ctx: IosSuiteContext, args: readonly string[]): CliResult { + return watch(ctx, invokeCli(expandStatic(args, ctx), baseFlags(ctx))); +} + +function watch(ctx: IosSuiteContext, result: CliResult): CliResult { + if (daemonTimedOut(result)) ctx.daemonStuck = true; + return result; +} + +function expandStatic(args: readonly string[], ctx: IosSuiteContext): string[] { + return args.map((arg) => + arg === '{app}' + ? ctx.appId + : arg.replace(/^\{out:(.+)\}$/, (_, file: string) => path.join(ctx.artifactsDir, file)), + ); +} + +type SnapshotNode = { + ref?: string; + identifier?: string; + rect?: { x: number; y: number; width: number; height: number }; +}; + +const TARGET_TOKEN = /^\{(ref|point):(.+)\}$/; + +/** Expands `{ref:id}` and `{point:id}` from one untimed interactive snapshot. */ +function resolveTargets(args: string[], ctx: IosSuiteContext): string[] | { error: string } { + if (!args.some((arg) => TARGET_TOKEN.test(arg))) return args; + const snapshot = run(ctx, ['snapshot', '-i']); + const nodes = ((snapshot.json as { data?: { nodes?: SnapshotNode[] } })?.data?.nodes ?? + []) as SnapshotNode[]; + const expanded: string[] = []; + for (const arg of args) { + const match = TARGET_TOKEN.exec(arg); + if (!match) { + expanded.push(arg); + continue; + } + const node = nodes.find((candidate) => candidate.identifier === match[2]); + if (!node?.ref || !node.rect) return { error: `target ${match[2]} is not in snapshot -i` }; + if (match[1] === 'ref') { + expanded.push(`@${node.ref}`); + } else { + expanded.push( + String(Math.round(node.rect.x + node.rect.width / 2)), + String(Math.round(node.rect.y + node.rect.height / 2)), + ); + } + } + return expanded; +} + +/** Runs untimed calls in order and describes the first failure. */ +function untimed( + ctx: IosSuiteContext, + calls: readonly (readonly string[])[] = [], +): string | undefined { + for (const args of calls) { + const result = run(ctx, args); + if (!result.ok) return `${args.join(' ')}: ${errorOf(result).errorMessage}`; + } + return undefined; +} + +/** True when the client gave up on a daemon request that is still running. */ +function daemonTimedOut(result: CliResult): boolean { + const details = (result.json as { error?: { details?: { reason?: unknown } } } | undefined)?.error + ?.details; + return details?.reason === 'daemon_transport_timeout'; +} + +function errorOf(result: CliResult): Pick { + const error = (result.json as { error?: { code?: string; message?: string } } | undefined)?.error; + return { + errorCode: error?.code ?? `exit:${result.exitCode}`, + errorMessage: (error?.message ?? result.stderr.trim().split('\n').pop() ?? '').slice(0, 300), + }; +} + +function readProfile(ctx: IosSuiteContext, requestId: string | undefined) { + if (!requestId) return undefined; + const sessionDir = resolveSessionDir(path.join(ctx.stateDir, 'sessions'), SESSION); + const logPath = resolveSessionRequestLogPath(sessionDir, requestId); + return fs.existsSync(logPath) ? profileRequestLog(fs.readFileSync(logPath, 'utf8')) : undefined; +} + +function recordFailure(ctx: IosSuiteContext, id: string, round: number, detail: unknown): void { + const dir = path.join(ctx.rawDir, 'failures'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, `${id.replace('/', '__')}.round-${round}.json`), + JSON.stringify(detail, null, 2), + ); +} + +async function runTimedStep( + scenario: SuiteScenario, + step: SuiteStep, + ctx: IosSuiteContext, + round: number, +): Promise { + const id = `${scenario.id}/${step.id}`; + const before = untimed(ctx, step.before); + if (before) { + log(` ✗ ${id} (before_failed: ${before})`); + return { round, ok: false, wallClockMs: 0, errorCode: 'before_failed', errorMessage: before }; + } + let sample: Sample; + if (step.via === 'sdk-forced-tree') { + const { result: forced, requestId } = await captureWithForcedBackend({ + stateDir: ctx.stateDir, + session: SESSION, + udid: ctx.udid, + backend: 'tree', + }); + const result = watch(ctx, forced); + const profile = readProfile(ctx, requestId); + sample = { + round, + ok: result.ok, + wallClockMs: result.wallClockMs, + daemonMs: profile?.requestSpanMs, + requestId, + snapshot: readSnapshotFacts(result.json), + profile, + }; + if (!result.ok) Object.assign(sample, errorOf(result)); + } else { + const args = resolveTargets(expandStatic(step.args, ctx), ctx); + if ('error' in args) { + sample = { + round, + ok: false, + wallClockMs: 0, + errorCode: 'target_unresolved', + errorMessage: args.error, + }; + } else { + const result = watch(ctx, invokeCli([...args, '--debug'], baseFlags(ctx))); + const diagnostics = readClientDiagnostics(result.stderr); + sample = { + round, + ok: result.ok, + wallClockMs: result.wallClockMs, + daemonMs: diagnostics.daemonMs, + requestId: diagnostics.requestId, + snapshot: step.command === 'snapshot' ? readSnapshotFacts(result.json) : undefined, + settle: readSettleFacts(result.json), + profile: readProfile(ctx, diagnostics.requestId), + }; + if (!result.ok) { + Object.assign(sample, errorOf(result)); + recordFailure(ctx, id, round, { + args, + json: result.json, + stderrTail: result.stderr.slice(-4000), + }); + } + } + } + const violation = sample.ok ? expectationViolation(step.expect, sample.snapshot) : undefined; + if (violation) + Object.assign(sample, { ok: false, errorCode: 'expectation_failed', errorMessage: violation }); + const after = untimed(ctx, step.after); + if (after) log(` ! ${id} after: ${after}`); + if (!sample.ok) log(` ✗ ${id} (${sample.errorCode}: ${sample.errorMessage})`); + return sample; +} + +function sessionError(result: CliResult): string | undefined { + if (result.ok) return undefined; + const message = `open: ${errorOf(result).errorMessage}`; + log(` ✗ session failed: ${message}`); + return message; +} + +function stopStuckDaemon(ctx: IosSuiteContext): void { + log(' ! a request outlived its timeout; restarting the daemon'); + invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); + ctx.daemonStuck = false; +} + +/** Opens the session with device selectors; later calls address it by session name alone. */ +function establishSession(ctx: IosSuiteContext): CliResult { + if (ctx.daemonStuck) stopStuckDaemon(ctx); + return watch( + ctx, + invokeCli(['open', ctx.appId, '--relaunch', '--udid', ctx.udid], baseFlags(ctx)), + ); +} + +export async function runIosSuite( + ctx: IosSuiteContext, + cfg: PerfConfig, + scenarios: readonly SuiteScenario[], +): Promise { + const samples = new Map(); + const totalRounds = cfg.warmup + cfg.rounds; + for (let round = 0; round < totalRounds; round++) { + const measured = round >= cfg.warmup; + log(`round ${round + 1}/${totalRounds}${measured ? '' : ' (warmup, dropped)'}`); + let sessionFailure = sessionError(establishSession(ctx)); + for (const scenario of scenarios) { + // A request that outlived its timeout still holds the session; reopen before the next scenario. + if (ctx.daemonStuck) sessionFailure = sessionError(establishSession(ctx)); + log(` scenario ${scenario.id}`); + const setupFailure = sessionFailure ?? untimed(ctx, scenario.setup); + if (setupFailure) log(` ✗ setup failed: ${setupFailure}`); + for (const step of scenario.steps) { + // CLI calls block the event loop; yield so an interrupt can stop the run between steps. + await new Promise((resolve) => setImmediate(resolve)); + const id = `${scenario.id}/${step.id}`; + const skipped = setupFailure + ? { errorCode: 'setup_failed', errorMessage: setupFailure } + : ctx.daemonStuck + ? { + errorCode: 'daemon_stuck', + errorMessage: 'an earlier request still holds the session', + } + : undefined; + const sample: Sample = skipped + ? { round, ok: false, wallClockMs: 0, ...skipped } + : await runTimedStep(scenario, step, ctx, round); + if (measured) samples.set(id, [...(samples.get(id) ?? []), sample]); + } + } + } + return scenarios.flatMap((scenario) => + scenario.steps.map((step) => + buildMeasurement({ + id: `${scenario.id}/${step.id}`, + scenario: scenario.id, + label: step.args.join(' '), + command: step.command, + args: [...step.args], + platform: 'ios', + execMode: step.via === 'sdk-forced-tree' ? 'sdk' : 'standalone', + samples: samples.get(`${scenario.id}/${step.id}`) ?? [], + warmupDropped: cfg.warmup, + }), + ), + ); +} + +// A stalled close or daemon stop must not keep the run, and its logs, from finishing. +const TEARDOWN_TIMEOUT_MS = 60_000; + +/** + * Returns the end of a run: save the logs, stop the session and the daemon within a deadline, save + * the logs again with the teardown, and remove the state directory unless it is kept. It runs once, + * so the interrupt handler and the normal finish can both call it. + */ +export function iosRunFinalizer(ctx: IosSuiteContext, keepArtifacts: boolean): () => void { + let finalized = false; + const saveLogs = (): void => { + try { + copyRawState(ctx); + } catch (error) { + log(`finalize: saving logs failed: ${(error as Error).message}`); + } + }; + const teardown = (args: string[], flags: string[]): void => { + const result = invokeCli(args, flags, { timeoutMs: TEARDOWN_TIMEOUT_MS }); + if (!result.ok) log(`finalize: ${args.join(' ')} failed: ${result.stderr.trim().slice(-400)}`); + }; + return () => { + if (finalized) return; + finalized = true; + saveLogs(); + teardown(['close'], ['--state-dir', ctx.stateDir, '--session', SESSION]); + teardown(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); + saveLogs(); + if (!keepArtifacts) fs.rmSync(ctx.stateDir, { recursive: true, force: true }); + }; +} + +/** Copies daemon and session logs into the raw artifact; the daemon token file stays behind. */ +function copyRawState(ctx: IosSuiteContext): void { + const target = path.join(ctx.rawDir, 'state'); + fs.mkdirSync(target, { recursive: true }); + const daemonLog = path.join(ctx.stateDir, 'daemon.log'); + if (fs.existsSync(daemonLog)) fs.copyFileSync(daemonLog, path.join(target, 'daemon.log')); + const sessions = path.join(ctx.stateDir, 'sessions'); + if (fs.existsSync(sessions)) + fs.cpSync(sessions, path.join(target, 'sessions'), { recursive: true }); +} diff --git a/scripts/perf/ios-suite.test.ts b/scripts/perf/ios-suite.test.ts new file mode 100644 index 0000000000..e9d58ece1f --- /dev/null +++ b/scripts/perf/ios-suite.test.ts @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict'; +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import { test } from 'vitest'; +import { + describeCoverage, + expectationViolation, + IOS_SUITE, + IOS_SUITE_EXCLUSIONS, +} from './ios-suite.ts'; + +test('every public command is either measured by the iOS suite or excluded with a reason', () => { + const { measured, excluded } = describeCoverage(IOS_SUITE, IOS_SUITE_EXCLUSIONS); + const classified = [...measured, ...excluded.map((e) => e.command)].sort(); + assert.deepEqual(classified, Object.values(PUBLIC_COMMANDS).sort()); + for (const { reason } of excluded) assert.ok(reason.length > 0); +}); + +test('step ids are unique, so comparisons key each step once', () => { + const ids = IOS_SUITE.flatMap((scenario) => scenario.steps.map((s) => `${scenario.id}/${s.id}`)); + assert.equal(new Set(ids).size, ids.length); +}); + +test('a heavy-feed capture breaks its expectation when the tree flattens', () => { + const depth = { maxDepthAtLeast: 64 }; + assert.equal(expectationViolation(depth, { maxDepth: 64 }), undefined); + assert.match(expectationViolation(depth, { maxDepth: 63 })!, /below 64/); + assert.match(expectationViolation(depth, undefined)!, /unknown/); +}); + +test('a pinned capture breaks its expectation unless the pinned backend served it', () => { + const pinned = { pinnedBackend: 'tree' } as const; + const served = { backend: 'tree', state: 'healthy', reasonCode: 'requested-backend' }; + assert.equal(expectationViolation(pinned, served), undefined); + // The raw plan ignores the pin and recovers through private AX on the heavy feed. + assert.match( + expectationViolation(pinned, { + backend: 'private-ax', + state: 'recovered', + reasonCode: 'ax-rejected', + })!, + /private-ax backend \(ax-rejected\)/, + ); + // An unpinned capture can still be tree-first on a light screen. + assert.match( + expectationViolation(pinned, { backend: 'tree', state: 'healthy' })!, + /no reason code/, + ); + assert.match(expectationViolation(pinned, undefined)!, /unknown backend/); +}); diff --git a/scripts/perf/ios-suite.ts b/scripts/perf/ios-suite.ts new file mode 100644 index 0000000000..efc7795de1 --- /dev/null +++ b/scripts/perf/ios-suite.ts @@ -0,0 +1,388 @@ +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { + SnapshotPreferredBackend, + SnapshotQualityVerdict, +} from '@agent-device/kernel/snapshot'; +import type { SnapshotFacts } from './types.ts'; + +/** Routes `trigger-app-event` into the fixture's automation screen. */ +export const FIXTURE_APP_EVENT_URL_TEMPLATE = + 'agent-device-test-app:///automation?event={event}&payload={payload}'; + +export type PublicCommand = (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS]; + +const C = PUBLIC_COMMANDS; + +type Cli = readonly string[]; + +/** + * One timed command. `args` tokens expand before each run: `{app}` is the fixture bundle id, + * `{out:}` a path in the run's artifact directory, and `{ref:}` / `{point:}` the ref + * or center point of the element with that identifier in an untimed `snapshot -i` taken just + * before the step. `before` and `after` run untimed around the step in every round. + */ +export type SuiteStep = { + id: string; + command: PublicCommand; + args: Cli; + before?: readonly Cli[]; + after?: readonly Cli[]; + // `sdk-forced-tree` captures a snapshot through the daemon with the XCTest tree backend pinned; + // the CLI has no flag for it. + via?: 'cli' | 'sdk-forced-tree'; + // A sample that succeeds but breaks the expectation fails with `expectation_failed`, so a + // fixture that stops reproducing its shape shows up as a new failure. + expect?: StepExpectation; +}; + +export type StepExpectation = { + // Deepest raw node depth, counted from the application at 0. + maxDepthAtLeast?: number; + // The pinned backend that must serve the capture. The runner marks a pinned capture with the + // reason code `requested-backend`; any other backend or code means the pin was not honored. + pinnedBackend?: SnapshotPreferredBackend; +}; + +const PINNED_REASON_CODE: SnapshotQualityVerdict['reasonCode'] = 'requested-backend'; + +/** Describes how a snapshot breaks the step expectation, or returns undefined. */ +export function expectationViolation( + expect: StepExpectation | undefined, + snapshot: SnapshotFacts | undefined, +): string | undefined { + if (!expect) return undefined; + if (expect.maxDepthAtLeast !== undefined && (snapshot?.maxDepth ?? -1) < expect.maxDepthAtLeast) { + return `max depth ${snapshot?.maxDepth ?? 'unknown'} is below ${expect.maxDepthAtLeast}`; + } + if ( + expect.pinnedBackend !== undefined && + (snapshot?.backend !== expect.pinnedBackend || snapshot.reasonCode !== PINNED_REASON_CODE) + ) { + return `the ${snapshot?.backend ?? 'unknown'} backend (${snapshot?.reasonCode ?? 'no reason code'}) served the capture, not the pinned ${expect.pinnedBackend}`; + } + return undefined; +} + +/** A scenario starts from `setup` (untimed, every round) and runs its steps in order. */ +export type SuiteScenario = { + id: string; + title: string; + setup: readonly Cli[]; + steps: SuiteStep[]; +}; + +// A cold relaunch on a hosted runner can take longer than 15 s before the first landmark. +const LANDMARK_WAIT_MS = '60000'; +const relaunch: Cli = ['open', '{app}', '--relaunch']; +const toTab = (label: string, landmark: string): Cli[] => [ + relaunch, + ['wait', `label="${label}"`, LANDMARK_WAIT_MS], + ['click', `label="${label}"`], + ['wait', landmark, LANDMARK_WAIT_MS], +]; +const toSettingsLab = (button: string, landmark: string): Cli[] => [ + ...toTab('Settings', 'id="settings-title"'), + ['scroll', 'down', '--until', `id="${button}"`], + ['click', `id="${button}"`], + ['wait', landmark, LANDMARK_WAIT_MS], +]; +const step = ( + id: string, + command: PublicCommand, + args: Cli, + extra: Omit = {}, +): SuiteStep => ({ id, command, args, ...extra }); + +export const IOS_SUITE: readonly SuiteScenario[] = [ + { + id: 'lifecycle', + title: 'App lifecycle, device state, and inventory', + setup: [relaunch, ['wait', 'text', 'Agent Device Tester', LANDMARK_WAIT_MS]], + steps: [ + step('open-relaunch', C.open, relaunch), + step('appstate', C.appState, ['appstate']), + step('home', C.home, ['home']), + step('open-resume', C.open, ['open', '{app}']), + step('app-switcher', C.appSwitcher, ['app-switcher'], { after: [['open', '{app}']] }), + step('orientation-landscape', C.orientation, ['orientation', 'landscape-left']), + step('orientation-portrait', C.orientation, ['orientation', 'portrait']), + step('settings-appearance', C.settings, ['settings', 'appearance', 'light']), + step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf']), + step('clipboard-read', C.clipboard, ['clipboard', 'read']), + step('action-button', C.actionButton, ['action-button']), + step('prepare-runner', C.prepare, ['prepare', 'ios-runner']), + step('devices', C.devices, ['devices', '--platform', 'ios']), + step('apps', C.apps, ['apps']), + step('capabilities', C.capabilities, ['capabilities']), + step('doctor', C.doctor, ['doctor']), + // Some runtimes ask "Open in Agent Device Tester?" for the event URL, and the prompt outlives a + // relaunch. The answer opens the URL seconds later, so wait for its route before the next + // relaunch. The untimed calls stop at alert get when no prompt is up. + step( + 'trigger-app-event', + C.triggerAppEvent, + ['trigger-app-event', 'fixture.ready', '{"source":"perf"}'], + { + after: [ + ['alert', 'get'], + ['alert', 'accept'], + ['wait', 'id="automation-title"', LANDMARK_WAIT_MS], + ], + }, + ), + ], + }, + { + id: 'home-reads', + title: 'Reads on the home screen (light tree)', + setup: [relaunch, ['wait', 'text', 'Agent Device Tester', LANDMARK_WAIT_MS]], + steps: [ + step('snapshot-i', C.snapshot, ['snapshot', '-i']), + step('snapshot', C.snapshot, ['snapshot']), + step('snapshot-raw', C.snapshot, ['snapshot', '--raw']), + // Only the regular capture plan honors the pin; the raw plan runs its own backend ladder. The + // XCTest tree rejects the heavy feed, so the pinned capture runs on this light screen. + step('snapshot-forced-tree', C.snapshot, ['snapshot'], { + via: 'sdk-forced-tree', + expect: { pinnedBackend: 'tree' }, + }), + step('snapshot-diff', C.snapshot, ['snapshot', '--diff']), + step('diff-snapshot', C.diff, ['diff', 'snapshot']), + step('wait-text', C.wait, ['wait', 'text', 'Agent Device Tester']), + step('wait-selector', C.wait, ['wait', 'id="gesture-canary-ready"']), + step('wait-absent', C.wait, ['wait', 'absent', 'id="heavy-feed-title"']), + step('wait-stable', C.wait, ['wait', 'stable', '300', '5000']), + step('is-visible', C.is, ['is', 'visible', 'id="gesture-canary-ready"']), + step('is-exists', C.is, ['is', 'exists', 'text="Gesture lab"']), + step('get-text', C.get, ['get', 'text', 'id="gesture-fling-status"']), + step('get-attrs', C.get, ['get', 'attrs', 'id="drag-source"']), + step('find-text', C.find, ['find', 'text', 'Gesture lab', 'exists']), + step('batch-reads', C.batch, [ + 'batch', + '--steps', + JSON.stringify([ + { + command: 'get', + input: { + format: 'text', + target: { kind: 'selector', selector: 'id="gesture-fling-status"' }, + }, + }, + { command: 'is', input: { predicate: 'visible', selector: 'id="gesture-canary-ready"' } }, + ]), + ]), + step('gesture-pan', C.gesture, [ + 'gesture', + 'pan', + '{point:gesture-target}', + '40', + '0', + '300', + ]), + step('gesture-pinch', C.gesture, ['gesture', 'pinch', '1.4', '{point:gesture-target}']), + step('screenshot', C.screenshot, ['screenshot', '{out:home.png}']), + step('screenshot-overlay', C.screenshot, [ + 'screenshot', + '{out:home-refs.png}', + '--overlay-refs', + ]), + step('events', C.events, ['events']), + step('logs-mark', C.logs, ['logs', 'mark', 'agent-device perf']), + step('perf-memory', C.perf, ['perf', 'memory', 'sample']), + step('react-native', C.reactNative, ['react-native', 'dismiss-overlay']), + step('artifacts', C.artifacts, ['artifacts']), + ], + }, + { + id: 'catalog', + title: 'Catalog list: press forms, text entry, and scrolling', + setup: toTab('Catalog', 'id="catalog-title"'), + steps: [ + step('press-selector', C.press, ['press', 'id="category-produce"']), + step('press-selector-settle', C.press, ['press', 'id="category-bakery"', '--settle']), + step('press-ref', C.press, ['press', '{ref:category-pantry}']), + step('press-ref-settle', C.press, ['press', '{ref:category-produce}', '--settle']), + step('press-point', C.press, ['press', '{point:category-bakery}']), + step('press-point-settle', C.press, ['press', '{point:category-pantry}', '--settle']), + step('click-selector', C.click, ['click', 'id="category-all"']), + step('click-ref', C.click, ['click', '{ref:category-bakery}']), + step('click-point', C.click, ['click', '{point:category-pantry}']), + step('click-settle', C.click, ['click', 'id="category-all"', '--settle']), + step('press-count', C.press, [ + 'press', + '{point:category-produce}', + '--count', + '2', + '--interval-ms', + '50', + ]), + step('fill', C.fill, ['fill', 'id="catalog-search"', 'tart']), + step('fill-settle', C.fill, ['fill', 'id="catalog-search"', 'kit', '--settle']), + step('focus', C.focus, ['focus', '{point:catalog-search}']), + step('type', C.type, ['type', ' box']), + step('keyboard-return', C.keyboard, ['keyboard', 'return']), + step('fill-clear', C.fill, ['fill', 'id="catalog-search"', ''], { + after: [['keyboard', 'return']], + }), + step('scroll-down', C.scroll, ['scroll', 'down']), + step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), + step('scroll-top', C.scroll, ['scroll', 'top']), + step('scroll-until', C.scroll, ['scroll', 'down', '--until', 'id="catalog-footer"']), + step('swipe', C.swipe, ['swipe', '200', '300', '200', '650']), + step('swipe-count', C.swipe, [ + 'swipe', + '200', + '650', + '200', + '450', + '--count', + '2', + '--pause-ms', + '50', + '--pattern', + 'ping-pong', + ]), + ], + }, + { + id: 'form', + title: 'Form fields, keyboard, and submit', + setup: toTab('Form', 'id="form-title"'), + steps: [ + step('fill-name', C.fill, ['fill', 'id="field-name"', 'Ada Lovelace']), + step('fill-email', C.fill, ['fill', 'id="field-email"', 'ada@example']), + step('type-suffix', C.type, ['type', '.test']), + step('keyboard-dismiss', C.keyboard, ['keyboard', 'dismiss'], { + before: [['wait', 'label="Done" visible=true', '10000']], + }), + step('press-checkbox', C.press, ['press', 'id="checkbox-agree"'], { + before: [['scroll', 'down', '--until', 'id="checkbox-agree"']], + }), + step('press-submit-settle', C.press, ['press', 'id="submit-order"', '--settle'], { + before: [['scroll', 'down', '--until', 'id="submit-order"']], + }), + ], + }, + { + id: 'automation', + title: 'Long press, native alerts, recording, tracing, and back navigation', + setup: toSettingsLab('open-automation-lab', 'id="automation-title"'), + steps: [ + step('record-start', C.record, ['record', 'start', '{out:automation.mp4}', '--hide-touches']), + step('record-stop', C.record, ['record', 'stop'], { + before: [ + ['scroll', 'down', '--until', 'id="automation-press"'], + ['press', 'id="automation-press"'], + ['wait', '1200'], + ], + }), + step('trace-start', C.trace, ['trace', 'start', '{out:automation.trace}']), + step('trace-stop', C.trace, ['trace', 'stop', '{out:automation.trace}'], { + before: [['press', 'id="automation-press"']], + }), + step('longpress', C.longPress, ['longpress', 'id="automation-longpress"', '800'], { + before: [['scroll', 'down', '--until', 'id="automation-longpress"']], + }), + step('longpress-settle', C.longPress, [ + 'longpress', + 'id="automation-longpress"', + '800', + '--settle', + ]), + step('alert-open', C.press, ['press', 'id="automation-open-alert"'], { + before: [['scroll', 'down', '--until', 'id="automation-open-alert"']], + }), + step('alert-wait', C.alert, ['alert', 'wait', '10000']), + step('alert-get', C.alert, ['alert', 'get']), + step('alert-accept', C.alert, ['alert', 'accept']), + step('alert-dismiss', C.alert, ['alert', 'dismiss'], { + before: [ + ['press', 'id="automation-open-alert"'], + ['alert', 'wait', '10000'], + ], + }), + step('logs-start', C.logs, ['logs', 'start']), + step('logs-stop', C.logs, ['logs', 'stop']), + step('network-dump', C.network, ['network', 'dump', '10']), + step('back', C.back, ['back']), + step('back-settle', C.back, ['back', '--settle'], { + before: [ + ['click', 'id="open-automation-lab"'], + ['wait', 'id="automation-title"', LANDMARK_WAIT_MS], + ], + }), + ], + }, + { + id: 'heavy-feed', + title: 'Heavy feed: Bluesky-depth tree (≥65 native levels) and virtualized list', + setup: toSettingsLab('open-heavy-feed', 'id="heavy-feed-title"'), + steps: [ + step('snapshot-i', C.snapshot, ['snapshot', '-i']), + step('snapshot', C.snapshot, ['snapshot']), + step('snapshot-raw', C.snapshot, ['snapshot', '--raw'], { expect: { maxDepthAtLeast: 64 } }), + step('press-like-selector', C.press, ['press', 'id="heavy-post-0-like"']), + step('press-like-settle', C.press, ['press', 'id="heavy-post-1-like"', '--settle']), + step('press-like-ref', C.press, ['press', '{ref:heavy-post-0-like}']), + step('press-like-point', C.press, ['press', '{point:heavy-post-0-like}']), + step('wait-text', C.wait, ['wait', 'text', 'Liked posts: 2']), + step('is-visible', C.is, ['is', 'visible', 'id="heavy-post-1-like"']), + step('get-text', C.get, ['get', 'text', 'id="heavy-feed-last-action"']), + step('find-text', C.find, ['find', 'text', 'Post 1:', 'exists']), + step('longpress-post', C.longPress, ['longpress', 'id="heavy-post-1"', '800']), + step('scroll-down', C.scroll, ['scroll', 'down']), + step('snapshot-i-scrolled', C.snapshot, ['snapshot', '-i']), + step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), + step('swipe', C.swipe, ['swipe', '200', '650', '200', '300']), + step('back', C.back, ['back']), + ], + }, + { + id: 'session', + title: 'Session teardown', + setup: [], + steps: [step('close', C.close, ['close'])], + }, +]; + +/** + * Catalog commands the iOS suite does not time, with the reason. Every public command is either + * measured by a step above or listed here; `ios-suite.test.ts` holds that partition. + */ +export const IOS_SUITE_EXCLUSIONS: Partial> = { + [C.boot]: + 'Simulator power cycle: CoreSimulator boot time dominates and it ends the session under test.', + [C.shutdown]: 'Simulator power cycle: it ends the session and the runner under test.', + [C.install]: + 'Bundle provisioning: simctl copy time dominates; the job installs the fixture once.', + [C.reinstall]: + 'Bundle provisioning: simctl copy time dominates and it resets the app under test.', + [C.uninstall]: 'Bundle provisioning: it removes the app under test.', + [C.installFromSource]: + 'Bundle provisioning from a source build; the job installs a cached Release fixture.', + [C.replay]: 'Composite runner: a replay is the sum of the measured commands it runs.', + [C.test]: 'Composite runner: a test suite is the sum of the measured commands it runs.', + [C.push]: + 'The fixture holds no notification authorization, so simctl refuses the payload; iOS push coverage is contract-only.', + [C.audio]: 'Host audio capture needs a macOS audio permission that hosted runners do not grant.', + [C.hover]: 'The iOS runtime refuses hover; it is a web pointer contract.', + [C.viewport]: 'The iOS runtime refuses viewport; it resizes web viewports only.', + [C.tvRemote]: 'tvOS only.', + [C.fold]: 'iPhone Duo only; the suite device has one panel.', + [C.telephony]: 'The iOS simulator runtime refuses simulated calls and SMS.', + [C.hardwareButton]: 'The iOS simulator runtime refuses hardware-button presses.', +}; + +/** Partitions the measured and excluded catalog commands for the report. */ +export function describeCoverage( + scenarios: readonly SuiteScenario[], + exclusions: Partial>, +): { measured: string[]; excluded: { command: string; reason: string }[] } { + const measured = [...new Set(scenarios.flatMap((s) => s.steps.map((step) => step.command)))]; + return { + measured: measured.sort(), + excluded: Object.entries(exclusions) + .map(([command, reason]) => ({ command, reason: reason! })) + .sort((a, b) => a.command.localeCompare(b.command)), + }; +} diff --git a/scripts/perf/measurement.ts b/scripts/perf/measurement.ts new file mode 100644 index 0000000000..1641361a3a --- /dev/null +++ b/scripts/perf/measurement.ts @@ -0,0 +1,32 @@ +import { aggregateProfiles } from './profile.ts'; +import { summarize } from './stats.ts'; +import type { Measurement } from './types.ts'; + +type MeasurementInput = Omit< + Measurement, + 'wallClock' | 'daemon' | 'elementCount' | 'profile' | 'failures' | 'notes' +>; + +function numbers(values: (number | undefined)[]): number[] { + return values.filter((n): n is number => typeof n === 'number'); +} + +/** Summarizes the successful samples; failed samples count only toward `failures`. */ +export function buildMeasurement(input: MeasurementInput): Measurement { + const ok = input.samples.filter((s) => s.ok); + const failures = input.samples.length - ok.length; + const notes: string[] = []; + if (failures > 0) { + const codes = [...new Set(input.samples.filter((s) => !s.ok).map((s) => s.errorCode))]; + notes.push(`${failures}/${input.samples.length} failed: ${codes.join(', ')}`); + } + return { + ...input, + wallClock: summarize(ok.map((s) => s.wallClockMs)), + daemon: summarize(numbers(ok.map((s) => s.daemonMs))), + elementCount: summarize(numbers(ok.map((s) => s.elementCount ?? s.snapshot?.nodeCount))), + profile: aggregateProfiles(input.samples), + failures, + notes, + }; +} diff --git a/scripts/perf/metadata.ts b/scripts/perf/metadata.ts new file mode 100644 index 0000000000..47f25b9098 --- /dev/null +++ b/scripts/perf/metadata.ts @@ -0,0 +1,44 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { readGitRevision, readTarget, readToolchain } from '../lib/bench-host.ts'; +import { REPO_ROOT, type PerfConfig } from './config.ts'; +import type { RunMetadata } from './types.ts'; + +function readVersion(): string { + try { + const pkg = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')); + return typeof pkg.version === 'string' ? pkg.version : 'unknown'; + } catch { + return 'unknown'; + } +} + +/** Records what the numbers depend on: source, toolchain, host, and device. */ +export function collectMetadata(cfg: PerfConfig, deviceName: string): RunMetadata { + const revision = readGitRevision(REPO_ROOT); + const toolchain = readToolchain(); + const target = cfg.platform === 'ios' ? readTarget(cfg.udid!, cfg.appId, cfg.appPath) : undefined; + const cpus = os.cpus(); + return { + commit: revision.commit, + ref: revision.branch, + dirty: revision.dirty, + runId: process.env.GITHUB_RUN_ID, + runAttempt: process.env.GITHUB_RUN_ATTEMPT, + agentDeviceVersion: readVersion(), + host: { + os: toolchain.os, + arch: toolchain.arch, + cpu: cpus[0]?.model ?? 'unknown', + cores: cpus.length, + memoryGb: Math.round(os.totalmem() / 1024 ** 3), + }, + xcode: target ? toolchain.xcode.replaceAll('\n', ' ') : undefined, + node: toolchain.node, + device: target + ? { name: target.name, udid: target.udid, runtime: target.runtime.split('.').pop() } + : { name: deviceName, serial: cfg.serial }, + fixture: target ? { appId: target.appId, appPath: target.appPath } : undefined, + }; +} diff --git a/scripts/perf/nightly-artifacts.test.ts b/scripts/perf/nightly-artifacts.test.ts new file mode 100644 index 0000000000..34a1aac17a --- /dev/null +++ b/scripts/perf/nightly-artifacts.test.ts @@ -0,0 +1,113 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + coversTarget, + resolveRunRef, + selectNightlyBaseline, + selectRange, + type WorkflowRun, +} from './nightly-artifacts.ts'; +import { summaryFixture } from './summary.fixtures.ts'; + +const run = (id: string, createdAt: string, event = 'schedule'): WorkflowRun => ({ + id, + createdAt, + headSha: `sha-${id}`, + event, +}); + +// Deliberately unsorted, with two runs on 2026-10-19. +const RUNS = [ + run('101', '2026-10-17T03:00:00Z'), + run('104', '2026-10-20T03:00:00Z'), + run('102', '2026-10-19T03:00:00Z'), + run('103', '2026-10-19T15:30:00Z'), +]; + +test('resolves latest, run ids, and dates to the newest matching run', () => { + assert.equal(resolveRunRef('latest', RUNS)?.id, '104'); + assert.equal(resolveRunRef('102', RUNS)?.id, '102'); + assert.equal(resolveRunRef('2026-10-19', RUNS)?.id, '103'); + assert.equal(resolveRunRef('2026-10-18', RUNS), undefined); + assert.throws(() => resolveRunRef('yesterday', RUNS), /Unknown run reference/); +}); + +const baselineIds = ( + runs: Iterable, + options: { nights?: number; exclude?: string[]; reject?: string[] } = {}, +) => + selectNightlyBaseline( + runs, + { + before: '2026-10-20T03:00:00Z', + exclude: new Set(options.exclude), + nights: options.nights ?? 3, + }, + (r) => (options.reject?.includes(r.id) ? undefined : r.id), + ).map(({ value }) => value); + +test('a rolling baseline takes one scheduled run per UTC night and skips dispatches', () => { + const newestFirst = [ + run('20', '2026-10-19T16:00:00Z', 'workflow_dispatch'), + run('19', '2026-10-19T15:00:00Z', 'workflow_dispatch'), + run('18', '2026-10-19T14:00:00Z', 'workflow_dispatch'), + run('17', '2026-10-19T02:50:00Z'), + run('16', '2026-10-19T02:40:00Z'), + run('15', '2026-10-18T02:40:00Z'), + run('14', '2026-10-17T02:40:00Z'), + ]; + // Three dispatches in one afternoon are not three nights; 2026-10-19 counts its newest run once. + assert.deepEqual(baselineIds(newestFirst), ['17', '15', '14']); +}); + +test('a night without a usable summary gives way to an older scheduled run of that night', () => { + const newestFirst = [ + run('17', '2026-10-19T02:50:00Z'), + run('16', '2026-10-19T02:40:00Z'), + run('15', '2026-10-18T02:40:00Z'), + run('14', '2026-10-17T02:40:00Z'), + ]; + assert.deepEqual(baselineIds(newestFirst, { reject: ['17'], exclude: ['15'] }), ['16', '14']); +}); + +test('a rolling baseline reads past 100 runs and stops once it has enough nights', () => { + function* history(): Generator { + for (let i = 0; i < 150; i++) { + yield run(`d${i}`, '2026-10-19T12:00:00Z', 'workflow_dispatch'); + } + yield run('n3', '2026-10-18T02:40:00Z'); + yield run('n2', '2026-10-17T02:40:00Z'); + yield run('n1', '2026-10-16T02:40:00Z'); + throw new Error('read past the last needed night'); + } + assert.deepEqual(baselineIds(history()), ['n3', 'n2', 'n1']); +}); + +test('a range includes both ends whichever order they are given in', () => { + const ids = (from: string, to: string) => selectRange(RUNS, from, to).map((r) => r.id); + assert.deepEqual(ids('101', '102'), ['102', '101']); + assert.deepEqual(ids('102', '101'), ['102', '101']); +}); + +test('a date end of a range covers its whole UTC day', () => { + // 2026-10-19 has a 03:00 run and a later 15:30 run; both belong to the range. + assert.deepEqual( + selectRange(RUNS, '2026-10-17', '2026-10-19').map((r) => r.id), + ['103', '102', '101'], + ); + assert.deepEqual( + selectRange(RUNS, '2026-10-19', '104').map((r) => r.id), + ['104', '103', '102'], + ); +}); + +test('a rolling baseline accepts only runs that measured every target scenario', () => { + const full = summaryFixture([ + { id: 'a/x', daemonMs: [1] }, + { id: 'b/y', daemonMs: [1] }, + ]); + const partial = summaryFixture([{ id: 'b/y', daemonMs: [1] }]); + assert.equal(coversTarget(full, full), true); + assert.equal(coversTarget(partial, full), false); + assert.equal(coversTarget(full, partial), true); +}); diff --git a/scripts/perf/nightly-artifacts.ts b/scripts/perf/nightly-artifacts.ts new file mode 100644 index 0000000000..8f0525f077 --- /dev/null +++ b/scripts/perf/nightly-artifacts.ts @@ -0,0 +1,189 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { SUMMARY_FILE } from './report.ts'; +import { parsePerfSummary } from './summary.ts'; +import type { PerfSummary } from './types.ts'; + +/** The stable name of the summary artifact that the nightly workflow uploads for every run. */ +export const SUMMARY_ARTIFACT = 'perf-ios-summary'; + +export type WorkflowRun = { + id: string; + createdAt: string; + headSha: string; + event: string; +}; + +export type NightlySource = { + repo: string; + workflow: string; + branch: string; + artifact: string; + cacheDir: string; +}; + +const DATE_REF = /^\d{4}-\d{2}-\d{2}$/; + +function gh(args: string[]): { ok: boolean; stdout: string; stderr: string } { + const result = runCmdSync('gh', args, { allowFailure: true, maxBuffer: 64 * 1024 * 1024 }); + return { ok: result.exitCode === 0, stdout: result.stdout, stderr: result.stderr }; +} + +const RUNS_PER_PAGE = 100; + +/** + * Yields the workflow's completed runs on one branch, newest first, one API page at a time. A + * caller that stops early fetches only the pages that it read. + */ +export function* listRuns(source: NightlySource): Generator { + const seen = new Set(); + for (let page = 1; ; page++) { + const result = gh([ + 'api', + '--method', + 'GET', + `repos/${source.repo}/actions/workflows/${source.workflow}/runs`, + '-f', + `branch=${source.branch}`, + '-f', + 'status=completed', + '-f', + `per_page=${RUNS_PER_PAGE}`, + '-f', + `page=${page}`, + '--jq', + '[.workflow_runs[] | {id, created_at, head_sha, event}]', + ]); + if (!result.ok) throw new Error(`gh api workflow runs failed: ${result.stderr.trim()}`); + const rows = JSON.parse(result.stdout) as { + id: number; + created_at: string; + head_sha: string; + event: string; + }[]; + for (const row of rows) { + const id = String(row.id); + // A run created during paging moves later pages by one place. + if (seen.has(id)) continue; + seen.add(id); + yield { id, createdAt: row.created_at, headSha: row.head_sha, event: row.event }; + } + if (rows.length < RUNS_PER_PAGE) return; + } +} + +function sortNewestFirst(runs: WorkflowRun[]): WorkflowRun[] { + return [...runs].sort((a, b) => b.createdAt.localeCompare(a.createdAt)); +} + +/** + * Resolves one run reference: `latest`, a numeric run id, or a `YYYY-MM-DD` date (the newest run + * created that UTC day). Returns undefined when no listed run matches. + */ +export function resolveRunRef(ref: string, runs: WorkflowRun[]): WorkflowRun | undefined { + const newestFirst = sortNewestFirst(runs); + if (ref === 'latest') return newestFirst[0]; + if (/^\d+$/.test(ref)) return newestFirst.find((run) => run.id === ref); + if (DATE_REF.test(ref)) return newestFirst.find((run) => run.createdAt.slice(0, 10) === ref); + throw new Error( + `Unknown run reference ${JSON.stringify(ref)}: use latest, a run id, a YYYY-MM-DD date, or a path.`, + ); +} + +/** + * Runs between two references inclusive, newest first. A date end covers its whole UTC day, so a + * dispatch later that day does not hide the night's scheduled run. + */ +export function selectRange(runs: WorkflowRun[], fromRef: string, toRef: string): WorkflowRun[] { + const bounds = (ref: string): [string, string] | undefined => { + if (DATE_REF.test(ref)) return [`${ref}T00:00:00Z`, `${ref}T23:59:59.999Z`]; + const run = resolveRunRef(ref, runs); + return run && [run.createdAt, run.createdAt]; + }; + const from = bounds(fromRef); + const to = bounds(toRef); + if (!from || !to) throw new Error(`No runs match the range ${fromRef}..${toRef}.`); + const low = from[0] < to[0] ? from[0] : to[0]; + const high = from[1] > to[1] ? from[1] : to[1]; + return sortNewestFirst(runs).filter((run) => run.createdAt >= low && run.createdAt <= high); +} + +/** + * A rolling-baseline run must have run every scenario of the target on the same platform, so a + * partial dispatch does not take the place of a full night. + */ +export function coversTarget(candidate: PerfSummary, target: PerfSummary): boolean { + return ( + candidate.platform === target.platform && + target.config.scenarios.every((scenario) => candidate.config.scenarios.includes(scenario)) + ); +} + +/** + * Selects the rolling baseline: one scheduled run for each UTC night before the target, newest night + * first. A night counts the newest of its scheduled runs that `load` accepts, so manual dispatches + * cannot fill the baseline with measurements from one afternoon. `runs` is newest first and is read + * only until `nights` nights are found. + */ +export function selectNightlyBaseline( + runs: Iterable, + options: { before: string; exclude: ReadonlySet; nights: number }, + load: (run: WorkflowRun) => T | undefined, +): { run: WorkflowRun; value: T }[] { + const selected: { run: WorkflowRun; value: T }[] = []; + const nights = new Set(); + for (const run of runs) { + const night = run.createdAt.slice(0, 10); + if ( + run.event !== 'schedule' || + run.createdAt >= options.before || + options.exclude.has(run.id) || + nights.has(night) + ) + continue; + const value = load(run); + if (value === undefined) continue; + nights.add(night); + selected.push({ run, value }); + if (selected.length >= options.nights) break; + } + return selected; +} + +/** + * Returns a run's summary, downloading the summary artifact once into the cache. Returns undefined + * when the run has no such artifact (it failed before upload, or the artifact expired). + */ +export function fetchSummary(source: NightlySource, run: WorkflowRun): PerfSummary | undefined { + const dir = path.join(source.cacheDir, run.id); + const file = path.join(dir, SUMMARY_FILE); + if (!fs.existsSync(file)) { + fs.rmSync(dir, { recursive: true, force: true }); + const result = gh([ + 'run', + 'download', + run.id, + '--repo', + source.repo, + '--name', + source.artifact, + '--dir', + dir, + ]); + if (!result.ok || !fs.existsSync(file)) { + process.stderr.write( + `[perf] run ${run.id}: no ${source.artifact} artifact (${result.stderr.trim()})\n`, + ); + fs.rmSync(dir, { recursive: true, force: true }); + return undefined; + } + } + return parsePerfSummary(JSON.parse(fs.readFileSync(file, 'utf8')), file); +} + +/** Reads a local run: a `summary.json` path or a run output directory. */ +export function readLocalSummary(location: string): PerfSummary { + const file = fs.statSync(location).isDirectory() ? path.join(location, SUMMARY_FILE) : location; + return parsePerfSummary(JSON.parse(fs.readFileSync(file, 'utf8')), file); +} diff --git a/scripts/perf/platform-profiles.ts b/scripts/perf/platform-profiles.ts index c082cdab06..447d3b310f 100644 --- a/scripts/perf/platform-profiles.ts +++ b/scripts/perf/platform-profiles.ts @@ -1,10 +1,7 @@ import type { PerfConfig } from './config.ts'; -import type { Platform } from './types.ts'; -// Local-convenience defaults for ad-hoc runs; CI always overrides them (--device / --serial). -// The iOS UDID is a specific local "iPhone 17" sim; the Android default is an AVD name that -// `boot` can launch. Pass --udid/--device/--serial to target your own device. -const DEFAULT_IOS_UDID = 'D74E0B66-57EB-4EC1-92DC-DA0A30581FE7'; +// Local-convenience default for ad-hoc Android runs: an AVD name that `boot` can launch. CI +// passes --serial. The iOS suite drives the fixture app instead (see ios-suite.ts). const DEFAULT_ANDROID_AVD = 'Pixel_9_Pro_XL_API_37'; export type ProfileSelectors = { @@ -14,10 +11,6 @@ export type ProfileSelectors = { searchField: string; // A selector that uniquely targets the EDITABLE search field (for fill). searchFieldEditable: string; - // iOS exposes an editable search field at the Settings root (fill works without focusing - // first; focusing then filling can hang). Android only reveals the editable after tapping - // the search card, so it must press the search entry before fill/type. - searchEditableAtRoot: boolean; // A label reliably visible on the Settings root, for get/is (selector form). anchorLabel: string; // Plain text of the anchor, for wait text / find (not a selector). @@ -25,9 +18,8 @@ export type ProfileSelectors = { }; export type ResolvedProfile = { - platform: Platform; + platform: 'android'; deviceName: string; - udid?: string; serial?: string; platformFlags: string[]; // --platform; applied to every call (only conflicts if it mismatches a locked session) selectorFlags: string[]; // device selectors — ONLY on the session-establishing open / selectorless boot @@ -36,27 +28,6 @@ export type ResolvedProfile = { }; export function resolveProfile(cfg: PerfConfig): ResolvedProfile { - if (cfg.platform === 'ios') { - // Prefer targeting by device name (CI boots a named simulator); fall back to a UDID. - const useName = cfg.device !== undefined; - const udid = useName ? undefined : (cfg.udid ?? DEFAULT_IOS_UDID); - return { - platform: 'ios', - deviceName: cfg.device ?? 'iPhone 17', - udid, - platformFlags: ['--platform', 'ios'], - selectorFlags: useName ? ['--device', cfg.device!] : ['--udid', udid!], - appTarget: 'settings', - selectors: { - deepScreen: 'label="General"', - searchField: 'label="Search"', - searchFieldEditable: 'label="Search" editable', - searchEditableAtRoot: true, - anchorLabel: 'label="General"', - anchorText: 'General', - }, - }; - } const avdName = cfg.device ?? DEFAULT_ANDROID_AVD; const serialFlags = cfg.serial ? ['--serial', cfg.serial, '--android-device-allowlist', cfg.serial] @@ -72,7 +43,6 @@ export function resolveProfile(cfg: PerfConfig): ResolvedProfile { deepScreen: 'text="Network & internet"', searchField: 'text="Search Settings"', searchFieldEditable: 'editable', - searchEditableAtRoot: false, anchorLabel: 'label="Network & internet"', anchorText: 'Network & internet', }, diff --git a/scripts/perf/profile.test.ts b/scripts/perf/profile.test.ts new file mode 100644 index 0000000000..9e9b9e61c1 --- /dev/null +++ b/scripts/perf/profile.test.ts @@ -0,0 +1,119 @@ +import assert from 'node:assert/strict'; +import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; +import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; +import { RETRY_PHASE } from '@agent-device/host-kit/retry'; +import { test } from 'vitest'; +import { + aggregateProfiles, + profileRequestLog, + readClientDiagnostics, + readSettleFacts, +} from './profile.ts'; +import type { Sample } from './types.ts'; + +const line = (event: Record) => JSON.stringify({ level: 'info', ...event }); + +// Trimmed from a heavy-feed `press` request log. +const PRESS_LOG = [ + line({ ts: '2026-10-21T19:00:00.000Z', phase: 'request_start', data: { command: 'press' } }), + line({ + phase: EXEC_COMMAND_PHASE, + durationMs: 0.02, + data: { command: 'xcrun', event: 'spawn_start' }, + }), + line({ phase: EXEC_COMMAND_PHASE, durationMs: 9, data: { command: 'xcrun', event: 'spawn' } }), + line({ phase: EXEC_COMMAND_PHASE, durationMs: 120, data: { command: 'xcrun', event: 'exit' } }), + line({ + phase: EXEC_COMMAND_PHASE, + durationMs: 121, + data: { command: 'xcrun', event: 'settled' }, + }), + line({ + level: 'error', + phase: 'ios.snapshot-source.acquire', + durationMs: 80, + data: { producer: 'simulator-ax-bridge', error: 'unsupported: foreground-owner-unverified' }, + }), + line({ + phase: 'ios.snapshot-source.acquire', + durationMs: 160, + data: { producer: 'simulator-ax-bridge' }, + }), + line({ + phase: 'ios_snapshot_source_recovery', + data: { producer: 'simulator-ax-bridge', truncated: true, acceptedLevels: 32 }, + }), + line({ phase: RETRY_PHASE, data: { event: 'attempt_failed', attempt: 1 } }), + line({ phase: 'ios_runner_command_send', durationMs: 900, data: { command: 'tap' } }), + line({ phase: 'post_gesture_snapshot_stabilized', data: { attempts: 2, durationMs: 300 } }), + line({ + ts: '2026-10-21T19:00:01.250Z', + level: 'warn', + phase: 'ios_runner_session_invalidated', + data: { reason: 'stale' }, + }), + 'not json', +].join('\n'); + +test('profiles runner, spawn, snapshot-source, retry, and settle phases of one request', () => { + const profile = profileRequestLog(PRESS_LOG); + assert.deepEqual(profile.spawns, { xcrun: { count: 1, totalMs: 121 } }); + assert.deepEqual(profile.runnerCommands, { tap: { count: 1, totalMs: 900 } }); + assert.deepEqual(profile.snapshotSources, { + 'simulator-ax-bridge': { count: 2, totalMs: 240, failures: 1 }, + }); + assert.equal(profile.retries, 1); + assert.equal(profile.requestSpanMs, 1250); + assert.deepEqual(profile.stabilization, { count: 1, totalMs: 300 }); + assert.deepEqual(profile.fallbacks, [ + 'ios_runner_session_invalidated: stale', + 'simulator-ax-bridge: truncated at 32 levels', + 'simulator-ax-bridge: unsupported: foreground-owner-unverified', + ]); +}); + +test('counts a producer only for samples where it answered', () => { + const failedOnly = profileRequestLog( + line({ + level: 'error', + phase: 'ios.snapshot-source.acquire', + durationMs: 5, + data: { producer: 'simulator-ax-bridge', error: 'cancelled' }, + }), + ); + const runnerSnapshot = profileRequestLog( + line({ phase: 'ios_runner_command_send', durationMs: 4000, data: { command: 'snapshot' } }), + ); + const samples: Sample[] = [ + { round: 0, ok: true, wallClockMs: 1, profile: failedOnly }, + { + round: 1, + ok: true, + wallClockMs: 1, + profile: runnerSnapshot, + snapshot: { backend: 'private-ax', state: 'recovered', reasonCode: 'ax-rejected' }, + }, + { round: 2, ok: false, wallClockMs: 1, profile: profileRequestLog(PRESS_LOG) }, + ]; + const aggregate = aggregateProfiles(samples)!; + assert.deepEqual(aggregate.producers, { 'apple-runner': 1 }); + assert.deepEqual(aggregate.backends, { 'private-ax/recovered/ax-rejected': 1 }); +}); + +test('reads the client daemon timer from --debug stderr', () => { + const stderr = [ + `${DIAGNOSTIC_STDERR_PREFIX}{"phase":"daemon_startup","durationMs":30,"requestId":"r1"}`, + 'noise', + `${DIAGNOSTIC_STDERR_PREFIX}{"phase":"daemon_request","durationMs":812,"requestId":"r1"}`, + ].join('\n'); + assert.deepEqual(readClientDiagnostics(stderr), { daemonMs: 812, requestId: 'r1' }); + assert.deepEqual(readClientDiagnostics(`${DIAGNOSTIC_STDERR_PREFIX}{"phase":"daemon_req`), {}); +}); + +test('reads the settle wait of a --settle response and nothing from other responses', () => { + assert.deepEqual( + readSettleFacts({ data: { settle: { settled: true, waitedMs: 1665, captures: 2 } } }), + { waitedMs: 1665, captures: 2, settled: true }, + ); + assert.equal(readSettleFacts({ data: { message: 'Tapped' } }), undefined); +}); diff --git a/scripts/perf/profile.ts b/scripts/perf/profile.ts new file mode 100644 index 0000000000..96a4a188f6 --- /dev/null +++ b/scripts/perf/profile.ts @@ -0,0 +1,233 @@ +import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; +import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; +import { RETRY_PHASE } from '@agent-device/host-kit/retry'; +import { median } from './stats.ts'; +import type { + MeasurementProfile, + Sample, + SampleProfile, + SettleFacts, + SnapshotFacts, + Timing, +} from './types.ts'; + +// The platform-apple façades and the root client export only what production code consumes, so +// these phase names stay literal here. +const RUNNER_COMMAND_SEND_PHASE = 'ios_runner_command_send'; +const SNAPSHOT_SOURCE_ACQUIRE_PHASE = 'ios.snapshot-source.acquire'; +const SNAPSHOT_SOURCE_RECOVERY_PHASE = 'ios_snapshot_source_recovery'; +const DAEMON_REQUEST_PHASE = 'daemon_request'; + +type DiagnosticEvent = { + ts?: string; + level?: string; + phase?: string; + durationMs?: number; + data?: Record; +}; + +const STABILIZATION_PHASE = /settle|stabiliz/i; +const FALLBACK_TEXT_LIMIT = 140; + +function addTiming(map: Record, key: string, durationMs: number | undefined): void { + const entry = (map[key] ??= { count: 0, totalMs: 0 }); + entry.count += 1; + entry.totalMs += durationMs ?? 0; +} + +function text(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 + ? value.slice(0, FALLBACK_TEXT_LIMIT) + : undefined; +} + +function eventDuration(event: DiagnosticEvent): number | undefined { + if (typeof event.durationMs === 'number') return event.durationMs; + const nested = event.data?.durationMs; + return typeof nested === 'number' ? nested : undefined; +} + +/** + * Reads one daemon request log (`--debug` ndjson). Process spawns count once, at their `settled` + * event. Fallbacks name every degraded path the request took: snapshot-source errors, truncated + * recovery, and warn- or error-level phases. + */ +export function profileRequestLog(ndjson: string): SampleProfile { + const profile: SampleProfile = { + phases: {}, + runnerCommands: {}, + spawns: {}, + snapshotSources: {}, + retries: 0, + stabilization: { count: 0, totalMs: 0 }, + fallbacks: [], + }; + const fallbacks = new Set(); + let firstTs: number | undefined; + let lastTs: number | undefined; + for (const line of ndjson.split('\n')) { + if (!line.trim()) continue; + let event: DiagnosticEvent; + try { + event = JSON.parse(line) as DiagnosticEvent; + } catch { + continue; + } + const phase = event.phase; + if (!phase) continue; + const ts = typeof event.ts === 'string' ? Date.parse(event.ts) : Number.NaN; + if (Number.isFinite(ts)) { + firstTs ??= ts; + lastTs = ts; + } + const data = event.data ?? {}; + const duration = eventDuration(event); + if (phase === EXEC_COMMAND_PHASE) { + if (data.event !== 'settled') continue; + addTiming(profile.spawns, text(data.command) ?? 'unknown', duration); + } else if (phase === RUNNER_COMMAND_SEND_PHASE) { + addTiming(profile.runnerCommands, text(data.command) ?? 'unknown', duration); + } else if (phase === SNAPSHOT_SOURCE_ACQUIRE_PHASE) { + const producer = text(data.producer) ?? 'unknown'; + const entry = (profile.snapshotSources[producer] ??= { count: 0, totalMs: 0, failures: 0 }); + entry.count += 1; + entry.totalMs += duration ?? 0; + const error = text(data.error); + if (error || event.level === 'error') { + entry.failures += 1; + fallbacks.add(`${producer}: ${error ?? 'failed'}`); + } + } else if (phase === SNAPSHOT_SOURCE_RECOVERY_PHASE && data.truncated === true) { + fallbacks.add( + `${text(data.producer) ?? 'unknown'}: truncated at ${String(data.acceptedLevels)} levels`, + ); + } else if (phase === RETRY_PHASE && data.event === 'attempt_failed') { + profile.retries += 1; + } else if (event.level === 'warn' || event.level === 'error') { + fallbacks.add(`${phase}${text(data.reason) ? `: ${text(data.reason)}` : ''}`); + } + if (STABILIZATION_PHASE.test(phase)) { + profile.stabilization.count += 1; + profile.stabilization.totalMs += duration ?? 0; + } + addTiming(profile.phases, phase, duration); + } + profile.fallbacks = [...fallbacks].sort(); + if (firstTs !== undefined && lastTs !== undefined) profile.requestSpanMs = lastTs - firstTs; + return profile; +} + +/** Reads the client's `daemon_request` timer and request id from `--debug` stderr. */ +export function readClientDiagnostics(stderr: string): { daemonMs?: number; requestId?: string } { + for (const line of stderr.split('\n')) { + const start = line.indexOf(DIAGNOSTIC_STDERR_PREFIX); + if (start < 0) continue; + try { + const event = JSON.parse(line.slice(start + DIAGNOSTIC_STDERR_PREFIX.length)) as { + phase?: string; + durationMs?: number; + requestId?: string; + }; + if (event.phase === DAEMON_REQUEST_PHASE) { + return { daemonMs: event.durationMs, requestId: event.requestId }; + } + } catch { + // A truncated diagnostic line carries no timer. + } + } + return {}; +} + +/** Reads the settle outcome of a `--settle` response. */ +export function readSettleFacts(json: unknown): SettleFacts | undefined { + const settle = (json as { data?: { settle?: Record } } | undefined)?.data + ?.settle; + if (!settle || typeof settle.waitedMs !== 'number') return undefined; + return { + waitedMs: settle.waitedMs, + captures: typeof settle.captures === 'number' ? settle.captures : undefined, + settled: typeof settle.settled === 'boolean' ? settle.settled : undefined, + }; +} + +/** Reads snapshot quality facts from a `--json` snapshot response. */ +export function readSnapshotFacts(json: unknown): SnapshotFacts | undefined { + const data = (json as { data?: Record } | undefined)?.data; + if (!data || !Array.isArray(data.nodes)) return undefined; + const nodes = data.nodes as { depth?: unknown }[]; + const quality = (data.snapshotQuality ?? {}) as Record; + const depths = nodes.map((node) => (typeof node.depth === 'number' ? node.depth : 0)); + return { + backend: text(quality.backend), + state: text(quality.state), + reasonCode: text(quality.reasonCode), + nodeCount: nodes.length, + maxDepth: depths.length > 0 ? Math.max(...depths) : undefined, + truncated: typeof data.truncated === 'boolean' ? data.truncated : undefined, + }; +} + +function medianOrNull(values: number[]): number | null { + return values.length === 0 ? null : median(values); +} + +function sumTimings(map: Record): Timing { + return Object.values(map).reduce( + (sum, t) => ({ count: sum.count + t.count, totalMs: sum.totalMs + t.totalMs }), + { count: 0, totalMs: 0 }, + ); +} + +function tally(counts: Record, keys: Iterable): void { + for (const key of new Set(keys)) counts[key] = (counts[key] ?? 0) + 1; +} + +/** + * Summarizes the successful samples' profiles. Medians describe a typical request; producers, + * backends, and fallbacks count how many samples took each path. + */ +export function aggregateProfiles(samples: Sample[]): MeasurementProfile | null { + const profiled = samples.filter((s) => s.ok && s.profile); + if (profiled.length === 0) return null; + const profiles = profiled.map((s) => s.profile!); + const producers: Record = {}; + const backends: Record = {}; + const fallbacks: Record = {}; + for (const sample of profiled) { + const profile = sample.profile!; + tally( + producers, + Object.entries(profile.snapshotSources) + .filter(([, source]) => source.count > source.failures) + .map(([producer]) => producer), + ); + if (profile.runnerCommands.snapshot) tally(producers, ['apple-runner']); + if (sample.snapshot?.backend) { + const { backend, state, reasonCode } = sample.snapshot; + tally(backends, [[backend, state, reasonCode].filter(Boolean).join('/')]); + } + tally(fallbacks, profile.fallbacks); + } + const phaseNames = new Set(profiles.flatMap((p) => Object.keys(p.phases))); + const topPhases = [...phaseNames] + .map((phase) => ({ + phase, + medianMs: median(profiles.map((p) => p.phases[phase]?.totalMs ?? 0)), + })) + .filter((entry) => entry.medianMs > 0) + .sort((a, b) => b.medianMs - a.medianMs) + .slice(0, 5); + return { + runnerMs: medianOrNull(profiles.map((p) => sumTimings(p.runnerCommands).totalMs)), + spawnCount: medianOrNull(profiles.map((p) => sumTimings(p.spawns).count)), + spawnMs: medianOrNull(profiles.map((p) => sumTimings(p.spawns).totalMs)), + snapshotSourceMs: medianOrNull(profiles.map((p) => sumTimings(p.snapshotSources).totalMs)), + stabilizationMs: medianOrNull(profiles.map((p) => p.stabilization.totalMs)), + settleWaitMs: medianOrNull(profiled.flatMap((s) => (s.settle ? [s.settle.waitedMs] : []))), + retries: profiles.reduce((sum, p) => sum + p.retries, 0), + producers, + backends, + fallbacks, + topPhases, + }; +} diff --git a/scripts/perf/report.ts b/scripts/perf/report.ts index 536639eadb..8ff0e69b35 100644 --- a/scripts/perf/report.ts +++ b/scripts/perf/report.ts @@ -1,73 +1,134 @@ import fs from 'node:fs'; import path from 'node:path'; -import type { Measurement, RunResult, Stat } from './types.ts'; +import type { Measurement, PerfSummary, Stat } from './types.ts'; -function ms(n: number | undefined): string { +export const SUMMARY_FILE = 'summary.json'; +export const REPORT_FILE = 'report.md'; + +function ms(n: number | null | undefined): string { return typeof n === 'number' && Number.isFinite(n) ? n.toFixed(0) : '–'; } +function pct(n: number | undefined): string { + return typeof n === 'number' && Number.isFinite(n) ? `${(n * 100).toFixed(0)}%` : '–'; +} + +function keys(counts: Record | undefined): string { + const entries = Object.entries(counts ?? {}); + return entries.length === 0 ? '–' : entries.map(([key, n]) => `${key}×${n}`).join(', '); +} + function wallCells(s: Stat | null): string { - if (!s) return '– | – | – | –'; - return `${ms(s.min)} | ${ms(s.median)} | ${ms(s.p95)} | ${ms(s.max)}`; + if (!s) return '– | – | – | – | – | –'; + return `${ms(s.mean)} ± ${ms(s.stddev)} | ${ms(s.median)} | ${ms(s.p95)} | ${ms(s.min)} | ${ms(s.max)} | ${pct(s.cv)}`; } -function stampName(platform: string, startedAt: string): string { - return `perf-${platform}-${startedAt.replaceAll(/[:.]/g, '-')}`; +function cell(text: string): string { + return text.replaceAll('|', String.raw`\|`); } -function measurementRow(m: Measurement): string { - const daemon = m.daemonDuration ? ms(m.daemonDuration.median) : '–'; - const elements = m.elementCount ? ms(m.elementCount.median) : '–'; - const n = m.wallClock?.n ?? 0; - return `| ${m.label} | ${m.command} | ${m.execMode} | ${n} | ${wallCells(m.wallClock)} | ${daemon} | ${elements} | ${m.notes.join('; ')} |`; +function timingRow(m: Measurement): string { + return [ + m.id.slice(m.scenario.length + 1), + `\`${cell(m.label)}\`${m.execMode === 'sdk' ? ' (SDK, in process; daemon = log span)' : ''}`, + String(m.samples.length), + wallCells(m.wallClock), + ms(m.daemon?.median), + String(m.failures), + ].join(' | '); } -function toMarkdown(run: RunResult): string { - const lines: string[] = []; - lines.push(`# agent-device command perf — ${run.platform}`); - lines.push(''); - lines.push(`- **Device**: ${run.device.name} (${run.device.udid ?? run.device.serial ?? '?'})`); - lines.push(`- **agent-device**: ${run.agentDeviceVersion}`); - lines.push(`- **Rounds**: ${run.config.rounds} (warmup ${run.config.warmup} dropped)`); - lines.push(`- **Started**: ${run.startedAt}`); - lines.push(`- **Finished**: ${run.finishedAt}`); - lines.push(''); - lines.push('All times in milliseconds. `wall-clock` includes process spawn + socket overhead;'); - lines.push( - '`daemon` is the batch step round-trip (spawn overhead ≈ wall-median − daemon-median).', - ); - lines.push('`elements` = node count in the snapshot payload (tree-size proxy).'); - lines.push('An untimed warmup interaction runs after each open/relaunch, so measured commands'); - lines.push('do not pay the one-time iOS-runner startup or post-relaunch first-AX-query cost.'); - lines.push(''); - lines.push( - '| command | cli | mode | n | wall min | wall median | wall p95 | wall max | daemon median | elements | notes |', - ); - lines.push('|---|---|---|---|---|---|---|---|---|---|---|'); - for (const m of run.measurements) lines.push(measurementRow(m)); - lines.push(''); +function profileRow(m: Measurement): string { + const p = m.profile; + return [ + m.id.slice(m.scenario.length + 1), + ms(p?.runnerMs), + p?.spawnCount == null ? '–' : `${p.spawnCount} / ${ms(p.spawnMs)}`, + ms(p?.snapshotSourceMs), + ms(p?.settleWaitMs), + ms(p?.stabilizationMs), + String(p?.retries ?? '–'), + cell(keys(p?.producers)), + cell(p?.topPhases.map((t) => `${t.phase} ${ms(t.medianMs)}`).join(', ') || '–'), + ].join(' | '); +} - const failed = run.measurements.filter((m) => m.failures > 0); - if (failed.length > 0) { - lines.push('## Failures'); +/** Renders the human report for one run. */ +export function renderReport(summary: PerfSummary): string { + const meta = summary.metadata; + const lines: string[] = [ + `# agent-device perf — ${summary.platform}`, + '', + `- **Commit**: ${meta.commit.slice(0, 12)}${meta.dirty ? ' (dirty)' : ''}${meta.ref ? ` (${meta.ref})` : ''}${meta.runId ? ` · run ${meta.runId}` : ''}`, + `- **Device**: ${meta.device.name}${meta.device.runtime ? ` · ${meta.device.runtime}` : ''}`, + `- **Toolchain**: ${meta.xcode ?? 'no Xcode'} · Node ${meta.node} · agent-device ${meta.agentDeviceVersion}`, + `- **Host**: ${meta.host.os} · ${meta.host.cpu} · ${meta.host.cores} cores · ${meta.host.memoryGb} GB`, + `- **Rounds**: ${summary.config.rounds} measured after ${summary.config.warmup} warmup · ${summary.startedAt} → ${summary.finishedAt}`, + '', + 'Times are milliseconds. See scripts/perf/README.md for each metric. The second table of each', + 'scenario holds per-request medians from the daemon `--debug` request logs.', + '', + ]; + const scenarios = [...new Set(summary.measurements.map((m) => m.scenario))]; + for (const scenario of scenarios) { + const measurements = summary.measurements.filter((m) => m.scenario === scenario); + lines.push( + `## ${scenario}`, + '', + '| step | command | n | wall mean ± sd | median | p95 | min | max | cv | daemon | fail |', + '|---|---|---|---|---|---|---|---|---|---|---|', + ...measurements.map((m) => `| ${timingRow(m)} |`), + '', + '| step | runner | spawns | snapshot source | settle wait | stabilization | retries | producer | top phases |', + '|---|---|---|---|---|---|---|---|---|', + ...measurements.map((m) => `| ${profileRow(m)} |`), + '', + ); + } + const degraded = summary.measurements.filter( + (m) => + Object.keys(m.profile?.fallbacks ?? {}).length > 0 || + Object.keys(m.profile?.backends ?? {}).length > 0, + ); + if (degraded.length > 0) { + lines.push('## Backends and fallbacks', ''); + for (const m of degraded) { + lines.push( + `- **${m.id}** — backends ${keys(m.profile?.backends)}; fallbacks ${keys(m.profile?.fallbacks)}`, + ); + } lines.push(''); + } + const failed = summary.measurements.filter((m) => m.failures > 0); + if (failed.length > 0) { + lines.push('## Failures', ''); for (const m of failed) { const sample = m.samples.find((s) => !s.ok); lines.push( - `- **${m.label}** — ${m.notes.join('; ')}${sample?.errorMessage ? ` — ${sample.errorMessage}` : ''}`, + `- **${m.id}** — ${m.notes.join('; ')}${sample?.errorMessage ? ` — ${sample.errorMessage}` : ''}`, ); } lines.push(''); } + lines.push( + '## Coverage', + '', + `Measured commands (${summary.coverage.measured.length}): ${summary.coverage.measured.join(', ')}.`, + '', + ...summary.coverage.excluded.map((e) => `- \`${e.command}\` not measured: ${e.reason}`), + '', + ); return lines.join('\n'); } -export function writeReports(run: RunResult, outDir: string): { jsonPath: string; mdPath: string } { +export function writeRunOutput( + summary: PerfSummary, + outDir: string, +): { jsonPath: string; mdPath: string } { fs.mkdirSync(outDir, { recursive: true }); - const base = stampName(run.platform, run.startedAt); - const jsonPath = path.join(outDir, `${base}.json`); - const mdPath = path.join(outDir, `${base}.md`); - fs.writeFileSync(jsonPath, JSON.stringify(run, null, 2)); - fs.writeFileSync(mdPath, toMarkdown(run)); + const jsonPath = path.join(outDir, SUMMARY_FILE); + const mdPath = path.join(outDir, REPORT_FILE); + fs.writeFileSync(jsonPath, `${JSON.stringify(summary, null, 2)}\n`); + fs.writeFileSync(mdPath, renderReport(summary)); return { jsonPath, mdPath }; } diff --git a/scripts/perf/run.ts b/scripts/perf/run.ts index f1f390cc4e..cd712056ae 100644 --- a/scripts/perf/run.ts +++ b/scripts/perf/run.ts @@ -1,72 +1,123 @@ import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; -import { parseConfig, REPO_ROOT, usesSourceCli } from './config.ts'; +import { parseConfig, REPO_ROOT, usesSourceCli, type PerfConfig } from './config.ts'; +import { runScenario, setupIsolation, teardownIsolation } from './harness.ts'; +import { iosRunFinalizer, runIosSuite, type IosSuiteContext } from './ios-harness.ts'; import { - runScenario, - setupIsolation, - teardownIsolation, - type IsolationContext, -} from './harness.ts'; -import { writeReports } from './report.ts'; -import type { RunResult } from './types.ts'; + describeCoverage, + FIXTURE_APP_EVENT_URL_TEMPLATE, + IOS_SUITE, + IOS_SUITE_EXCLUSIONS, +} from './ios-suite.ts'; +import { collectMetadata } from './metadata.ts'; +import { writeRunOutput } from './report.ts'; +import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary } from './types.ts'; -function readVersion(): string { - try { - const pkg = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')); - return typeof pkg.version === 'string' ? pkg.version : 'unknown'; - } catch { - return 'unknown'; - } +function log(msg: string): void { + process.stderr.write(`[perf] ${msg}\n`); } -function main(): void { - const cfg = parseConfig(process.argv.slice(2)); - // The dist binary needs a build; running from source (AGENT_DEVICE_PERF_CLI) does not. - if (!usesSourceCli() && !fs.existsSync(path.join(REPO_ROOT, 'dist', 'src'))) { - process.stderr.write('[perf] dist/ is missing — run `pnpm build` first.\n'); - process.exit(1); - } - - const startedAt = new Date().toISOString(); - let ctx: IsolationContext | null = null; - let exitCode = 0; - - const cleanup = (): void => { - if (ctx) { - teardownIsolation(ctx, cfg); - ctx = null; - } - }; - process.on('SIGINT', () => { +function onInterrupt(cleanup: () => void): void { + process.once('SIGINT', () => { cleanup(); process.exit(130); }); - process.on('SIGTERM', () => { + process.once('SIGTERM', () => { cleanup(); process.exit(143); }); +} + +function selectScenarios(cfg: PerfConfig) { + if (!cfg.scenarios) return IOS_SUITE; + const unknown = cfg.scenarios.filter((id) => !IOS_SUITE.some((s) => s.id === id)); + if (unknown.length > 0) { + throw new Error( + `Unknown scenarios: ${unknown.join(', ')} (known: ${IOS_SUITE.map((s) => s.id).join(', ')})`, + ); + } + return IOS_SUITE.filter((s) => cfg.scenarios!.includes(s.id)); +} + +async function runIos( + cfg: PerfConfig, +): Promise<{ measurements: Measurement[]; coverage: PerfSummary['coverage'] }> { + const scenarios = selectScenarios(cfg); + // The daemon inherits this from the first CLI call; it routes trigger-app-event to the fixture. + process.env.AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE ??= FIXTURE_APP_EVENT_URL_TEMPLATE; + const rawDir = path.join(cfg.outDir, 'raw'); + const ctx: IosSuiteContext = { + stateDir: fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-perf-')), + udid: cfg.udid!, + appId: cfg.appId, + artifactsDir: path.join(rawDir, 'artifacts'), + rawDir, + }; + fs.mkdirSync(ctx.artifactsDir, { recursive: true }); + log(`state-dir: ${ctx.stateDir}`); + const finalize = iosRunFinalizer(ctx, cfg.keepArtifacts); + onInterrupt(finalize); + try { + const measurements = await runIosSuite(ctx, cfg, scenarios); + return { measurements, coverage: describeCoverage(scenarios, IOS_SUITE_EXCLUSIONS) }; + } finally { + finalize(); + } +} +function runAndroid(cfg: PerfConfig): { + measurements: Measurement[]; + coverage: PerfSummary['coverage']; +} { + const ctx = setupIsolation(cfg); + onInterrupt(() => teardownIsolation(ctx, cfg)); try { - ctx = setupIsolation(cfg); const measurements = runScenario(ctx, cfg); - const run: RunResult = { - startedAt, - finishedAt: new Date().toISOString(), - platform: cfg.platform, - device: { udid: ctx.profile.udid, serial: ctx.profile.serial, name: ctx.profile.deviceName }, - config: { rounds: cfg.rounds, warmup: cfg.warmup, keepArtifacts: cfg.keepArtifacts }, - agentDeviceVersion: readVersion(), + return { measurements, + coverage: { measured: [...new Set(measurements.map((m) => m.command))].sort(), excluded: [] }, }; - const { jsonPath, mdPath } = writeReports(run, cfg.outDir); - process.stderr.write(`\n[perf] report: ${mdPath}\n[perf] json: ${jsonPath}\n`); - } catch (error) { - process.stderr.write(`[perf] error: ${(error as Error).stack ?? String(error)}\n`); - exitCode = 1; } finally { - cleanup(); + teardownIsolation(ctx, cfg); } - process.exit(exitCode); } -main(); +async function main(): Promise { + const cfg = parseConfig(process.argv.slice(2)); + // The iOS forced-backend probe loads the built SDK; the dist CLI needs the build as well. + if ( + (cfg.platform === 'ios' || !usesSourceCli()) && + !fs.existsSync(path.join(REPO_ROOT, 'dist', 'src')) + ) { + throw new Error('dist/ is missing — run `pnpm build` first.'); + } + const startedAt = new Date().toISOString(); + // Read before the run, so a checkout change during a long run cannot relabel it. + const metadata = collectMetadata(cfg, cfg.serial ?? cfg.device ?? 'unknown'); + const { measurements, coverage } = cfg.platform === 'ios' ? await runIos(cfg) : runAndroid(cfg); + const summary: PerfSummary = { + schema: PERF_SUMMARY_SCHEMA, + platform: cfg.platform, + startedAt, + finishedAt: new Date().toISOString(), + config: { + rounds: cfg.rounds, + warmup: cfg.warmup, + scenarios: [...new Set(measurements.map((m) => m.scenario))], + }, + metadata, + coverage, + measurements, + }; + const { jsonPath, mdPath } = writeRunOutput(summary, cfg.outDir); + log(`report: ${mdPath}`); + log(`summary: ${jsonPath}`); +} + +main() + .then(() => process.exit(0)) + .catch((error: unknown) => { + process.stderr.write(`[perf] error: ${(error as Error).stack ?? String(error)}\n`); + process.exit(1); + }); diff --git a/scripts/perf/scenario.ts b/scripts/perf/scenario.ts index d99ddd2bba..1e34cad240 100644 --- a/scripts/perf/scenario.ts +++ b/scripts/perf/scenario.ts @@ -37,7 +37,7 @@ function bat( return { label, command, execMode: 'batch' as const, step, ...opts }; } -// One ordered pass over Settings. The harness repeats this N (+warmup) times; +// One ordered pass over Android Settings. The harness repeats this N (+warmup) times; // the leading `open --relaunch` resets the app to its root each round, so every // round starts from a known state while commands run in their natural order. export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): ScenarioStep[] { @@ -46,92 +46,33 @@ export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): Scenari const rec = path.join(ctx.artifactsDir, 'rec.mp4'); const trace = path.join(ctx.artifactsDir, 'trace.log'); - // Text entry differs per platform: iOS fills the root search field directly (focusing it - // first can hang); Android must open the search screen before an editable field exists. - const textEntry: ScenarioStep[] = p.selectors.searchEditableAtRoot - ? [ - // iOS: editable search field exists at root; fill it directly (freshRoot resets scroll). - bat( - 'fill search', - 'fill', - { - command: 'fill', - input: { - target: { kind: 'selector', selector: s.searchFieldEditable }, - text: 'general', - }, - }, - { freshRoot: true }, - ), - bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), - bat('get editable text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.searchFieldEditable }, - }, - }), - bat('keyboard return', 'keyboard', { command: 'keyboard', input: { action: 'return' } }), - ] - : [ - // Android: tap the search entry first to reveal the editable, then type/fill it. - bat( - 'press search field', - 'press', - { - command: 'press', - input: { target: { kind: 'selector', selector: s.searchField } }, - }, - { freshRoot: true }, - ), - bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), - bat('fill search', 'fill', { - command: 'fill', - input: { - target: { kind: 'selector', selector: s.searchFieldEditable }, - text: 'general', - }, - }), - bat('get editable text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.searchFieldEditable }, - }, - }), - ]; - - // These iOS-only repeated gesture forms fuse into `sequence` runner requests: - // press --count > 1 and swipe --count > 1 both batch their steps into one command. - const iosRunnerSeries: ScenarioStep[] = - p.platform === 'ios' - ? [ - bat( - 'press series (sequence)', - 'press', - { - command: 'press', - input: { target: { kind: 'point', x: 200, y: 95 }, count: 2, intervalMs: 50 }, - }, - { freshRoot: true }, - ), - bat( - 'swipe series (sequence)', - 'swipe', - { - command: 'swipe', - input: { - from: { x: 200, y: 650 }, - to: { x: 200, y: 450 }, - count: 2, - pauseMs: 50, - pattern: 'ping-pong', - }, - }, - { freshRoot: true }, - ), - ] - : []; + // Android reveals the editable search field only after a tap on the search card. + const textEntry: ScenarioStep[] = [ + bat( + 'press search field', + 'press', + { + command: 'press', + input: { target: { kind: 'selector', selector: s.searchField } }, + }, + { freshRoot: true }, + ), + bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), + bat('fill search', 'fill', { + command: 'fill', + input: { + target: { kind: 'selector', selector: s.searchFieldEditable }, + text: 'general', + }, + }), + bat('get editable text', 'get', { + command: 'get', + input: { + format: 'text', + target: { kind: 'selector', selector: s.searchFieldEditable }, + }, + }), + ]; return [ // --- reset to root via relaunch --- @@ -166,9 +107,6 @@ export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): Scenari ), bat('back', 'back', { command: 'back', input: {} }), - // --- iOS runner series commands surfaced by PR #643 --- - ...iosRunnerSeries, - // --- targeted reads against the visible anchor (freshRoot so the anchor is on screen) --- bat( 'wait text', diff --git a/scripts/perf/stats.test.ts b/scripts/perf/stats.test.ts new file mode 100644 index 0000000000..9b5d61ac46 --- /dev/null +++ b/scripts/perf/stats.test.ts @@ -0,0 +1,33 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { scaledMad, summarize } from './stats.ts'; + +test('summarize reports sample spread, the middle median, and nearest-rank p95', () => { + const stat = summarize([40, 10, 30, 20, Number.NaN])!; + assert.equal(stat.n, 4); + assert.equal(stat.mean, 25); + // Sample (n − 1) deviation: sqrt((15² + 5² + 5² + 15²) / 3). + assert.ok(Math.abs(stat.stddev - Math.sqrt(500 / 3)) < 1e-9); + assert.ok(Math.abs(stat.cv - Math.sqrt(500 / 3) / 25) < 1e-9); + assert.deepEqual([stat.min, stat.median, stat.p95, stat.max], [10, 25, 40, 40]); +}); + +test('summarize gives a single sample zero spread and no samples no summary', () => { + assert.deepEqual(summarize([7]), { + n: 1, + mean: 7, + stddev: 0, + cv: 0, + median: 7, + min: 7, + max: 7, + p95: 7, + }); + assert.equal(summarize([]), null); + assert.equal(summarize([0, 0])?.cv, 0); +}); + +test('scaledMad ignores one outlying night', () => { + // Deviations from the median 101 are 49, 1, 1, 3; their median is 2. + assert.ok(Math.abs(scaledMad([150, 100, 98, 102]) - 1.4826 * 2) < 1e-9); +}); diff --git a/scripts/perf/stats.ts b/scripts/perf/stats.ts index 14b846cac2..e4941f4608 100644 --- a/scripts/perf/stats.ts +++ b/scripts/perf/stats.ts @@ -1,22 +1,49 @@ import type { Stat } from './types.ts'; -// Nearest-rank percentile over a copy of the values. +// Nearest-rank percentile over sorted values. function percentile(sorted: number[], p: number): number { - if (sorted.length === 0) return Number.NaN; const rank = Math.ceil((p / 100) * sorted.length); const idx = Math.min(sorted.length - 1, Math.max(0, rank - 1)); - return sorted[idx]; + return sorted[idx]!; } +/** + * Describes one sample set. `stddev` is the sample standard deviation (n − 1), so it is 0 for a + * single value; `cv` is stddev / mean and is 0 when the mean is 0. + */ export function summarize(values: number[]): Stat | null { const clean = values.filter((v) => Number.isFinite(v)); if (clean.length === 0) return null; const sorted = [...clean].sort((a, b) => a - b); + const n = sorted.length; + const mean = sorted.reduce((sum, v) => sum + v, 0) / n; + const variance = n > 1 ? sorted.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (n - 1) : 0; + const stddev = Math.sqrt(variance); return { - n: sorted.length, - min: sorted[0], - median: percentile(sorted, 50), + n, + mean, + stddev, + cv: mean === 0 ? 0 : stddev / mean, + median: middle(sorted), + min: sorted[0]!, + max: sorted.at(-1)!, p95: percentile(sorted, 95), - max: sorted.at(-1), }; } + +// The median of sorted values; an even count averages the two middle values. +function middle(sorted: number[]): number { + const half = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[half]! : (sorted[half - 1]! + sorted[half]!) / 2; +} + +export function median(values: number[]): number { + const sorted = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b); + return sorted.length === 0 ? Number.NaN : middle(sorted); +} + +/** Median absolute deviation scaled to estimate a normal standard deviation. */ +export function scaledMad(values: number[]): number { + const center = median(values); + return 1.4826 * median(values.map((v) => Math.abs(v - center))); +} diff --git a/scripts/perf/summary.fixtures.ts b/scripts/perf/summary.fixtures.ts new file mode 100644 index 0000000000..5ad12534e6 --- /dev/null +++ b/scripts/perf/summary.fixtures.ts @@ -0,0 +1,65 @@ +import { buildMeasurement } from './measurement.ts'; +import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary, type Sample } from './types.ts'; + +export type StepFixture = { + id: string; + // Daemon round trips of the successful samples; wall adds a fixed CLI overhead. + daemonMs?: number[]; + wallMs?: number[]; + failures?: number; +}; + +function measurement(step: StepFixture): Measurement { + const timings = step.daemonMs ?? step.wallMs ?? []; + const samples: Sample[] = timings.map((ms, round) => ({ + round, + ok: true, + wallClockMs: step.daemonMs ? ms + 300 : ms, + daemonMs: step.daemonMs ? ms : undefined, + })); + for (let i = 0; i < (step.failures ?? 0); i++) { + samples.push({ round: samples.length, ok: false, wallClockMs: 0, errorCode: 'COMMAND_FAILED' }); + } + const [scenario] = step.id.split('/'); + return buildMeasurement({ + id: step.id, + scenario: scenario!, + label: step.id, + command: 'snapshot', + args: [], + platform: 'ios', + execMode: 'standalone', + samples, + warmupDropped: 1, + }); +} + +/** A schema-valid summary whose steps carry the given timings. */ +export function summaryFixture( + steps: StepFixture[], + overrides: { startedAt?: string; xcode?: string; runId?: string } = {}, +): PerfSummary { + return { + schema: PERF_SUMMARY_SCHEMA, + platform: 'ios', + startedAt: overrides.startedAt ?? '2026-10-20T03:00:00.000Z', + finishedAt: '2026-10-20T03:30:00.000Z', + config: { + rounds: 5, + warmup: 1, + scenarios: [...new Set(steps.map((s) => s.id.split('/')[0]!))], + }, + metadata: { + commit: '0123456789abcdef', + dirty: false, + runId: overrides.runId, + agentDeviceVersion: '0.0.0', + host: { os: 'Darwin', arch: 'arm64', cpu: 'Apple M1 (Virtual)', cores: 3, memoryGb: 7 }, + xcode: overrides.xcode ?? 'Xcode 26.2', + node: 'v24.0.0', + device: { name: 'iPhone 17 Pro', runtime: 'iOS-26-2' }, + }, + coverage: { measured: ['snapshot'], excluded: [] }, + measurements: steps.map(measurement), + }; +} diff --git a/scripts/perf/summary.test.ts b/scripts/perf/summary.test.ts new file mode 100644 index 0000000000..70d199ef9e --- /dev/null +++ b/scripts/perf/summary.test.ts @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { parsePerfSummary } from './summary.ts'; +import { summaryFixture } from './summary.fixtures.ts'; + +const valid = () => + JSON.parse(JSON.stringify(summaryFixture([{ id: 's/step', daemonMs: [10, 20] }]))) as Record< + string, + unknown + >; + +test('accepts a summary that carries every field the comparison reads', () => { + const summary = parsePerfSummary(valid(), 'summary.json'); + assert.equal(summary.measurements[0]?.daemon?.median, 15); +}); + +test('rejects another schema version and names the field', () => { + assert.throws( + () => parsePerfSummary({ ...valid(), schema: 'agent-device-perf-summary/v0' }, 'a.json'), + /a\.json: schema must be "agent-device-perf-summary\/v1"/, + ); +}); + +test('rejects a measurement whose timing summary lost a statistic', () => { + const summary = valid(); + const [measurement] = summary.measurements as { daemon: Record }[]; + delete measurement!.daemon.cv; + assert.throws( + () => parsePerfSummary(summary, 'b.json'), + /b\.json: measurements\[0\]\.daemon\.cv must be a finite number/, + ); +}); diff --git a/scripts/perf/summary.ts b/scripts/perf/summary.ts new file mode 100644 index 0000000000..2feff211aa --- /dev/null +++ b/scripts/perf/summary.ts @@ -0,0 +1,63 @@ +import { PERF_SUMMARY_SCHEMA, type PerfSummary } from './types.ts'; + +type Json = Record; + +const STAT_FIELDS = ['n', 'mean', 'stddev', 'cv', 'median', 'min', 'max', 'p95'] as const; + +function fail(source: string, at: string, expected: string): never { + throw new Error(`${source}: ${at} must be ${expected}`); +} + +function object(value: unknown, source: string, at: string): Json { + if (value === null || typeof value !== 'object' || Array.isArray(value)) + fail(source, at, 'an object'); + return value as Json; +} + +function string(value: unknown, source: string, at: string): string { + if (typeof value !== 'string') fail(source, at, 'a string'); + return value; +} + +function number(value: unknown, source: string, at: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) fail(source, at, 'a finite number'); + return value; +} + +function stat(value: unknown, source: string, at: string): void { + if (value === null) return; + const s = object(value, source, at); + for (const field of STAT_FIELDS) number(s[field], source, `${at}.${field}`); +} + +/** + * Validates a `summary.json` read from disk or a downloaded artifact. Only the fields that the + * comparison and report read are checked; a run from another schema version is rejected. + */ +export function parsePerfSummary(value: unknown, source: string): PerfSummary { + const root = object(value, source, 'summary'); + if (root.schema !== PERF_SUMMARY_SCHEMA) fail(source, 'schema', `"${PERF_SUMMARY_SCHEMA}"`); + if (root.platform !== 'ios' && root.platform !== 'android') + fail(source, 'platform', 'ios or android'); + string(root.startedAt, source, 'startedAt'); + const config = object(root.config, source, 'config'); + number(config.rounds, source, 'config.rounds'); + const metadata = object(root.metadata, source, 'metadata'); + string(metadata.commit, source, 'metadata.commit'); + string(object(metadata.host, source, 'metadata.host').cpu, source, 'metadata.host.cpu'); + string(object(metadata.device, source, 'metadata.device').name, source, 'metadata.device.name'); + if (!Array.isArray(config.scenarios)) fail(source, 'config.scenarios', 'an array'); + if (!Array.isArray(root.measurements)) fail(source, 'measurements', 'an array'); + root.measurements.forEach((entry, index) => { + const at = `measurements[${index}]`; + const m = object(entry, source, at); + string(m.id, source, `${at}.id`); + string(m.scenario, source, `${at}.scenario`); + string(m.command, source, `${at}.command`); + number(m.failures, source, `${at}.failures`); + if (!Array.isArray(m.samples)) fail(source, `${at}.samples`, 'an array'); + stat(m.wallClock, source, `${at}.wallClock`); + stat(m.daemon, source, `${at}.daemon`); + }); + return value as PerfSummary; +} diff --git a/scripts/perf/types.ts b/scripts/perf/types.ts index 175481a889..cfb14ffd2f 100644 --- a/scripts/perf/types.ts +++ b/scripts/perf/types.ts @@ -1,8 +1,8 @@ -// Shared data shapes for the e2e perf benchmark harness. +// Shared data shapes for the perf suite and its night-over-night comparison. export type Platform = 'ios' | 'android'; -export type ExecMode = 'batch' | 'standalone'; +export type ExecMode = 'batch' | 'standalone' | 'sdk'; export type CliResult = { exitCode: number; @@ -13,38 +13,116 @@ export type CliResult = { ok: boolean; // exit 0 AND (json.ok !== false) }; +export type Stat = { + n: number; + mean: number; + stddev: number; + cv: number; + median: number; + min: number; + max: number; + p95: number; +}; + +export type Timing = { count: number; totalMs: number }; + +/** Phase breakdown of one request, read from its daemon `--debug` request log. */ +export type SampleProfile = { + phases: Record; + runnerCommands: Record; + spawns: Record; + snapshotSources: Record; + retries: number; + stabilization: Timing; + fallbacks: string[]; + // Daemon-side request time: from `request_start` to the last event of the request log. + requestSpanMs?: number; +}; + +/** The settle outcome that a `--settle` response reports. */ +export type SettleFacts = { waitedMs: number; captures?: number; settled?: boolean }; + +export type SnapshotFacts = { + backend?: string; + state?: string; + reasonCode?: string; + nodeCount?: number; + maxDepth?: number; + truncated?: boolean; +}; + export type Sample = { round: number; wallClockMs: number; - daemonDurationMs?: number; // from batch results[0].durationMs (batch mode only) - elementCount?: number; // for snapshot rows: parsed @eN count, a tree-size proxy + // Daemon request round trip: the client's `daemon_request` timer for CLI calls, the batch step + // duration for batch calls, or the request-log span for SDK calls. + daemonMs?: number; + elementCount?: number; + requestId?: string; + snapshot?: SnapshotFacts; + settle?: SettleFacts; + profile?: SampleProfile; ok: boolean; errorCode?: string; errorMessage?: string; }; -export type Stat = { n: number; min: number; median: number; p95: number; max: number }; +export type MeasurementProfile = { + runnerMs: number | null; + spawnCount: number | null; + spawnMs: number | null; + snapshotSourceMs: number | null; + stabilizationMs: number | null; + settleWaitMs: number | null; + retries: number; + producers: Record; + backends: Record; + fallbacks: Record; + topPhases: { phase: string; medianMs: number }[]; +}; export type Measurement = { - command: string; + id: string; // stable `/` comparison key + scenario: string; label: string; + command: string; // catalog command name + args: string[]; platform: Platform; execMode: ExecMode; - samples: Sample[]; // kept samples only (warmup rounds dropped) + samples: Sample[]; // measured rounds only (warmup rounds dropped) warmupDropped: number; wallClock: Stat | null; - daemonDuration: Stat | null; // null for standalone or when no ok samples - elementCount: Stat | null; // null unless snapshot row + daemon: Stat | null; + elementCount: Stat | null; + profile: MeasurementProfile | null; failures: number; notes: string[]; }; -export type RunResult = { +export type RunMetadata = { + commit: string; + ref?: string; + // Uncommitted changes in the checkout at the start of the run. + dirty: boolean; + runId?: string; + runAttempt?: string; + agentDeviceVersion: string; + host: { os: string; arch: string; cpu: string; cores: number; memoryGb: number }; + xcode?: string; + node: string; + device: { name: string; udid?: string; serial?: string; runtime?: string }; + fixture?: { appId: string; appPath?: string }; +}; + +export const PERF_SUMMARY_SCHEMA = 'agent-device-perf-summary/v1'; + +export type PerfSummary = { + schema: typeof PERF_SUMMARY_SCHEMA; + platform: Platform; startedAt: string; finishedAt: string; - platform: Platform; - device: { udid?: string; serial?: string; name: string }; - config: { rounds: number; warmup: number; keepArtifacts: boolean }; - agentDeviceVersion: string; + config: { rounds: number; warmup: number; scenarios: string[] }; + metadata: RunMetadata; + coverage: { measured: string[]; excluded: { command: string; reason: string }[] }; measurements: Measurement[]; }; diff --git a/test/repo-guards/npm-package-scripts.test.ts b/test/repo-guards/npm-package-scripts.test.ts index 7f35bea71e..9b12030361 100644 --- a/test/repo-guards/npm-package-scripts.test.ts +++ b/test/repo-guards/npm-package-scripts.test.ts @@ -36,7 +36,7 @@ test('perf uses one platform-selectable entry point', () => { assert.equal(script('perf'), 'node --experimental-strip-types scripts/perf/run.ts'); assert.equal(packageJson.scripts['perf:ios'], undefined); assert.equal(packageJson.scripts['perf:android'], undefined); - assert.doesNotMatch(perfNightlyWorkflow, /pnpm perf:/); + assert.doesNotMatch(perfNightlyWorkflow, /pnpm perf:(ios|android)\b/); assert.match(perfNightlyWorkflow, /pnpm perf --platform ios/); assert.match(perfNightlyWorkflow, /pnpm perf --platform android/); }); diff --git a/vitest.config.ts b/vitest.config.ts index b5a54e80f7..5da1a3c156 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -196,6 +196,8 @@ export default defineConfig({ 'scripts/__tests__/release-version.test.ts', 'scripts/__tests__/release-plan.test.ts', 'scripts/ios-snapshot-benchmark/*.test.ts', + // The perf suite's pure logic, without a device. + 'scripts/perf/*.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts', 'test/ci/upload-artifact-hidden-paths.test.ts',