feat(perf): time every iOS command against the fixture app

.github/workflows/perf-nightly.yml

@@ -1,35 +1,65 @@
 name: Perf Nightly
 
-# Parked to workflow_dispatch by #1781 A3: it only writes a report and compares nothing, so it
-# structurally cannot catch a regression, and iOS wall-clock medians swing up to +122%
-# night-to-night at n=5 with no consumer reading the report. `pnpm perf` / scripts/perf are
-# unaffected — still runnable on demand. Un-park once a comparison step with multi-night
-# smoothing (someone reading it) exists, or a perf arc needs nightly data.
+# Nightly iOS command performance against the fixture app, compared night over night (#1781 A3).
+# - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts
+#   failing. PR lanes check behavior, not time.
+# - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians
+#   swing up to +122% between single nights, so a step is judged against the median of up to seven
+#   earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. A night
+#   runs 3 rounds, the minimum for a timing verdict, so one failed sample withholds that step's
+#   timing verdict.
+# - Cost: one macos-26 job of about 45 minutes a night; summaries are kept 90 days, raw logs 30.
+# - Delete or park when: no flagged change has been acted on for 60 nights.
+# Android stays report-only and runs only on request.
 
 on:
+  schedule:
+    - cron: '40 2 * * *'
   workflow_dispatch:
     inputs:
       rounds:
-        description: 'Measured rounds per command (samples)'
+        description: 'Measured rounds per step (samples, at least 3)'
         required: false
-        default: '5'
+        default: '3'
+      warmup:
+        description: 'Warmup rounds dropped before measuring'
+        required: false
+        default: '1'
+      scenarios:
+        description: 'Comma-separated iOS scenario ids (empty runs all)'
+        required: false
+        default: ''
+      baseline-nights:
+        description: 'Earlier scheduled nights in the rolling baseline'
+        required: false
+        default: '7'
+      baseline-branch:
+        description: 'Branch whose runs form the baseline'
+        required: false
+        default: 'main'
+      android:
+        description: 'Also run the report-only Android Settings tour'
+        type: boolean
+        required: false
+        default: true
 
 permissions:
   contents: read
+  actions: read
 
 concurrency:
-  group: ci-${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  group: perf-nightly-${{ github.ref }}
+  cancel-in-progress: false
 
 env:
-  AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts'
-  PERF_ROUNDS: ${{ github.event.inputs.rounds || '5' }}
+  PERF_ROUNDS: ${{ github.event.inputs.rounds || '3' }}
+  PERF_WARMUP: ${{ github.event.inputs.warmup || '1' }}
 
 jobs:
   perf-ios:
     name: iOS Command Perf
     runs-on: macos-26
-    timeout-minutes: 80
+    timeout-minutes: 100
     env:
       IOS_RUNTIME_VERSION: '26.2'
       AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'
@@ -42,6 +72,22 @@ jobs:
 
       - name: Setup toolchain
         uses: ./.github/actions/setup-node-pnpm
+        with:
+          cache-dependency-path: |
+            pnpm-lock.yaml
+            examples/test-app/pnpm-lock.yaml
+
+      - name: Build the shared CLI
+        uses: ./.github/actions/run-gate
+        with:
+          gate: build
+
+      - name: Boot iOS test simulator
+        id: ios-simulator
+        uses: ./.github/actions/boot-ios-test-simulator
+        with:
+          runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
+          preferred-device-name: iPhone 17 Pro
 
       - name: Restore and build iOS XCTest runner
         uses: ./.github/actions/setup-apple-runner-build
@@ -53,37 +99,76 @@ jobs:
           xcuitest-platform: ios
           xcuitest-destination: generic/platform=iOS Simulator
 
-      - name: Boot iOS test simulator
-        uses: ./.github/actions/boot-ios-test-simulator
-        with:
-          runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
-          preferred-device-name: iPhone 17 Pro
+      - name: Install the fixture app
+        id: fixture-app
+        uses: ./.github/actions/setup-fixture-app
 
       - name: Preflight iOS runner through public CLI
+        env:
+          UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}
         run: |
+          # The keyboard must stay expanded for keyboard dismiss to find its Done key.
+          xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false
           pnpm clean:daemon
-          node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS"
+          node bin/agent-device.mjs prepare ios-runner --platform ios --udid "$UDID" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS"
           pnpm clean:daemon
 
-      - name: Run iOS command perf benchmark
+      - name: Run iOS command perf suite
+        env:
+          UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}
+          APP_ID: ${{ steps.fixture-app.outputs.app-id }}
+          APP_PATH: ${{ steps.fixture-app.outputs.app-path }}
+          SCENARIOS: ${{ github.event.inputs.scenarios }}
         run: |
-          pnpm perf --platform ios \
-            --device "iPhone 17 Pro" \
-            --n "$PERF_ROUNDS" --warmup 1 \
+          pnpm perf --platform ios --udid "$UDID" \
+            --app-id "$APP_ID" --app-path "$APP_PATH" \
+            --rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP" \
+            ${SCENARIOS:+--scenarios "$SCENARIOS"} \
             --out-dir "$GITHUB_WORKSPACE/perf-results"
 
-      - name: Upload iOS perf report
-        if: always()
+      - name: Compare with earlier nights
+        if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }}
+          BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }}
+        run: |
+          pnpm perf:compare --to perf-results \
+            --branch "$BASELINE_BRANCH" \
+            --nights "$BASELINE_NIGHTS" \
+            --exclude-run "$GITHUB_RUN_ID" \
+            --out perf-results --github
+          echo "Full report: the perf-ios-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload iOS perf summary
+        if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: perf-ios
-          path: perf-results/
+          name: perf-ios-summary
+          path: |
+            perf-results/summary.json
+            perf-results/report.md
+            perf-results/comparison.json
+            perf-results/comparison.md
+          retention-days: 90
+          if-no-files-found: error
+
+      - name: Upload iOS perf raw logs
+        if: ${{ always() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: perf-ios-raw
+          path: perf-results/raw/
+          retention-days: 30
           if-no-files-found: warn
 
   perf-android:
     name: Android Command Perf
+    if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.android == 'true' }}
     runs-on: ubuntu-latest
     timeout-minutes: 80
+    env:
+      AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts'
     steps:
       - name: Checkout
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -110,7 +195,7 @@ jobs:
             adb -s emulator-5554 shell settings put global window_animation_scale 0 || true;
             adb -s emulator-5554 shell settings put global transition_animation_scale 0 || true;
             adb -s emulator-5554 shell settings put global animator_duration_scale 0 || true;
-            pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup 1 --out-dir "$GITHUB_WORKSPACE/perf-results"
+            pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup "$PERF_WARMUP" --out-dir "$GITHUB_WORKSPACE/perf-results"
 
       - name: Upload Android perf report
         if: always()

package.json

@@ -131,6 +131,7 @@
     "test:ios-snapshot-differential": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift test --package-path apple/snapshot-presentation && node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/ios-snapshot-differential.test.ts",
     "size": "node scripts/size-report.mjs",
     "perf": "node --experimental-strip-types scripts/perf/run.ts",
+    "perf:compare": "node --experimental-strip-types scripts/perf/compare-cli.ts",
     "bench:ios-snapshot": "node --experimental-strip-types scripts/ios-snapshot-benchmark/run.ts",
     "bench:ios-snapshot:deep-button": "node --experimental-strip-types scripts/ios-snapshot-benchmark/deep-button.ts",
     "bench:ios-snapshot:evidence": "node --experimental-strip-types scripts/ios-snapshot-benchmark/evidence.ts",

packages/host-kit/src/command.ts

@@ -26,3 +26,4 @@ export {
   withCommandHost,
   withoutCommandExecutorOverride,
 } from './internal/exec.ts';
+export { EXEC_COMMAND_PHASE } from './internal/exec-trace.ts';

packages/host-kit/src/diagnostics.ts

@@ -1,6 +1,7 @@
 export {
   countDiagnosticEventsByPhase,
   createRequestId,
+  DIAGNOSTIC_STDERR_PREFIX,
   type DiagnosticEventInput,
   emitDiagnostic,
   flushDiagnosticsToSessionFile,

packages/host-kit/src/internal/diagnostics.ts

@@ -53,6 +53,9 @@ type DiagnosticsScope = DiagnosticsScopeOptions & {
   sortedSensitiveValues?: string[];
 };
 
+/** Prefixes each diagnostic event that a `--debug` client without a log file writes to stderr. */
+export const DIAGNOSTIC_STDERR_PREFIX = '[agent-device][diag] ';
+
 const diagnosticsStorage = new AsyncLocalStorage<DiagnosticsScope>();
 
 export function createRequestId(): string {
@@ -164,7 +167,7 @@ export function emitDiagnostic(event: DiagnosticEventInput): void {
       appendDiagnosticLine(scope, scope.traceLogPath, fileLine);
     }
     if (scope.debug && !scope.logPath && !scope.traceLogPath) {
-      process.stderr.write(`[agent-device][diag] ${fileLine}`);
+      process.stderr.write(`${DIAGNOSTIC_STDERR_PREFIX}${fileLine}`);
     }
   } catch {
     // Best-effort diagnostics should not break request flow.

packages/host-kit/src/internal/exec-trace.ts

@@ -9,6 +9,9 @@ import {
   updateDiagnosticsScope,
 } from './diagnostics.ts';
 
+/** The diagnostic phase of each host process lifecycle event. */
+export const EXEC_COMMAND_PHASE = 'exec_command';
+
 export type ExecSettlementReason =
   | 'success'
   | 'exit_failure'
@@ -52,7 +55,7 @@ export function createExecTraceContext(
       pid = data.pid ?? pid;
       emitDiagnostic({
         level: 'debug',
-        phase: 'exec_command',
+        phase: EXEC_COMMAND_PHASE,
         durationMs: performance.now() - started,
         data: { execId, command, event, pid, timeoutMs, ...data },
       });

packages/host-kit/src/internal/retry.ts

@@ -9,6 +9,9 @@ type RetryPolicy = {
   shouldRetry?: (error: unknown, attempt: number) => boolean;
 };
 
+/** The diagnostic phase of each retry attempt and outcome. */
+export const RETRY_PHASE = 'retry';
+
 /**
  * The read side of a deadline. Holders are typed against this rather than the class so a caller
  * can hand one a substitute clock; {@link Deadline} is the production implementation.
@@ -281,7 +284,7 @@ export function sleep(ms: number, signal?: AbortSignal): Promise<void> {
 function publishRetryEvent(event: RetryTelemetryEvent): void {
   emitDiagnostic({
     level: event.event === 'attempt_failed' || event.event === 'exhausted' ? 'warn' : 'debug',
-    phase: 'retry',
+    phase: RETRY_PHASE,
     data: {
       ...event,
     },

packages/host-kit/src/retry.ts

@@ -5,4 +5,5 @@ export {
   pollUntilSettled,
   retryWithPolicy,
   sleep,
+  RETRY_PHASE,
 } from './internal/retry.ts';

scripts/ios-snapshot-benchmark/benchmark-config.ts

@@ -1,6 +1,6 @@
 import path from 'node:path';
 import { parseLocalStates, parseRtt, parseSampleCount, parseScreenIds } from './definitions.ts';
-import { resolveRepoRoot } from './host.ts';
+import { resolveRepoRoot } from '../lib/bench-host.ts';
 import {
   assertBenchmarkOwner,
   assertOwnedDerivedPath,

scripts/ios-snapshot-benchmark/run.test.ts

@@ -4,7 +4,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { test } from 'vitest';
 import { mkdtempForTest } from '@agent-device/test-kit';
-import { resolveRepoRoot } from './host.ts';
+import { resolveRepoRoot } from '../lib/bench-host.ts';
 
 test('CLI refuses an unmarked existing root before any descendant can be cleared', async () => {
   const repoRoot = resolveRepoRoot();

scripts/ios-snapshot-benchmark/run.ts

@@ -10,7 +10,7 @@ import { classifyFailure, formatCliFailure, openFixture, type CliContext } from
 import { BenchmarkControlError, runDeepButtonControls } from './deep-control.ts';
 import { CONTRACT, screenFixture } from './definitions.ts';
 import { deepButtonFixtureEvidence } from './deep-button.ts';
-import { readGitRevision, readHostIdentity, readTarget, readToolchain } from './host.ts';
+import { readGitRevision, readHostIdentity, readTarget, readToolchain } from '../lib/bench-host.ts';
 import {
   BenchmarkCellAdmissionError,
   BenchmarkContentionError,

scripts/ios-snapshot-benchmark/types.ts

@@ -1,3 +1,5 @@
+import type { GitRevision, HostIdentity, Target, Toolchain } from '../lib/bench-host.ts';
+
 export const BENCHMARK_SCHEMA_VERSION = 'ios-snapshot-convergence.v1' as const;
 export const ISSUE_ID = '#2189';
 export const PARENT_ISSUE_ID = '#2188';
@@ -95,38 +97,6 @@ export type Measurement = {
   failureCategories: Partial<Record<FailureCategory, number>>;
 };
 
-export type Toolchain = {
-  node: string;
-  pnpm: string;
-  xcode: string;
-  simctl: string;
-  os: string;
-  arch: string;
-};
-
-export type HostIdentity = {
-  model: string;
-  modelIdentifier: string;
-  cpu: string;
-  cpuCores: number;
-};
-
-export type GitRevision = {
-  commit: string;
-  branch: string;
-  dirty: boolean;
-};
-
-export type Target = {
-  platform: 'ios';
-  kind: 'simulator';
-  udid: string;
-  name: string;
-  runtime: string;
-  appId: string;
-  appPath?: string;
-};
-
 export type PackageSize = {
   status: 'measured' | 'not-run';
   revision: string;

scripts/ios-snapshot-benchmark/host.test.ts → scripts/lib/bench-host.test.ts

@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict';
 import { test } from 'vitest';
-import { parseHardwareProfile } from './host.ts';
+import { parseHardwareProfile } from './bench-host.ts';
 
 test('parses the Mac model and CPU identity from system_profiler output', () => {
   assert.deepEqual(