From d17b8ead563120a8461fa966bf1a0bbc773c9c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 21 Oct 2026 21:33:03 +0200 Subject: [PATCH 01/15] feat(test-app): add a Bluesky-depth heavy feed screen A JavaScript-only route that reproduces a heavy React Native home feed: 30 nested containers above a virtualized list of composite posts with image, link, quote, and quote-with-media embeds. Its native accessibility tree is deeper than 65 levels, so XCTest tree capture fails with kAXErrorIllegalArgument and recovers through private AX, as on Bluesky 1.136. Settings links to it with open-heavy-feed. Co-Authored-By: Claude Opus 5.5 --- examples/test-app/README.md | 18 + examples/test-app/app/(tabs)/settings.tsx | 1 + examples/test-app/app/_layout.tsx | 1 + examples/test-app/app/heavy-feed.tsx | 5 + .../test-app/src/screens/HeavyFeedScreen.tsx | 362 ++++++++++++++++++ .../test-app/src/screens/SettingsScreen.tsx | 6 + 6 files changed, 393 insertions(+) create mode 100644 examples/test-app/app/heavy-feed.tsx create mode 100644 examples/test-app/src/screens/HeavyFeedScreen.tsx diff --git a/examples/test-app/README.md b/examples/test-app/README.md index 50604c65f1..854a7fda64 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -19,6 +19,7 @@ It is intentionally small, but each surface is dense with durable accessibility - `Automation lab`: long-press, alert-result, app-event, app-state, appearance, orientation, permission-recovery, log canaries, a flattened (`accessible={true}`) text input, and an Apple Pay sheet hosted in `com.apple.PassbookUIService` - `WebView accessibility`: a deterministic semantic fixture plus live websites with varied HTML for native accessibility snapshot verification - `Perf lab`: seeded JavaScript render problems for `react-devtools profile`, one per scenario (see [Perf lab](#perf-lab)) +- `Heavy feed`: a Bluesky-shaped home feed whose accessibility tree is deeper than 65 levels (see [Heavy feed](#heavy-feed)) Navigation uses Expo Router native bottom tabs, so the tab bar itself is also part of the test surface. @@ -261,6 +262,23 @@ agent-device react-devtools profile stop `src/screens/PerfLabScreen.tsx` imports only React and React Native, so it can be copied into a bare React Native app to check the profiler on other React Native versions. +## Heavy feed + +**Settings → Open heavy feed** (`open-heavy-feed`) opens a JavaScript-only reproduction of a heavy +React Native home feed, shaped after Bluesky 1.136. Thirty nested containers stand in for its +navigation and provider layers. Below them a virtualized list renders posts with avatar, text, +image, link, quote, and quote-with-media embeds. Every level opts out of view flattening, so the +native accessibility tree is deeper than 65 levels and holds about 360 raw nodes. + +On this screen the XCTest tree capture fails with `kAXErrorIllegalArgument` and recovers through +private AX, while the simulator accessibility bridge still answers. Use it to reproduce deep-tree +capture, recovery, and gesture-routing behavior without a third-party app. `wrappers` (default 30) +and `posts` (default 60) route parameters change the shape. + +The stable ids are `heavy-feed-title`, `heavy-feed-liked-count`, `heavy-feed-last-action`, and per +post `heavy-post-`, `heavy-post--like`, `-reply`, `-repost`, and `-more`. A like press updates +`Liked posts: `, and a long press on a post sets `Last action: held post `. + ## Local Agent Device suites The repo includes two local suites for iterating on the fixture app: diff --git a/examples/test-app/app/(tabs)/settings.tsx b/examples/test-app/app/(tabs)/settings.tsx index 7f7ca57180..e8e39b4796 100644 --- a/examples/test-app/app/(tabs)/settings.tsx +++ b/examples/test-app/app/(tabs)/settings.tsx @@ -19,6 +19,7 @@ export default function SettingsRoute() { onLoadDiagnostics={state.loadDiagnostics} onOpenAccessorySetup={() => router.push('/accessory-setup')} onOpenAutomationLab={() => router.push('/automation')} + onOpenHeavyFeed={() => router.push('/heavy-feed')} onOpenInertSurface={() => router.push('/inert')} onOpenSnapshotDepth={() => router.push('/snapshot-depth')} onOpenWebViewLab={() => router.push('/webview')} diff --git a/examples/test-app/app/_layout.tsx b/examples/test-app/app/_layout.tsx index e64205f5f4..e452a411d2 100644 --- a/examples/test-app/app/_layout.tsx +++ b/examples/test-app/app/_layout.tsx @@ -28,6 +28,7 @@ function RootLayoutContent() { + diff --git a/examples/test-app/app/heavy-feed.tsx b/examples/test-app/app/heavy-feed.tsx new file mode 100644 index 0000000000..12dbd4d8fa --- /dev/null +++ b/examples/test-app/app/heavy-feed.tsx @@ -0,0 +1,5 @@ +import { HeavyFeedScreen } from '../src/screens/HeavyFeedScreen'; + +export default function HeavyFeedRoute() { + return ; +} diff --git a/examples/test-app/src/screens/HeavyFeedScreen.tsx b/examples/test-app/src/screens/HeavyFeedScreen.tsx new file mode 100644 index 0000000000..0ac110a54a --- /dev/null +++ b/examples/test-app/src/screens/HeavyFeedScreen.tsx @@ -0,0 +1,362 @@ +import { useLocalSearchParams } from 'expo-router'; +import { useCallback, useMemo, useState, type ReactNode } from 'react'; +import { FlatList, Pressable, StyleSheet, Text, View } from 'react-native'; + +import { useAppColors, type AppColors } from '../theme'; + +/** + * A JavaScript-only reproduction of a heavy React Native home feed, shaped after Bluesky 1.136. + * `wrappers` nested containers stand in for its navigation, provider, and screen layers. Below them + * a virtualized list renders composite posts five to ten levels deep, and quote-with-media embeds + * add more. With the defaults the deepest post node sits more than 65 levels below the application, + * so the XCTest tree capture degrades as it does on Bluesky. Every level opts out of view + * flattening, so the native hierarchy is as deep as the React tree. + */ +export function HeavyFeedScreen() { + const colors = useAppColors(); + const styles = useMemo(() => createStyles(colors), [colors]); + const params = useLocalSearchParams<{ wrappers?: string; posts?: string }>(); + const wrappers = clampInteger(params.wrappers, 30, 0, 80); + const postCount = clampInteger(params.posts, 60, 1, 300); + const posts = useMemo(() => buildPosts(postCount), [postCount]); + const [liked, setLiked] = useState>(new Set()); + const [lastAction, setLastAction] = useState('none'); + const [tab, setTab] = useState<'following' | 'discover'>('following'); + + const toggleLike = useCallback((id: number) => { + setLiked((current) => { + const next = new Set(current); + if (next.has(id)) next.delete(id); + else next.add(id); + setLastAction(`${next.has(id) ? 'liked' : 'unliked'} post ${id}`); + return next; + }); + }, []); + const holdPost = useCallback((id: number) => setLastAction(`held post ${id}`), []); + + const feed = ( + + + + Heavy feed + + + {(['following', 'discover'] as const).map((name) => ( + { + setTab(name); + setLastAction(`tab ${name}`); + }} + style={[styles.tab, tab === name ? styles.tabSelected : null]} + testID={`heavy-feed-tab-${name}`} + > + {name === 'following' ? 'Following' : 'Discover'} + + ))} + + + Liked posts: {liked.size} + + + Last action: {lastAction} + + + String(post.id)} + renderItem={({ item }) => ( + + )} + testID="heavy-feed-list" + windowSize={5} + /> + + ); + + return ( + + {feed} + + ); +} + +type Embed = 'none' | 'image' | 'link' | 'quote' | 'quote-media'; + +type Post = { + id: number; + author: string; + handle: string; + age: string; + text: string; + embed: Embed; + replies: number; + reposts: number; + likes: number; +}; + +const AUTHORS = ['Ada Byron', 'Grace Hopper', 'Alan Turing', 'Edsger Dijkstra', 'Barbara Liskov']; +const EMBEDS: readonly Embed[] = ['quote-media', 'none', 'image', 'quote', 'link']; + +function buildPosts(count: number): Post[] { + return Array.from({ length: count }, (_, id) => { + const author = AUTHORS[id % AUTHORS.length]!; + return { + id, + author, + handle: `@${author.toLowerCase().replace(' ', '.')}.example`, + age: `${(id % 23) + 1}h`, + text: `Post ${id}: a deterministic feed entry with enough copy to wrap onto a second line.`, + embed: EMBEDS[id % EMBEDS.length]!, + replies: (id * 7) % 31, + reposts: (id * 5) % 17, + likes: (id * 11) % 97, + }; + }); +} + +function ShellLayers({ + count, + styles, + children, +}: { + count: number; + styles: Styles; + children: ReactNode; +}) { + let tree = children; + for (let level = count - 1; level >= 0; level -= 1) { + tree = ( + + {tree} + + ); + } + return tree; +} + +function PostCard({ + post, + liked, + onLike, + onHold, + styles, +}: { + post: Post; + liked: boolean; + onLike: (id: number) => void; + onHold: (id: number) => void; + styles: Styles; +}) { + const id = `heavy-post-${post.id}`; + return ( + + onHold(post.id)}> + + + + + {post.author} + + {post.handle} · {post.age} + + + + {post.text} + + + + + + onLike(post.id)} + selected={liked} + styles={styles} + testID={`${id}-like`} + /> + + + + + + + ); +} + +function ActionButton({ + label, + count, + onPress, + selected = false, + styles, + testID, +}: { + label: string; + count?: number; + onPress?: () => void; + selected?: boolean; + styles: Styles; + testID: string; +}) { + return ( + + + + {count === undefined ? '…' : count} + + + + ); +} + +function PostEmbed({ embed, postId, styles }: { embed: Embed; postId: number; styles: Styles }) { + if (embed === 'none') return null; + if (embed === 'image') return ; + if (embed === 'link') { + return ( + + + + example.com + A link card with a title and a description. + + + ); + } + return ( + + + + + + + Quoted author + + + + A quoted post embedded inside post {postId}. + + + {embed === 'quote-media' ? ( + + ) : null} + + + + + ); +} + +function MediaGrid({ postId, tiles, styles }: { postId: number; tiles: number; styles: Styles }) { + return ( + + + + {Array.from({ length: tiles }, (_, index) => ( + + + + + + + + ))} + + + + ); +} + +function clampInteger(value: string | undefined, fallback: number, min: number, max: number) { + const parsed = Number.parseInt(value ?? '', 10); + if (!Number.isFinite(parsed)) return fallback; + return Math.min(max, Math.max(min, parsed)); +} + +type Styles = ReturnType; + +function createStyles(colors: AppColors) { + return StyleSheet.create({ + shell: { flex: 1 }, + frame: { flex: 1, backgroundColor: colors.surface }, + header: { + borderBottomColor: colors.line, + borderBottomWidth: 1, + gap: 4, + paddingHorizontal: 16, + paddingVertical: 8, + }, + title: { color: colors.text, fontSize: 20, fontWeight: '700' }, + tabs: { flexDirection: 'row', gap: 8 }, + tab: { borderRadius: 14, paddingHorizontal: 12, paddingVertical: 6 }, + tabSelected: { backgroundColor: colors.accentSoft }, + tabLabel: { color: colors.text, fontWeight: '600' }, + status: { color: colors.textSoft, fontSize: 12 }, + post: { borderBottomColor: colors.line, borderBottomWidth: 1, padding: 12 }, + postRow: { flexDirection: 'row', gap: 10 }, + avatar: { backgroundColor: colors.accentSoft, borderRadius: 20, height: 40, width: 40 }, + postContent: { flex: 1, gap: 6 }, + postMeta: { alignItems: 'center', flexDirection: 'row', flexWrap: 'wrap', gap: 6 }, + author: { color: colors.text, fontWeight: '600' }, + soft: { color: colors.textSoft, fontSize: 12 }, + body: { color: colors.text }, + embed: { + borderColor: colors.lineStrong, + borderRadius: 10, + borderWidth: 1, + gap: 6, + padding: 8, + }, + quoteBody: { gap: 6 }, + quoteAvatar: { backgroundColor: colors.cardStrong, borderRadius: 8, height: 16, width: 16 }, + linkThumb: { backgroundColor: colors.cardStrong, borderRadius: 6, height: 48 }, + media: { borderRadius: 8, overflow: 'hidden' }, + mediaRow: { flexDirection: 'row', gap: 2 }, + mediaCell: { flex: 1 }, + mediaFrame: { aspectRatio: 1 }, + mediaImage: { backgroundColor: colors.accentSoft, flex: 1 }, + actions: { flexDirection: 'row', justifyContent: 'space-between' }, + action: { minWidth: 44, paddingVertical: 4 }, + actionInner: { alignItems: 'center' }, + actionSelected: { color: colors.accent, fontSize: 12, fontWeight: '700' }, + }); +} diff --git a/examples/test-app/src/screens/SettingsScreen.tsx b/examples/test-app/src/screens/SettingsScreen.tsx index c580c3f21a..5d099275df 100644 --- a/examples/test-app/src/screens/SettingsScreen.tsx +++ b/examples/test-app/src/screens/SettingsScreen.tsx @@ -20,6 +20,7 @@ export interface SettingsScreenProps { onLoadDiagnostics: () => void; onOpenAccessorySetup: () => void; onOpenAutomationLab: () => void; + onOpenHeavyFeed: () => void; onOpenInertSurface: () => void; onOpenSnapshotDepth: () => void; onOpenWebViewLab: () => void; @@ -81,6 +82,11 @@ export function SettingsScreen(props: SettingsScreenProps) { onPress={props.onOpenSnapshotDepth} testID="open-snapshot-depth-lab" /> + Date: Wed, 21 Oct 2026 21:42:25 +0200 Subject: [PATCH 02/15] feat(perf): time every iOS command against the fixture app pnpm perf --platform ios now drives the test app through seven scenarios that time each catalog command the iOS simulator serves, including the heavy feed and a raw capture with the XCTest tree backend pinned. The suite records a reason for every command it does not time, and a unit test keeps that partition complete. Each sample keeps the client daemon round trip and a phase profile from its --debug request log: runner commands, host process spawns, snapshot sources, retries, stabilization, and fallbacks. A run writes summary.json, report.md, and the raw daemon and session logs. The iOS Settings tour is removed; the Android tour stays as it was. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/perf-nightly.yml | 42 +++- scripts/perf/README.md | 51 ++++ scripts/perf/config.ts | 27 +- scripts/perf/forced-backend.ts | 90 +++++++ scripts/perf/harness.ts | 48 ++-- scripts/perf/ios-harness.ts | 294 ++++++++++++++++++++++ scripts/perf/ios-suite.test.ts | 39 +++ scripts/perf/ios-suite.ts | 380 +++++++++++++++++++++++++++++ scripts/perf/measurement.ts | 32 +++ scripts/perf/metadata.ts | 69 ++++++ scripts/perf/platform-profiles.ts | 36 +-- scripts/perf/profile.test.ts | 112 +++++++++ scripts/perf/profile.ts | 228 +++++++++++++++++ scripts/perf/report.ts | 151 ++++++++---- scripts/perf/run.ts | 159 ++++++++---- scripts/perf/scenario.ts | 118 +++------ scripts/perf/stats.test.ts | 28 +++ scripts/perf/stats.ts | 35 ++- scripts/perf/types.ts | 104 ++++++-- vitest.config.ts | 2 + 20 files changed, 1763 insertions(+), 282 deletions(-) create mode 100644 scripts/perf/README.md create mode 100644 scripts/perf/forced-backend.ts create mode 100644 scripts/perf/ios-harness.ts create mode 100644 scripts/perf/ios-suite.test.ts create mode 100644 scripts/perf/ios-suite.ts create mode 100644 scripts/perf/measurement.ts create mode 100644 scripts/perf/metadata.ts create mode 100644 scripts/perf/profile.test.ts create mode 100644 scripts/perf/profile.ts create mode 100644 scripts/perf/stats.test.ts diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index 5814855ead..1d6c4938a5 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -12,24 +12,24 @@ on: rounds: description: 'Measured rounds per command (samples)' required: false - default: '5' + default: '3' permissions: contents: read + actions: read concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: - AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts' - PERF_ROUNDS: ${{ github.event.inputs.rounds || '5' }} + PERF_ROUNDS: ${{ github.event.inputs.rounds || '3' }} jobs: perf-ios: name: iOS Command Perf runs-on: macos-26 - timeout-minutes: 80 + timeout-minutes: 100 env: IOS_RUNTIME_VERSION: '26.2' AGENT_DEVICE_XCUITEST_ARCHS: 'arm64' @@ -42,6 +42,15 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml + + - name: Build the shared CLI + uses: ./.github/actions/run-gate + with: + gate: build - name: Restore and build iOS XCTest runner uses: ./.github/actions/setup-apple-runner-build @@ -54,22 +63,35 @@ jobs: xcuitest-destination: generic/platform=iOS Simulator - name: Boot iOS test simulator + id: ios-simulator uses: ./.github/actions/boot-ios-test-simulator with: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro + - name: Install the fixture app + id: fixture-app + uses: ./.github/actions/setup-fixture-app + - name: Preflight iOS runner through public CLI + env: + UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} run: | + # The keyboard must stay expanded for keyboard dismiss to find its Done key. + xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "$UDID" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" pnpm clean:daemon - - name: Run iOS command perf benchmark + - name: Run iOS command perf suite + env: + UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + APP_ID: ${{ steps.fixture-app.outputs.app-id }} + APP_PATH: ${{ steps.fixture-app.outputs.app-path }} run: | - pnpm perf --platform ios \ - --device "iPhone 17 Pro" \ - --n "$PERF_ROUNDS" --warmup 1 \ + pnpm perf --platform ios --udid "$UDID" \ + --app-id "$APP_ID" --app-path "$APP_PATH" \ + --rounds "$PERF_ROUNDS" --warmup 1 \ --out-dir "$GITHUB_WORKSPACE/perf-results" - name: Upload iOS perf report @@ -84,6 +106,8 @@ jobs: name: Android Command Perf runs-on: ubuntu-latest timeout-minutes: 80 + env: + AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts' steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/scripts/perf/README.md b/scripts/perf/README.md new file mode 100644 index 0000000000..42d61d4301 --- /dev/null +++ b/scripts/perf/README.md @@ -0,0 +1,51 @@ +# Command performance suite + +`pnpm perf --platform ios` times every interaction command against the fixture app (`examples/test-app`). +The steps are in `ios-suite.ts`. Each step belongs to one scenario. +Each scenario starts from an untimed setup in every round. +The heavy-feed scenario uses a screen whose accessibility tree is deeper than 65 levels. +The suite records a reason for each catalog command that it does not time. `ios-suite.test.ts` keeps that list complete. +`pnpm perf --platform android` runs the older report-only Settings tour. + +## Run locally + +1. Run `pnpm build`. The suite drives the built CLI and the built SDK. +2. Use your own booted simulator. Install a current Release fixture on it. For the cached CI build and the JavaScript repack, see `examples/test-app/README.md`. +3. Set `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH` to a path in your worktree. +4. Run the suite: + +```sh +pnpm perf --platform ios --udid --rounds 3 --warmup 1 --out-dir .tmp/perf/local +``` + +`--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id. +The suite uses its own temporary state directory and daemon. It stops them at the end. + +## Output + +- `summary.json`: the machine-readable run (`agent-device-perf-summary/v1`). It holds the metadata, the coverage, and every sample. +- `report.md`: one table per scenario, the backends and fallbacks, the failures, and the coverage. +- `raw/state/`: the daemon log, and for each session `runner.log`, `events.ndjson`, and the `--debug` request log of each request. `requestId` in a sample names its request log. +- `raw/failures/`: the JSON response and stderr of each failed sample. + +## Metrics + +All times are milliseconds. Statistics use only the successful samples of a step. + +| Metric | Meaning | +| --- | --- | +| `wall` | The CLI process time, from spawn to exit. | +| `daemon` | The daemon request round trip that the client measures (`daemon_request`). For the pinned-backend capture, the span of its request log. Comparisons use this metric. | +| `mean ± sd`, `cv` | The mean, the sample standard deviation, and the standard deviation divided by the mean. | +| `median`, `p95`, `min`, `max` | The median (the mean of the two middle values for an even count), the nearest-rank 95th percentile, and the extremes. | +| `runner` | The median sum of XCTest runner command times in one request. | +| `spawns` | The median count and time of host processes that one request started. | +| `snapshot source` | The median time of host accessibility-bridge acquisitions in one request. | +| `settle wait` | The median `settle.waitedMs` that a `--settle` response reports. | +| `stabilization` | The median time of post-gesture stabilization phases in one request. | +| `retries` | The failed attempts of retried operations in all successful samples. | +| `top phases` | The five phases with the largest median time per request. | +| `producer` | The number of samples that each snapshot producer answered (`simulator-ax-bridge`, `apple-runner`). | +| backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected`. | +| fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. | +| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth. | diff --git a/scripts/perf/config.ts b/scripts/perf/config.ts index 10c3e84630..1db472fa88 100644 --- a/scripts/perf/config.ts +++ b/scripts/perf/config.ts @@ -13,11 +13,16 @@ export type PerfConfig = { warmup: number; // leading rounds dropped from stats keepArtifacts: boolean; // keep temp state dir + leave device booted outDir: string; - udid?: string; // iOS device override (UDID) - device?: string; // device override by name (e.g. "iPhone 17 Pro"); preferred over udid + udid?: string; // iOS simulator UDID (required for iOS) + device?: string; // Android AVD name for boot serial?: string; // Android device override + appId: string; // iOS fixture bundle id + appPath?: string; // iOS fixture bundle path, recorded in the run metadata + scenarios?: string[]; // iOS scenario ids; all when omitted }; +export const DEFAULT_IOS_APP_ID = 'com.callstack.agentdevicelab'; + // How to invoke the CLI. Defaults to the built dist binary (bin/agent-device.mjs). // Set AGENT_DEVICE_PERF_CLI to run from source instead, e.g. on CI: // AGENT_DEVICE_PERF_CLI="--experimental-strip-types src/bin.ts" @@ -50,10 +55,11 @@ function readIntValue(argv: string[], i: number, flag: string, min: number): num export function parseConfig(argv: string[]): PerfConfig { const cfg: PerfConfig = { platform: 'ios', - rounds: 5, + rounds: 3, warmup: 1, keepArtifacts: false, - outDir: DEFAULT_OUT_DIR, + outDir: path.join(DEFAULT_OUT_DIR, new Date().toISOString().replaceAll(/[:.]/g, '-')), + appId: DEFAULT_IOS_APP_ID, }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; @@ -86,9 +92,22 @@ export function parseConfig(argv: string[]): PerfConfig { case '--serial': cfg.serial = readValue(argv, i++, a); break; + case '--app-id': + cfg.appId = readValue(argv, i++, a); + break; + case '--app-path': + cfg.appPath = path.resolve(readValue(argv, i++, a)); + break; + case '--scenarios': + cfg.scenarios = readValue(argv, i++, a) + .split(',') + .map((id) => id.trim()) + .filter(Boolean); + break; default: throw new Error(`Unknown flag: ${a}`); } } + if (cfg.platform === 'ios' && !cfg.udid) throw new Error('--udid is required for --platform ios'); return cfg; } diff --git a/scripts/perf/forced-backend.ts b/scripts/perf/forced-backend.ts new file mode 100644 index 0000000000..387b22fea2 --- /dev/null +++ b/scripts/perf/forced-backend.ts @@ -0,0 +1,90 @@ +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { pathToFileURL } from 'node:url'; +import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { createRequestId } from '@agent-device/host-kit/diagnostics'; +import { captureHostEnvironment } from '@agent-device/host-kit/process'; +import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot'; +import { readDaemonInfo } from '../../src/daemon-client/daemon-client-metadata.ts'; +import { sendRequest } from '../../src/daemon-client/daemon-client-transport.ts'; +import { REPO_ROOT } from './config.ts'; +import type { CliResult } from './types.ts'; + +type AgentDeviceSdk = typeof import('../../src/sdk/index.ts'); + +const FORCED_CAPTURE_TIMEOUT_MS = 90_000; + +/** + * Captures one raw snapshot with the internal backend pin that the CLI does not expose. The + * transport talks to the daemon that the built CLI already started and never starts or replaces + * it, so the measured daemon stays the daemon under test. + */ +export async function captureWithForcedBackend(options: { + stateDir: string; + session: string; + udid: string; + backend: SnapshotPreferredBackend; +}): Promise<{ result: CliResult; requestId: string }> { + const requestId = createRequestId(); + const sdk = (await import( + pathToFileURL(path.join(REPO_ROOT, 'dist/src/index.js')).href + )) as AgentDeviceSdk; + const client = sdk.createAgentDeviceClient( + { session: options.session, stateDir: options.stateDir }, + { transport: forcedBackendTransport(options.backend, requestId) }, + ); + const t0 = performance.now(); + try { + const snapshot = await client.capture.snapshot({ + raw: true, + platform: 'ios', + udid: options.udid, + timeoutMs: FORCED_CAPTURE_TIMEOUT_MS, + }); + const json = { success: true, data: snapshot }; + const wallClockMs = performance.now() - t0; + return { + result: { exitCode: 0, wallClockMs, stdout: '', stderr: '', json, ok: true }, + requestId, + }; + } catch (error) { + const normalized = sdk.normalizeAgentDeviceError(error); + const json = { success: false, error: normalized }; + const wallClockMs = performance.now() - t0; + return { + result: { exitCode: 1, wallClockMs, stdout: '', stderr: normalized.message, json, ok: false }, + requestId, + }; + } +} + +function forcedBackendTransport( + backend: SnapshotPreferredBackend, + requestId: string, +): AgentDeviceDaemonTransport { + const host = captureHostEnvironment(process); + return async (request, context) => { + const stateDir = request.flags?.stateDir; + if (request.session === undefined || typeof stateDir !== 'string') { + throw new Error('A forced-backend capture requires an explicit session and state dir.'); + } + const paths = resolveDaemonPaths(stateDir, { env: process.env }); + const info = readDaemonInfo(paths.infoPath); + if (!info) throw new Error(`No daemon is running for state dir ${stateDir}.`); + return await sendRequest( + info, + { + ...request, + session: request.session, + token: info.token, + flags: { ...request.flags, snapshotPreferredBackend: backend }, + meta: { ...request.meta, debug: true, requestId }, + }, + 'auto', + paths, + FORCED_CAPTURE_TIMEOUT_MS, + { host, signal: context?.signal, timeoutRecovery: 'none' }, + ); + }; +} diff --git a/scripts/perf/harness.ts b/scripts/perf/harness.ts index 49f0bb6f93..09063440a7 100644 --- a/scripts/perf/harness.ts +++ b/scripts/perf/harness.ts @@ -10,8 +10,8 @@ import { } from './cli.ts'; import type { PerfConfig } from './config.ts'; import { resolveProfile, type ResolvedProfile } from './platform-profiles.ts'; +import { buildMeasurement } from './measurement.ts'; import { buildSettingsTour, type ScenarioStep } from './scenario.ts'; -import { summarize } from './stats.ts'; import type { CliResult, Measurement, Sample } from './types.ts'; export type IsolationContext = { @@ -32,7 +32,7 @@ export function setupIsolation(cfg: PerfConfig): IsolationContext { fs.mkdirSync(artifactsDir, { recursive: true }); const baseFlags = ['--state-dir', stateDir, '--session', 'perf', ...profile.platformFlags]; log(`state-dir: ${stateDir}`); - log(`device: ${profile.deviceName} (${profile.udid ?? profile.serial})`); + log(`device: ${profile.deviceName} (${profile.serial ?? 'default'})`); return { stateDir, artifactsDir, baseFlags, profile }; } @@ -72,8 +72,8 @@ function toSample(r: CliResult, round: number): Sample { return sample; } -// The first interaction after open/relaunch pays the iOS XCUITest runner startup (~10s+ cold) -// and a per-relaunch first-AX-query settle cost. Run an untimed throwaway interaction so that +// The first interaction after open/relaunch pays the Android helper startup and a per-relaunch +// first-query settle cost. Run an untimed throwaway interaction so that // cost is never attributed to a measured command. function warmRunner(ctx: IsolationContext): void { invokeCli(['snapshot', '-i'], ctx.baseFlags); @@ -91,39 +91,31 @@ function runStep(step: ScenarioStep, ctx: IsolationContext, round: number): Samp : invokeBatchStep(step.step, ctx.baseFlags); const sample = toSample(r, round); if (step.execMode === 'batch') { - sample.daemonDurationMs = readBatchStepDurationMs(r); + sample.daemonMs = readBatchStepDurationMs(r); if (step.isSnapshot) sample.elementCount = countElements(r); } return sample; } -function buildMeasurement( +const TOUR_SCENARIO = 'settings-tour'; + +function tourMeasurement( step: Pick, platform: ResolvedProfile['platform'], samples: Sample[], warmupDropped: number, ): Measurement { - const ok = samples.filter((s) => s.ok); - const failures = samples.length - ok.length; - const notes: string[] = []; - if (failures > 0) { - const codes = [...new Set(samples.filter((s) => !s.ok).map((s) => s.errorCode))].join(', '); - notes.push(`${failures}/${samples.length} samples failed: ${codes}`); - } - const num = (xs: (number | undefined)[]) => xs.filter((n): n is number => typeof n === 'number'); - return { - command: step.command, + return buildMeasurement({ + id: `${TOUR_SCENARIO}/${step.label}`, + scenario: TOUR_SCENARIO, label: step.label, + command: step.command, + args: [], platform, execMode: step.execMode, samples, warmupDropped, - wallClock: summarize(ok.map((s) => s.wallClockMs)), - daemonDuration: summarize(num(ok.map((s) => s.daemonDurationMs))), - elementCount: summarize(num(ok.map((s) => s.elementCount))), - failures, - notes, - }; + }); } function setupMeasurementFailed(measurement: Measurement): boolean { @@ -137,7 +129,7 @@ function bootOnce(ctx: IsolationContext): Measurement { const bootFlags = ['--state-dir', ctx.stateDir, ...ctx.profile.platformFlags]; const r = invokeCli(['boot', ...ctx.profile.selectorFlags], bootFlags); const sample = toSample(r, 0); - return buildMeasurement( + return tourMeasurement( { command: 'boot', label: 'boot device', execMode: 'standalone' }, ctx.profile.platform, [sample], @@ -152,7 +144,7 @@ function establishSession(ctx: IsolationContext): Measurement { log('establishing session (open with device selectors)'); const r = invokeCli(['open', ctx.profile.appTarget, ...ctx.profile.selectorFlags], ctx.baseFlags); const sample = toSample(r, 0); - return buildMeasurement( + return tourMeasurement( { command: 'open', label: 'open (establish + cold)', execMode: 'standalone' }, ctx.profile.platform, [sample], @@ -176,10 +168,8 @@ export function runScenario(ctx: IsolationContext, cfg: PerfConfig): Measurement // Android accessibility dumps time out while the UI is animating; disable animations // up front (untimed) so snapshot/get/is/fill can read an idle hierarchy. - if (ctx.profile.platform === 'android') { - log('disabling animations (android)'); - invokeCli(['settings', 'animations', 'off'], ctx.baseFlags); - } + log('disabling animations'); + invokeCli(['settings', 'animations', 'off'], ctx.baseFlags); const totalRounds = cfg.warmup + cfg.rounds; for (let round = 0; round < totalRounds; round++) { @@ -197,7 +187,7 @@ export function runScenario(ctx: IsolationContext, cfg: PerfConfig): Measurement } const tourMeasurements = steps.map((step) => - buildMeasurement(step, ctx.profile.platform, acc.get(step.label)!, cfg.warmup), + tourMeasurement(step, ctx.profile.platform, acc.get(step.label)!, cfg.warmup), ); return [boot, establish, ...tourMeasurements]; } diff --git a/scripts/perf/ios-harness.ts b/scripts/perf/ios-harness.ts new file mode 100644 index 0000000000..bcd2c3f4a6 --- /dev/null +++ b/scripts/perf/ios-harness.ts @@ -0,0 +1,294 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + resolveSessionDir, + resolveSessionRequestLogPath, +} from '@agent-device/daemon-contracts/session-artifact-paths'; +import { invokeCli } from './cli.ts'; +import type { PerfConfig } from './config.ts'; +import { captureWithForcedBackend } from './forced-backend.ts'; +import { expectationViolation, type SuiteScenario, type SuiteStep } from './ios-suite.ts'; +import { buildMeasurement } from './measurement.ts'; +import { + profileRequestLog, + readClientDiagnostics, + readSettleFacts, + readSnapshotFacts, +} from './profile.ts'; +import type { CliResult, Measurement, Sample } from './types.ts'; + +const SESSION = 'perf'; + +export type IosSuiteContext = { + stateDir: string; + udid: string; + appId: string; + artifactsDir: string; + rawDir: string; + // Set when a request outlived the client timeout. That request still holds the session, so + // the suite restarts the daemon before the next scenario. + daemonStuck?: boolean; +}; + +function log(msg: string): void { + process.stderr.write(`[perf] ${msg}\n`); +} + +function baseFlags(ctx: IosSuiteContext): string[] { + return ['--state-dir', ctx.stateDir, '--session', SESSION, '--platform', 'ios']; +} + +function run(ctx: IosSuiteContext, args: readonly string[]): CliResult { + return watch(ctx, invokeCli(expandStatic(args, ctx), baseFlags(ctx))); +} + +function watch(ctx: IosSuiteContext, result: CliResult): CliResult { + if (daemonTimedOut(result)) ctx.daemonStuck = true; + return result; +} + +function expandStatic(args: readonly string[], ctx: IosSuiteContext): string[] { + return args.map((arg) => + arg === '{app}' + ? ctx.appId + : arg.replace(/^\{out:(.+)\}$/, (_, file: string) => path.join(ctx.artifactsDir, file)), + ); +} + +type SnapshotNode = { + ref?: string; + identifier?: string; + rect?: { x: number; y: number; width: number; height: number }; +}; + +const TARGET_TOKEN = /^\{(ref|point):(.+)\}$/; + +/** Expands `{ref:id}` and `{point:id}` from one untimed interactive snapshot. */ +function resolveTargets(args: string[], ctx: IosSuiteContext): string[] | { error: string } { + if (!args.some((arg) => TARGET_TOKEN.test(arg))) return args; + const snapshot = run(ctx, ['snapshot', '-i']); + const nodes = ((snapshot.json as { data?: { nodes?: SnapshotNode[] } })?.data?.nodes ?? + []) as SnapshotNode[]; + const expanded: string[] = []; + for (const arg of args) { + const match = TARGET_TOKEN.exec(arg); + if (!match) { + expanded.push(arg); + continue; + } + const node = nodes.find((candidate) => candidate.identifier === match[2]); + if (!node?.ref || !node.rect) return { error: `target ${match[2]} is not in snapshot -i` }; + if (match[1] === 'ref') { + expanded.push(`@${node.ref}`); + } else { + expanded.push( + String(Math.round(node.rect.x + node.rect.width / 2)), + String(Math.round(node.rect.y + node.rect.height / 2)), + ); + } + } + return expanded; +} + +/** Runs untimed calls in order and describes the first failure. */ +function untimed( + ctx: IosSuiteContext, + calls: readonly (readonly string[])[] = [], +): string | undefined { + for (const args of calls) { + const result = run(ctx, args); + if (!result.ok) return `${args.join(' ')}: ${errorOf(result).errorMessage}`; + } + return undefined; +} + +/** True when the client gave up on a daemon request that is still running. */ +function daemonTimedOut(result: CliResult): boolean { + const details = (result.json as { error?: { details?: { reason?: unknown } } } | undefined)?.error + ?.details; + return details?.reason === 'daemon_transport_timeout'; +} + +function errorOf(result: CliResult): Pick { + const error = (result.json as { error?: { code?: string; message?: string } } | undefined)?.error; + return { + errorCode: error?.code ?? `exit:${result.exitCode}`, + errorMessage: (error?.message ?? result.stderr.trim().split('\n').pop() ?? '').slice(0, 300), + }; +} + +function readProfile(ctx: IosSuiteContext, requestId: string | undefined) { + if (!requestId) return undefined; + const sessionDir = resolveSessionDir(path.join(ctx.stateDir, 'sessions'), SESSION); + const logPath = resolveSessionRequestLogPath(sessionDir, requestId); + return fs.existsSync(logPath) ? profileRequestLog(fs.readFileSync(logPath, 'utf8')) : undefined; +} + +function recordFailure(ctx: IosSuiteContext, id: string, round: number, detail: unknown): void { + const dir = path.join(ctx.rawDir, 'failures'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, `${id.replace('/', '__')}.round-${round}.json`), + JSON.stringify(detail, null, 2), + ); +} + +async function runTimedStep( + scenario: SuiteScenario, + step: SuiteStep, + ctx: IosSuiteContext, + round: number, +): Promise { + const id = `${scenario.id}/${step.id}`; + const before = untimed(ctx, step.before); + if (before) { + log(` ✗ ${id} (before_failed: ${before})`); + return { round, ok: false, wallClockMs: 0, errorCode: 'before_failed', errorMessage: before }; + } + let sample: Sample; + if (step.via === 'sdk-forced-tree') { + const { result: forced, requestId } = await captureWithForcedBackend({ + stateDir: ctx.stateDir, + session: SESSION, + udid: ctx.udid, + backend: 'tree', + }); + const result = watch(ctx, forced); + const profile = readProfile(ctx, requestId); + sample = { + round, + ok: result.ok, + wallClockMs: result.wallClockMs, + daemonMs: profile?.requestSpanMs, + requestId, + snapshot: readSnapshotFacts(result.json), + profile, + }; + if (!result.ok) Object.assign(sample, errorOf(result)); + } else { + const args = resolveTargets(expandStatic(step.args, ctx), ctx); + if ('error' in args) { + sample = { + round, + ok: false, + wallClockMs: 0, + errorCode: 'target_unresolved', + errorMessage: args.error, + }; + } else { + const result = watch(ctx, invokeCli([...args, '--debug'], baseFlags(ctx))); + const diagnostics = readClientDiagnostics(result.stderr); + sample = { + round, + ok: result.ok, + wallClockMs: result.wallClockMs, + daemonMs: diagnostics.daemonMs, + requestId: diagnostics.requestId, + snapshot: step.command === 'snapshot' ? readSnapshotFacts(result.json) : undefined, + settle: readSettleFacts(result.json), + profile: readProfile(ctx, diagnostics.requestId), + }; + if (!result.ok) { + Object.assign(sample, errorOf(result)); + recordFailure(ctx, id, round, { + args, + json: result.json, + stderrTail: result.stderr.slice(-4000), + }); + } + } + } + const violation = sample.ok ? expectationViolation(step.expect, sample.snapshot) : undefined; + if (violation) + Object.assign(sample, { ok: false, errorCode: 'expectation_failed', errorMessage: violation }); + const after = untimed(ctx, step.after); + if (after) log(` ! ${id} after: ${after}`); + if (!sample.ok) log(` ✗ ${id} (${sample.errorCode}: ${sample.errorMessage})`); + return sample; +} + +function sessionError(result: CliResult): string | undefined { + if (result.ok) return undefined; + const message = `open: ${errorOf(result).errorMessage}`; + log(` ✗ session failed: ${message}`); + return message; +} + +function stopStuckDaemon(ctx: IosSuiteContext): void { + log(' ! a request outlived its timeout; restarting the daemon'); + invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); + ctx.daemonStuck = false; +} + +/** Opens the session with device selectors; later calls address it by session name alone. */ +function establishSession(ctx: IosSuiteContext): CliResult { + if (ctx.daemonStuck) stopStuckDaemon(ctx); + return watch( + ctx, + invokeCli(['open', ctx.appId, '--relaunch', '--udid', ctx.udid], baseFlags(ctx)), + ); +} + +export async function runIosSuite( + ctx: IosSuiteContext, + cfg: PerfConfig, + scenarios: readonly SuiteScenario[], +): Promise { + const samples = new Map(); + const totalRounds = cfg.warmup + cfg.rounds; + for (let round = 0; round < totalRounds; round++) { + const measured = round >= cfg.warmup; + log(`round ${round + 1}/${totalRounds}${measured ? '' : ' (warmup, dropped)'}`); + let sessionFailure = sessionError(establishSession(ctx)); + for (const scenario of scenarios) { + // A request that outlived its timeout still holds the session; reopen before the next scenario. + if (ctx.daemonStuck) sessionFailure = sessionError(establishSession(ctx)); + log(` scenario ${scenario.id}`); + const setupFailure = sessionFailure ?? untimed(ctx, scenario.setup); + if (setupFailure) log(` ✗ setup failed: ${setupFailure}`); + for (const step of scenario.steps) { + // CLI calls block the event loop; yield so an interrupt can stop the run between steps. + await new Promise((resolve) => setImmediate(resolve)); + const id = `${scenario.id}/${step.id}`; + const skipped = setupFailure + ? { errorCode: 'setup_failed', errorMessage: setupFailure } + : ctx.daemonStuck + ? { + errorCode: 'daemon_stuck', + errorMessage: 'an earlier request still holds the session', + } + : undefined; + const sample: Sample = skipped + ? { round, ok: false, wallClockMs: 0, ...skipped } + : await runTimedStep(scenario, step, ctx, round); + if (measured) samples.set(id, [...(samples.get(id) ?? []), sample]); + } + } + } + return scenarios.flatMap((scenario) => + scenario.steps.map((step) => + buildMeasurement({ + id: `${scenario.id}/${step.id}`, + scenario: scenario.id, + label: step.args.join(' '), + command: step.command, + args: [...step.args], + platform: 'ios', + execMode: step.via === 'sdk-forced-tree' ? 'sdk' : 'standalone', + samples: samples.get(`${scenario.id}/${step.id}`) ?? [], + warmupDropped: cfg.warmup, + }), + ), + ); +} + +/** Copies daemon and session logs into the raw artifact; the daemon token file stays behind. */ +export function copyRawState(ctx: IosSuiteContext): void { + const target = path.join(ctx.rawDir, 'state'); + fs.mkdirSync(target, { recursive: true }); + const daemonLog = path.join(ctx.stateDir, 'daemon.log'); + if (fs.existsSync(daemonLog)) fs.copyFileSync(daemonLog, path.join(target, 'daemon.log')); + const sessions = path.join(ctx.stateDir, 'sessions'); + if (fs.existsSync(sessions)) + fs.cpSync(sessions, path.join(target, 'sessions'), { recursive: true }); +} diff --git a/scripts/perf/ios-suite.test.ts b/scripts/perf/ios-suite.test.ts new file mode 100644 index 0000000000..cc92840b1e --- /dev/null +++ b/scripts/perf/ios-suite.test.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import { test } from 'vitest'; +import { + describeCoverage, + expectationViolation, + IOS_SUITE, + IOS_SUITE_EXCLUSIONS, +} from './ios-suite.ts'; + +test('every public command is either measured by the iOS suite or excluded with a reason', () => { + const { measured, excluded } = describeCoverage(IOS_SUITE, IOS_SUITE_EXCLUSIONS); + const classified = [...measured, ...excluded.map((e) => e.command)].sort(); + assert.deepEqual(classified, Object.values(PUBLIC_COMMANDS).sort()); + for (const { reason } of excluded) assert.ok(reason.length > 0); +}); + +test('step ids are unique, so comparisons key each step once', () => { + const ids = IOS_SUITE.flatMap((scenario) => scenario.steps.map((s) => `${scenario.id}/${s.id}`)); + assert.equal(new Set(ids).size, ids.length); +}); + +test('a heavy-feed capture breaks its expectation when the tree flattens or the tree backend copes', () => { + const depth = { maxDepthAtLeast: 64 }; + assert.equal(expectationViolation(depth, { maxDepth: 64 }), undefined); + assert.match(expectationViolation(depth, { maxDepth: 63 })!, /below 64/); + assert.match(expectationViolation(depth, undefined)!, /unknown/); + + const degraded = { treeBackendDegraded: true }; + assert.equal( + expectationViolation(degraded, { backend: 'private-ax', state: 'recovered' }), + undefined, + ); + assert.equal(expectationViolation(degraded, { backend: 'tree', state: 'sparse' }), undefined); + assert.match( + expectationViolation(degraded, { backend: 'tree', state: 'healthy' })!, + /non-sparse/, + ); +}); diff --git a/scripts/perf/ios-suite.ts b/scripts/perf/ios-suite.ts new file mode 100644 index 0000000000..bde981db05 --- /dev/null +++ b/scripts/perf/ios-suite.ts @@ -0,0 +1,380 @@ +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { SnapshotFacts } from './types.ts'; + +/** Routes `trigger-app-event` into the fixture's automation screen. */ +export const FIXTURE_APP_EVENT_URL_TEMPLATE = + 'agent-device-test-app:///automation?event={event}&payload={payload}'; + +export type PublicCommand = (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS]; + +const C = PUBLIC_COMMANDS; + +type Cli = readonly string[]; + +/** + * One timed command. `args` tokens expand before each run: `{app}` is the fixture bundle id, + * `{out:}` a path in the run's artifact directory, and `{ref:}` / `{point:}` the ref + * or center point of the element with that identifier in an untimed `snapshot -i` taken just + * before the step. `before` and `after` run untimed around the step in every round. + */ +export type SuiteStep = { + id: string; + command: PublicCommand; + args: Cli; + before?: readonly Cli[]; + after?: readonly Cli[]; + // `sdk-forced-tree` captures a raw snapshot through the daemon with the XCTest tree backend + // pinned; the CLI has no flag for it. + via?: 'cli' | 'sdk-forced-tree'; + // A sample that succeeds but breaks the expectation fails with `expectation_failed`, so a + // fixture that stops reproducing its shape shows up as a new failure. + expect?: StepExpectation; +}; + +export type StepExpectation = { + // Deepest raw node depth, counted from the application at 0. + maxDepthAtLeast?: number; + // The pinned XCTest tree backend did not serve a healthy capture. + treeBackendDegraded?: boolean; +}; + +/** Describes how a snapshot breaks the step expectation, or returns undefined. */ +export function expectationViolation( + expect: StepExpectation | undefined, + snapshot: SnapshotFacts | undefined, +): string | undefined { + if (!expect) return undefined; + if (expect.maxDepthAtLeast !== undefined && (snapshot?.maxDepth ?? -1) < expect.maxDepthAtLeast) { + return `max depth ${snapshot?.maxDepth ?? 'unknown'} is below ${expect.maxDepthAtLeast}`; + } + if (expect.treeBackendDegraded && snapshot?.backend === 'tree' && snapshot.state !== 'sparse') { + return 'the pinned tree backend served a non-sparse capture'; + } + return undefined; +} + +/** A scenario starts from `setup` (untimed, every round) and runs its steps in order. */ +export type SuiteScenario = { + id: string; + title: string; + setup: readonly Cli[]; + steps: SuiteStep[]; +}; + +// A cold relaunch on a hosted runner can take longer than 15 s before the first landmark. +const LANDMARK_WAIT_MS = '60000'; +const relaunch: Cli = ['open', '{app}', '--relaunch']; +const toTab = (label: string, landmark: string): Cli[] => [ + relaunch, + ['wait', `label="${label}"`, LANDMARK_WAIT_MS], + ['click', `label="${label}"`], + ['wait', landmark, LANDMARK_WAIT_MS], +]; +const toSettingsLab = (button: string, landmark: string): Cli[] => [ + ...toTab('Settings', 'id="settings-title"'), + ['scroll', 'down', '--until', `id="${button}"`], + ['click', `id="${button}"`], + ['wait', landmark, LANDMARK_WAIT_MS], +]; +const step = ( + id: string, + command: PublicCommand, + args: Cli, + extra: Omit = {}, +): SuiteStep => ({ id, command, args, ...extra }); + +export const IOS_SUITE: readonly SuiteScenario[] = [ + { + id: 'lifecycle', + title: 'App lifecycle, device state, and inventory', + setup: [relaunch, ['wait', 'text', 'Agent Device Tester', LANDMARK_WAIT_MS]], + steps: [ + step('open-relaunch', C.open, relaunch), + step('appstate', C.appState, ['appstate']), + step('home', C.home, ['home']), + step('open-resume', C.open, ['open', '{app}']), + step('app-switcher', C.appSwitcher, ['app-switcher'], { after: [['open', '{app}']] }), + step('orientation-landscape', C.orientation, ['orientation', 'landscape-left']), + step('orientation-portrait', C.orientation, ['orientation', 'portrait']), + step('settings-appearance', C.settings, ['settings', 'appearance', 'light']), + step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf']), + step('clipboard-read', C.clipboard, ['clipboard', 'read']), + step('action-button', C.actionButton, ['action-button']), + step('prepare-runner', C.prepare, ['prepare', 'ios-runner']), + step('devices', C.devices, ['devices', '--platform', 'ios']), + step('apps', C.apps, ['apps']), + step('capabilities', C.capabilities, ['capabilities']), + step('doctor', C.doctor, ['doctor']), + // Some runtimes ask "Open in Agent Device Tester?" for the event URL, and the prompt outlives a + // relaunch. The answer opens the URL seconds later, so wait for its route before the next + // relaunch. The untimed calls stop at alert get when no prompt is up. + step( + 'trigger-app-event', + C.triggerAppEvent, + ['trigger-app-event', 'fixture.ready', '{"source":"perf"}'], + { + after: [ + ['alert', 'get'], + ['alert', 'accept'], + ['wait', 'id="automation-title"', LANDMARK_WAIT_MS], + ], + }, + ), + ], + }, + { + id: 'home-reads', + title: 'Reads on the home screen (light tree)', + setup: [relaunch, ['wait', 'text', 'Agent Device Tester', LANDMARK_WAIT_MS]], + steps: [ + step('snapshot-i', C.snapshot, ['snapshot', '-i']), + step('snapshot', C.snapshot, ['snapshot']), + step('snapshot-raw', C.snapshot, ['snapshot', '--raw']), + step('snapshot-diff', C.snapshot, ['snapshot', '--diff']), + step('diff-snapshot', C.diff, ['diff', 'snapshot']), + step('wait-text', C.wait, ['wait', 'text', 'Agent Device Tester']), + step('wait-selector', C.wait, ['wait', 'id="gesture-canary-ready"']), + step('wait-absent', C.wait, ['wait', 'absent', 'id="heavy-feed-title"']), + step('wait-stable', C.wait, ['wait', 'stable', '300', '5000']), + step('is-visible', C.is, ['is', 'visible', 'id="gesture-canary-ready"']), + step('is-exists', C.is, ['is', 'exists', 'text="Gesture lab"']), + step('get-text', C.get, ['get', 'text', 'id="gesture-fling-status"']), + step('get-attrs', C.get, ['get', 'attrs', 'id="drag-source"']), + step('find-text', C.find, ['find', 'text', 'Gesture lab', 'exists']), + step('batch-reads', C.batch, [ + 'batch', + '--steps', + JSON.stringify([ + { + command: 'get', + input: { + format: 'text', + target: { kind: 'selector', selector: 'id="gesture-fling-status"' }, + }, + }, + { command: 'is', input: { predicate: 'visible', selector: 'id="gesture-canary-ready"' } }, + ]), + ]), + step('gesture-pan', C.gesture, [ + 'gesture', + 'pan', + '{point:gesture-target}', + '40', + '0', + '300', + ]), + step('gesture-pinch', C.gesture, ['gesture', 'pinch', '1.4', '{point:gesture-target}']), + step('screenshot', C.screenshot, ['screenshot', '{out:home.png}']), + step('screenshot-overlay', C.screenshot, [ + 'screenshot', + '{out:home-refs.png}', + '--overlay-refs', + ]), + step('events', C.events, ['events']), + step('logs-mark', C.logs, ['logs', 'mark', 'agent-device perf']), + step('perf-memory', C.perf, ['perf', 'memory', 'sample']), + step('react-native', C.reactNative, ['react-native', 'dismiss-overlay']), + step('artifacts', C.artifacts, ['artifacts']), + ], + }, + { + id: 'catalog', + title: 'Catalog list: press forms, text entry, and scrolling', + setup: toTab('Catalog', 'id="catalog-title"'), + steps: [ + step('press-selector', C.press, ['press', 'id="category-produce"']), + step('press-selector-settle', C.press, ['press', 'id="category-bakery"', '--settle']), + step('press-ref', C.press, ['press', '{ref:category-pantry}']), + step('press-ref-settle', C.press, ['press', '{ref:category-produce}', '--settle']), + step('press-point', C.press, ['press', '{point:category-bakery}']), + step('press-point-settle', C.press, ['press', '{point:category-pantry}', '--settle']), + step('click-selector', C.click, ['click', 'id="category-all"']), + step('click-ref', C.click, ['click', '{ref:category-bakery}']), + step('click-point', C.click, ['click', '{point:category-pantry}']), + step('click-settle', C.click, ['click', 'id="category-all"', '--settle']), + step('press-count', C.press, [ + 'press', + '{point:category-produce}', + '--count', + '2', + '--interval-ms', + '50', + ]), + step('fill', C.fill, ['fill', 'id="catalog-search"', 'tart']), + step('fill-settle', C.fill, ['fill', 'id="catalog-search"', 'kit', '--settle']), + step('focus', C.focus, ['focus', '{point:catalog-search}']), + step('type', C.type, ['type', ' box']), + step('keyboard-return', C.keyboard, ['keyboard', 'return']), + step('fill-clear', C.fill, ['fill', 'id="catalog-search"', ''], { + after: [['keyboard', 'return']], + }), + step('scroll-down', C.scroll, ['scroll', 'down']), + step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), + step('scroll-top', C.scroll, ['scroll', 'top']), + step('scroll-until', C.scroll, ['scroll', 'down', '--until', 'id="catalog-footer"']), + step('swipe', C.swipe, ['swipe', '200', '300', '200', '650']), + step('swipe-count', C.swipe, [ + 'swipe', + '200', + '650', + '200', + '450', + '--count', + '2', + '--pause-ms', + '50', + '--pattern', + 'ping-pong', + ]), + ], + }, + { + id: 'form', + title: 'Form fields, keyboard, and submit', + setup: toTab('Form', 'id="form-title"'), + steps: [ + step('fill-name', C.fill, ['fill', 'id="field-name"', 'Ada Lovelace']), + step('fill-email', C.fill, ['fill', 'id="field-email"', 'ada@example']), + step('type-suffix', C.type, ['type', '.test']), + step('keyboard-dismiss', C.keyboard, ['keyboard', 'dismiss'], { + before: [['wait', 'label="Done" visible=true', '10000']], + }), + step('press-checkbox', C.press, ['press', 'id="checkbox-agree"'], { + before: [['scroll', 'down', '--until', 'id="checkbox-agree"']], + }), + step('press-submit-settle', C.press, ['press', 'id="submit-order"', '--settle'], { + before: [['scroll', 'down', '--until', 'id="submit-order"']], + }), + ], + }, + { + id: 'automation', + title: 'Long press, native alerts, recording, tracing, and back navigation', + setup: toSettingsLab('open-automation-lab', 'id="automation-title"'), + steps: [ + step('record-start', C.record, ['record', 'start', '{out:automation.mp4}', '--hide-touches']), + step('record-stop', C.record, ['record', 'stop'], { + before: [ + ['scroll', 'down', '--until', 'id="automation-press"'], + ['press', 'id="automation-press"'], + ['wait', '1200'], + ], + }), + step('trace-start', C.trace, ['trace', 'start', '{out:automation.trace}']), + step('trace-stop', C.trace, ['trace', 'stop', '{out:automation.trace}'], { + before: [['press', 'id="automation-press"']], + }), + step('longpress', C.longPress, ['longpress', 'id="automation-longpress"', '800'], { + before: [['scroll', 'down', '--until', 'id="automation-longpress"']], + }), + step('longpress-settle', C.longPress, [ + 'longpress', + 'id="automation-longpress"', + '800', + '--settle', + ]), + step('alert-open', C.press, ['press', 'id="automation-open-alert"'], { + before: [['scroll', 'down', '--until', 'id="automation-open-alert"']], + }), + step('alert-wait', C.alert, ['alert', 'wait', '10000']), + step('alert-get', C.alert, ['alert', 'get']), + step('alert-accept', C.alert, ['alert', 'accept']), + step('alert-dismiss', C.alert, ['alert', 'dismiss'], { + before: [ + ['press', 'id="automation-open-alert"'], + ['alert', 'wait', '10000'], + ], + }), + step('logs-start', C.logs, ['logs', 'start']), + // A stop issued 400 ms after start has hung its request in local runs; let the stream run. + step('logs-stop', C.logs, ['logs', 'stop'], { before: [['wait', '1000']] }), + step('network-dump', C.network, ['network', 'dump', '10']), + step('back', C.back, ['back']), + step('back-settle', C.back, ['back', '--settle'], { + before: [ + ['click', 'id="open-automation-lab"'], + ['wait', 'id="automation-title"', LANDMARK_WAIT_MS], + ], + }), + ], + }, + { + id: 'heavy-feed', + title: 'Heavy feed: Bluesky-depth tree (≥65 native levels) and virtualized list', + setup: toSettingsLab('open-heavy-feed', 'id="heavy-feed-title"'), + steps: [ + step('snapshot-i', C.snapshot, ['snapshot', '-i']), + step('snapshot', C.snapshot, ['snapshot']), + step('snapshot-raw', C.snapshot, ['snapshot', '--raw'], { expect: { maxDepthAtLeast: 64 } }), + step('press-like-selector', C.press, ['press', 'id="heavy-post-0-like"']), + step('press-like-settle', C.press, ['press', 'id="heavy-post-1-like"', '--settle']), + step('press-like-ref', C.press, ['press', '{ref:heavy-post-0-like}']), + step('press-like-point', C.press, ['press', '{point:heavy-post-0-like}']), + step('wait-text', C.wait, ['wait', 'text', 'Liked posts: 2']), + step('is-visible', C.is, ['is', 'visible', 'id="heavy-post-1-like"']), + step('get-text', C.get, ['get', 'text', 'id="heavy-feed-last-action"']), + step('find-text', C.find, ['find', 'text', 'Post 1:', 'exists']), + step('longpress-post', C.longPress, ['longpress', 'id="heavy-post-1"', '800']), + step('scroll-down', C.scroll, ['scroll', 'down']), + step('snapshot-i-scrolled', C.snapshot, ['snapshot', '-i']), + step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), + step('swipe', C.swipe, ['swipe', '200', '650', '200', '300']), + // The pinned XCTest capture outlives its watchdog by about a second, and the next runner + // command refuses until it ends; the untimed wait keeps that cost out of the next step. + step('snapshot-raw-forced-tree', C.snapshot, ['snapshot', '--raw'], { + via: 'sdk-forced-tree', + expect: { treeBackendDegraded: true }, + after: [['wait', '2000']], + }), + step('back', C.back, ['back']), + ], + }, + { + id: 'session', + title: 'Session teardown', + setup: [], + steps: [step('close', C.close, ['close'])], + }, +]; + +/** + * Catalog commands the iOS suite does not time, with the reason. Every public command is either + * measured by a step above or listed here; `ios-suite.test.ts` holds that partition. + */ +export const IOS_SUITE_EXCLUSIONS: Partial> = { + [C.boot]: + 'Simulator power cycle: CoreSimulator boot time dominates and it ends the session under test.', + [C.shutdown]: 'Simulator power cycle: it ends the session and the runner under test.', + [C.install]: + 'Bundle provisioning: simctl copy time dominates; the job installs the fixture once.', + [C.reinstall]: + 'Bundle provisioning: simctl copy time dominates and it resets the app under test.', + [C.uninstall]: 'Bundle provisioning: it removes the app under test.', + [C.installFromSource]: + 'Bundle provisioning from a source build; the job installs a cached Release fixture.', + [C.replay]: 'Composite runner: a replay is the sum of the measured commands it runs.', + [C.test]: 'Composite runner: a test suite is the sum of the measured commands it runs.', + [C.push]: + 'The fixture holds no notification authorization, so simctl refuses the payload; iOS push coverage is contract-only.', + [C.audio]: 'Host audio capture needs a macOS audio permission that hosted runners do not grant.', + [C.hover]: 'The iOS runtime refuses hover; it is a web pointer contract.', + [C.viewport]: 'The iOS runtime refuses viewport; it resizes web viewports only.', + [C.tvRemote]: 'tvOS only.', + [C.fold]: 'iPhone Duo only; the suite device has one panel.', + [C.telephony]: 'The iOS simulator runtime refuses simulated calls and SMS.', + [C.hardwareButton]: 'The iOS simulator runtime refuses hardware-button presses.', +}; + +/** Partitions the measured and excluded catalog commands for the report. */ +export function describeCoverage( + scenarios: readonly SuiteScenario[], + exclusions: Partial>, +): { measured: string[]; excluded: { command: string; reason: string }[] } { + const measured = [...new Set(scenarios.flatMap((s) => s.steps.map((step) => step.command)))]; + return { + measured: measured.sort(), + excluded: Object.entries(exclusions) + .map(([command, reason]) => ({ command, reason: reason! })) + .sort((a, b) => a.command.localeCompare(b.command)), + }; +} diff --git a/scripts/perf/measurement.ts b/scripts/perf/measurement.ts new file mode 100644 index 0000000000..1641361a3a --- /dev/null +++ b/scripts/perf/measurement.ts @@ -0,0 +1,32 @@ +import { aggregateProfiles } from './profile.ts'; +import { summarize } from './stats.ts'; +import type { Measurement } from './types.ts'; + +type MeasurementInput = Omit< + Measurement, + 'wallClock' | 'daemon' | 'elementCount' | 'profile' | 'failures' | 'notes' +>; + +function numbers(values: (number | undefined)[]): number[] { + return values.filter((n): n is number => typeof n === 'number'); +} + +/** Summarizes the successful samples; failed samples count only toward `failures`. */ +export function buildMeasurement(input: MeasurementInput): Measurement { + const ok = input.samples.filter((s) => s.ok); + const failures = input.samples.length - ok.length; + const notes: string[] = []; + if (failures > 0) { + const codes = [...new Set(input.samples.filter((s) => !s.ok).map((s) => s.errorCode))]; + notes.push(`${failures}/${input.samples.length} failed: ${codes.join(', ')}`); + } + return { + ...input, + wallClock: summarize(ok.map((s) => s.wallClockMs)), + daemon: summarize(numbers(ok.map((s) => s.daemonMs))), + elementCount: summarize(numbers(ok.map((s) => s.elementCount ?? s.snapshot?.nodeCount))), + profile: aggregateProfiles(input.samples), + failures, + notes, + }; +} diff --git a/scripts/perf/metadata.ts b/scripts/perf/metadata.ts new file mode 100644 index 0000000000..6bb4e7d861 --- /dev/null +++ b/scripts/perf/metadata.ts @@ -0,0 +1,69 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { REPO_ROOT, type PerfConfig } from './config.ts'; +import type { RunMetadata } from './types.ts'; + +function capture(cmd: string, args: string[]): string | undefined { + try { + const result = runCmdSync(cmd, args, { cwd: REPO_ROOT, allowFailure: true }); + return result.exitCode === 0 ? result.stdout.trim() : undefined; + } catch { + return undefined; + } +} + +function readVersion(): string { + try { + const pkg = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')); + return typeof pkg.version === 'string' ? pkg.version : 'unknown'; + } catch { + return 'unknown'; + } +} + +function simulator(udid: string): { name: string; runtime?: string } | undefined { + const raw = capture('xcrun', ['simctl', 'list', 'devices', '--json']); + if (!raw) return undefined; + const devices = (JSON.parse(raw) as { devices: Record }) + .devices; + for (const [runtime, list] of Object.entries(devices)) { + const device = list.find((entry) => entry.udid === udid); + if (device) return { name: device.name, runtime: runtime.split('.').pop() }; + } + return undefined; +} + +/** Records what the numbers depend on: source, toolchain, host, and device. */ +export function collectMetadata(cfg: PerfConfig, deviceName: string): RunMetadata { + const env = process.env; + const sim = cfg.platform === 'ios' && cfg.udid ? simulator(cfg.udid) : undefined; + const cpus = os.cpus(); + return { + commit: env.GITHUB_SHA ?? capture('git', ['rev-parse', 'HEAD']) ?? 'unknown', + ref: env.GITHUB_REF_NAME ?? capture('git', ['rev-parse', '--abbrev-ref', 'HEAD']), + runId: env.GITHUB_RUN_ID, + runAttempt: env.GITHUB_RUN_ATTEMPT, + agentDeviceVersion: readVersion(), + host: { + os: `${os.type()} ${os.release()}`, + arch: os.arch(), + cpu: cpus[0]?.model ?? 'unknown', + cores: cpus.length, + memoryGb: Math.round(os.totalmem() / 1024 ** 3), + }, + xcode: + cfg.platform === 'ios' + ? capture('xcodebuild', ['-version'])?.replaceAll('\n', ' ') + : undefined, + node: process.version, + device: { + name: sim?.name ?? deviceName, + udid: cfg.udid, + serial: cfg.serial, + runtime: sim?.runtime, + }, + fixture: cfg.platform === 'ios' ? { appId: cfg.appId, appPath: cfg.appPath } : undefined, + }; +} diff --git a/scripts/perf/platform-profiles.ts b/scripts/perf/platform-profiles.ts index c082cdab06..447d3b310f 100644 --- a/scripts/perf/platform-profiles.ts +++ b/scripts/perf/platform-profiles.ts @@ -1,10 +1,7 @@ import type { PerfConfig } from './config.ts'; -import type { Platform } from './types.ts'; -// Local-convenience defaults for ad-hoc runs; CI always overrides them (--device / --serial). -// The iOS UDID is a specific local "iPhone 17" sim; the Android default is an AVD name that -// `boot` can launch. Pass --udid/--device/--serial to target your own device. -const DEFAULT_IOS_UDID = 'D74E0B66-57EB-4EC1-92DC-DA0A30581FE7'; +// Local-convenience default for ad-hoc Android runs: an AVD name that `boot` can launch. CI +// passes --serial. The iOS suite drives the fixture app instead (see ios-suite.ts). const DEFAULT_ANDROID_AVD = 'Pixel_9_Pro_XL_API_37'; export type ProfileSelectors = { @@ -14,10 +11,6 @@ export type ProfileSelectors = { searchField: string; // A selector that uniquely targets the EDITABLE search field (for fill). searchFieldEditable: string; - // iOS exposes an editable search field at the Settings root (fill works without focusing - // first; focusing then filling can hang). Android only reveals the editable after tapping - // the search card, so it must press the search entry before fill/type. - searchEditableAtRoot: boolean; // A label reliably visible on the Settings root, for get/is (selector form). anchorLabel: string; // Plain text of the anchor, for wait text / find (not a selector). @@ -25,9 +18,8 @@ export type ProfileSelectors = { }; export type ResolvedProfile = { - platform: Platform; + platform: 'android'; deviceName: string; - udid?: string; serial?: string; platformFlags: string[]; // --platform; applied to every call (only conflicts if it mismatches a locked session) selectorFlags: string[]; // device selectors — ONLY on the session-establishing open / selectorless boot @@ -36,27 +28,6 @@ export type ResolvedProfile = { }; export function resolveProfile(cfg: PerfConfig): ResolvedProfile { - if (cfg.platform === 'ios') { - // Prefer targeting by device name (CI boots a named simulator); fall back to a UDID. - const useName = cfg.device !== undefined; - const udid = useName ? undefined : (cfg.udid ?? DEFAULT_IOS_UDID); - return { - platform: 'ios', - deviceName: cfg.device ?? 'iPhone 17', - udid, - platformFlags: ['--platform', 'ios'], - selectorFlags: useName ? ['--device', cfg.device!] : ['--udid', udid!], - appTarget: 'settings', - selectors: { - deepScreen: 'label="General"', - searchField: 'label="Search"', - searchFieldEditable: 'label="Search" editable', - searchEditableAtRoot: true, - anchorLabel: 'label="General"', - anchorText: 'General', - }, - }; - } const avdName = cfg.device ?? DEFAULT_ANDROID_AVD; const serialFlags = cfg.serial ? ['--serial', cfg.serial, '--android-device-allowlist', cfg.serial] @@ -72,7 +43,6 @@ export function resolveProfile(cfg: PerfConfig): ResolvedProfile { deepScreen: 'text="Network & internet"', searchField: 'text="Search Settings"', searchFieldEditable: 'editable', - searchEditableAtRoot: false, anchorLabel: 'label="Network & internet"', anchorText: 'Network & internet', }, diff --git a/scripts/perf/profile.test.ts b/scripts/perf/profile.test.ts new file mode 100644 index 0000000000..91e16e4627 --- /dev/null +++ b/scripts/perf/profile.test.ts @@ -0,0 +1,112 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + aggregateProfiles, + profileRequestLog, + readClientDiagnostics, + readSettleFacts, +} from './profile.ts'; +import type { Sample } from './types.ts'; + +const line = (event: Record) => JSON.stringify({ level: 'info', ...event }); + +// Trimmed from a heavy-feed `press` request log. +const PRESS_LOG = [ + line({ ts: '2026-10-21T19:00:00.000Z', phase: 'request_start', data: { command: 'press' } }), + line({ + phase: 'exec_command', + durationMs: 0.02, + data: { command: 'xcrun', event: 'spawn_start' }, + }), + line({ phase: 'exec_command', durationMs: 9, data: { command: 'xcrun', event: 'spawn' } }), + line({ phase: 'exec_command', durationMs: 120, data: { command: 'xcrun', event: 'exit' } }), + line({ phase: 'exec_command', durationMs: 121, data: { command: 'xcrun', event: 'settled' } }), + line({ + level: 'error', + phase: 'ios.snapshot-source.acquire', + durationMs: 80, + data: { producer: 'simulator-ax-bridge', error: 'unsupported: foreground-owner-unverified' }, + }), + line({ + phase: 'ios.snapshot-source.acquire', + durationMs: 160, + data: { producer: 'simulator-ax-bridge' }, + }), + line({ + phase: 'ios_snapshot_source_recovery', + data: { producer: 'simulator-ax-bridge', truncated: true, acceptedLevels: 32 }, + }), + line({ phase: 'retry', data: { event: 'attempt_failed', attempt: 1 } }), + line({ phase: 'ios_runner_command_send', durationMs: 900, data: { command: 'tap' } }), + line({ phase: 'post_gesture_snapshot_stabilized', data: { attempts: 2, durationMs: 300 } }), + line({ + ts: '2026-10-21T19:00:01.250Z', + level: 'warn', + phase: 'ios_runner_session_invalidated', + data: { reason: 'stale' }, + }), + 'not json', +].join('\n'); + +test('profiles runner, spawn, snapshot-source, retry, and settle phases of one request', () => { + const profile = profileRequestLog(PRESS_LOG); + assert.deepEqual(profile.spawns, { xcrun: { count: 1, totalMs: 121 } }); + assert.deepEqual(profile.runnerCommands, { tap: { count: 1, totalMs: 900 } }); + assert.deepEqual(profile.snapshotSources, { + 'simulator-ax-bridge': { count: 2, totalMs: 240, failures: 1 }, + }); + assert.equal(profile.retries, 1); + assert.equal(profile.requestSpanMs, 1250); + assert.deepEqual(profile.stabilization, { count: 1, totalMs: 300 }); + assert.deepEqual(profile.fallbacks, [ + 'ios_runner_session_invalidated: stale', + 'simulator-ax-bridge: truncated at 32 levels', + 'simulator-ax-bridge: unsupported: foreground-owner-unverified', + ]); +}); + +test('counts a producer only for samples where it answered', () => { + const failedOnly = profileRequestLog( + line({ + level: 'error', + phase: 'ios.snapshot-source.acquire', + durationMs: 5, + data: { producer: 'simulator-ax-bridge', error: 'cancelled' }, + }), + ); + const runnerSnapshot = profileRequestLog( + line({ phase: 'ios_runner_command_send', durationMs: 4000, data: { command: 'snapshot' } }), + ); + const samples: Sample[] = [ + { round: 0, ok: true, wallClockMs: 1, profile: failedOnly }, + { + round: 1, + ok: true, + wallClockMs: 1, + profile: runnerSnapshot, + snapshot: { backend: 'private-ax', state: 'recovered', reasonCode: 'ax-rejected' }, + }, + { round: 2, ok: false, wallClockMs: 1, profile: profileRequestLog(PRESS_LOG) }, + ]; + const aggregate = aggregateProfiles(samples)!; + assert.deepEqual(aggregate.producers, { 'apple-runner': 1 }); + assert.deepEqual(aggregate.backends, { 'private-ax/recovered/ax-rejected': 1 }); +}); + +test('reads the client daemon timer from --debug stderr', () => { + const stderr = [ + '[agent-device][diag] {"phase":"daemon_startup","durationMs":30,"requestId":"r1"}', + 'noise', + '[agent-device][diag] {"phase":"daemon_request","durationMs":812,"requestId":"r1"}', + ].join('\n'); + assert.deepEqual(readClientDiagnostics(stderr), { daemonMs: 812, requestId: 'r1' }); + assert.deepEqual(readClientDiagnostics('[agent-device][diag] {"phase":"daemon_req'), {}); +}); + +test('reads the settle wait of a --settle response and nothing from other responses', () => { + assert.deepEqual( + readSettleFacts({ data: { settle: { settled: true, waitedMs: 1665, captures: 2 } } }), + { waitedMs: 1665, captures: 2, settled: true }, + ); + assert.equal(readSettleFacts({ data: { message: 'Tapped' } }), undefined); +}); diff --git a/scripts/perf/profile.ts b/scripts/perf/profile.ts new file mode 100644 index 0000000000..a869d74c32 --- /dev/null +++ b/scripts/perf/profile.ts @@ -0,0 +1,228 @@ +import { median } from './stats.ts'; +import type { + MeasurementProfile, + Sample, + SampleProfile, + SettleFacts, + SnapshotFacts, + Timing, +} from './types.ts'; + +type DiagnosticEvent = { + ts?: string; + level?: string; + phase?: string; + durationMs?: number; + data?: Record; +}; + +const RUNNER_COMMAND_PHASE = 'ios_runner_command_send'; +const SPAWN_PHASE = 'exec_command'; +const SNAPSHOT_SOURCE_PHASE = 'ios.snapshot-source.acquire'; +const SNAPSHOT_RECOVERY_PHASE = 'ios_snapshot_source_recovery'; +const STABILIZATION_PHASE = /settle|stabiliz/i; +const FALLBACK_TEXT_LIMIT = 140; +const CLIENT_DIAG_PREFIX = '[agent-device][diag] '; + +function addTiming(map: Record, key: string, durationMs: number | undefined): void { + const entry = (map[key] ??= { count: 0, totalMs: 0 }); + entry.count += 1; + entry.totalMs += durationMs ?? 0; +} + +function text(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 + ? value.slice(0, FALLBACK_TEXT_LIMIT) + : undefined; +} + +function eventDuration(event: DiagnosticEvent): number | undefined { + if (typeof event.durationMs === 'number') return event.durationMs; + const nested = event.data?.durationMs; + return typeof nested === 'number' ? nested : undefined; +} + +/** + * Reads one daemon request log (`--debug` ndjson). Process spawns count once, at their `settled` + * event. Fallbacks name every degraded path the request took: snapshot-source errors, truncated + * recovery, and warn- or error-level phases. + */ +export function profileRequestLog(ndjson: string): SampleProfile { + const profile: SampleProfile = { + phases: {}, + runnerCommands: {}, + spawns: {}, + snapshotSources: {}, + retries: 0, + stabilization: { count: 0, totalMs: 0 }, + fallbacks: [], + }; + const fallbacks = new Set(); + let firstTs: number | undefined; + let lastTs: number | undefined; + for (const line of ndjson.split('\n')) { + if (!line.trim()) continue; + let event: DiagnosticEvent; + try { + event = JSON.parse(line) as DiagnosticEvent; + } catch { + continue; + } + const phase = event.phase; + if (!phase) continue; + const ts = typeof event.ts === 'string' ? Date.parse(event.ts) : Number.NaN; + if (Number.isFinite(ts)) { + firstTs ??= ts; + lastTs = ts; + } + const data = event.data ?? {}; + const duration = eventDuration(event); + if (phase === SPAWN_PHASE) { + if (data.event !== 'settled') continue; + addTiming(profile.spawns, text(data.command) ?? 'unknown', duration); + } else if (phase === RUNNER_COMMAND_PHASE) { + addTiming(profile.runnerCommands, text(data.command) ?? 'unknown', duration); + } else if (phase === SNAPSHOT_SOURCE_PHASE) { + const producer = text(data.producer) ?? 'unknown'; + const entry = (profile.snapshotSources[producer] ??= { count: 0, totalMs: 0, failures: 0 }); + entry.count += 1; + entry.totalMs += duration ?? 0; + const error = text(data.error); + if (error || event.level === 'error') { + entry.failures += 1; + fallbacks.add(`${producer}: ${error ?? 'failed'}`); + } + } else if (phase === SNAPSHOT_RECOVERY_PHASE && data.truncated === true) { + fallbacks.add( + `${text(data.producer) ?? 'unknown'}: truncated at ${String(data.acceptedLevels)} levels`, + ); + } else if (phase === 'retry' && data.event === 'attempt_failed') { + profile.retries += 1; + } else if (event.level === 'warn' || event.level === 'error') { + fallbacks.add(`${phase}${text(data.reason) ? `: ${text(data.reason)}` : ''}`); + } + if (STABILIZATION_PHASE.test(phase)) { + profile.stabilization.count += 1; + profile.stabilization.totalMs += duration ?? 0; + } + addTiming(profile.phases, phase, duration); + } + profile.fallbacks = [...fallbacks].sort(); + if (firstTs !== undefined && lastTs !== undefined) profile.requestSpanMs = lastTs - firstTs; + return profile; +} + +/** Reads the client's `daemon_request` timer and request id from `--debug` stderr. */ +export function readClientDiagnostics(stderr: string): { daemonMs?: number; requestId?: string } { + for (const line of stderr.split('\n')) { + const start = line.indexOf(CLIENT_DIAG_PREFIX); + if (start < 0) continue; + try { + const event = JSON.parse(line.slice(start + CLIENT_DIAG_PREFIX.length)) as { + phase?: string; + durationMs?: number; + requestId?: string; + }; + if (event.phase === 'daemon_request') { + return { daemonMs: event.durationMs, requestId: event.requestId }; + } + } catch { + // A truncated diagnostic line carries no timer. + } + } + return {}; +} + +/** Reads the settle outcome of a `--settle` response. */ +export function readSettleFacts(json: unknown): SettleFacts | undefined { + const settle = (json as { data?: { settle?: Record } } | undefined)?.data + ?.settle; + if (!settle || typeof settle.waitedMs !== 'number') return undefined; + return { + waitedMs: settle.waitedMs, + captures: typeof settle.captures === 'number' ? settle.captures : undefined, + settled: typeof settle.settled === 'boolean' ? settle.settled : undefined, + }; +} + +/** Reads snapshot quality facts from a `--json` snapshot response. */ +export function readSnapshotFacts(json: unknown): SnapshotFacts | undefined { + const data = (json as { data?: Record } | undefined)?.data; + if (!data || !Array.isArray(data.nodes)) return undefined; + const nodes = data.nodes as { depth?: unknown }[]; + const quality = (data.snapshotQuality ?? {}) as Record; + const depths = nodes.map((node) => (typeof node.depth === 'number' ? node.depth : 0)); + return { + backend: text(quality.backend), + state: text(quality.state), + reasonCode: text(quality.reasonCode), + nodeCount: nodes.length, + maxDepth: depths.length > 0 ? Math.max(...depths) : undefined, + truncated: typeof data.truncated === 'boolean' ? data.truncated : undefined, + }; +} + +function medianOrNull(values: number[]): number | null { + return values.length === 0 ? null : median(values); +} + +function sumTimings(map: Record): Timing { + return Object.values(map).reduce( + (sum, t) => ({ count: sum.count + t.count, totalMs: sum.totalMs + t.totalMs }), + { count: 0, totalMs: 0 }, + ); +} + +function tally(counts: Record, keys: Iterable): void { + for (const key of new Set(keys)) counts[key] = (counts[key] ?? 0) + 1; +} + +/** + * Summarizes the successful samples' profiles. Medians describe a typical request; producers, + * backends, and fallbacks count how many samples took each path. + */ +export function aggregateProfiles(samples: Sample[]): MeasurementProfile | null { + const profiled = samples.filter((s) => s.ok && s.profile); + if (profiled.length === 0) return null; + const profiles = profiled.map((s) => s.profile!); + const producers: Record = {}; + const backends: Record = {}; + const fallbacks: Record = {}; + for (const sample of profiled) { + const profile = sample.profile!; + tally( + producers, + Object.entries(profile.snapshotSources) + .filter(([, source]) => source.count > source.failures) + .map(([producer]) => producer), + ); + if (profile.runnerCommands.snapshot) tally(producers, ['apple-runner']); + if (sample.snapshot?.backend) { + const { backend, state, reasonCode } = sample.snapshot; + tally(backends, [[backend, state, reasonCode].filter(Boolean).join('/')]); + } + tally(fallbacks, profile.fallbacks); + } + const phaseNames = new Set(profiles.flatMap((p) => Object.keys(p.phases))); + const topPhases = [...phaseNames] + .map((phase) => ({ + phase, + medianMs: median(profiles.map((p) => p.phases[phase]?.totalMs ?? 0)), + })) + .filter((entry) => entry.medianMs > 0) + .sort((a, b) => b.medianMs - a.medianMs) + .slice(0, 5); + return { + runnerMs: medianOrNull(profiles.map((p) => sumTimings(p.runnerCommands).totalMs)), + spawnCount: medianOrNull(profiles.map((p) => sumTimings(p.spawns).count)), + spawnMs: medianOrNull(profiles.map((p) => sumTimings(p.spawns).totalMs)), + snapshotSourceMs: medianOrNull(profiles.map((p) => sumTimings(p.snapshotSources).totalMs)), + stabilizationMs: medianOrNull(profiles.map((p) => p.stabilization.totalMs)), + settleWaitMs: medianOrNull(profiled.flatMap((s) => (s.settle ? [s.settle.waitedMs] : []))), + retries: profiles.reduce((sum, p) => sum + p.retries, 0), + producers, + backends, + fallbacks, + topPhases, + }; +} diff --git a/scripts/perf/report.ts b/scripts/perf/report.ts index 536639eadb..480978638d 100644 --- a/scripts/perf/report.ts +++ b/scripts/perf/report.ts @@ -1,73 +1,134 @@ import fs from 'node:fs'; import path from 'node:path'; -import type { Measurement, RunResult, Stat } from './types.ts'; +import type { Measurement, PerfSummary, Stat } from './types.ts'; -function ms(n: number | undefined): string { +export const SUMMARY_FILE = 'summary.json'; +export const REPORT_FILE = 'report.md'; + +function ms(n: number | null | undefined): string { return typeof n === 'number' && Number.isFinite(n) ? n.toFixed(0) : '–'; } +function pct(n: number | undefined): string { + return typeof n === 'number' && Number.isFinite(n) ? `${(n * 100).toFixed(0)}%` : '–'; +} + +function keys(counts: Record | undefined): string { + const entries = Object.entries(counts ?? {}); + return entries.length === 0 ? '–' : entries.map(([key, n]) => `${key}×${n}`).join(', '); +} + function wallCells(s: Stat | null): string { - if (!s) return '– | – | – | –'; - return `${ms(s.min)} | ${ms(s.median)} | ${ms(s.p95)} | ${ms(s.max)}`; + if (!s) return '– | – | – | – | – | –'; + return `${ms(s.mean)} ± ${ms(s.stddev)} | ${ms(s.median)} | ${ms(s.p95)} | ${ms(s.min)} | ${ms(s.max)} | ${pct(s.cv)}`; } -function stampName(platform: string, startedAt: string): string { - return `perf-${platform}-${startedAt.replaceAll(/[:.]/g, '-')}`; +function cell(text: string): string { + return text.replaceAll('|', String.raw`\|`); } -function measurementRow(m: Measurement): string { - const daemon = m.daemonDuration ? ms(m.daemonDuration.median) : '–'; - const elements = m.elementCount ? ms(m.elementCount.median) : '–'; - const n = m.wallClock?.n ?? 0; - return `| ${m.label} | ${m.command} | ${m.execMode} | ${n} | ${wallCells(m.wallClock)} | ${daemon} | ${elements} | ${m.notes.join('; ')} |`; +function timingRow(m: Measurement): string { + return [ + m.id.slice(m.scenario.length + 1), + `\`${cell(m.label)}\``, + String(m.samples.length), + wallCells(m.wallClock), + ms(m.daemon?.median), + String(m.failures), + ].join(' | '); } -function toMarkdown(run: RunResult): string { - const lines: string[] = []; - lines.push(`# agent-device command perf — ${run.platform}`); - lines.push(''); - lines.push(`- **Device**: ${run.device.name} (${run.device.udid ?? run.device.serial ?? '?'})`); - lines.push(`- **agent-device**: ${run.agentDeviceVersion}`); - lines.push(`- **Rounds**: ${run.config.rounds} (warmup ${run.config.warmup} dropped)`); - lines.push(`- **Started**: ${run.startedAt}`); - lines.push(`- **Finished**: ${run.finishedAt}`); - lines.push(''); - lines.push('All times in milliseconds. `wall-clock` includes process spawn + socket overhead;'); - lines.push( - '`daemon` is the batch step round-trip (spawn overhead ≈ wall-median − daemon-median).', - ); - lines.push('`elements` = node count in the snapshot payload (tree-size proxy).'); - lines.push('An untimed warmup interaction runs after each open/relaunch, so measured commands'); - lines.push('do not pay the one-time iOS-runner startup or post-relaunch first-AX-query cost.'); - lines.push(''); - lines.push( - '| command | cli | mode | n | wall min | wall median | wall p95 | wall max | daemon median | elements | notes |', - ); - lines.push('|---|---|---|---|---|---|---|---|---|---|---|'); - for (const m of run.measurements) lines.push(measurementRow(m)); - lines.push(''); +function profileRow(m: Measurement): string { + const p = m.profile; + return [ + m.id.slice(m.scenario.length + 1), + ms(p?.runnerMs), + p?.spawnCount == null ? '–' : `${p.spawnCount} / ${ms(p.spawnMs)}`, + ms(p?.snapshotSourceMs), + ms(p?.settleWaitMs), + ms(p?.stabilizationMs), + String(p?.retries ?? '–'), + cell(keys(p?.producers)), + cell(p?.topPhases.map((t) => `${t.phase} ${ms(t.medianMs)}`).join(', ') || '–'), + ].join(' | '); +} - const failed = run.measurements.filter((m) => m.failures > 0); - if (failed.length > 0) { - lines.push('## Failures'); +/** Renders the human report for one run. */ +export function renderReport(summary: PerfSummary): string { + const meta = summary.metadata; + const lines: string[] = [ + `# agent-device perf — ${summary.platform}`, + '', + `- **Commit**: ${meta.commit.slice(0, 12)}${meta.ref ? ` (${meta.ref})` : ''}${meta.runId ? ` · run ${meta.runId}` : ''}`, + `- **Device**: ${meta.device.name}${meta.device.runtime ? ` · ${meta.device.runtime}` : ''}`, + `- **Toolchain**: ${meta.xcode ?? 'no Xcode'} · Node ${meta.node} · agent-device ${meta.agentDeviceVersion}`, + `- **Host**: ${meta.host.os} · ${meta.host.cpu} · ${meta.host.cores} cores · ${meta.host.memoryGb} GB`, + `- **Rounds**: ${summary.config.rounds} measured after ${summary.config.warmup} warmup · ${summary.startedAt} → ${summary.finishedAt}`, + '', + 'Times are milliseconds. See scripts/perf/README.md for each metric. The second table of each', + 'scenario holds per-request medians from the daemon `--debug` request logs.', + '', + ]; + const scenarios = [...new Set(summary.measurements.map((m) => m.scenario))]; + for (const scenario of scenarios) { + const measurements = summary.measurements.filter((m) => m.scenario === scenario); + lines.push( + `## ${scenario}`, + '', + '| step | command | n | wall mean ± sd | median | p95 | min | max | cv | daemon | fail |', + '|---|---|---|---|---|---|---|---|---|---|---|', + ...measurements.map((m) => `| ${timingRow(m)} |`), + '', + '| step | runner | spawns | snapshot source | settle wait | stabilization | retries | producer | top phases |', + '|---|---|---|---|---|---|---|---|---|', + ...measurements.map((m) => `| ${profileRow(m)} |`), + '', + ); + } + const degraded = summary.measurements.filter( + (m) => + Object.keys(m.profile?.fallbacks ?? {}).length > 0 || + Object.keys(m.profile?.backends ?? {}).length > 0, + ); + if (degraded.length > 0) { + lines.push('## Backends and fallbacks', ''); + for (const m of degraded) { + lines.push( + `- **${m.id}** — backends ${keys(m.profile?.backends)}; fallbacks ${keys(m.profile?.fallbacks)}`, + ); + } lines.push(''); + } + const failed = summary.measurements.filter((m) => m.failures > 0); + if (failed.length > 0) { + lines.push('## Failures', ''); for (const m of failed) { const sample = m.samples.find((s) => !s.ok); lines.push( - `- **${m.label}** — ${m.notes.join('; ')}${sample?.errorMessage ? ` — ${sample.errorMessage}` : ''}`, + `- **${m.id}** — ${m.notes.join('; ')}${sample?.errorMessage ? ` — ${sample.errorMessage}` : ''}`, ); } lines.push(''); } + lines.push( + '## Coverage', + '', + `Measured commands (${summary.coverage.measured.length}): ${summary.coverage.measured.join(', ')}.`, + '', + ...summary.coverage.excluded.map((e) => `- \`${e.command}\` not measured: ${e.reason}`), + '', + ); return lines.join('\n'); } -export function writeReports(run: RunResult, outDir: string): { jsonPath: string; mdPath: string } { +export function writeRunOutput( + summary: PerfSummary, + outDir: string, +): { jsonPath: string; mdPath: string } { fs.mkdirSync(outDir, { recursive: true }); - const base = stampName(run.platform, run.startedAt); - const jsonPath = path.join(outDir, `${base}.json`); - const mdPath = path.join(outDir, `${base}.md`); - fs.writeFileSync(jsonPath, JSON.stringify(run, null, 2)); - fs.writeFileSync(mdPath, toMarkdown(run)); + const jsonPath = path.join(outDir, SUMMARY_FILE); + const mdPath = path.join(outDir, REPORT_FILE); + fs.writeFileSync(jsonPath, `${JSON.stringify(summary, null, 2)}\n`); + fs.writeFileSync(mdPath, renderReport(summary)); return { jsonPath, mdPath }; } diff --git a/scripts/perf/run.ts b/scripts/perf/run.ts index f1f390cc4e..51cff7c18d 100644 --- a/scripts/perf/run.ts +++ b/scripts/perf/run.ts @@ -1,72 +1,129 @@ import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; -import { parseConfig, REPO_ROOT, usesSourceCli } from './config.ts'; +import { invokeCli } from './cli.ts'; +import { parseConfig, REPO_ROOT, usesSourceCli, type PerfConfig } from './config.ts'; +import { runScenario, setupIsolation, teardownIsolation } from './harness.ts'; +import { copyRawState, runIosSuite, type IosSuiteContext } from './ios-harness.ts'; import { - runScenario, - setupIsolation, - teardownIsolation, - type IsolationContext, -} from './harness.ts'; -import { writeReports } from './report.ts'; -import type { RunResult } from './types.ts'; + describeCoverage, + FIXTURE_APP_EVENT_URL_TEMPLATE, + IOS_SUITE, + IOS_SUITE_EXCLUSIONS, +} from './ios-suite.ts'; +import { collectMetadata } from './metadata.ts'; +import { writeRunOutput } from './report.ts'; +import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary } from './types.ts'; -function readVersion(): string { - try { - const pkg = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')); - return typeof pkg.version === 'string' ? pkg.version : 'unknown'; - } catch { - return 'unknown'; - } +function log(msg: string): void { + process.stderr.write(`[perf] ${msg}\n`); } -function main(): void { - const cfg = parseConfig(process.argv.slice(2)); - // The dist binary needs a build; running from source (AGENT_DEVICE_PERF_CLI) does not. - if (!usesSourceCli() && !fs.existsSync(path.join(REPO_ROOT, 'dist', 'src'))) { - process.stderr.write('[perf] dist/ is missing — run `pnpm build` first.\n'); - process.exit(1); - } - - const startedAt = new Date().toISOString(); - let ctx: IsolationContext | null = null; - let exitCode = 0; - - const cleanup = (): void => { - if (ctx) { - teardownIsolation(ctx, cfg); - ctx = null; - } - }; - process.on('SIGINT', () => { +function onInterrupt(cleanup: () => void): void { + process.once('SIGINT', () => { cleanup(); process.exit(130); }); - process.on('SIGTERM', () => { + process.once('SIGTERM', () => { cleanup(); process.exit(143); }); +} + +function selectScenarios(cfg: PerfConfig) { + if (!cfg.scenarios) return IOS_SUITE; + const unknown = cfg.scenarios.filter((id) => !IOS_SUITE.some((s) => s.id === id)); + if (unknown.length > 0) { + throw new Error( + `Unknown scenarios: ${unknown.join(', ')} (known: ${IOS_SUITE.map((s) => s.id).join(', ')})`, + ); + } + return IOS_SUITE.filter((s) => cfg.scenarios!.includes(s.id)); +} +async function runIos( + cfg: PerfConfig, +): Promise<{ measurements: Measurement[]; coverage: PerfSummary['coverage'] }> { + const scenarios = selectScenarios(cfg); + // The daemon inherits this from the first CLI call; it routes trigger-app-event to the fixture. + process.env.AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE ??= FIXTURE_APP_EVENT_URL_TEMPLATE; + const rawDir = path.join(cfg.outDir, 'raw'); + const ctx: IosSuiteContext = { + stateDir: fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-perf-')), + udid: cfg.udid!, + appId: cfg.appId, + artifactsDir: path.join(rawDir, 'artifacts'), + rawDir, + }; + fs.mkdirSync(ctx.artifactsDir, { recursive: true }); + log(`state-dir: ${ctx.stateDir}`); + const stop = (): void => { + invokeCli(['close'], ['--state-dir', ctx.stateDir, '--session', 'perf']); + invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); + }; + onInterrupt(stop); + try { + const measurements = await runIosSuite(ctx, cfg, scenarios); + return { measurements, coverage: describeCoverage(scenarios, IOS_SUITE_EXCLUSIONS) }; + } finally { + stop(); + copyRawState(ctx); + if (!cfg.keepArtifacts) fs.rmSync(ctx.stateDir, { recursive: true, force: true }); + } +} + +function runAndroid(cfg: PerfConfig): { + measurements: Measurement[]; + coverage: PerfSummary['coverage']; +} { + const ctx = setupIsolation(cfg); + onInterrupt(() => teardownIsolation(ctx, cfg)); try { - ctx = setupIsolation(cfg); const measurements = runScenario(ctx, cfg); - const run: RunResult = { - startedAt, - finishedAt: new Date().toISOString(), - platform: cfg.platform, - device: { udid: ctx.profile.udid, serial: ctx.profile.serial, name: ctx.profile.deviceName }, - config: { rounds: cfg.rounds, warmup: cfg.warmup, keepArtifacts: cfg.keepArtifacts }, - agentDeviceVersion: readVersion(), + return { measurements, + coverage: { measured: [...new Set(measurements.map((m) => m.command))].sort(), excluded: [] }, }; - const { jsonPath, mdPath } = writeReports(run, cfg.outDir); - process.stderr.write(`\n[perf] report: ${mdPath}\n[perf] json: ${jsonPath}\n`); - } catch (error) { - process.stderr.write(`[perf] error: ${(error as Error).stack ?? String(error)}\n`); - exitCode = 1; } finally { - cleanup(); + teardownIsolation(ctx, cfg); } - process.exit(exitCode); } -main(); +async function main(): Promise { + const cfg = parseConfig(process.argv.slice(2)); + // The iOS forced-backend probe loads the built SDK; the dist CLI needs the build as well. + if ( + (cfg.platform === 'ios' || !usesSourceCli()) && + !fs.existsSync(path.join(REPO_ROOT, 'dist', 'src')) + ) { + throw new Error('dist/ is missing — run `pnpm build` first.'); + } + const startedAt = new Date().toISOString(); + // Read before the run, so a checkout change during a long run cannot relabel it. + const metadata = collectMetadata(cfg, cfg.serial ?? cfg.device ?? 'unknown'); + const { measurements, coverage } = cfg.platform === 'ios' ? await runIos(cfg) : runAndroid(cfg); + const summary: PerfSummary = { + schema: PERF_SUMMARY_SCHEMA, + platform: cfg.platform, + startedAt, + finishedAt: new Date().toISOString(), + config: { + rounds: cfg.rounds, + warmup: cfg.warmup, + scenarios: [...new Set(measurements.map((m) => m.scenario))], + }, + metadata, + coverage, + measurements, + }; + const { jsonPath, mdPath } = writeRunOutput(summary, cfg.outDir); + log(`report: ${mdPath}`); + log(`summary: ${jsonPath}`); +} + +main() + .then(() => process.exit(0)) + .catch((error: unknown) => { + process.stderr.write(`[perf] error: ${(error as Error).stack ?? String(error)}\n`); + process.exit(1); + }); diff --git a/scripts/perf/scenario.ts b/scripts/perf/scenario.ts index d99ddd2bba..1e34cad240 100644 --- a/scripts/perf/scenario.ts +++ b/scripts/perf/scenario.ts @@ -37,7 +37,7 @@ function bat( return { label, command, execMode: 'batch' as const, step, ...opts }; } -// One ordered pass over Settings. The harness repeats this N (+warmup) times; +// One ordered pass over Android Settings. The harness repeats this N (+warmup) times; // the leading `open --relaunch` resets the app to its root each round, so every // round starts from a known state while commands run in their natural order. export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): ScenarioStep[] { @@ -46,92 +46,33 @@ export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): Scenari const rec = path.join(ctx.artifactsDir, 'rec.mp4'); const trace = path.join(ctx.artifactsDir, 'trace.log'); - // Text entry differs per platform: iOS fills the root search field directly (focusing it - // first can hang); Android must open the search screen before an editable field exists. - const textEntry: ScenarioStep[] = p.selectors.searchEditableAtRoot - ? [ - // iOS: editable search field exists at root; fill it directly (freshRoot resets scroll). - bat( - 'fill search', - 'fill', - { - command: 'fill', - input: { - target: { kind: 'selector', selector: s.searchFieldEditable }, - text: 'general', - }, - }, - { freshRoot: true }, - ), - bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), - bat('get editable text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.searchFieldEditable }, - }, - }), - bat('keyboard return', 'keyboard', { command: 'keyboard', input: { action: 'return' } }), - ] - : [ - // Android: tap the search entry first to reveal the editable, then type/fill it. - bat( - 'press search field', - 'press', - { - command: 'press', - input: { target: { kind: 'selector', selector: s.searchField } }, - }, - { freshRoot: true }, - ), - bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), - bat('fill search', 'fill', { - command: 'fill', - input: { - target: { kind: 'selector', selector: s.searchFieldEditable }, - text: 'general', - }, - }), - bat('get editable text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.searchFieldEditable }, - }, - }), - ]; - - // These iOS-only repeated gesture forms fuse into `sequence` runner requests: - // press --count > 1 and swipe --count > 1 both batch their steps into one command. - const iosRunnerSeries: ScenarioStep[] = - p.platform === 'ios' - ? [ - bat( - 'press series (sequence)', - 'press', - { - command: 'press', - input: { target: { kind: 'point', x: 200, y: 95 }, count: 2, intervalMs: 50 }, - }, - { freshRoot: true }, - ), - bat( - 'swipe series (sequence)', - 'swipe', - { - command: 'swipe', - input: { - from: { x: 200, y: 650 }, - to: { x: 200, y: 450 }, - count: 2, - pauseMs: 50, - pattern: 'ping-pong', - }, - }, - { freshRoot: true }, - ), - ] - : []; + // Android reveals the editable search field only after a tap on the search card. + const textEntry: ScenarioStep[] = [ + bat( + 'press search field', + 'press', + { + command: 'press', + input: { target: { kind: 'selector', selector: s.searchField } }, + }, + { freshRoot: true }, + ), + bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), + bat('fill search', 'fill', { + command: 'fill', + input: { + target: { kind: 'selector', selector: s.searchFieldEditable }, + text: 'general', + }, + }), + bat('get editable text', 'get', { + command: 'get', + input: { + format: 'text', + target: { kind: 'selector', selector: s.searchFieldEditable }, + }, + }), + ]; return [ // --- reset to root via relaunch --- @@ -166,9 +107,6 @@ export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): Scenari ), bat('back', 'back', { command: 'back', input: {} }), - // --- iOS runner series commands surfaced by PR #643 --- - ...iosRunnerSeries, - // --- targeted reads against the visible anchor (freshRoot so the anchor is on screen) --- bat( 'wait text', diff --git a/scripts/perf/stats.test.ts b/scripts/perf/stats.test.ts new file mode 100644 index 0000000000..088c7e1806 --- /dev/null +++ b/scripts/perf/stats.test.ts @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { summarize } from './stats.ts'; + +test('summarize reports sample spread, the middle median, and nearest-rank p95', () => { + const stat = summarize([40, 10, 30, 20, Number.NaN])!; + assert.equal(stat.n, 4); + assert.equal(stat.mean, 25); + // Sample (n − 1) deviation: sqrt((15² + 5² + 5² + 15²) / 3). + assert.ok(Math.abs(stat.stddev - Math.sqrt(500 / 3)) < 1e-9); + assert.ok(Math.abs(stat.cv - Math.sqrt(500 / 3) / 25) < 1e-9); + assert.deepEqual([stat.min, stat.median, stat.p95, stat.max], [10, 25, 40, 40]); +}); + +test('summarize gives a single sample zero spread and no samples no summary', () => { + assert.deepEqual(summarize([7]), { + n: 1, + mean: 7, + stddev: 0, + cv: 0, + median: 7, + min: 7, + max: 7, + p95: 7, + }); + assert.equal(summarize([]), null); + assert.equal(summarize([0, 0])?.cv, 0); +}); diff --git a/scripts/perf/stats.ts b/scripts/perf/stats.ts index 14b846cac2..f972ec4e37 100644 --- a/scripts/perf/stats.ts +++ b/scripts/perf/stats.ts @@ -1,22 +1,43 @@ import type { Stat } from './types.ts'; -// Nearest-rank percentile over a copy of the values. +// Nearest-rank percentile over sorted values. function percentile(sorted: number[], p: number): number { - if (sorted.length === 0) return Number.NaN; const rank = Math.ceil((p / 100) * sorted.length); const idx = Math.min(sorted.length - 1, Math.max(0, rank - 1)); - return sorted[idx]; + return sorted[idx]!; } +/** + * Describes one sample set. `stddev` is the sample standard deviation (n − 1), so it is 0 for a + * single value; `cv` is stddev / mean and is 0 when the mean is 0. + */ export function summarize(values: number[]): Stat | null { const clean = values.filter((v) => Number.isFinite(v)); if (clean.length === 0) return null; const sorted = [...clean].sort((a, b) => a - b); + const n = sorted.length; + const mean = sorted.reduce((sum, v) => sum + v, 0) / n; + const variance = n > 1 ? sorted.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (n - 1) : 0; + const stddev = Math.sqrt(variance); return { - n: sorted.length, - min: sorted[0], - median: percentile(sorted, 50), + n, + mean, + stddev, + cv: mean === 0 ? 0 : stddev / mean, + median: middle(sorted), + min: sorted[0]!, + max: sorted.at(-1)!, p95: percentile(sorted, 95), - max: sorted.at(-1), }; } + +// The median of sorted values; an even count averages the two middle values. +function middle(sorted: number[]): number { + const half = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[half]! : (sorted[half - 1]! + sorted[half]!) / 2; +} + +export function median(values: number[]): number { + const sorted = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b); + return sorted.length === 0 ? Number.NaN : middle(sorted); +} diff --git a/scripts/perf/types.ts b/scripts/perf/types.ts index 175481a889..a3cc091be5 100644 --- a/scripts/perf/types.ts +++ b/scripts/perf/types.ts @@ -1,8 +1,8 @@ -// Shared data shapes for the e2e perf benchmark harness. +// Shared data shapes for the perf suite and its night-over-night comparison. export type Platform = 'ios' | 'android'; -export type ExecMode = 'batch' | 'standalone'; +export type ExecMode = 'batch' | 'standalone' | 'sdk'; export type CliResult = { exitCode: number; @@ -13,38 +13,114 @@ export type CliResult = { ok: boolean; // exit 0 AND (json.ok !== false) }; +export type Stat = { + n: number; + mean: number; + stddev: number; + cv: number; + median: number; + min: number; + max: number; + p95: number; +}; + +export type Timing = { count: number; totalMs: number }; + +/** Phase breakdown of one request, read from its daemon `--debug` request log. */ +export type SampleProfile = { + phases: Record; + runnerCommands: Record; + spawns: Record; + snapshotSources: Record; + retries: number; + stabilization: Timing; + fallbacks: string[]; + // Daemon-side request time: from `request_start` to the last event of the request log. + requestSpanMs?: number; +}; + +/** The settle outcome that a `--settle` response reports. */ +export type SettleFacts = { waitedMs: number; captures?: number; settled?: boolean }; + +export type SnapshotFacts = { + backend?: string; + state?: string; + reasonCode?: string; + nodeCount?: number; + maxDepth?: number; + truncated?: boolean; +}; + export type Sample = { round: number; wallClockMs: number; - daemonDurationMs?: number; // from batch results[0].durationMs (batch mode only) - elementCount?: number; // for snapshot rows: parsed @eN count, a tree-size proxy + // Daemon request round trip: the client's `daemon_request` timer for CLI calls, the batch step + // duration for batch calls, or the request-log span for SDK calls. + daemonMs?: number; + elementCount?: number; + requestId?: string; + snapshot?: SnapshotFacts; + settle?: SettleFacts; + profile?: SampleProfile; ok: boolean; errorCode?: string; errorMessage?: string; }; -export type Stat = { n: number; min: number; median: number; p95: number; max: number }; +export type MeasurementProfile = { + runnerMs: number | null; + spawnCount: number | null; + spawnMs: number | null; + snapshotSourceMs: number | null; + stabilizationMs: number | null; + settleWaitMs: number | null; + retries: number; + producers: Record; + backends: Record; + fallbacks: Record; + topPhases: { phase: string; medianMs: number }[]; +}; export type Measurement = { - command: string; + id: string; // stable `/` comparison key + scenario: string; label: string; + command: string; // catalog command name + args: string[]; platform: Platform; execMode: ExecMode; - samples: Sample[]; // kept samples only (warmup rounds dropped) + samples: Sample[]; // measured rounds only (warmup rounds dropped) warmupDropped: number; wallClock: Stat | null; - daemonDuration: Stat | null; // null for standalone or when no ok samples - elementCount: Stat | null; // null unless snapshot row + daemon: Stat | null; + elementCount: Stat | null; + profile: MeasurementProfile | null; failures: number; notes: string[]; }; -export type RunResult = { +export type RunMetadata = { + commit: string; + ref?: string; + runId?: string; + runAttempt?: string; + agentDeviceVersion: string; + host: { os: string; arch: string; cpu: string; cores: number; memoryGb: number }; + xcode?: string; + node: string; + device: { name: string; udid?: string; serial?: string; runtime?: string }; + fixture?: { appId: string; appPath?: string }; +}; + +export const PERF_SUMMARY_SCHEMA = 'agent-device-perf-summary/v1'; + +export type PerfSummary = { + schema: typeof PERF_SUMMARY_SCHEMA; + platform: Platform; startedAt: string; finishedAt: string; - platform: Platform; - device: { udid?: string; serial?: string; name: string }; - config: { rounds: number; warmup: number; keepArtifacts: boolean }; - agentDeviceVersion: string; + config: { rounds: number; warmup: number; scenarios: string[] }; + metadata: RunMetadata; + coverage: { measured: string[]; excluded: { command: string; reason: string }[] }; measurements: Measurement[]; }; diff --git a/vitest.config.ts b/vitest.config.ts index b5a54e80f7..5da1a3c156 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -196,6 +196,8 @@ export default defineConfig({ 'scripts/__tests__/release-version.test.ts', 'scripts/__tests__/release-plan.test.ts', 'scripts/ios-snapshot-benchmark/*.test.ts', + // The perf suite's pure logic, without a device. + 'scripts/perf/*.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts', 'test/ci/upload-artifact-hidden-paths.test.ts', From 00d3d0cc405017dded24c1371f74a17d4582a0da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 04:52:54 +0200 Subject: [PATCH 03/15] feat(perf): compare nightly iOS runs and run the suite every night (#3886) pnpm perf:compare judges one run against earlier runs: by default the seven nights before it, or a chosen run, date, range, or local output. A step's baseline is the median of its nightly medians. A change is flagged only when it is larger than 20%, three times the night-to-night scaled MAD, twice the run's own cv/sqrt(n), and 50 ms; new failures and fixed steps are flagged too. perf-nightly runs the iOS suite at 02:40 UTC again (#1781 A3). Each run uploads perf-ios-summary (90 days) and perf-ios-raw (30 days), and writes the comparison to the job summary with warnings for regressions and new failures. Android stays report-only behind a dispatch input. Co-authored-by: Claude Opus 5.5 --- .github/workflows/perf-nightly.yml | 101 +++++-- package.json | 1 + scripts/perf/README.md | 36 +++ scripts/perf/compare-cli.ts | 196 ++++++++++++++ scripts/perf/compare.test.ts | 104 +++++++ scripts/perf/compare.ts | 270 +++++++++++++++++++ scripts/perf/nightly-artifacts.test.ts | 70 +++++ scripts/perf/nightly-artifacts.ts | 159 +++++++++++ scripts/perf/stats.test.ts | 7 +- scripts/perf/stats.ts | 6 + scripts/perf/summary-fixture.ts | 64 +++++ scripts/perf/summary.test.ts | 32 +++ scripts/perf/summary.ts | 63 +++++ test/repo-guards/npm-package-scripts.test.ts | 2 +- 14 files changed, 1088 insertions(+), 23 deletions(-) create mode 100644 scripts/perf/compare-cli.ts create mode 100644 scripts/perf/compare.test.ts create mode 100644 scripts/perf/compare.ts create mode 100644 scripts/perf/nightly-artifacts.test.ts create mode 100644 scripts/perf/nightly-artifacts.ts create mode 100644 scripts/perf/summary-fixture.ts create mode 100644 scripts/perf/summary.test.ts create mode 100644 scripts/perf/summary.ts diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index 1d6c4938a5..243cd08c17 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -1,29 +1,57 @@ name: Perf Nightly -# Parked to workflow_dispatch by #1781 A3: it only writes a report and compares nothing, so it -# structurally cannot catch a regression, and iOS wall-clock medians swing up to +122% -# night-to-night at n=5 with no consumer reading the report. `pnpm perf` / scripts/perf are -# unaffected — still runnable on demand. Un-park once a comparison step with multi-night -# smoothing (someone reading it) exists, or a perf arc needs nightly data. +# Nightly iOS command performance against the fixture app, compared night over night (#1781 A3). +# - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts +# failing. PR lanes check behavior, not time. +# - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians +# swing up to +122% between single nights at n=5, so a step is judged against the median of up to +# seven earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. +# - Cost: one macos-26 job of about 45 minutes a night; summaries are kept 90 days, raw logs 30. +# - Delete or park when: no flagged change has been acted on for 60 nights. +# Android stays report-only and runs only on request. on: + schedule: + - cron: '40 2 * * *' workflow_dispatch: inputs: rounds: - description: 'Measured rounds per command (samples)' + description: 'Measured rounds per step (samples)' required: false default: '3' + warmup: + description: 'Warmup rounds dropped before measuring' + required: false + default: '1' + scenarios: + description: 'Comma-separated iOS scenario ids (empty runs all)' + required: false + default: '' + baseline-nights: + description: 'Earlier runs in the rolling baseline' + required: false + default: '7' + baseline-branch: + description: 'Branch whose runs form the baseline' + required: false + default: 'main' + android: + description: 'Also run the report-only Android Settings tour' + type: boolean + required: false + default: true permissions: contents: read actions: read concurrency: - group: ci-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: perf-nightly-${{ github.ref }} + cancel-in-progress: false env: PERF_ROUNDS: ${{ github.event.inputs.rounds || '3' }} + PERF_WARMUP: ${{ github.event.inputs.warmup || '1' }} jobs: perf-ios: @@ -52,6 +80,13 @@ jobs: with: gate: build + - name: Boot iOS test simulator + id: ios-simulator + uses: ./.github/actions/boot-ios-test-simulator + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + preferred-device-name: iPhone 17 Pro + - name: Restore and build iOS XCTest runner uses: ./.github/actions/setup-apple-runner-build with: @@ -62,13 +97,6 @@ jobs: xcuitest-platform: ios xcuitest-destination: generic/platform=iOS Simulator - - name: Boot iOS test simulator - id: ios-simulator - uses: ./.github/actions/boot-ios-test-simulator - with: - runtime-version: ${{ env.IOS_RUNTIME_VERSION }} - preferred-device-name: iPhone 17 Pro - - name: Install the fixture app id: fixture-app uses: ./.github/actions/setup-fixture-app @@ -88,22 +116,53 @@ jobs: UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} APP_ID: ${{ steps.fixture-app.outputs.app-id }} APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + SCENARIOS: ${{ github.event.inputs.scenarios }} run: | pnpm perf --platform ios --udid "$UDID" \ --app-id "$APP_ID" --app-path "$APP_PATH" \ - --rounds "$PERF_ROUNDS" --warmup 1 \ + --rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP" \ + ${SCENARIOS:+--scenarios "$SCENARIOS"} \ --out-dir "$GITHUB_WORKSPACE/perf-results" - - name: Upload iOS perf report - if: always() + - name: Compare with earlier nights + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} + env: + GH_TOKEN: ${{ github.token }} + BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }} + BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }} + run: | + pnpm perf:compare --to perf-results \ + --branch "$BASELINE_BRANCH" \ + --nights "$BASELINE_NIGHTS" \ + --exclude-run "$GITHUB_RUN_ID" \ + --out perf-results --github + echo "Full report: the perf-ios-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY" + + - name: Upload iOS perf summary + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: perf-ios - path: perf-results/ + name: perf-ios-summary + path: | + perf-results/summary.json + perf-results/report.md + perf-results/comparison.json + perf-results/comparison.md + retention-days: 90 + if-no-files-found: error + + - name: Upload iOS perf raw logs + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: perf-ios-raw + path: perf-results/raw/ + retention-days: 30 if-no-files-found: warn perf-android: name: Android Command Perf + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.android == 'true' }} runs-on: ubuntu-latest timeout-minutes: 80 env: @@ -134,7 +193,7 @@ jobs: adb -s emulator-5554 shell settings put global window_animation_scale 0 || true; adb -s emulator-5554 shell settings put global transition_animation_scale 0 || true; adb -s emulator-5554 shell settings put global animator_duration_scale 0 || true; - pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup 1 --out-dir "$GITHUB_WORKSPACE/perf-results" + pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup "$PERF_WARMUP" --out-dir "$GITHUB_WORKSPACE/perf-results" - name: Upload Android perf report if: always() diff --git a/package.json b/package.json index 07a4584c2b..b19f477964 100644 --- a/package.json +++ b/package.json @@ -131,6 +131,7 @@ "test:ios-snapshot-differential": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift test --package-path apple/snapshot-presentation && node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/ios-snapshot-differential.test.ts", "size": "node scripts/size-report.mjs", "perf": "node --experimental-strip-types scripts/perf/run.ts", + "perf:compare": "node --experimental-strip-types scripts/perf/compare-cli.ts", "bench:ios-snapshot": "node --experimental-strip-types scripts/ios-snapshot-benchmark/run.ts", "bench:ios-snapshot:deep-button": "node --experimental-strip-types scripts/ios-snapshot-benchmark/deep-button.ts", "bench:ios-snapshot:evidence": "node --experimental-strip-types scripts/ios-snapshot-benchmark/evidence.ts", diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 42d61d4301..4f088d0aea 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -49,3 +49,39 @@ All times are milliseconds. Statistics use only the successful samples of a step | backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected`. | | fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. | | `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth. | + +## Nightly results + +`.github/workflows/perf-nightly.yml` runs the iOS suite every night on `main`. +Each run uploads `perf-ios-summary` (summary, report, comparison; 90 days) and `perf-ios-raw` (logs; 30 days). +The job compares the run with up to seven earlier runs and writes the result to the job summary. + +To fetch and compare runs on your machine, use `gh` with access to the repository: + +```sh +pnpm perf:compare # latest night against the 7 nights before it +pnpm perf:compare --to 2026-10-28 # that night against the 7 nights before it +pnpm perf:compare --to latest --from 2026-10-21 # latest against one night +pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare +pnpm perf:compare --to .tmp/perf/local # a local run against recent nights +``` + +A reference is `latest`, a run id, a `YYYY-MM-DD` date, or a local output path. +Downloads stay in `.tmp/perf-nightly//`. +Do not judge a local run against CI nights. The comparison warns when the device, Xcode, or CPU differs. + +## Comparison rule + +The baseline of a step is the median of its nightly medians. +The noise of a step is the scaled median absolute deviation of those medians. +A step is a `regression` or an `improvement` only when its change is more than all of these: + +- 20% of the baseline. +- Three times the night-to-night noise. +- Two times the coefficient of variation of the run divided by √n. +- 50 ms. + +The rolling baseline skips runs that did not run every scenario of the judged run, such as a partial dispatch. +Fewer than three baseline nights give `insufficient-baseline`. +`new-failures` means that the step failed and no baseline night failed. `fixed` means the reverse. +The thresholds are in `compare.ts`. diff --git a/scripts/perf/compare-cli.ts b/scripts/perf/compare-cli.ts new file mode 100644 index 0000000000..0336beca96 --- /dev/null +++ b/scripts/perf/compare-cli.ts @@ -0,0 +1,196 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { compareRuns, renderComparison, type Comparison } from './compare.ts'; +import { REPO_ROOT } from './config.ts'; +import { + baselineCandidates, + coversTarget, + fetchSummary, + listRuns, + readLocalSummary, + resolveRunRef, + selectRange, + SUMMARY_ARTIFACT, + type NightlySource, + type WorkflowRun, +} from './nightly-artifacts.ts'; +import type { PerfSummary } from './types.ts'; + +const USAGE = `Usage: pnpm perf:compare [--to ] [--from |..] [--nights ] [options] + +Compares one perf run with earlier runs and prints a Markdown verdict per step. + +A is latest, a workflow run id, a YYYY-MM-DD date (the newest run that UTC day), or a local +path to a summary.json or a \`pnpm perf\` output directory. + + --to Run to judge (default: latest). + --from Baseline run, or an inclusive range ... Without it the baseline is + the --nights runs before --to. + --nights Rolling baseline size (default: 7). + --branch Branch whose nightly runs are listed (default: main). + --repo Repository (default: $GITHUB_REPOSITORY or callstack/agent-device). + --workflow Workflow file (default: perf-nightly.yml). + --cache-dir Download cache (default: .tmp/perf-nightly). + --exclude-run Never use this run as baseline (repeatable). + --out Also write comparison.json and comparison.md there. + --github Append the Markdown to $GITHUB_STEP_SUMMARY and annotate flagged steps.`; + +type Options = { + to: string; + from?: string; + nights: number; + out?: string; + github: boolean; + exclude: Set; + source: NightlySource; +}; + +function parse(argv: string[]): Options { + const options: Options = { + to: 'latest', + nights: 7, + github: false, + exclude: new Set(), + source: { + repo: process.env.GITHUB_REPOSITORY ?? 'callstack/agent-device', + workflow: 'perf-nightly.yml', + branch: 'main', + artifact: SUMMARY_ARTIFACT, + cacheDir: path.join(REPO_ROOT, '.tmp', 'perf-nightly'), + }, + }; + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]!; + const value = (): string => { + const next = argv[++i]; + if (next === undefined) throw new Error(`Missing value for ${flag}`); + return next; + }; + switch (flag) { + case '--to': + options.to = value(); + break; + case '--from': + options.from = value(); + break; + case '--nights': { + const n = Number(value()); + if (!Number.isInteger(n) || n < 1) throw new Error('--nights must be a positive integer'); + options.nights = n; + break; + } + case '--branch': + options.source.branch = value(); + break; + case '--repo': + options.source.repo = value(); + break; + case '--workflow': + options.source.workflow = value(); + break; + case '--cache-dir': + options.source.cacheDir = path.resolve(value()); + break; + case '--exclude-run': + options.exclude.add(value()); + break; + case '--out': + options.out = path.resolve(value()); + break; + case '--github': + options.github = true; + break; + case '--help': + case '-h': + process.stdout.write(`${USAGE}\n`); + process.exit(0); + break; + default: + throw new Error(`Unknown flag: ${flag}\n\n${USAGE}`); + } + } + return options; +} + +type Loaded = { summary: PerfSummary; label: string; run?: WorkflowRun }; + +function main(): void { + const options = parse(process.argv.slice(2)); + let runs: WorkflowRun[] | undefined; + const allRuns = (): WorkflowRun[] => (runs ??= listRuns(options.source)); + const runLabel = (run: WorkflowRun) => `run ${run.id} (${run.createdAt.slice(0, 10)})`; + const load = (ref: string): Loaded => { + if (fs.existsSync(ref)) + return { summary: readLocalSummary(ref), label: path.basename(path.resolve(ref)) }; + const run = resolveRunRef(ref, allRuns()); + if (!run) + throw new Error( + `No completed ${options.source.workflow} run on ${options.source.branch} matches ${ref}.`, + ); + const summary = fetchSummary(options.source, run); + if (!summary) throw new Error(`Run ${run.id} has no ${options.source.artifact} artifact.`); + return { summary, label: runLabel(run), run }; + }; + const loadRuns = ( + candidates: WorkflowRun[], + limit: number, + accept: (summary: PerfSummary) => boolean = () => true, + ): Loaded[] => { + const loaded: Loaded[] = []; + for (const run of candidates) { + if (loaded.length >= limit) break; + const summary = fetchSummary(options.source, run); + if (summary && accept(summary)) loaded.push({ summary, label: runLabel(run), run }); + } + return loaded; + }; + + const target = load(options.to); + const exclude = new Set([...options.exclude, ...(target.run ? [target.run.id] : [])]); + let baseline: Loaded[]; + if (options.from?.includes('..')) { + const [fromRef, toRef] = options.from.split('..'); + const range = selectRange(allRuns(), fromRef!, toRef!).filter((run) => !exclude.has(run.id)); + baseline = loadRuns(range, range.length); + } else if (options.from) { + baseline = [load(options.from)]; + } else { + const before = target.run?.createdAt ?? target.summary.startedAt; + baseline = loadRuns(baselineCandidates(allRuns(), before, exclude), options.nights, (summary) => + coversTarget(summary, target.summary), + ); + } + + const comparison = compareRuns(target, baseline); + const markdown = renderComparison(comparison); + process.stdout.write(`${markdown}\n`); + if (options.out) writeComparison(comparison, markdown, options.out); + if (options.github) publishToGitHub(comparison, markdown); +} + +function writeComparison(comparison: Comparison, markdown: string, dir: string): void { + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'comparison.json'), `${JSON.stringify(comparison, null, 2)}\n`); + fs.writeFileSync(path.join(dir, 'comparison.md'), markdown); +} + +function publishToGitHub(comparison: Comparison, markdown: string): void { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (summaryPath) fs.appendFileSync(summaryPath, `${markdown}\n`); + for (const row of comparison.rows) { + if (row.verdict === 'regression' || row.verdict === 'new-failures') { + const change = + row.verdict === 'regression' + ? `median ${row.target?.median.toFixed(0)} ms vs ${row.baseline?.median.toFixed(0)} ms baseline` + : `${row.target?.failures}/${row.target?.samples} samples failed`; + process.stdout.write(`::warning title=perf ${row.verdict}::${row.id}: ${change}\n`); + } + } +} + +try { + main(); +} catch (error) { + process.stderr.write(`[perf:compare] ${(error as Error).message}\n`); + process.exit(1); +} diff --git a/scripts/perf/compare.test.ts b/scripts/perf/compare.test.ts new file mode 100644 index 0000000000..d7046d1860 --- /dev/null +++ b/scripts/perf/compare.test.ts @@ -0,0 +1,104 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { compareRuns, type ComparisonRow } from './compare.ts'; +import { summaryFixture, type StepFixture } from './summary-fixture.ts'; + +const STEP = 'heavy-feed/snapshot'; + +function nights(...steps: StepFixture[][]) { + return steps.map((night, index) => ({ summary: summaryFixture(night), label: `night ${index}` })); +} + +function row(target: StepFixture[], baseline: StepFixture[][], id = STEP): ComparisonRow { + const comparison = compareRuns( + { summary: summaryFixture(target), label: 'tonight' }, + nights(...baseline), + ); + return comparison.rows.find((r) => r.id === id)!; +} + +const flat = (ms: number, extra: Partial = {}): StepFixture[] => [ + { id: STEP, daemonMs: [ms, ms, ms], ...extra }, +]; + +test('flags a slowdown that exceeds a quiet baseline and the relative floor', () => { + const result = row( + [{ id: STEP, daemonMs: [1450, 1500, 1550] }], + [flat(1000), flat(1010), flat(990)], + ); + assert.equal(result.verdict, 'regression'); + assert.equal(result.metric, 'daemon'); + assert.equal(result.baseline?.median, 1000); + assert.equal(result.deltaMs, 500); + assert.equal(result.threshold, 0.2); +}); + +test('does not flag the same slowdown when nightly medians already swing that much', () => { + const result = row( + [{ id: STEP, daemonMs: [1450, 1500, 1550] }], + [flat(500), flat(1000), flat(1500), flat(700), flat(1300)], + ); + assert.equal(result.verdict, 'unchanged'); + // 3 × scaled MAD: deviations from 1000 are 0, 300, 300, 500, 500. + assert.ok(Math.abs(result.threshold! - (3 * 1.4826 * 300) / 1000) < 1e-9); +}); + +test('does not flag a large relative change below the absolute floor', () => { + assert.equal(row(flat(60), [flat(40), flat(40), flat(40)]).verdict, 'unchanged'); +}); + +test('flags a speedup beyond noise as an improvement', () => { + assert.equal(row(flat(500), [flat(1000), flat(1010), flat(990)]).verdict, 'improvement'); +}); + +test('withholds a timing verdict until enough nights exist', () => { + const result = row(flat(1500), [flat(1000), flat(1000)]); + assert.equal(result.verdict, 'insufficient-baseline'); + assert.equal(result.deltaRelative, 0.5); +}); + +test('reports failures that no baseline night had, and failures that went away', () => { + const clean = [flat(1000), flat(1000), flat(1000)]; + assert.equal(row(flat(1000, { failures: 1 }), clean).verdict, 'new-failures'); + const failing = [ + flat(1000, { failures: 1 }), + flat(1000, { failures: 2 }), + flat(1000, { failures: 1 }), + ]; + assert.equal(row(flat(1000), failing).verdict, 'fixed'); + // A step that already failed on some baseline night is judged on timing instead. + const flaky = [flat(1000, { failures: 1 }), flat(1000), flat(1000)]; + assert.equal(row(flat(1000, { failures: 1 }), flaky).verdict, 'unchanged'); +}); + +test('falls back to wall time when any run lacks the daemon timer', () => { + const result = row(flat(1000), [ + flat(1000), + [{ id: STEP, wallMs: [1300, 1300, 1300] }], + flat(1000), + ]); + assert.equal(result.metric, 'wall'); +}); + +test('marks steps that only one side measured', () => { + const comparison = compareRuns( + { summary: summaryFixture([{ id: 'a/new', daemonMs: [1] }]), label: 't' }, + [{ summary: summaryFixture([{ id: 'a/old', daemonMs: [1] }]), label: 'b' }], + ); + assert.deepEqual( + comparison.rows.map((r) => [r.id, r.verdict]), + [ + ['a/new', 'new'], + ['a/old', 'missing'], + ], + ); +}); + +test('warns when the baseline ran on another toolchain', () => { + const comparison = compareRuns( + { summary: summaryFixture(flat(1000), { xcode: 'Xcode 26.3' }), label: 't' }, + [{ summary: summaryFixture(flat(1000)), label: 'b' }], + ); + assert.equal(comparison.warnings.length, 1); + assert.match(comparison.warnings[0]!, /xcode/); +}); diff --git a/scripts/perf/compare.ts b/scripts/perf/compare.ts new file mode 100644 index 0000000000..11dae30520 --- /dev/null +++ b/scripts/perf/compare.ts @@ -0,0 +1,270 @@ +import { median, scaledMad } from './stats.ts'; +import type { Measurement, PerfSummary, Stat } from './types.ts'; + +export type ComparisonThresholds = { + // A change must exceed every one of these to be flagged. + minRelative: number; // fraction of the baseline median + minAbsoluteMs: number; + spreadFactor: number; // × night-to-night relative spread (scaled MAD of nightly medians) + withinFactor: number; // × tonight's cv / √n (standard error of tonight's median) + minBaselineNights: number; // fewer nights give `insufficient-baseline` +}; + +const DEFAULT_THRESHOLDS: ComparisonThresholds = { + minRelative: 0.2, + minAbsoluteMs: 50, + spreadFactor: 3, + withinFactor: 2, + minBaselineNights: 3, +}; + +export type Verdict = + | 'regression' + | 'improvement' + | 'unchanged' + | 'new-failures' + | 'fixed' + | 'failing' + | 'insufficient-baseline' + | 'new' + | 'missing'; + +/** Verdicts that need a reader's attention, in report order. */ +const FLAGGED_VERDICTS: readonly Verdict[] = ['new-failures', 'regression', 'fixed', 'improvement']; + +export type ComparisonRow = { + id: string; + scenario: string; + command: string; + verdict: Verdict; + metric: 'daemon' | 'wall'; + target: { median: number; n: number; cv: number; failures: number; samples: number } | null; + baseline: { median: number; nights: number; spread: number; failingNights: number } | null; + deltaMs: number | null; + deltaRelative: number | null; + threshold: number | null; +}; + +export type RunRef = { + label: string; + runId?: string; + commit: string; + startedAt: string; + device: string; + xcode?: string; + rounds: number; +}; + +const COMPARISON_SCHEMA = 'agent-device-perf-comparison/v1'; + +export type Comparison = { + schema: typeof COMPARISON_SCHEMA; + target: RunRef; + baseline: RunRef[]; + thresholds: ComparisonThresholds; + counts: Partial>; + warnings: string[]; + rows: ComparisonRow[]; +}; + +function describeRun(summary: PerfSummary, label: string): RunRef { + return { + label, + runId: summary.metadata.runId, + commit: summary.metadata.commit, + startedAt: summary.startedAt, + device: `${summary.metadata.device.name}${summary.metadata.device.runtime ? ` ${summary.metadata.device.runtime}` : ''}`, + xcode: summary.metadata.xcode, + rounds: summary.config.rounds, + }; +} + +function metricStat(m: Measurement, metric: 'daemon' | 'wall'): Stat | null { + return metric === 'daemon' ? m.daemon : m.wallClock; +} + +function compareStep( + id: string, + target: Measurement | undefined, + history: Measurement[], + thresholds: ComparisonThresholds, +): ComparisonRow { + const any = target ?? history[0]!; + const base = { id, scenario: any.scenario, command: any.command }; + const metric: 'daemon' | 'wall' = [target, ...history].every( + (m) => !m || !m.wallClock || m.daemon, + ) + ? 'daemon' + : 'wall'; + const empty = { deltaMs: null, deltaRelative: null, threshold: null }; + const okNights = history + .map((m) => metricStat(m, metric)) + .filter((s): s is Stat => s !== null) + .map((s) => s.median); + const failingNights = history.filter((m) => m.failures > 0).length; + const baselineMedian = okNights.length > 0 ? median(okNights) : null; + const baseline = + history.length === 0 + ? null + : { + median: baselineMedian ?? Number.NaN, + nights: history.length, + spread: baselineMedian ? scaledMad(okNights) / baselineMedian : 0, + failingNights, + }; + if (!target) return { ...base, verdict: 'missing', metric, target: null, baseline, ...empty }; + const stat = metricStat(target, metric); + const targetFacts = stat + ? { + median: stat.median, + n: stat.n, + cv: stat.cv, + failures: target.failures, + samples: target.samples.length, + } + : { + median: Number.NaN, + n: 0, + cv: 0, + failures: target.failures, + samples: target.samples.length, + }; + const row = { ...base, metric, target: targetFacts, baseline }; + if (!baseline) return { ...row, verdict: 'new', ...empty }; + if (target.failures > 0 && failingNights === 0) + return { ...row, verdict: 'new-failures', ...empty }; + if (target.failures === 0 && failingNights === history.length) + return { ...row, verdict: 'fixed', ...empty }; + if (!stat || baselineMedian === null) return { ...row, verdict: 'failing', ...empty }; + + const deltaMs = stat.median - baselineMedian; + const deltaRelative = baselineMedian === 0 ? 0 : deltaMs / baselineMedian; + const threshold = Math.max( + thresholds.minRelative, + thresholds.spreadFactor * baseline.spread, + (thresholds.withinFactor * stat.cv) / Math.sqrt(stat.n), + ); + const measured = { deltaMs, deltaRelative, threshold }; + if (okNights.length < thresholds.minBaselineNights) { + return { ...row, verdict: 'insufficient-baseline', ...measured }; + } + const beyondNoise = + Math.abs(deltaRelative) > threshold && Math.abs(deltaMs) > thresholds.minAbsoluteMs; + const verdict: Verdict = !beyondNoise ? 'unchanged' : deltaMs > 0 ? 'regression' : 'improvement'; + return { ...row, verdict, ...measured }; +} + +function environmentWarnings(target: PerfSummary, baseline: PerfSummary[]): string[] { + const facts = (s: PerfSummary) => ({ + device: `${s.metadata.device.name} ${s.metadata.device.runtime ?? ''}`.trim(), + xcode: s.metadata.xcode ?? 'none', + cpu: s.metadata.host.cpu, + }); + const mine = facts(target); + const warnings: string[] = []; + for (const key of ['device', 'xcode', 'cpu'] as const) { + const differing = baseline.filter((s) => facts(s)[key] !== mine[key]).length; + if (differing > 0) { + warnings.push( + `${differing}/${baseline.length} baseline runs used a different ${key} than tonight (${mine[key]}); their timings are not like for like.`, + ); + } + } + return warnings; +} + +/** + * Compares one run against a rolling baseline of earlier runs. The baseline value of a step is the + * median of its nightly medians, and its noise is the scaled MAD of those medians. A step is flagged + * only when its change exceeds the relative floor, the night-to-night spread, tonight's own + * sampling error, and an absolute floor together. + */ +export function compareRuns( + target: { summary: PerfSummary; label: string }, + baseline: { summary: PerfSummary; label: string }[], + thresholds: ComparisonThresholds = DEFAULT_THRESHOLDS, +): Comparison { + const targetById = new Map(target.summary.measurements.map((m) => [m.id, m])); + const ids = [ + ...target.summary.measurements.map((m) => m.id), + ...baseline.flatMap((b) => b.summary.measurements.map((m) => m.id)), + ]; + const rows = [...new Set(ids)].map((id) => + compareStep( + id, + targetById.get(id), + baseline.flatMap((b) => b.summary.measurements.filter((m) => m.id === id)), + thresholds, + ), + ); + const counts: Partial> = {}; + for (const row of rows) counts[row.verdict] = (counts[row.verdict] ?? 0) + 1; + return { + schema: COMPARISON_SCHEMA, + target: describeRun(target.summary, target.label), + baseline: baseline.map((b) => describeRun(b.summary, b.label)), + thresholds, + counts, + warnings: environmentWarnings( + target.summary, + baseline.map((b) => b.summary), + ), + rows, + }; +} + +function ms(n: number | null | undefined): string { + return typeof n === 'number' && Number.isFinite(n) ? n.toFixed(0) : '–'; +} + +function pct(n: number | null | undefined, signed = false): string { + if (typeof n !== 'number' || !Number.isFinite(n)) return '–'; + const value = `${(n * 100).toFixed(0)}%`; + return signed && n > 0 ? `+${value}` : value; +} + +function tableRow(row: ComparisonRow): string { + const failures = row.target && row.target.failures > 0 ? ` (${row.target.failures} failed)` : ''; + return `| ${row.verdict} | \`${row.id}\` | ${row.metric} | ${ms(row.target?.median)}${failures} | ${ms(row.baseline?.median)} | ${ms(row.deltaMs)} | ${pct(row.deltaRelative, true)} | ${pct(row.threshold)} | ${row.baseline?.nights ?? 0} |`; +} + +const TABLE_HEAD = [ + '| verdict | step | metric | tonight median | baseline median | Δ ms | Δ | threshold | nights |', + '|---|---|---|---|---|---|---|---|---|', +]; + +/** Renders the comparison for the job summary and the artifact. */ +export function renderComparison(comparison: Comparison): string { + const t = comparison.thresholds; + const flagged = comparison.rows + .filter((row) => FLAGGED_VERDICTS.includes(row.verdict)) + .sort((a, b) => FLAGGED_VERDICTS.indexOf(a.verdict) - FLAGGED_VERDICTS.indexOf(b.verdict)); + const counts = Object.entries(comparison.counts) + .map(([verdict, n]) => `${n} ${verdict}`) + .join(', '); + const lines = [ + `## agent-device perf comparison`, + '', + `- **Tonight**: ${comparison.target.label} · ${comparison.target.commit.slice(0, 12)} · ${comparison.target.startedAt} · ${comparison.target.device}`, + `- **Baseline**: ${comparison.baseline.length === 0 ? 'none' : comparison.baseline.map((b) => `${b.label} (${b.commit.slice(0, 7)})`).join(', ')}`, + `- **Rule**: flag a step when its median moves by more than max(${pct(t.minRelative)}, ${t.spreadFactor}× night-to-night spread, ${t.withinFactor}× tonight cv/√n) and more than ${t.minAbsoluteMs} ms against the median of the baseline nights; at least ${t.minBaselineNights} nights are needed for a verdict.`, + `- **Steps**: ${counts || 'none'}`, + '', + ]; + for (const warning of comparison.warnings) lines.push(`> ⚠️ ${warning}`, ''); + if (flagged.length > 0) { + lines.push(...TABLE_HEAD, ...flagged.map(tableRow), ''); + } else { + lines.push('No step moved beyond its noise threshold.', ''); + } + lines.push( + '
All steps', + '', + ...TABLE_HEAD, + ...comparison.rows.map(tableRow), + '', + '
', + '', + ); + return lines.join('\n'); +} diff --git a/scripts/perf/nightly-artifacts.test.ts b/scripts/perf/nightly-artifacts.test.ts new file mode 100644 index 0000000000..74671a936d --- /dev/null +++ b/scripts/perf/nightly-artifacts.test.ts @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + baselineCandidates, + coversTarget, + resolveRunRef, + selectRange, + type WorkflowRun, +} from './nightly-artifacts.ts'; +import { summaryFixture } from './summary-fixture.ts'; + +const run = (id: string, createdAt: string): WorkflowRun => ({ + id, + createdAt, + headSha: `sha-${id}`, + event: 'schedule', +}); + +// Deliberately unsorted, with two runs on 2026-10-19. +const RUNS = [ + run('101', '2026-10-17T03:00:00Z'), + run('104', '2026-10-20T03:00:00Z'), + run('102', '2026-10-19T03:00:00Z'), + run('103', '2026-10-19T15:30:00Z'), +]; + +test('resolves latest, run ids, and dates to the newest matching run', () => { + assert.equal(resolveRunRef('latest', RUNS)?.id, '104'); + assert.equal(resolveRunRef('102', RUNS)?.id, '102'); + assert.equal(resolveRunRef('2026-10-19', RUNS)?.id, '103'); + assert.equal(resolveRunRef('2026-10-18', RUNS), undefined); + assert.throws(() => resolveRunRef('yesterday', RUNS), /Unknown run reference/); +}); + +test('baseline candidates are earlier runs, newest first, without excluded runs', () => { + const candidates = baselineCandidates(RUNS, '2026-10-20T03:00:00Z', new Set(['103'])); + assert.deepEqual( + candidates.map((r) => r.id), + ['102', '101'], + ); +}); + +test('a range includes both ends whichever order they are given in', () => { + const ids = (from: string, to: string) => selectRange(RUNS, from, to).map((r) => r.id); + assert.deepEqual(ids('101', '102'), ['102', '101']); + assert.deepEqual(ids('102', '101'), ['102', '101']); +}); + +test('a date end of a range covers its whole UTC day', () => { + // 2026-10-19 has a 03:00 run and a later 15:30 run; both belong to the range. + assert.deepEqual( + selectRange(RUNS, '2026-10-17', '2026-10-19').map((r) => r.id), + ['103', '102', '101'], + ); + assert.deepEqual( + selectRange(RUNS, '2026-10-19', '104').map((r) => r.id), + ['104', '103', '102'], + ); +}); + +test('a rolling baseline accepts only runs that measured every target scenario', () => { + const full = summaryFixture([ + { id: 'a/x', daemonMs: [1] }, + { id: 'b/y', daemonMs: [1] }, + ]); + const partial = summaryFixture([{ id: 'b/y', daemonMs: [1] }]); + assert.equal(coversTarget(full, full), true); + assert.equal(coversTarget(partial, full), false); + assert.equal(coversTarget(full, partial), true); +}); diff --git a/scripts/perf/nightly-artifacts.ts b/scripts/perf/nightly-artifacts.ts new file mode 100644 index 0000000000..08b00b7797 --- /dev/null +++ b/scripts/perf/nightly-artifacts.ts @@ -0,0 +1,159 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { SUMMARY_FILE } from './report.ts'; +import { parsePerfSummary } from './summary.ts'; +import type { PerfSummary } from './types.ts'; + +/** The stable name of the summary artifact that the nightly workflow uploads for every run. */ +export const SUMMARY_ARTIFACT = 'perf-ios-summary'; + +export type WorkflowRun = { + id: string; + createdAt: string; + headSha: string; + event: string; +}; + +export type NightlySource = { + repo: string; + workflow: string; + branch: string; + artifact: string; + cacheDir: string; +}; + +const DATE_REF = /^\d{4}-\d{2}-\d{2}$/; + +function gh(args: string[]): { ok: boolean; stdout: string; stderr: string } { + const result = runCmdSync('gh', args, { allowFailure: true, maxBuffer: 64 * 1024 * 1024 }); + return { ok: result.exitCode === 0, stdout: result.stdout, stderr: result.stderr }; +} + +/** Lists the workflow's completed runs on one branch, newest first. */ +export function listRuns(source: NightlySource, limit = 100): WorkflowRun[] { + const result = gh([ + 'run', + 'list', + '--repo', + source.repo, + '--workflow', + source.workflow, + '--branch', + source.branch, + '--status', + 'completed', + '--limit', + String(limit), + '--json', + 'databaseId,createdAt,headSha,event', + ]); + if (!result.ok) throw new Error(`gh run list failed: ${result.stderr.trim()}`); + const rows = JSON.parse(result.stdout) as { + databaseId: number; + createdAt: string; + headSha: string; + event: string; + }[]; + return sortNewestFirst( + rows.map((row) => ({ + id: String(row.databaseId), + createdAt: row.createdAt, + headSha: row.headSha, + event: row.event, + })), + ); +} + +function sortNewestFirst(runs: WorkflowRun[]): WorkflowRun[] { + return [...runs].sort((a, b) => b.createdAt.localeCompare(a.createdAt)); +} + +/** + * Resolves one run reference: `latest`, a numeric run id, or a `YYYY-MM-DD` date (the newest run + * created that UTC day). Returns undefined when no listed run matches. + */ +export function resolveRunRef(ref: string, runs: WorkflowRun[]): WorkflowRun | undefined { + const newestFirst = sortNewestFirst(runs); + if (ref === 'latest') return newestFirst[0]; + if (/^\d+$/.test(ref)) return newestFirst.find((run) => run.id === ref); + if (DATE_REF.test(ref)) return newestFirst.find((run) => run.createdAt.slice(0, 10) === ref); + throw new Error( + `Unknown run reference ${JSON.stringify(ref)}: use latest, a run id, a YYYY-MM-DD date, or a path.`, + ); +} + +/** + * Runs between two references inclusive, newest first. A date end covers its whole UTC day, so a + * dispatch later that day does not hide the night's scheduled run. + */ +export function selectRange(runs: WorkflowRun[], fromRef: string, toRef: string): WorkflowRun[] { + const bounds = (ref: string): [string, string] | undefined => { + if (DATE_REF.test(ref)) return [`${ref}T00:00:00Z`, `${ref}T23:59:59.999Z`]; + const run = resolveRunRef(ref, runs); + return run && [run.createdAt, run.createdAt]; + }; + const from = bounds(fromRef); + const to = bounds(toRef); + if (!from || !to) throw new Error(`No runs match the range ${fromRef}..${toRef}.`); + const low = from[0] < to[0] ? from[0] : to[0]; + const high = from[1] > to[1] ? from[1] : to[1]; + return sortNewestFirst(runs).filter((run) => run.createdAt >= low && run.createdAt <= high); +} + +/** + * A rolling-baseline run must have run every scenario of the target on the same platform, so a + * partial dispatch does not take the place of a full night. + */ +export function coversTarget(candidate: PerfSummary, target: PerfSummary): boolean { + return ( + candidate.platform === target.platform && + target.config.scenarios.every((scenario) => candidate.config.scenarios.includes(scenario)) + ); +} + +/** Baseline candidates: runs created before the target, newest first, never the target itself. */ +export function baselineCandidates( + runs: WorkflowRun[], + before: string, + exclude: ReadonlySet, +): WorkflowRun[] { + return sortNewestFirst(runs).filter((run) => run.createdAt < before && !exclude.has(run.id)); +} + +/** + * Returns a run's summary, downloading the summary artifact once into the cache. Returns undefined + * when the run has no such artifact (it failed before upload, or the artifact expired). + */ +export function fetchSummary(source: NightlySource, run: WorkflowRun): PerfSummary | undefined { + const dir = path.join(source.cacheDir, run.id); + const file = path.join(dir, SUMMARY_FILE); + if (!fs.existsSync(file)) { + fs.rmSync(dir, { recursive: true, force: true }); + const result = gh([ + 'run', + 'download', + run.id, + '--repo', + source.repo, + '--name', + source.artifact, + '--dir', + dir, + ]); + if (!result.ok || !fs.existsSync(file)) { + process.stderr.write( + `[perf] run ${run.id}: no ${source.artifact} artifact (${result.stderr.trim()})\n`, + ); + fs.rmSync(dir, { recursive: true, force: true }); + return undefined; + } + } + return parsePerfSummary(JSON.parse(fs.readFileSync(file, 'utf8')), file); +} + +/** Reads a local run: a `summary.json` path or a run output directory. */ +export function readLocalSummary(location: string): PerfSummary { + const file = fs.statSync(location).isDirectory() ? path.join(location, SUMMARY_FILE) : location; + return parsePerfSummary(JSON.parse(fs.readFileSync(file, 'utf8')), file); +} diff --git a/scripts/perf/stats.test.ts b/scripts/perf/stats.test.ts index 088c7e1806..9b5d61ac46 100644 --- a/scripts/perf/stats.test.ts +++ b/scripts/perf/stats.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { summarize } from './stats.ts'; +import { scaledMad, summarize } from './stats.ts'; test('summarize reports sample spread, the middle median, and nearest-rank p95', () => { const stat = summarize([40, 10, 30, 20, Number.NaN])!; @@ -26,3 +26,8 @@ test('summarize gives a single sample zero spread and no samples no summary', () assert.equal(summarize([]), null); assert.equal(summarize([0, 0])?.cv, 0); }); + +test('scaledMad ignores one outlying night', () => { + // Deviations from the median 101 are 49, 1, 1, 3; their median is 2. + assert.ok(Math.abs(scaledMad([150, 100, 98, 102]) - 1.4826 * 2) < 1e-9); +}); diff --git a/scripts/perf/stats.ts b/scripts/perf/stats.ts index f972ec4e37..e4941f4608 100644 --- a/scripts/perf/stats.ts +++ b/scripts/perf/stats.ts @@ -41,3 +41,9 @@ export function median(values: number[]): number { const sorted = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b); return sorted.length === 0 ? Number.NaN : middle(sorted); } + +/** Median absolute deviation scaled to estimate a normal standard deviation. */ +export function scaledMad(values: number[]): number { + const center = median(values); + return 1.4826 * median(values.map((v) => Math.abs(v - center))); +} diff --git a/scripts/perf/summary-fixture.ts b/scripts/perf/summary-fixture.ts new file mode 100644 index 0000000000..b86ad31ef8 --- /dev/null +++ b/scripts/perf/summary-fixture.ts @@ -0,0 +1,64 @@ +import { buildMeasurement } from './measurement.ts'; +import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary, type Sample } from './types.ts'; + +export type StepFixture = { + id: string; + // Daemon round trips of the successful samples; wall adds a fixed CLI overhead. + daemonMs?: number[]; + wallMs?: number[]; + failures?: number; +}; + +function measurement(step: StepFixture): Measurement { + const timings = step.daemonMs ?? step.wallMs ?? []; + const samples: Sample[] = timings.map((ms, round) => ({ + round, + ok: true, + wallClockMs: step.daemonMs ? ms + 300 : ms, + daemonMs: step.daemonMs ? ms : undefined, + })); + for (let i = 0; i < (step.failures ?? 0); i++) { + samples.push({ round: samples.length, ok: false, wallClockMs: 0, errorCode: 'COMMAND_FAILED' }); + } + const [scenario] = step.id.split('/'); + return buildMeasurement({ + id: step.id, + scenario: scenario!, + label: step.id, + command: 'snapshot', + args: [], + platform: 'ios', + execMode: 'standalone', + samples, + warmupDropped: 1, + }); +} + +/** A schema-valid summary whose steps carry the given timings. */ +export function summaryFixture( + steps: StepFixture[], + overrides: { startedAt?: string; xcode?: string; runId?: string } = {}, +): PerfSummary { + return { + schema: PERF_SUMMARY_SCHEMA, + platform: 'ios', + startedAt: overrides.startedAt ?? '2026-10-20T03:00:00.000Z', + finishedAt: '2026-10-20T03:30:00.000Z', + config: { + rounds: 5, + warmup: 1, + scenarios: [...new Set(steps.map((s) => s.id.split('/')[0]!))], + }, + metadata: { + commit: '0123456789abcdef', + runId: overrides.runId, + agentDeviceVersion: '0.0.0', + host: { os: 'Darwin', arch: 'arm64', cpu: 'Apple M1 (Virtual)', cores: 3, memoryGb: 7 }, + xcode: overrides.xcode ?? 'Xcode 26.2', + node: 'v24.0.0', + device: { name: 'iPhone 17 Pro', runtime: 'iOS-26-2' }, + }, + coverage: { measured: ['snapshot'], excluded: [] }, + measurements: steps.map(measurement), + }; +} diff --git a/scripts/perf/summary.test.ts b/scripts/perf/summary.test.ts new file mode 100644 index 0000000000..421f234f17 --- /dev/null +++ b/scripts/perf/summary.test.ts @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { parsePerfSummary } from './summary.ts'; +import { summaryFixture } from './summary-fixture.ts'; + +const valid = () => + JSON.parse(JSON.stringify(summaryFixture([{ id: 's/step', daemonMs: [10, 20] }]))) as Record< + string, + unknown + >; + +test('accepts a summary that carries every field the comparison reads', () => { + const summary = parsePerfSummary(valid(), 'summary.json'); + assert.equal(summary.measurements[0]?.daemon?.median, 15); +}); + +test('rejects another schema version and names the field', () => { + assert.throws( + () => parsePerfSummary({ ...valid(), schema: 'agent-device-perf-summary/v0' }, 'a.json'), + /a\.json: schema must be "agent-device-perf-summary\/v1"/, + ); +}); + +test('rejects a measurement whose timing summary lost a statistic', () => { + const summary = valid(); + const [measurement] = summary.measurements as { daemon: Record }[]; + delete measurement!.daemon.cv; + assert.throws( + () => parsePerfSummary(summary, 'b.json'), + /b\.json: measurements\[0\]\.daemon\.cv must be a finite number/, + ); +}); diff --git a/scripts/perf/summary.ts b/scripts/perf/summary.ts new file mode 100644 index 0000000000..2feff211aa --- /dev/null +++ b/scripts/perf/summary.ts @@ -0,0 +1,63 @@ +import { PERF_SUMMARY_SCHEMA, type PerfSummary } from './types.ts'; + +type Json = Record; + +const STAT_FIELDS = ['n', 'mean', 'stddev', 'cv', 'median', 'min', 'max', 'p95'] as const; + +function fail(source: string, at: string, expected: string): never { + throw new Error(`${source}: ${at} must be ${expected}`); +} + +function object(value: unknown, source: string, at: string): Json { + if (value === null || typeof value !== 'object' || Array.isArray(value)) + fail(source, at, 'an object'); + return value as Json; +} + +function string(value: unknown, source: string, at: string): string { + if (typeof value !== 'string') fail(source, at, 'a string'); + return value; +} + +function number(value: unknown, source: string, at: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) fail(source, at, 'a finite number'); + return value; +} + +function stat(value: unknown, source: string, at: string): void { + if (value === null) return; + const s = object(value, source, at); + for (const field of STAT_FIELDS) number(s[field], source, `${at}.${field}`); +} + +/** + * Validates a `summary.json` read from disk or a downloaded artifact. Only the fields that the + * comparison and report read are checked; a run from another schema version is rejected. + */ +export function parsePerfSummary(value: unknown, source: string): PerfSummary { + const root = object(value, source, 'summary'); + if (root.schema !== PERF_SUMMARY_SCHEMA) fail(source, 'schema', `"${PERF_SUMMARY_SCHEMA}"`); + if (root.platform !== 'ios' && root.platform !== 'android') + fail(source, 'platform', 'ios or android'); + string(root.startedAt, source, 'startedAt'); + const config = object(root.config, source, 'config'); + number(config.rounds, source, 'config.rounds'); + const metadata = object(root.metadata, source, 'metadata'); + string(metadata.commit, source, 'metadata.commit'); + string(object(metadata.host, source, 'metadata.host').cpu, source, 'metadata.host.cpu'); + string(object(metadata.device, source, 'metadata.device').name, source, 'metadata.device.name'); + if (!Array.isArray(config.scenarios)) fail(source, 'config.scenarios', 'an array'); + if (!Array.isArray(root.measurements)) fail(source, 'measurements', 'an array'); + root.measurements.forEach((entry, index) => { + const at = `measurements[${index}]`; + const m = object(entry, source, at); + string(m.id, source, `${at}.id`); + string(m.scenario, source, `${at}.scenario`); + string(m.command, source, `${at}.command`); + number(m.failures, source, `${at}.failures`); + if (!Array.isArray(m.samples)) fail(source, `${at}.samples`, 'an array'); + stat(m.wallClock, source, `${at}.wallClock`); + stat(m.daemon, source, `${at}.daemon`); + }); + return value as PerfSummary; +} diff --git a/test/repo-guards/npm-package-scripts.test.ts b/test/repo-guards/npm-package-scripts.test.ts index 7f35bea71e..9b12030361 100644 --- a/test/repo-guards/npm-package-scripts.test.ts +++ b/test/repo-guards/npm-package-scripts.test.ts @@ -36,7 +36,7 @@ test('perf uses one platform-selectable entry point', () => { assert.equal(script('perf'), 'node --experimental-strip-types scripts/perf/run.ts'); assert.equal(packageJson.scripts['perf:ios'], undefined); assert.equal(packageJson.scripts['perf:android'], undefined); - assert.doesNotMatch(perfNightlyWorkflow, /pnpm perf:/); + assert.doesNotMatch(perfNightlyWorkflow, /pnpm perf:(ios|android)\b/); assert.match(perfNightlyWorkflow, /pnpm perf --platform ios/); assert.match(perfNightlyWorkflow, /pnpm perf --platform android/); }); From 558da5cdbbb3e830c8e70c93d69511dc817757bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 05:53:53 +0200 Subject: [PATCH 04/15] fix(perf): judge nights, samples, and failures soundly; keep logs on interrupt - Rolling baselines take one scheduled run per UTC night (the newest usable one) and page through workflow runs until enough nights are found. Explicit --to/--from references still accept any trigger. - A run measures at least three rounds. A timing verdict needs three successful samples tonight and on each counted baseline night. - Failures get their own verdict. more-failures flags tonight failing more than every baseline night, in count and share; a faster median with worse failures is not an improvement. - The interrupt handler and the normal finish share one finalizer: save logs, stop session and daemon within a deadline, save again. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/perf-nightly.yml | 4 +- scripts/perf/README.md | 30 +++-- scripts/perf/cli.ts | 7 +- scripts/perf/compare-cli.ts | 54 ++++---- scripts/perf/compare.test.ts | 46 +++++-- scripts/perf/compare.ts | 167 ++++++++++++++++--------- scripts/perf/config.test.ts | 9 ++ scripts/perf/config.ts | 7 +- scripts/perf/ios-harness.test.ts | 57 +++++++++ scripts/perf/ios-harness.ts | 34 ++++- scripts/perf/nightly-artifacts.test.ts | 61 +++++++-- scripts/perf/nightly-artifacts.ts | 110 ++++++++++------ scripts/perf/run.ts | 14 +-- 13 files changed, 435 insertions(+), 165 deletions(-) create mode 100644 scripts/perf/config.test.ts create mode 100644 scripts/perf/ios-harness.test.ts diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index 243cd08c17..f4d7cd4863 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -16,7 +16,7 @@ on: workflow_dispatch: inputs: rounds: - description: 'Measured rounds per step (samples)' + description: 'Measured rounds per step (samples, at least 3)' required: false default: '3' warmup: @@ -28,7 +28,7 @@ on: required: false default: '' baseline-nights: - description: 'Earlier runs in the rolling baseline' + description: 'Earlier scheduled nights in the rolling baseline' required: false default: '7' baseline-branch: diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 4f088d0aea..0715aa7061 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -18,8 +18,9 @@ The suite records a reason for each catalog command that it does not time. `ios- pnpm perf --platform ios --udid --rounds 3 --warmup 1 --out-dir .tmp/perf/local ``` -`--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id. +`--rounds` is at least 3. `--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id. The suite uses its own temporary state directory and daemon. It stops them at the end. +At the end, or on SIGINT or SIGTERM, the suite saves the logs to `raw/state/`, stops the session and daemon within a deadline, and saves the logs again. ## Output @@ -54,19 +55,22 @@ All times are milliseconds. Statistics use only the successful samples of a step `.github/workflows/perf-nightly.yml` runs the iOS suite every night on `main`. Each run uploads `perf-ios-summary` (summary, report, comparison; 90 days) and `perf-ios-raw` (logs; 30 days). -The job compares the run with up to seven earlier runs and writes the result to the job summary. +The job compares the run with up to seven earlier nights and writes the result to the job summary. +A night is the newest scheduled run of one UTC day that ran every scenario of the judged run. +Manual dispatches are never part of the rolling baseline. To fetch and compare runs on your machine, use `gh` with access to the repository: ```sh pnpm perf:compare # latest night against the 7 nights before it pnpm perf:compare --to 2026-10-28 # that night against the 7 nights before it -pnpm perf:compare --to latest --from 2026-10-21 # latest against one night -pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare +pnpm perf:compare --to latest --from 2026-10-21 # latest against the newest run of that day +pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare # every run in the range pnpm perf:compare --to .tmp/perf/local # a local run against recent nights ``` A reference is `latest`, a run id, a `YYYY-MM-DD` date, or a local output path. +An explicit `--to` or `--from` reference accepts runs of any trigger, so it can compare dispatches and local runs. Downloads stay in `.tmp/perf-nightly//`. Do not judge a local run against CI nights. The comparison warns when the device, Xcode, or CPU differs. @@ -81,7 +85,19 @@ A step is a `regression` or an `improvement` only when its change is more than a - Two times the coefficient of variation of the run divided by √n. - 50 ms. -The rolling baseline skips runs that did not run every scenario of the judged run, such as a partial dispatch. -Fewer than three baseline nights give `insufficient-baseline`. -`new-failures` means that the step failed and no baseline night failed. `fixed` means the reverse. +Each step gets a timing verdict and a failure verdict. +A timing verdict uses only successful samples: + +- `insufficient-samples`: the judged run has fewer than three successful samples of the step. +- `insufficient-baseline`: fewer than three baseline nights have three successful samples of the step. Nights with fewer do not count. + +The failure verdict compares failed samples with the worst baseline night: + +- `new-failures`: the step failed and no baseline night failed. +- `more-failures`: the step failed more often than on every baseline night, both in count and in share of its samples. +- `fixed`: the step did not fail and every baseline night failed. + +When failures are new or more, a faster median is `insufficient-samples`, not `improvement`. +The samples that still pass do not show that the step got faster. +Regressions and worse failures are workflow warnings. The job stays green. The thresholds are in `compare.ts`. diff --git a/scripts/perf/cli.ts b/scripts/perf/cli.ts index 57ee22c56a..cada28867d 100644 --- a/scripts/perf/cli.ts +++ b/scripts/perf/cli.ts @@ -33,7 +33,11 @@ function jsonOk(json: unknown): boolean { // Invoke the built CLI once. `args` includes the command + positionals + dash-flags; // `baseFlags` carries the isolation + device flags shared by every call. -export function invokeCli(args: string[], baseFlags: string[]): CliResult { +export function invokeCli( + args: string[], + baseFlags: string[], + options: { timeoutMs?: number } = {}, +): CliResult { const full = [...CLI_ARGV, ...args, ...baseFlags, '--json']; const t0 = performance.now(); let stdout = ''; @@ -46,6 +50,7 @@ export function invokeCli(args: string[], baseFlags: string[]): CliResult { cwd: REPO_ROOT, maxBuffer: MAX_BUFFER, allowFailure: true, + timeoutMs: options.timeoutMs, }); stdout = r.stdout; stderr = r.stderr; diff --git a/scripts/perf/compare-cli.ts b/scripts/perf/compare-cli.ts index 0336beca96..3b3593acfe 100644 --- a/scripts/perf/compare-cli.ts +++ b/scripts/perf/compare-cli.ts @@ -3,12 +3,12 @@ import path from 'node:path'; import { compareRuns, renderComparison, type Comparison } from './compare.ts'; import { REPO_ROOT } from './config.ts'; import { - baselineCandidates, coversTarget, fetchSummary, listRuns, readLocalSummary, resolveRunRef, + selectNightlyBaseline, selectRange, SUMMARY_ARTIFACT, type NightlySource, @@ -24,9 +24,9 @@ A is latest, a workflow run id, a YYYY-MM-DD date (the newest run that UTC path to a summary.json or a \`pnpm perf\` output directory. --to Run to judge (default: latest). - --from Baseline run, or an inclusive range ... Without it the baseline is - the --nights runs before --to. - --nights Rolling baseline size (default: 7). + --from Baseline run, or an inclusive range .. of runs of any trigger. + Without it the baseline is the --nights scheduled nights before --to. + --nights Rolling baseline size in UTC nights, one scheduled run each (default: 7). --branch Branch whose nightly runs are listed (default: main). --repo Repository (default: $GITHUB_REPOSITORY or callstack/agent-device). --workflow Workflow file (default: perf-nightly.yml). @@ -117,7 +117,7 @@ type Loaded = { summary: PerfSummary; label: string; run?: WorkflowRun }; function main(): void { const options = parse(process.argv.slice(2)); let runs: WorkflowRun[] | undefined; - const allRuns = (): WorkflowRun[] => (runs ??= listRuns(options.source)); + const allRuns = (): WorkflowRun[] => (runs ??= [...listRuns(options.source)]); const runLabel = (run: WorkflowRun) => `run ${run.id} (${run.createdAt.slice(0, 10)})`; const load = (ref: string): Loaded => { if (fs.existsSync(ref)) @@ -131,19 +131,11 @@ function main(): void { if (!summary) throw new Error(`Run ${run.id} has no ${options.source.artifact} artifact.`); return { summary, label: runLabel(run), run }; }; - const loadRuns = ( - candidates: WorkflowRun[], - limit: number, - accept: (summary: PerfSummary) => boolean = () => true, - ): Loaded[] => { - const loaded: Loaded[] = []; - for (const run of candidates) { - if (loaded.length >= limit) break; + const loadRuns = (candidates: WorkflowRun[]): Loaded[] => + candidates.flatMap((run) => { const summary = fetchSummary(options.source, run); - if (summary && accept(summary)) loaded.push({ summary, label: runLabel(run), run }); - } - return loaded; - }; + return summary ? [{ summary, label: runLabel(run), run }] : []; + }); const target = load(options.to); const exclude = new Set([...options.exclude, ...(target.run ? [target.run.id] : [])]); @@ -151,14 +143,19 @@ function main(): void { if (options.from?.includes('..')) { const [fromRef, toRef] = options.from.split('..'); const range = selectRange(allRuns(), fromRef!, toRef!).filter((run) => !exclude.has(run.id)); - baseline = loadRuns(range, range.length); + baseline = loadRuns(range); } else if (options.from) { baseline = [load(options.from)]; } else { const before = target.run?.createdAt ?? target.summary.startedAt; - baseline = loadRuns(baselineCandidates(allRuns(), before, exclude), options.nights, (summary) => - coversTarget(summary, target.summary), - ); + baseline = selectNightlyBaseline( + runs ?? listRuns(options.source), + { before, exclude, nights: options.nights }, + (run) => { + const summary = fetchSummary(options.source, run); + return summary && coversTarget(summary, target.summary) ? summary : undefined; + }, + ).map(({ run, value }) => ({ summary: value, label: runLabel(run), run })); } const comparison = compareRuns(target, baseline); @@ -178,12 +175,15 @@ function publishToGitHub(comparison: Comparison, markdown: string): void { const summaryPath = process.env.GITHUB_STEP_SUMMARY; if (summaryPath) fs.appendFileSync(summaryPath, `${markdown}\n`); for (const row of comparison.rows) { - if (row.verdict === 'regression' || row.verdict === 'new-failures') { - const change = - row.verdict === 'regression' - ? `median ${row.target?.median.toFixed(0)} ms vs ${row.baseline?.median.toFixed(0)} ms baseline` - : `${row.target?.failures}/${row.target?.samples} samples failed`; - process.stdout.write(`::warning title=perf ${row.verdict}::${row.id}: ${change}\n`); + if (row.timing === 'regression') { + process.stdout.write( + `::warning title=perf regression::${row.id}: median ${row.target?.median.toFixed(0)} ms vs ${row.baseline?.median.toFixed(0)} ms baseline\n`, + ); + } + if (row.failures === 'new-failures' || row.failures === 'more-failures') { + process.stdout.write( + `::warning title=perf ${row.failures}::${row.id}: ${row.target?.failures}/${row.target?.samples} samples failed\n`, + ); } } } diff --git a/scripts/perf/compare.test.ts b/scripts/perf/compare.test.ts index d7046d1860..51fb75d55c 100644 --- a/scripts/perf/compare.test.ts +++ b/scripts/perf/compare.test.ts @@ -26,7 +26,7 @@ test('flags a slowdown that exceeds a quiet baseline and the relative floor', () [{ id: STEP, daemonMs: [1450, 1500, 1550] }], [flat(1000), flat(1010), flat(990)], ); - assert.equal(result.verdict, 'regression'); + assert.equal(result.timing, 'regression'); assert.equal(result.metric, 'daemon'); assert.equal(result.baseline?.median, 1000); assert.equal(result.deltaMs, 500); @@ -38,37 +38,63 @@ test('does not flag the same slowdown when nightly medians already swing that mu [{ id: STEP, daemonMs: [1450, 1500, 1550] }], [flat(500), flat(1000), flat(1500), flat(700), flat(1300)], ); - assert.equal(result.verdict, 'unchanged'); + assert.equal(result.timing, 'unchanged'); // 3 × scaled MAD: deviations from 1000 are 0, 300, 300, 500, 500. assert.ok(Math.abs(result.threshold! - (3 * 1.4826 * 300) / 1000) < 1e-9); }); test('does not flag a large relative change below the absolute floor', () => { - assert.equal(row(flat(60), [flat(40), flat(40), flat(40)]).verdict, 'unchanged'); + assert.equal(row(flat(60), [flat(40), flat(40), flat(40)]).timing, 'unchanged'); }); test('flags a speedup beyond noise as an improvement', () => { - assert.equal(row(flat(500), [flat(1000), flat(1010), flat(990)]).verdict, 'improvement'); + assert.equal(row(flat(500), [flat(1000), flat(1010), flat(990)]).timing, 'improvement'); +}); + +test('withholds a timing verdict when tonight or a baseline night has fewer than three successes', () => { + const one = (ms: number): StepFixture[] => [{ id: STEP, daemonMs: [ms] }]; + const fromOneSample = row(one(1500), [one(1000), one(1000), one(1000)]); + assert.equal(fromOneSample.timing, 'insufficient-samples'); + assert.equal(fromOneSample.baseline?.timedNights, 0); + // Three successes tonight still need three nights that each had three successes. + const thinNights = row(flat(1500), [flat(1000), flat(1000), one(1000)]); + assert.equal(thinNights.timing, 'insufficient-baseline'); + assert.equal(thinNights.baseline?.timedNights, 2); + assert.equal(thinNights.deltaMs, 500); }); test('withholds a timing verdict until enough nights exist', () => { const result = row(flat(1500), [flat(1000), flat(1000)]); - assert.equal(result.verdict, 'insufficient-baseline'); + assert.equal(result.timing, 'insufficient-baseline'); assert.equal(result.deltaRelative, 0.5); }); test('reports failures that no baseline night had, and failures that went away', () => { const clean = [flat(1000), flat(1000), flat(1000)]; - assert.equal(row(flat(1000, { failures: 1 }), clean).verdict, 'new-failures'); + assert.equal(row(flat(1000, { failures: 1 }), clean).failures, 'new-failures'); const failing = [ flat(1000, { failures: 1 }), flat(1000, { failures: 2 }), flat(1000, { failures: 1 }), ]; - assert.equal(row(flat(1000), failing).verdict, 'fixed'); - // A step that already failed on some baseline night is judged on timing instead. + assert.equal(row(flat(1000), failing).failures, 'fixed'); + // As many failures as the worst baseline night is not a change; timing is judged as usual. + const flaky = [flat(1000, { failures: 1 }), flat(1000), flat(1000)]; + const steady = row(flat(1000, { failures: 1 }), flaky); + assert.equal(steady.failures, 'unchanged'); + assert.equal(steady.timing, 'unchanged'); +}); + +test('reports worse failures over a flaky baseline and never as an improvement', () => { const flaky = [flat(1000, { failures: 1 }), flat(1000), flat(1000)]; - assert.equal(row(flat(1000, { failures: 1 }), flaky).verdict, 'unchanged'); + const mostlyFailing = row([{ id: STEP, daemonMs: [500], failures: 2 }], flaky); + assert.equal(mostlyFailing.failures, 'more-failures'); + assert.equal(mostlyFailing.timing, 'insufficient-samples'); + // Enough successes, but they are the subset that still passes, so the speedup is withheld. + const faster = row(flat(500, { failures: 2 }), flaky); + assert.equal(faster.failures, 'more-failures'); + assert.equal(faster.timing, 'insufficient-samples'); + assert.equal(faster.deltaMs, -500); }); test('falls back to wall time when any run lacks the daemon timer', () => { @@ -86,7 +112,7 @@ test('marks steps that only one side measured', () => { [{ summary: summaryFixture([{ id: 'a/old', daemonMs: [1] }]), label: 'b' }], ); assert.deepEqual( - comparison.rows.map((r) => [r.id, r.verdict]), + comparison.rows.map((r) => [r.id, r.timing]), [ ['a/new', 'new'], ['a/old', 'missing'], diff --git a/scripts/perf/compare.ts b/scripts/perf/compare.ts index 11dae30520..22b1f28a61 100644 --- a/scripts/perf/compare.ts +++ b/scripts/perf/compare.ts @@ -1,3 +1,4 @@ +import { MIN_MEASURED_ROUNDS } from './config.ts'; import { median, scaledMad } from './stats.ts'; import type { Measurement, PerfSummary, Stat } from './types.ts'; @@ -7,7 +8,8 @@ export type ComparisonThresholds = { minAbsoluteMs: number; spreadFactor: number; // × night-to-night relative spread (scaled MAD of nightly medians) withinFactor: number; // × tonight's cv / √n (standard error of tonight's median) - minBaselineNights: number; // fewer nights give `insufficient-baseline` + minBaselineNights: number; // fewer timed nights give `insufficient-baseline` + minSamples: number; // successful samples that tonight and each timed night need }; const DEFAULT_THRESHOLDS: ComparisonThresholds = { @@ -16,30 +18,47 @@ const DEFAULT_THRESHOLDS: ComparisonThresholds = { spreadFactor: 3, withinFactor: 2, minBaselineNights: 3, + minSamples: MIN_MEASURED_ROUNDS, }; -export type Verdict = +export type TimingVerdict = | 'regression' | 'improvement' | 'unchanged' - | 'new-failures' - | 'fixed' - | 'failing' + | 'insufficient-samples' | 'insufficient-baseline' | 'new' | 'missing'; +export type FailureVerdict = 'new-failures' | 'more-failures' | 'fixed' | 'unchanged'; + +type Flagged = Exclude; + /** Verdicts that need a reader's attention, in report order. */ -const FLAGGED_VERDICTS: readonly Verdict[] = ['new-failures', 'regression', 'fixed', 'improvement']; +const FLAGGED_VERDICTS: readonly Flagged[] = [ + 'new-failures', + 'more-failures', + 'regression', + 'fixed', + 'improvement', +]; export type ComparisonRow = { id: string; scenario: string; command: string; - verdict: Verdict; + timing: TimingVerdict; + // Null when only one side measured the step. + failures: FailureVerdict | null; metric: 'daemon' | 'wall'; target: { median: number; n: number; cv: number; failures: number; samples: number } | null; - baseline: { median: number; nights: number; spread: number; failingNights: number } | null; + baseline: { + median: number; + nights: number; + timedNights: number; + spread: number; + failingNights: number; + } | null; deltaMs: number | null; deltaRelative: number | null; threshold: number | null; @@ -62,7 +81,10 @@ export type Comparison = { target: RunRef; baseline: RunRef[]; thresholds: ComparisonThresholds; - counts: Partial>; + counts: { + timing: Partial>; + failures: Partial>; + }; warnings: string[]; rows: ComparisonRow[]; }; @@ -83,6 +105,24 @@ function metricStat(m: Measurement, metric: 'daemon' | 'wall'): Stat | null { return metric === 'daemon' ? m.daemon : m.wallClock; } +function failureRate(m: Measurement): number { + return m.samples.length === 0 ? 0 : m.failures / m.samples.length; +} + +/** + * Failures worsen when tonight has more failed samples than every baseline night, both as a count + * and as a share of samples, so a different round count cannot fake a change. + */ +function judgeFailures(target: Measurement, history: Measurement[]): FailureVerdict { + const failingNights = history.filter((m) => m.failures > 0).length; + const worstCount = Math.max(...history.map((m) => m.failures)); + const worstRate = Math.max(...history.map(failureRate)); + if (target.failures > worstCount && failureRate(target) > worstRate) + return failingNights === 0 ? 'new-failures' : 'more-failures'; + if (target.failures === 0 && failingNights === history.length) return 'fixed'; + return 'unchanged'; +} + function compareStep( id: string, target: Measurement | undefined, @@ -97,45 +137,39 @@ function compareStep( ? 'daemon' : 'wall'; const empty = { deltaMs: null, deltaRelative: null, threshold: null }; - const okNights = history + const timed = (s: Stat | null): s is Stat => s !== null && s.n >= thresholds.minSamples; + const timedNights = history .map((m) => metricStat(m, metric)) - .filter((s): s is Stat => s !== null) + .filter(timed) .map((s) => s.median); - const failingNights = history.filter((m) => m.failures > 0).length; - const baselineMedian = okNights.length > 0 ? median(okNights) : null; + const baselineMedian = timedNights.length > 0 ? median(timedNights) : null; const baseline = history.length === 0 ? null : { median: baselineMedian ?? Number.NaN, nights: history.length, - spread: baselineMedian ? scaledMad(okNights) / baselineMedian : 0, - failingNights, + timedNights: timedNights.length, + spread: baselineMedian ? scaledMad(timedNights) / baselineMedian : 0, + failingNights: history.filter((m) => m.failures > 0).length, }; - if (!target) return { ...base, verdict: 'missing', metric, target: null, baseline, ...empty }; + if (!target) + return { ...base, timing: 'missing', failures: null, metric, target: null, baseline, ...empty }; const stat = metricStat(target, metric); - const targetFacts = stat - ? { - median: stat.median, - n: stat.n, - cv: stat.cv, - failures: target.failures, - samples: target.samples.length, - } - : { - median: Number.NaN, - n: 0, - cv: 0, - failures: target.failures, - samples: target.samples.length, - }; + const targetFacts = { + median: stat?.median ?? Number.NaN, + n: stat?.n ?? 0, + cv: stat?.cv ?? 0, + failures: target.failures, + samples: target.samples.length, + }; const row = { ...base, metric, target: targetFacts, baseline }; - if (!baseline) return { ...row, verdict: 'new', ...empty }; - if (target.failures > 0 && failingNights === 0) - return { ...row, verdict: 'new-failures', ...empty }; - if (target.failures === 0 && failingNights === history.length) - return { ...row, verdict: 'fixed', ...empty }; - if (!stat || baselineMedian === null) return { ...row, verdict: 'failing', ...empty }; + if (!baseline) return { ...row, timing: 'new', failures: null, ...empty }; + const failures = judgeFailures(target, history); + if (!stat || baselineMedian === null) { + const timing = timed(stat) ? 'insufficient-baseline' : 'insufficient-samples'; + return { ...row, timing, failures, ...empty }; + } const deltaMs = stat.median - baselineMedian; const deltaRelative = baselineMedian === 0 ? 0 : deltaMs / baselineMedian; @@ -144,14 +178,18 @@ function compareStep( thresholds.spreadFactor * baseline.spread, (thresholds.withinFactor * stat.cv) / Math.sqrt(stat.n), ); - const measured = { deltaMs, deltaRelative, threshold }; - if (okNights.length < thresholds.minBaselineNights) { - return { ...row, verdict: 'insufficient-baseline', ...measured }; + const measured = { ...row, failures, deltaMs, deltaRelative, threshold }; + if (!timed(stat)) return { ...measured, timing: 'insufficient-samples' }; + if (timedNights.length < thresholds.minBaselineNights) { + return { ...measured, timing: 'insufficient-baseline' }; } const beyondNoise = Math.abs(deltaRelative) > threshold && Math.abs(deltaMs) > thresholds.minAbsoluteMs; - const verdict: Verdict = !beyondNoise ? 'unchanged' : deltaMs > 0 ? 'regression' : 'improvement'; - return { ...row, verdict, ...measured }; + if (!beyondNoise) return { ...measured, timing: 'unchanged' }; + if (deltaMs > 0) return { ...measured, timing: 'regression' }; + // The samples that still succeed while failures worsen do not show that the step got faster. + const worsened = failures === 'new-failures' || failures === 'more-failures'; + return { ...measured, timing: worsened ? 'insufficient-samples' : 'improvement' }; } function environmentWarnings(target: PerfSummary, baseline: PerfSummary[]): string[] { @@ -177,7 +215,8 @@ function environmentWarnings(target: PerfSummary, baseline: PerfSummary[]): stri * Compares one run against a rolling baseline of earlier runs. The baseline value of a step is the * median of its nightly medians, and its noise is the scaled MAD of those medians. A step is flagged * only when its change exceeds the relative floor, the night-to-night spread, tonight's own - * sampling error, and an absolute floor together. + * sampling error, and an absolute floor together. Tonight and every night that counts need + * `minSamples` successful samples. Failures are judged apart from timing. */ export function compareRuns( target: { summary: PerfSummary; label: string }, @@ -197,8 +236,11 @@ export function compareRuns( thresholds, ), ); - const counts: Partial> = {}; - for (const row of rows) counts[row.verdict] = (counts[row.verdict] ?? 0) + 1; + const counts: Comparison['counts'] = { timing: {}, failures: {} }; + for (const row of rows) { + counts.timing[row.timing] = (counts.timing[row.timing] ?? 0) + 1; + if (row.failures) counts.failures[row.failures] = (counts.failures[row.failures] ?? 0) + 1; + } return { schema: COMPARISON_SCHEMA, target: describeRun(target.summary, target.label), @@ -223,32 +265,45 @@ function pct(n: number | null | undefined, signed = false): string { return signed && n > 0 ? `+${value}` : value; } +function flagRank(row: ComparisonRow): number { + const ranks = [row.failures, row.timing].map((v) => FLAGGED_VERDICTS.indexOf(v as Flagged)); + const flagged = ranks.filter((rank) => rank >= 0); + return flagged.length === 0 ? -1 : Math.min(...flagged); +} + function tableRow(row: ComparisonRow): string { - const failures = row.target && row.target.failures > 0 ? ` (${row.target.failures} failed)` : ''; - return `| ${row.verdict} | \`${row.id}\` | ${row.metric} | ${ms(row.target?.median)}${failures} | ${ms(row.baseline?.median)} | ${ms(row.deltaMs)} | ${pct(row.deltaRelative, true)} | ${pct(row.threshold)} | ${row.baseline?.nights ?? 0} |`; + const failures = row.target ? `${row.target.failures}/${row.target.samples}` : '–'; + const nights = row.baseline ? `${row.baseline.timedNights}/${row.baseline.nights}` : '0'; + return `| ${row.timing} | ${row.failures ?? '–'} | \`${row.id}\` | ${row.metric} | ${ms(row.target?.median)} | ${ms(row.baseline?.median)} | ${ms(row.deltaMs)} | ${pct(row.deltaRelative, true)} | ${pct(row.threshold)} | ${failures} | ${nights} |`; } const TABLE_HEAD = [ - '| verdict | step | metric | tonight median | baseline median | Δ ms | Δ | threshold | nights |', - '|---|---|---|---|---|---|---|---|---|', + '| timing | failures | step | metric | tonight median | baseline median | Δ ms | Δ | threshold | failed tonight | timed nights |', + '|---|---|---|---|---|---|---|---|---|---|---|', ]; +function describeCounts(counts: Partial>): string { + return ( + Object.entries(counts) + .map(([verdict, n]) => `${n} ${verdict}`) + .join(', ') || 'none' + ); +} + /** Renders the comparison for the job summary and the artifact. */ export function renderComparison(comparison: Comparison): string { const t = comparison.thresholds; const flagged = comparison.rows - .filter((row) => FLAGGED_VERDICTS.includes(row.verdict)) - .sort((a, b) => FLAGGED_VERDICTS.indexOf(a.verdict) - FLAGGED_VERDICTS.indexOf(b.verdict)); - const counts = Object.entries(comparison.counts) - .map(([verdict, n]) => `${n} ${verdict}`) - .join(', '); + .filter((row) => flagRank(row) >= 0) + .sort((a, b) => flagRank(a) - flagRank(b)); const lines = [ `## agent-device perf comparison`, '', `- **Tonight**: ${comparison.target.label} · ${comparison.target.commit.slice(0, 12)} · ${comparison.target.startedAt} · ${comparison.target.device}`, `- **Baseline**: ${comparison.baseline.length === 0 ? 'none' : comparison.baseline.map((b) => `${b.label} (${b.commit.slice(0, 7)})`).join(', ')}`, - `- **Rule**: flag a step when its median moves by more than max(${pct(t.minRelative)}, ${t.spreadFactor}× night-to-night spread, ${t.withinFactor}× tonight cv/√n) and more than ${t.minAbsoluteMs} ms against the median of the baseline nights; at least ${t.minBaselineNights} nights are needed for a verdict.`, - `- **Steps**: ${counts || 'none'}`, + `- **Rule**: flag a step when its median moves by more than max(${pct(t.minRelative)}, ${t.spreadFactor}× night-to-night spread, ${t.withinFactor}× tonight cv/√n) and more than ${t.minAbsoluteMs} ms against the median of the baseline nights; a timing verdict needs ${t.minBaselineNights} nights, and tonight and each of those nights need ${t.minSamples} successful samples. Failures are flagged when tonight has more failed samples than every baseline night, in count and in share.`, + `- **Timing**: ${describeCounts(comparison.counts.timing)}`, + `- **Failures**: ${describeCounts(comparison.counts.failures)}`, '', ]; for (const warning of comparison.warnings) lines.push(`> ⚠️ ${warning}`, ''); diff --git a/scripts/perf/config.test.ts b/scripts/perf/config.test.ts new file mode 100644 index 0000000000..d8bb3b363a --- /dev/null +++ b/scripts/perf/config.test.ts @@ -0,0 +1,9 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { parseConfig } from './config.ts'; + +test('a run measures at least three rounds, so every step can get a timing verdict', () => { + assert.equal(parseConfig(['--udid', 'sim', '--rounds', '3']).rounds, 3); + assert.throws(() => parseConfig(['--udid', 'sim', '--rounds', '2']), /--rounds must be .* >= 3/); + assert.throws(() => parseConfig(['--platform', 'android', '--n', '1']), /--n must be .* >= 3/); +}); diff --git a/scripts/perf/config.ts b/scripts/perf/config.ts index 1db472fa88..32623a8ebf 100644 --- a/scripts/perf/config.ts +++ b/scripts/perf/config.ts @@ -23,6 +23,9 @@ export type PerfConfig = { export const DEFAULT_IOS_APP_ID = 'com.callstack.agentdevicelab'; +/** The fewest measured rounds that a run accepts; a comparison needs as many successful samples. */ +export const MIN_MEASURED_ROUNDS = 3; + // How to invoke the CLI. Defaults to the built dist binary (bin/agent-device.mjs). // Set AGENT_DEVICE_PERF_CLI to run from source instead, e.g. on CI: // AGENT_DEVICE_PERF_CLI="--experimental-strip-types src/bin.ts" @@ -55,7 +58,7 @@ function readIntValue(argv: string[], i: number, flag: string, min: number): num export function parseConfig(argv: string[]): PerfConfig { const cfg: PerfConfig = { platform: 'ios', - rounds: 3, + rounds: MIN_MEASURED_ROUNDS, warmup: 1, keepArtifacts: false, outDir: path.join(DEFAULT_OUT_DIR, new Date().toISOString().replaceAll(/[:.]/g, '-')), @@ -72,7 +75,7 @@ export function parseConfig(argv: string[]): PerfConfig { } case '--n': case '--rounds': - cfg.rounds = readIntValue(argv, i++, a, 1); + cfg.rounds = readIntValue(argv, i++, a, MIN_MEASURED_ROUNDS); break; case '--warmup': cfg.warmup = readIntValue(argv, i++, a, 0); diff --git a/scripts/perf/ios-harness.test.ts b/scripts/perf/ios-harness.test.ts new file mode 100644 index 0000000000..e8e88f5129 --- /dev/null +++ b/scripts/perf/ios-harness.test.ts @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { test, vi } from 'vitest'; + +// Stands in for the CLI: records each teardown call, and whether the logs were already saved, +// then appends to the daemon log and fails like a teardown that went wrong. +const FAKE_CLI = ` +import fs from 'node:fs'; +const args = process.argv.slice(2); +const stateDir = args[args.indexOf('--state-dir') + 1]; +const command = args[0] === 'daemon' ? 'daemon stop' : args[0]; +const saved = fs.existsSync(process.env.PERF_FAKE_SAVED_LOG); +fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' saved=' + saved + '\\n'); +fs.appendFileSync(stateDir + '/daemon.log', command + '\\n'); +process.exit(1); +`; + +test('finalizing a run saves the logs before and after a failing teardown, once', async () => { + const root = mkdtempForTestSync('perf-finalize-'); + const fakeCli = path.join(root, 'fake-cli.mjs'); + fs.writeFileSync(fakeCli, FAKE_CLI); + const stateDir = path.join(root, 'state'); + const rawDir = path.join(root, 'out', 'raw'); + fs.mkdirSync(path.join(stateDir, 'sessions', 'perf'), { recursive: true }); + fs.writeFileSync(path.join(stateDir, 'daemon.log'), 'suite\n'); + fs.writeFileSync(path.join(stateDir, 'sessions', 'perf', 'runner.log'), 'runner\n'); + fs.writeFileSync(path.join(stateDir, 'daemon.json'), '{"token":"secret"}'); + const calls = path.join(root, 'calls.log'); + vi.stubEnv('AGENT_DEVICE_PERF_CLI', fakeCli); + vi.stubEnv('PERF_FAKE_CALLS', calls); + vi.stubEnv('PERF_FAKE_SAVED_LOG', path.join(rawDir, 'state', 'daemon.log')); + // The CLI command is read when the harness loads, so load it after the stub. + const { iosRunFinalizer } = await import('./ios-harness.ts'); + + const finalize = iosRunFinalizer( + { stateDir, udid: 'sim', appId: 'app', artifactsDir: path.join(rawDir, 'artifacts'), rawDir }, + false, + ); + finalize(); + // The interrupt handler and the normal finish both call it; the second call does nothing. + finalize(); + + assert.equal(fs.readFileSync(calls, 'utf8'), 'close saved=true\ndaemon stop saved=true\n'); + const saved = path.join(rawDir, 'state'); + assert.equal( + fs.readFileSync(path.join(saved, 'daemon.log'), 'utf8'), + 'suite\nclose\ndaemon stop\n', + ); + assert.equal( + fs.readFileSync(path.join(saved, 'sessions', 'perf', 'runner.log'), 'utf8'), + 'runner\n', + ); + assert.equal(fs.existsSync(path.join(saved, 'daemon.json')), false); + assert.equal(fs.existsSync(stateDir), false); +}); diff --git a/scripts/perf/ios-harness.ts b/scripts/perf/ios-harness.ts index bcd2c3f4a6..16579f7506 100644 --- a/scripts/perf/ios-harness.ts +++ b/scripts/perf/ios-harness.ts @@ -282,8 +282,40 @@ export async function runIosSuite( ); } +// A stalled close or daemon stop must not keep the run, and its logs, from finishing. +const TEARDOWN_TIMEOUT_MS = 60_000; + +/** + * Returns the end of a run: save the logs, stop the session and the daemon within a deadline, save + * the logs again with the teardown, and remove the state directory unless it is kept. It runs once, + * so the interrupt handler and the normal finish can both call it. + */ +export function iosRunFinalizer(ctx: IosSuiteContext, keepArtifacts: boolean): () => void { + let finalized = false; + const saveLogs = (): void => { + try { + copyRawState(ctx); + } catch (error) { + log(`finalize: saving logs failed: ${(error as Error).message}`); + } + }; + const teardown = (args: string[], flags: string[]): void => { + const result = invokeCli(args, flags, { timeoutMs: TEARDOWN_TIMEOUT_MS }); + if (!result.ok) log(`finalize: ${args.join(' ')} failed: ${result.stderr.trim().slice(-400)}`); + }; + return () => { + if (finalized) return; + finalized = true; + saveLogs(); + teardown(['close'], ['--state-dir', ctx.stateDir, '--session', SESSION]); + teardown(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); + saveLogs(); + if (!keepArtifacts) fs.rmSync(ctx.stateDir, { recursive: true, force: true }); + }; +} + /** Copies daemon and session logs into the raw artifact; the daemon token file stays behind. */ -export function copyRawState(ctx: IosSuiteContext): void { +function copyRawState(ctx: IosSuiteContext): void { const target = path.join(ctx.rawDir, 'state'); fs.mkdirSync(target, { recursive: true }); const daemonLog = path.join(ctx.stateDir, 'daemon.log'); diff --git a/scripts/perf/nightly-artifacts.test.ts b/scripts/perf/nightly-artifacts.test.ts index 74671a936d..d302a99a94 100644 --- a/scripts/perf/nightly-artifacts.test.ts +++ b/scripts/perf/nightly-artifacts.test.ts @@ -1,19 +1,19 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { - baselineCandidates, coversTarget, resolveRunRef, + selectNightlyBaseline, selectRange, type WorkflowRun, } from './nightly-artifacts.ts'; import { summaryFixture } from './summary-fixture.ts'; -const run = (id: string, createdAt: string): WorkflowRun => ({ +const run = (id: string, createdAt: string, event = 'schedule'): WorkflowRun => ({ id, createdAt, headSha: `sha-${id}`, - event: 'schedule', + event, }); // Deliberately unsorted, with two runs on 2026-10-19. @@ -32,12 +32,55 @@ test('resolves latest, run ids, and dates to the newest matching run', () => { assert.throws(() => resolveRunRef('yesterday', RUNS), /Unknown run reference/); }); -test('baseline candidates are earlier runs, newest first, without excluded runs', () => { - const candidates = baselineCandidates(RUNS, '2026-10-20T03:00:00Z', new Set(['103'])); - assert.deepEqual( - candidates.map((r) => r.id), - ['102', '101'], - ); +const baselineIds = ( + runs: Iterable, + options: { nights?: number; exclude?: string[]; reject?: string[] } = {}, +) => + selectNightlyBaseline( + runs, + { + before: '2026-10-20T03:00:00Z', + exclude: new Set(options.exclude), + nights: options.nights ?? 3, + }, + (r) => (options.reject?.includes(r.id) ? undefined : r.id), + ).map(({ value }) => value); + +test('a rolling baseline takes one scheduled run per UTC night and skips dispatches', () => { + const newestFirst = [ + run('20', '2026-10-19T16:00:00Z', 'workflow_dispatch'), + run('19', '2026-10-19T15:00:00Z', 'workflow_dispatch'), + run('18', '2026-10-19T14:00:00Z', 'workflow_dispatch'), + run('17', '2026-10-19T02:50:00Z'), + run('16', '2026-10-19T02:40:00Z'), + run('15', '2026-10-18T02:40:00Z'), + run('14', '2026-10-17T02:40:00Z'), + ]; + // Three dispatches in one afternoon are not three nights; 2026-10-19 counts its newest run once. + assert.deepEqual(baselineIds(newestFirst), ['17', '15', '14']); +}); + +test('a night without a usable summary gives way to an older scheduled run of that night', () => { + const newestFirst = [ + run('17', '2026-10-19T02:50:00Z'), + run('16', '2026-10-19T02:40:00Z'), + run('15', '2026-10-18T02:40:00Z'), + run('14', '2026-10-17T02:40:00Z'), + ]; + assert.deepEqual(baselineIds(newestFirst, { reject: ['17'], exclude: ['15'] }), ['16', '14']); +}); + +test('a rolling baseline reads past 100 runs and stops once it has enough nights', () => { + function* history(): Generator { + for (let i = 0; i < 150; i++) { + yield run(`d${i}`, '2026-10-19T12:00:00Z', 'workflow_dispatch'); + } + yield run('n3', '2026-10-18T02:40:00Z'); + yield run('n2', '2026-10-17T02:40:00Z'); + yield run('n1', '2026-10-16T02:40:00Z'); + throw new Error('read past the last needed night'); + } + assert.deepEqual(baselineIds(history()), ['n3', 'n2', 'n1']); }); test('a range includes both ends whichever order they are given in', () => { diff --git a/scripts/perf/nightly-artifacts.ts b/scripts/perf/nightly-artifacts.ts index 08b00b7797..8f0525f077 100644 --- a/scripts/perf/nightly-artifacts.ts +++ b/scripts/perf/nightly-artifacts.ts @@ -30,39 +30,47 @@ function gh(args: string[]): { ok: boolean; stdout: string; stderr: string } { return { ok: result.exitCode === 0, stdout: result.stdout, stderr: result.stderr }; } -/** Lists the workflow's completed runs on one branch, newest first. */ -export function listRuns(source: NightlySource, limit = 100): WorkflowRun[] { - const result = gh([ - 'run', - 'list', - '--repo', - source.repo, - '--workflow', - source.workflow, - '--branch', - source.branch, - '--status', - 'completed', - '--limit', - String(limit), - '--json', - 'databaseId,createdAt,headSha,event', - ]); - if (!result.ok) throw new Error(`gh run list failed: ${result.stderr.trim()}`); - const rows = JSON.parse(result.stdout) as { - databaseId: number; - createdAt: string; - headSha: string; - event: string; - }[]; - return sortNewestFirst( - rows.map((row) => ({ - id: String(row.databaseId), - createdAt: row.createdAt, - headSha: row.headSha, - event: row.event, - })), - ); +const RUNS_PER_PAGE = 100; + +/** + * Yields the workflow's completed runs on one branch, newest first, one API page at a time. A + * caller that stops early fetches only the pages that it read. + */ +export function* listRuns(source: NightlySource): Generator { + const seen = new Set(); + for (let page = 1; ; page++) { + const result = gh([ + 'api', + '--method', + 'GET', + `repos/${source.repo}/actions/workflows/${source.workflow}/runs`, + '-f', + `branch=${source.branch}`, + '-f', + 'status=completed', + '-f', + `per_page=${RUNS_PER_PAGE}`, + '-f', + `page=${page}`, + '--jq', + '[.workflow_runs[] | {id, created_at, head_sha, event}]', + ]); + if (!result.ok) throw new Error(`gh api workflow runs failed: ${result.stderr.trim()}`); + const rows = JSON.parse(result.stdout) as { + id: number; + created_at: string; + head_sha: string; + event: string; + }[]; + for (const row of rows) { + const id = String(row.id); + // A run created during paging moves later pages by one place. + if (seen.has(id)) continue; + seen.add(id); + yield { id, createdAt: row.created_at, headSha: row.head_sha, event: row.event }; + } + if (rows.length < RUNS_PER_PAGE) return; + } } function sortNewestFirst(runs: WorkflowRun[]): WorkflowRun[] { @@ -112,13 +120,35 @@ export function coversTarget(candidate: PerfSummary, target: PerfSummary): boole ); } -/** Baseline candidates: runs created before the target, newest first, never the target itself. */ -export function baselineCandidates( - runs: WorkflowRun[], - before: string, - exclude: ReadonlySet, -): WorkflowRun[] { - return sortNewestFirst(runs).filter((run) => run.createdAt < before && !exclude.has(run.id)); +/** + * Selects the rolling baseline: one scheduled run for each UTC night before the target, newest night + * first. A night counts the newest of its scheduled runs that `load` accepts, so manual dispatches + * cannot fill the baseline with measurements from one afternoon. `runs` is newest first and is read + * only until `nights` nights are found. + */ +export function selectNightlyBaseline( + runs: Iterable, + options: { before: string; exclude: ReadonlySet; nights: number }, + load: (run: WorkflowRun) => T | undefined, +): { run: WorkflowRun; value: T }[] { + const selected: { run: WorkflowRun; value: T }[] = []; + const nights = new Set(); + for (const run of runs) { + const night = run.createdAt.slice(0, 10); + if ( + run.event !== 'schedule' || + run.createdAt >= options.before || + options.exclude.has(run.id) || + nights.has(night) + ) + continue; + const value = load(run); + if (value === undefined) continue; + nights.add(night); + selected.push({ run, value }); + if (selected.length >= options.nights) break; + } + return selected; } /** diff --git a/scripts/perf/run.ts b/scripts/perf/run.ts index 51cff7c18d..cd712056ae 100644 --- a/scripts/perf/run.ts +++ b/scripts/perf/run.ts @@ -1,10 +1,9 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { invokeCli } from './cli.ts'; import { parseConfig, REPO_ROOT, usesSourceCli, type PerfConfig } from './config.ts'; import { runScenario, setupIsolation, teardownIsolation } from './harness.ts'; -import { copyRawState, runIosSuite, type IosSuiteContext } from './ios-harness.ts'; +import { iosRunFinalizer, runIosSuite, type IosSuiteContext } from './ios-harness.ts'; import { describeCoverage, FIXTURE_APP_EVENT_URL_TEMPLATE, @@ -57,18 +56,13 @@ async function runIos( }; fs.mkdirSync(ctx.artifactsDir, { recursive: true }); log(`state-dir: ${ctx.stateDir}`); - const stop = (): void => { - invokeCli(['close'], ['--state-dir', ctx.stateDir, '--session', 'perf']); - invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); - }; - onInterrupt(stop); + const finalize = iosRunFinalizer(ctx, cfg.keepArtifacts); + onInterrupt(finalize); try { const measurements = await runIosSuite(ctx, cfg, scenarios); return { measurements, coverage: describeCoverage(scenarios, IOS_SUITE_EXCLUSIONS) }; } finally { - stop(); - copyRawState(ctx); - if (!cfg.keepArtifacts) fs.rmSync(ctx.stateDir, { recursive: true, force: true }); + finalize(); } } From ea9af01814a7a717c923cab9ab2875e2a22903fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 08:48:56 +0200 Subject: [PATCH 05/15] feat(perf): time every Android command against the fixture app Run iOS and Android from one shared suite: platform deltas are data on each step, and one harness owns rounds, sessions, and daemon restarts. The heavy-feed scenario times the Android helper's one-shot instrumentation capture by restarting the daemon with the helper session off. Request profiles name the helper transport that answered, adb spawns, and helper gestures. The nightly runs both platforms and compares each against its own nights; the Settings tour is removed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/perf-nightly.yml | 95 +++++++-- scripts/perf/README.md | 91 +++++++-- scripts/perf/cli.ts | 43 +--- scripts/perf/compare-cli.ts | 25 ++- scripts/perf/compare.test.ts | 16 ++ scripts/perf/compare.ts | 16 +- scripts/perf/config.test.ts | 14 +- scripts/perf/config.ts | 28 +-- scripts/perf/harness.ts | 193 ------------------ scripts/perf/ios-harness.test.ts | 57 ------ scripts/perf/ios-suite.test.ts | 39 ---- scripts/perf/measurement.ts | 2 +- scripts/perf/metadata.ts | 23 ++- scripts/perf/nightly-artifacts.test.ts | 8 + scripts/perf/nightly-artifacts.ts | 11 +- scripts/perf/platform-profiles.ts | 50 ----- scripts/perf/profile.test.ts | 96 +++++++++ scripts/perf/profile.ts | 47 ++++- scripts/perf/run.ts | 70 +++---- scripts/perf/scenario.ts | 148 -------------- scripts/perf/suite-harness.test.ts | 134 ++++++++++++ .../perf/{ios-harness.ts => suite-harness.ts} | 96 ++++++--- scripts/perf/suite.test.ts | 92 +++++++++ scripts/perf/{ios-suite.ts => suite.ts} | 188 ++++++++++++----- scripts/perf/summary-fixture.ts | 12 +- scripts/perf/types.ts | 9 +- 26 files changed, 859 insertions(+), 744 deletions(-) delete mode 100644 scripts/perf/harness.ts delete mode 100644 scripts/perf/ios-harness.test.ts delete mode 100644 scripts/perf/ios-suite.test.ts delete mode 100644 scripts/perf/platform-profiles.ts delete mode 100644 scripts/perf/scenario.ts create mode 100644 scripts/perf/suite-harness.test.ts rename scripts/perf/{ios-harness.ts => suite-harness.ts} (77%) create mode 100644 scripts/perf/suite.test.ts rename scripts/perf/{ios-suite.ts => suite.ts} (67%) diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index f4d7cd4863..38eeb50112 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -1,14 +1,15 @@ name: Perf Nightly -# Nightly iOS command performance against the fixture app, compared night over night (#1781 A3). +# Nightly iOS and Android command performance against the fixture app, compared night over night +# per platform (#1781 A3). # - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts # failing. PR lanes check behavior, not time. # - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians # swing up to +122% between single nights at n=5, so a step is judged against the median of up to # seven earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. -# - Cost: one macos-26 job of about 45 minutes a night; summaries are kept 90 days, raw logs 30. +# - Cost: one macos-26 job of about 45 minutes and one ubuntu emulator job a night; summaries are +# kept 90 days, raw logs 30. # - Delete or park when: no flagged change has been acted on for 60 nights. -# Android stays report-only and runs only on request. on: schedule: @@ -24,7 +25,7 @@ on: required: false default: '1' scenarios: - description: 'Comma-separated iOS scenario ids (empty runs all)' + description: 'Comma-separated scenario ids (empty runs all)' required: false default: '' baseline-nights: @@ -35,8 +36,13 @@ on: description: 'Branch whose runs form the baseline' required: false default: 'main' + ios: + description: 'Run the iOS suite' + type: boolean + required: false + default: true android: - description: 'Also run the report-only Android Settings tour' + description: 'Run the Android suite' type: boolean required: false default: true @@ -56,6 +62,7 @@ env: jobs: perf-ios: name: iOS Command Perf + if: ${{ github.event_name == 'schedule' || github.event.inputs.ios == 'true' }} runs-on: macos-26 timeout-minutes: 100 env: @@ -131,7 +138,7 @@ jobs: BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }} BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }} run: | - pnpm perf:compare --to perf-results \ + pnpm perf:compare --platform ios --to perf-results \ --branch "$BASELINE_BRANCH" \ --nights "$BASELINE_NIGHTS" \ --exclude-run "$GITHUB_RUN_ID" \ @@ -162,26 +169,44 @@ jobs: perf-android: name: Android Command Perf - if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.android == 'true' }} + if: ${{ github.event_name == 'schedule' || github.event.inputs.android == 'true' }} runs-on: ubuntu-latest - timeout-minutes: 80 - env: - AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts' + timeout-minutes: 100 steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml + + - name: Build the shared CLI + uses: ./.github/actions/run-gate + with: + gate: build - name: Setup Android replay host - id: android-replay-host uses: ./.github/actions/setup-android-replay-host with: package-helpers: 'true' - - name: Run Android command perf benchmark + - name: Restore the fixture APK + id: fixture-app + uses: ./.github/actions/setup-fixture-app + with: + platform: android + install: 'false' + + # The emulator action runs each script line on its own, so the script is one folded line. + - name: Run Android command perf suite uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 + env: + APP_ID: ${{ steps.fixture-app.outputs.app-id }} + APP_PATH: ${{ steps.fixture-app.outputs.apk-path }} + SCENARIOS: ${{ github.event.inputs.scenarios }} with: api-level: 36 arch: x86_64 @@ -190,15 +215,45 @@ jobs: emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics script: >- set -e; - adb -s emulator-5554 shell settings put global window_animation_scale 0 || true; - adb -s emulator-5554 shell settings put global transition_animation_scale 0 || true; - adb -s emulator-5554 shell settings put global animator_duration_scale 0 || true; - pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup "$PERF_WARMUP" --out-dir "$GITHUB_WORKSPACE/perf-results" + adb -s emulator-5554 install -r "$APP_PATH"; + pnpm perf --platform android --serial emulator-5554 + --app-id "$APP_ID" --app-path "$APP_PATH" + --rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP" + ${SCENARIOS:+--scenarios "$SCENARIOS"} + --out-dir "$GITHUB_WORKSPACE/perf-results" + + - name: Compare with earlier nights + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} + env: + GH_TOKEN: ${{ github.token }} + BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }} + BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }} + run: | + pnpm perf:compare --platform android --to perf-results \ + --branch "$BASELINE_BRANCH" \ + --nights "$BASELINE_NIGHTS" \ + --exclude-run "$GITHUB_RUN_ID" \ + --out perf-results --github + echo "Full report: the perf-android-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Android perf summary + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: perf-android-summary + path: | + perf-results/summary.json + perf-results/report.md + perf-results/comparison.json + perf-results/comparison.md + retention-days: 90 + if-no-files-found: error - - name: Upload Android perf report - if: always() + - name: Upload Android perf raw logs + if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: perf-android - path: perf-results/ + name: perf-android-raw + path: perf-results/raw/ + retention-days: 30 if-no-files-found: warn diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 0715aa7061..66f4a04810 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -1,27 +1,73 @@ # Command performance suite -`pnpm perf --platform ios` times every interaction command against the fixture app (`examples/test-app`). -The steps are in `ios-suite.ts`. Each step belongs to one scenario. +`pnpm perf` times every interaction command against the fixture app (`examples/test-app`) on iOS and on Android. +The steps are in `suite.ts`. Each step belongs to one scenario. Both platforms run the same scenarios. +A step runs on both platforms unless it names one platform. A step can replace its arguments, waits, or expectations on one platform. Each scenario starts from an untimed setup in every round. -The heavy-feed scenario uses a screen whose accessibility tree is deeper than 65 levels. -The suite records a reason for each catalog command that it does not time. `ios-suite.test.ts` keeps that list complete. -`pnpm perf --platform android` runs the older report-only Settings tour. +The heavy-feed scenario uses a screen whose accessibility tree is deeper than 65 levels on iOS and 60 levels on Android. +For each platform, the suite records a reason for each catalog command that it does not time. `suite.test.ts` keeps those lists complete. ## Run locally 1. Run `pnpm build`. The suite drives the built CLI and the built SDK. -2. Use your own booted simulator. Install a current Release fixture on it. For the cached CI build and the JavaScript repack, see `examples/test-app/README.md`. -3. Set `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH` to a path in your worktree. -4. Run the suite: +2. Use your own booted simulator or emulator. Install a current Release fixture on it. For the cached CI build and the JavaScript repack, see `examples/test-app/README.md`. +3. For iOS, set `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH` to a path in your worktree. +4. For Android, run `pnpm build:android` once. The snapshot helper APK must be in `android/snapshot-helper/dist/`. +5. Run the suite: ```sh -pnpm perf --platform ios --udid --rounds 3 --warmup 1 --out-dir .tmp/perf/local +pnpm perf --platform ios --udid --rounds 3 --warmup 1 --out-dir .tmp/perf/ios +pnpm perf --platform android --serial --rounds 3 --warmup 1 --out-dir .tmp/perf/android ``` -`--rounds` is at least 3. `--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id. +`--rounds` is at least 3. `--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id or package name. +To install the cached Android Release fixture, use an empty download directory: + +```sh +pnpm test-app:install +NAME="$(sh .github/actions/setup-fixture-app/resolve-artifact-name.sh android)" +RUN="$(gh api "repos/callstack/agent-device/actions/artifacts?name=$NAME" --jq '.artifacts[0].workflow_run.id')" +gh run download "$RUN" --repo callstack/agent-device --name "$NAME" --dir +tar -xzf /binary.tar.gz -C +ANDROID_HOME= bash .github/actions/setup-fixture-app/repack-android-apk.sh /app-release.apk /fixture-app.apk +adb -s install -r /fixture-app.apk +``` + +The repack puts the current JavaScript into the cached native build. +If no artifact has the current fingerprint, build the APK as `.github/workflows/test-app-build-cache.yml` does. + The suite uses its own temporary state directory and daemon. It stops them at the end. At the end, or on SIGINT or SIGTERM, the suite saves the logs to `raw/state/`, stops the session and daemon within a deadline, and saves the logs again. +## Snapshot paths + +The heavy-feed scenario times the normal snapshot path and one forced path on each platform. + +- iOS: the simulator accessibility bridge answers normal captures. `snapshot-raw-forced-tree` pins the XCTest tree backend through the SDK, because the CLI has no flag for it. On this screen, the tree backend must degrade to `sparse`. +- Android: the snapshot helper APK is the only snapshot engine (ADR 0002). Its persistent session answers normal captures. `snapshot-raw-one-shot` restarts the daemon with `AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION=0`, so a one-shot `am instrument` capture answers. The harness restarts the daemon without the variable after the step. Both raw captures must reach depth 56. The forced capture must report the `instrumentation` transport. + +## Platform parity + +These commands differ between the platforms. All other catalog commands run the same steps on both platforms. +Some shared steps use other screen coordinates or waits on Android. The `on.android` fields in `suite.ts` list them. + +| Command | iOS | Android | +| --- | --- | --- | +| `action-button` | lifecycle | Not timed: Android has no key event for the Action button. | +| `prepare` | lifecycle (`prepare ios-runner`) | Not timed: `prepare` has only the `ios-runner` action. | +| `hardware-button` | Not timed: the simulator refuses hardware-button presses. | lifecycle (`hardware-button wake`) | +| `push` | Not timed: the fixture has no notification authorization. | lifecycle (adb broadcast) | +| `telephony` | Not timed: the simulator refuses calls and SMS. | lifecycle (`telephony sms`) | +| `snapshot` | heavy-feed: `snapshot-raw-forced-tree` | heavy-feed: `snapshot-raw-one-shot` | +| `audio` | Not timed: hosted runners grant no macOS audio permission. | Not timed: the Linux runner has no ScreenCaptureKit. | +| `fold` | Not timed: iPhone Duo only. | Not timed: foldable emulators only. | +| `tv-remote` | Not timed: tvOS only. | Not timed: Android TV only. | +| `hover`, `viewport` | Not timed: web targets only. | Not timed: web targets only. | +| `boot`, `shutdown`, `install`, `reinstall`, `uninstall`, `install-from-source` | Not timed: provisioning time dominates, or the command ends the session under test. | Same reason. | +| `replay`, `test` | Not timed: the sum of the commands that they run. | Same reason. | + +Each run lists the measured commands and every reason in the Coverage section of `report.md`. + ## Output - `summary.json`: the machine-readable run (`agent-device-perf-summary/v1`). It holds the metadata, the coverage, and every sample. @@ -39,30 +85,32 @@ All times are milliseconds. Statistics use only the successful samples of a step | `daemon` | The daemon request round trip that the client measures (`daemon_request`). For the pinned-backend capture, the span of its request log. Comparisons use this metric. | | `mean ± sd`, `cv` | The mean, the sample standard deviation, and the standard deviation divided by the mean. | | `median`, `p95`, `min`, `max` | The median (the mean of the two middle values for an even count), the nearest-rank 95th percentile, and the extremes. | -| `runner` | The median sum of XCTest runner command times in one request. | -| `spawns` | The median count and time of host processes that one request started. | -| `snapshot source` | The median time of host accessibility-bridge acquisitions in one request. | +| `runner` | The median sum of XCTest runner command times (iOS) or snapshot-helper gesture times (Android) in one request. | +| `spawns` | The median count and time of host processes, for example `xcrun` or `adb`, that one request started. | +| `snapshot source` | The median time of host accessibility-bridge acquisitions (iOS) or snapshot-helper captures (Android) in one request. | | `settle wait` | The median `settle.waitedMs` that a `--settle` response reports. | | `stabilization` | The median time of post-gesture stabilization phases in one request. | | `retries` | The failed attempts of retried operations in all successful samples. | | `top phases` | The five phases with the largest median time per request. | -| `producer` | The number of samples that each snapshot producer answered (`simulator-ax-bridge`, `apple-runner`). | -| backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected`. | +| `producer` | The number of samples that each snapshot producer answered: `simulator-ax-bridge` and `apple-runner` on iOS; `android-helper/persistent-session` and `android-helper/instrumentation` (one-shot `am instrument`) on Android. | +| backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected` or `android-helper/healthy`. | | fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. | -| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth. | +| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth, or that the forced path did not serve the capture. | ## Nightly results -`.github/workflows/perf-nightly.yml` runs the iOS suite every night on `main`. -Each run uploads `perf-ios-summary` (summary, report, comparison; 90 days) and `perf-ios-raw` (logs; 30 days). -The job compares the run with up to seven earlier nights and writes the result to the job summary. +`.github/workflows/perf-nightly.yml` runs the iOS suite and the Android suite every night on `main`. +iOS runs on a `macos-26` simulator. Android runs on an API 36 `pixel_7` emulator on a Linux KVM runner. +For each platform, a run uploads `perf--summary` (summary, report, comparison; 90 days) and `perf--raw` (logs; 30 days). +Each job compares its run with up to seven earlier nights of the same platform and writes the result to the job summary. A night is the newest scheduled run of one UTC day that ran every scenario of the judged run. Manual dispatches are never part of the rolling baseline. To fetch and compare runs on your machine, use `gh` with access to the repository: ```sh -pnpm perf:compare # latest night against the 7 nights before it +pnpm perf:compare # latest iOS night against the 7 nights before it +pnpm perf:compare --platform android # the same for Android pnpm perf:compare --to 2026-10-28 # that night against the 7 nights before it pnpm perf:compare --to latest --from 2026-10-21 # latest against the newest run of that day pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare # every run in the range @@ -71,7 +119,8 @@ pnpm perf:compare --to .tmp/perf/local # a local run against recent A reference is `latest`, a run id, a `YYYY-MM-DD` date, or a local output path. An explicit `--to` or `--from` reference accepts runs of any trigger, so it can compare dispatches and local runs. -Downloads stay in `.tmp/perf-nightly//`. +`--platform` (default `ios`) selects the artifact. Every compared run must be of that platform. +Downloads stay in `.tmp/perf-nightly///`. Do not judge a local run against CI nights. The comparison warns when the device, Xcode, or CPU differs. ## Comparison rule diff --git a/scripts/perf/cli.ts b/scripts/perf/cli.ts index cada28867d..e7ad1889b3 100644 --- a/scripts/perf/cli.ts +++ b/scripts/perf/cli.ts @@ -1,7 +1,6 @@ import { performance } from 'node:perf_hooks'; import { runCmdSync } from '@agent-device/host-kit/command'; import { resolveCliArgv, REPO_ROOT } from './config.ts'; -import type { BatchStepSpec } from './scenario.ts'; import type { CliResult } from './types.ts'; const MAX_BUFFER = 64 * 1024 * 1024; @@ -36,7 +35,7 @@ function jsonOk(json: unknown): boolean { export function invokeCli( args: string[], baseFlags: string[], - options: { timeoutMs?: number } = {}, + options: { timeoutMs?: number; env?: Readonly> } = {}, ): CliResult { const full = [...CLI_ARGV, ...args, ...baseFlags, '--json']; const t0 = performance.now(); @@ -51,6 +50,7 @@ export function invokeCli( maxBuffer: MAX_BUFFER, allowFailure: true, timeoutMs: options.timeoutMs, + env: options.env ? { ...process.env, ...options.env } : undefined, }); stdout = r.stdout; stderr = r.stderr; @@ -63,42 +63,3 @@ export function invokeCli( const json = tryParseJson(stdout); return { exitCode, wallClockMs, stdout, stderr, json, ok: exitCode === 0 && jsonOk(json) }; } - -// Wrap a single command in its own `batch` invocation to read per-step durationMs. -export function invokeBatchStep(spec: BatchStepSpec, baseFlags: string[]): CliResult { - const result = invokeCli(['batch', '--steps', JSON.stringify([spec])], baseFlags); - // Defensive: today's stop-only batch surfaces a failed step as a top-level non-zero/ok:false - // (already caught by invokeCli). But if a future on-error mode keeps the batch ok while a step - // fails, don't silently count that step as a success — downgrade ok from the step's own ok. - const stepOk = firstBatchResult(result.json)?.ok; - if (result.ok && stepOk === false) { - return { ...result, ok: false }; - } - return result; -} - -function firstBatchResult(json: unknown): Record | undefined { - const data = (json as { data?: { results?: unknown[] } } | undefined)?.data; - const first = data?.results?.[0]; - return first && typeof first === 'object' ? (first as Record) : undefined; -} - -export function readBatchStepDurationMs(result: CliResult): number | undefined { - const v = firstBatchResult(result.json)?.durationMs; - return typeof v === 'number' ? v : undefined; -} - -export function readBatchStepError(result: CliResult): { code?: string; message?: string } { - const err = (result.json as { error?: { code?: string; message?: string } } | undefined)?.error; - return { code: err?.code, message: err?.message }; -} - -// Proxy for a11y-tree size: snapshot node count (falls back to distinct @eN refs). -export function countElements(result: CliResult): number | undefined { - const stepData = firstBatchResult(result.json)?.data; - if (stepData === undefined || typeof stepData !== 'object') return undefined; - const nodes = (stepData as { nodes?: unknown }).nodes; - if (Array.isArray(nodes)) return nodes.length; - const matches = JSON.stringify(stepData).match(/@e\d+/g); - return matches ? new Set(matches).size : 0; -} diff --git a/scripts/perf/compare-cli.ts b/scripts/perf/compare-cli.ts index 3b3593acfe..8b130c457d 100644 --- a/scripts/perf/compare-cli.ts +++ b/scripts/perf/compare-cli.ts @@ -10,19 +10,21 @@ import { resolveRunRef, selectNightlyBaseline, selectRange, - SUMMARY_ARTIFACT, + summaryArtifact, type NightlySource, type WorkflowRun, } from './nightly-artifacts.ts'; -import type { PerfSummary } from './types.ts'; +import type { PerfSummary, Platform } from './types.ts'; -const USAGE = `Usage: pnpm perf:compare [--to ] [--from |..] [--nights ] [options] +const USAGE = `Usage: pnpm perf:compare [--platform ios|android] [--to ] [--from |..] [--nights ] [options] Compares one perf run with earlier runs and prints a Markdown verdict per step. A is latest, a workflow run id, a YYYY-MM-DD date (the newest run that UTC day), or a local path to a summary.json or a \`pnpm perf\` output directory. + --platform ios or android (default: ios). Nightly runs are read from + perf--summary; every compared run must be of this platform. --to Run to judge (default: latest). --from Baseline run, or an inclusive range .. of runs of any trigger. Without it the baseline is the --nights scheduled nights before --to. @@ -36,6 +38,7 @@ path to a summary.json or a \`pnpm perf\` output directory. --github Append the Markdown to $GITHUB_STEP_SUMMARY and annotate flagged steps.`; type Options = { + platform: Platform; to: string; from?: string; nights: number; @@ -47,6 +50,7 @@ type Options = { function parse(argv: string[]): Options { const options: Options = { + platform: 'ios', to: 'latest', nights: 7, github: false, @@ -55,7 +59,7 @@ function parse(argv: string[]): Options { repo: process.env.GITHUB_REPOSITORY ?? 'callstack/agent-device', workflow: 'perf-nightly.yml', branch: 'main', - artifact: SUMMARY_ARTIFACT, + artifact: summaryArtifact('ios'), cacheDir: path.join(REPO_ROOT, '.tmp', 'perf-nightly'), }, }; @@ -67,6 +71,14 @@ function parse(argv: string[]): Options { return next; }; switch (flag) { + case '--platform': { + const platform = value(); + if (platform !== 'ios' && platform !== 'android') + throw new Error('--platform must be ios or android'); + options.platform = platform; + options.source.artifact = summaryArtifact(platform); + break; + } case '--to': options.to = value(); break; @@ -138,6 +150,11 @@ function main(): void { }); const target = load(options.to); + if (target.summary.platform !== options.platform) { + throw new Error( + `${target.label} is an ${target.summary.platform} run; pass --platform ${target.summary.platform}.`, + ); + } const exclude = new Set([...options.exclude, ...(target.run ? [target.run.id] : [])]); let baseline: Loaded[]; if (options.from?.includes('..')) { diff --git a/scripts/perf/compare.test.ts b/scripts/perf/compare.test.ts index 51fb75d55c..760887d065 100644 --- a/scripts/perf/compare.test.ts +++ b/scripts/perf/compare.test.ts @@ -128,3 +128,19 @@ test('warns when the baseline ran on another toolchain', () => { assert.equal(comparison.warnings.length, 1); assert.match(comparison.warnings[0]!, /xcode/); }); + +test('refuses a baseline run from another platform', () => { + const android = summaryFixture(flat(1000), { platform: 'android' }); + assert.throws( + () => + compareRuns({ summary: summaryFixture(flat(1000)), label: 'tonight' }, [ + ...nights(flat(1000), flat(1000)), + { summary: android, label: 'android night' }, + ]), + /An ios run cannot be compared with android night \(android\)/, + ); + const comparison = compareRuns({ summary: android, label: 'tonight' }, [ + { summary: summaryFixture(flat(1000), { platform: 'android' }), label: 'night' }, + ]); + assert.equal(comparison.platform, 'android'); +}); diff --git a/scripts/perf/compare.ts b/scripts/perf/compare.ts index 22b1f28a61..4d9289e5a7 100644 --- a/scripts/perf/compare.ts +++ b/scripts/perf/compare.ts @@ -1,6 +1,6 @@ import { MIN_MEASURED_ROUNDS } from './config.ts'; import { median, scaledMad } from './stats.ts'; -import type { Measurement, PerfSummary, Stat } from './types.ts'; +import type { Measurement, PerfSummary, Platform, Stat } from './types.ts'; export type ComparisonThresholds = { // A change must exceed every one of these to be flagged. @@ -78,6 +78,7 @@ const COMPARISON_SCHEMA = 'agent-device-perf-comparison/v1'; export type Comparison = { schema: typeof COMPARISON_SCHEMA; + platform: Platform; target: RunRef; baseline: RunRef[]; thresholds: ComparisonThresholds; @@ -216,13 +217,21 @@ function environmentWarnings(target: PerfSummary, baseline: PerfSummary[]): stri * median of its nightly medians, and its noise is the scaled MAD of those medians. A step is flagged * only when its change exceeds the relative floor, the night-to-night spread, tonight's own * sampling error, and an absolute floor together. Tonight and every night that counts need - * `minSamples` successful samples. Failures are judged apart from timing. + * `minSamples` successful samples. Failures are judged apart from timing. Every baseline run must + * come from the platform of the target. */ export function compareRuns( target: { summary: PerfSummary; label: string }, baseline: { summary: PerfSummary; label: string }[], thresholds: ComparisonThresholds = DEFAULT_THRESHOLDS, ): Comparison { + const platform = target.summary.platform; + const foreign = baseline.filter((b) => b.summary.platform !== platform); + if (foreign.length > 0) { + throw new Error( + `An ${platform} run cannot be compared with ${foreign.map((b) => `${b.label} (${b.summary.platform})`).join(', ')}.`, + ); + } const targetById = new Map(target.summary.measurements.map((m) => [m.id, m])); const ids = [ ...target.summary.measurements.map((m) => m.id), @@ -243,6 +252,7 @@ export function compareRuns( } return { schema: COMPARISON_SCHEMA, + platform, target: describeRun(target.summary, target.label), baseline: baseline.map((b) => describeRun(b.summary, b.label)), thresholds, @@ -297,7 +307,7 @@ export function renderComparison(comparison: Comparison): string { .filter((row) => flagRank(row) >= 0) .sort((a, b) => flagRank(a) - flagRank(b)); const lines = [ - `## agent-device perf comparison`, + `## agent-device perf comparison — ${comparison.platform}`, '', `- **Tonight**: ${comparison.target.label} · ${comparison.target.commit.slice(0, 12)} · ${comparison.target.startedAt} · ${comparison.target.device}`, `- **Baseline**: ${comparison.baseline.length === 0 ? 'none' : comparison.baseline.map((b) => `${b.label} (${b.commit.slice(0, 7)})`).join(', ')}`, diff --git a/scripts/perf/config.test.ts b/scripts/perf/config.test.ts index d8bb3b363a..18408db309 100644 --- a/scripts/perf/config.test.ts +++ b/scripts/perf/config.test.ts @@ -5,5 +5,17 @@ import { parseConfig } from './config.ts'; test('a run measures at least three rounds, so every step can get a timing verdict', () => { assert.equal(parseConfig(['--udid', 'sim', '--rounds', '3']).rounds, 3); assert.throws(() => parseConfig(['--udid', 'sim', '--rounds', '2']), /--rounds must be .* >= 3/); - assert.throws(() => parseConfig(['--platform', 'android', '--n', '1']), /--n must be .* >= 3/); + assert.throws( + () => parseConfig(['--platform', 'android', '--serial', 'emulator-5554', '--n', '1']), + /--n must be .* >= 3/, + ); +}); + +test('each platform requires its device selector', () => { + assert.throws(() => parseConfig(['--platform', 'ios']), /--udid is required/); + assert.throws(() => parseConfig(['--platform', 'android']), /--serial is required/); + assert.equal( + parseConfig(['--platform', 'android', '--serial', 'emulator-5554']).serial, + 'emulator-5554', + ); }); diff --git a/scripts/perf/config.ts b/scripts/perf/config.ts index 32623a8ebf..ca5812e3a2 100644 --- a/scripts/perf/config.ts +++ b/scripts/perf/config.ts @@ -11,35 +11,28 @@ export type PerfConfig = { platform: Platform; rounds: number; // measured rounds (samples per command) warmup: number; // leading rounds dropped from stats - keepArtifacts: boolean; // keep temp state dir + leave device booted + keepArtifacts: boolean; // keep the temporary state dir outDir: string; udid?: string; // iOS simulator UDID (required for iOS) - device?: string; // Android AVD name for boot - serial?: string; // Android device override - appId: string; // iOS fixture bundle id - appPath?: string; // iOS fixture bundle path, recorded in the run metadata - scenarios?: string[]; // iOS scenario ids; all when omitted + serial?: string; // Android emulator serial (required for Android) + appId: string; // fixture bundle id or package name + appPath?: string; // fixture .app or .apk path, recorded in the run metadata + scenarios?: string[]; // scenario ids; all when omitted }; -export const DEFAULT_IOS_APP_ID = 'com.callstack.agentdevicelab'; +export const DEFAULT_APP_ID = 'com.callstack.agentdevicelab'; /** The fewest measured rounds that a run accepts; a comparison needs as many successful samples. */ export const MIN_MEASURED_ROUNDS = 3; // How to invoke the CLI. Defaults to the built dist binary (bin/agent-device.mjs). -// Set AGENT_DEVICE_PERF_CLI to run from source instead, e.g. on CI: -// AGENT_DEVICE_PERF_CLI="--experimental-strip-types src/bin.ts" -// (matches the device workflows, which run from source and skip the dist build). +// AGENT_DEVICE_PERF_CLI replaces it, for example with a fake CLI in tests. export function resolveCliArgv(): string[] { const override = process.env.AGENT_DEVICE_PERF_CLI?.trim(); if (override) return override.split(/\s+/); return [CLI_BIN]; } -export function usesSourceCli(): boolean { - return Boolean(process.env.AGENT_DEVICE_PERF_CLI?.trim()); -} - function readValue(argv: string[], i: number, flag: string): string { const v = argv[i + 1]; if (v === undefined) throw new Error(`Missing value for ${flag}`); @@ -62,7 +55,7 @@ export function parseConfig(argv: string[]): PerfConfig { warmup: 1, keepArtifacts: false, outDir: path.join(DEFAULT_OUT_DIR, new Date().toISOString().replaceAll(/[:.]/g, '-')), - appId: DEFAULT_IOS_APP_ID, + appId: DEFAULT_APP_ID, }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; @@ -89,9 +82,6 @@ export function parseConfig(argv: string[]): PerfConfig { case '--udid': cfg.udid = readValue(argv, i++, a); break; - case '--device': - cfg.device = readValue(argv, i++, a); - break; case '--serial': cfg.serial = readValue(argv, i++, a); break; @@ -112,5 +102,7 @@ export function parseConfig(argv: string[]): PerfConfig { } } if (cfg.platform === 'ios' && !cfg.udid) throw new Error('--udid is required for --platform ios'); + if (cfg.platform === 'android' && !cfg.serial) + throw new Error('--serial is required for --platform android'); return cfg; } diff --git a/scripts/perf/harness.ts b/scripts/perf/harness.ts deleted file mode 100644 index 09063440a7..0000000000 --- a/scripts/perf/harness.ts +++ /dev/null @@ -1,193 +0,0 @@ -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import { - countElements, - invokeBatchStep, - invokeCli, - readBatchStepDurationMs, - readBatchStepError, -} from './cli.ts'; -import type { PerfConfig } from './config.ts'; -import { resolveProfile, type ResolvedProfile } from './platform-profiles.ts'; -import { buildMeasurement } from './measurement.ts'; -import { buildSettingsTour, type ScenarioStep } from './scenario.ts'; -import type { CliResult, Measurement, Sample } from './types.ts'; - -export type IsolationContext = { - stateDir: string; - artifactsDir: string; - baseFlags: string[]; - profile: ResolvedProfile; -}; - -function log(msg: string): void { - process.stderr.write(`[perf] ${msg}\n`); -} - -export function setupIsolation(cfg: PerfConfig): IsolationContext { - const profile = resolveProfile(cfg); - const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-perf-')); - const artifactsDir = path.join(stateDir, 'artifacts'); - fs.mkdirSync(artifactsDir, { recursive: true }); - const baseFlags = ['--state-dir', stateDir, '--session', 'perf', ...profile.platformFlags]; - log(`state-dir: ${stateDir}`); - log(`device: ${profile.deviceName} (${profile.serial ?? 'default'})`); - return { stateDir, artifactsDir, baseFlags, profile }; -} - -export function teardownIsolation(ctx: IsolationContext, cfg: PerfConfig): void { - log('teardown: closing session'); - try { - const args = ['close']; - if (!cfg.keepArtifacts) args.push('--shutdown'); - invokeCli(args, ctx.baseFlags); - } catch { - /* best-effort */ - } - if (cfg.keepArtifacts) { - log(`keep-artifacts: leaving ${ctx.stateDir} and device in place`); - return; - } - try { - fs.rmSync(ctx.stateDir, { recursive: true, force: true }); - log('teardown: removed temp state dir'); - } catch { - /* best-effort */ - } -} - -function sampleError(r: CliResult): Pick { - const err = readBatchStepError(r); - return { - errorCode: err.code ?? `exit:${r.exitCode}`, - errorMessage: (err.message ?? r.stderr.trim().split('\n').pop() ?? '').slice(0, 200), - }; -} - -// Base sample (timing + ok + error note on failure) shared by every measured invocation. -function toSample(r: CliResult, round: number): Sample { - const sample: Sample = { round, wallClockMs: r.wallClockMs, ok: r.ok }; - if (!r.ok) Object.assign(sample, sampleError(r)); - return sample; -} - -// The first interaction after open/relaunch pays the Android helper startup and a per-relaunch -// first-query settle cost. Run an untimed throwaway interaction so that -// cost is never attributed to a measured command. -function warmRunner(ctx: IsolationContext): void { - invokeCli(['snapshot', '-i'], ctx.baseFlags); -} - -function runStep(step: ScenarioStep, ctx: IsolationContext, round: number): Sample { - // Untimed reset to root for steps whose precondition is a clean, top-of-list root. - if (step.freshRoot) { - invokeCli(['open', ctx.profile.appTarget, '--relaunch'], ctx.baseFlags); - warmRunner(ctx); - } - const r = - step.execMode === 'standalone' - ? invokeCli(step.args, ctx.baseFlags) - : invokeBatchStep(step.step, ctx.baseFlags); - const sample = toSample(r, round); - if (step.execMode === 'batch') { - sample.daemonMs = readBatchStepDurationMs(r); - if (step.isSnapshot) sample.elementCount = countElements(r); - } - return sample; -} - -const TOUR_SCENARIO = 'settings-tour'; - -function tourMeasurement( - step: Pick, - platform: ResolvedProfile['platform'], - samples: Sample[], - warmupDropped: number, -): Measurement { - return buildMeasurement({ - id: `${TOUR_SCENARIO}/${step.label}`, - scenario: TOUR_SCENARIO, - label: step.label, - command: step.command, - args: [], - platform, - execMode: step.execMode, - samples, - warmupDropped, - }); -} - -function setupMeasurementFailed(measurement: Measurement): boolean { - return measurement.samples.some((sample) => !sample.ok); -} - -// Boot the device once and time it. Runs WITHOUT --session so no session lock policy -// applies and the device selectors are honored (selectors are rejected on locked sessions). -function bootOnce(ctx: IsolationContext): Measurement { - log('booting device (no session lock; sampled once)'); - const bootFlags = ['--state-dir', ctx.stateDir, ...ctx.profile.platformFlags]; - const r = invokeCli(['boot', ...ctx.profile.selectorFlags], bootFlags); - const sample = toSample(r, 0); - return tourMeasurement( - { command: 'boot', label: 'boot device', execMode: 'standalone' }, - ctx.profile.platform, - [sample], - 0, - ); -} - -// Establish the session by opening Settings WITH device selectors (open is the only -// interaction command allowed to carry selectors on a fresh session). Locks the session -// to our device so every later call targets it via --session alone. -function establishSession(ctx: IsolationContext): Measurement { - log('establishing session (open with device selectors)'); - const r = invokeCli(['open', ctx.profile.appTarget, ...ctx.profile.selectorFlags], ctx.baseFlags); - const sample = toSample(r, 0); - return tourMeasurement( - { command: 'open', label: 'open (establish + cold)', execMode: 'standalone' }, - ctx.profile.platform, - [sample], - 0, - ); -} - -export function runScenario(ctx: IsolationContext, cfg: PerfConfig): Measurement[] { - const steps = buildSettingsTour(ctx.profile, { artifactsDir: ctx.artifactsDir }); - const acc = new Map(); - for (const step of steps) acc.set(step.label, []); - - const boot = bootOnce(ctx); - const establish = establishSession(ctx); - if (setupMeasurementFailed(boot) || setupMeasurementFailed(establish)) { - log('setup failed; skipping timed tour'); - return [boot, establish]; - } - // Absorb the one-time runner startup before any round so it isn't charged to a measurement. - warmRunner(ctx); - - // Android accessibility dumps time out while the UI is animating; disable animations - // up front (untimed) so snapshot/get/is/fill can read an idle hierarchy. - log('disabling animations'); - invokeCli(['settings', 'animations', 'off'], ctx.baseFlags); - - const totalRounds = cfg.warmup + cfg.rounds; - for (let round = 0; round < totalRounds; round++) { - const measured = round >= cfg.warmup; - log(`round ${round + 1}/${totalRounds}${measured ? '' : ' (warmup, dropped)'}`); - for (const step of steps) { - const sample = runStep(step, ctx, round); - if (measured) acc.get(step.label)!.push(sample); - // After the round's reset-open relaunch, warm the runner (untimed) so the first measured - // read (snapshot -i) doesn't pay the post-relaunch first-AX-query cost. - if (step.command === 'open' && step.execMode === 'standalone') { - warmRunner(ctx); - } - } - } - - const tourMeasurements = steps.map((step) => - tourMeasurement(step, ctx.profile.platform, acc.get(step.label)!, cfg.warmup), - ); - return [boot, establish, ...tourMeasurements]; -} diff --git a/scripts/perf/ios-harness.test.ts b/scripts/perf/ios-harness.test.ts deleted file mode 100644 index e8e88f5129..0000000000 --- a/scripts/perf/ios-harness.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import path from 'node:path'; -import { mkdtempForTestSync } from '@agent-device/test-kit'; -import { test, vi } from 'vitest'; - -// Stands in for the CLI: records each teardown call, and whether the logs were already saved, -// then appends to the daemon log and fails like a teardown that went wrong. -const FAKE_CLI = ` -import fs from 'node:fs'; -const args = process.argv.slice(2); -const stateDir = args[args.indexOf('--state-dir') + 1]; -const command = args[0] === 'daemon' ? 'daemon stop' : args[0]; -const saved = fs.existsSync(process.env.PERF_FAKE_SAVED_LOG); -fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' saved=' + saved + '\\n'); -fs.appendFileSync(stateDir + '/daemon.log', command + '\\n'); -process.exit(1); -`; - -test('finalizing a run saves the logs before and after a failing teardown, once', async () => { - const root = mkdtempForTestSync('perf-finalize-'); - const fakeCli = path.join(root, 'fake-cli.mjs'); - fs.writeFileSync(fakeCli, FAKE_CLI); - const stateDir = path.join(root, 'state'); - const rawDir = path.join(root, 'out', 'raw'); - fs.mkdirSync(path.join(stateDir, 'sessions', 'perf'), { recursive: true }); - fs.writeFileSync(path.join(stateDir, 'daemon.log'), 'suite\n'); - fs.writeFileSync(path.join(stateDir, 'sessions', 'perf', 'runner.log'), 'runner\n'); - fs.writeFileSync(path.join(stateDir, 'daemon.json'), '{"token":"secret"}'); - const calls = path.join(root, 'calls.log'); - vi.stubEnv('AGENT_DEVICE_PERF_CLI', fakeCli); - vi.stubEnv('PERF_FAKE_CALLS', calls); - vi.stubEnv('PERF_FAKE_SAVED_LOG', path.join(rawDir, 'state', 'daemon.log')); - // The CLI command is read when the harness loads, so load it after the stub. - const { iosRunFinalizer } = await import('./ios-harness.ts'); - - const finalize = iosRunFinalizer( - { stateDir, udid: 'sim', appId: 'app', artifactsDir: path.join(rawDir, 'artifacts'), rawDir }, - false, - ); - finalize(); - // The interrupt handler and the normal finish both call it; the second call does nothing. - finalize(); - - assert.equal(fs.readFileSync(calls, 'utf8'), 'close saved=true\ndaemon stop saved=true\n'); - const saved = path.join(rawDir, 'state'); - assert.equal( - fs.readFileSync(path.join(saved, 'daemon.log'), 'utf8'), - 'suite\nclose\ndaemon stop\n', - ); - assert.equal( - fs.readFileSync(path.join(saved, 'sessions', 'perf', 'runner.log'), 'utf8'), - 'runner\n', - ); - assert.equal(fs.existsSync(path.join(saved, 'daemon.json')), false); - assert.equal(fs.existsSync(stateDir), false); -}); diff --git a/scripts/perf/ios-suite.test.ts b/scripts/perf/ios-suite.test.ts deleted file mode 100644 index cc92840b1e..0000000000 --- a/scripts/perf/ios-suite.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import assert from 'node:assert/strict'; -import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import { test } from 'vitest'; -import { - describeCoverage, - expectationViolation, - IOS_SUITE, - IOS_SUITE_EXCLUSIONS, -} from './ios-suite.ts'; - -test('every public command is either measured by the iOS suite or excluded with a reason', () => { - const { measured, excluded } = describeCoverage(IOS_SUITE, IOS_SUITE_EXCLUSIONS); - const classified = [...measured, ...excluded.map((e) => e.command)].sort(); - assert.deepEqual(classified, Object.values(PUBLIC_COMMANDS).sort()); - for (const { reason } of excluded) assert.ok(reason.length > 0); -}); - -test('step ids are unique, so comparisons key each step once', () => { - const ids = IOS_SUITE.flatMap((scenario) => scenario.steps.map((s) => `${scenario.id}/${s.id}`)); - assert.equal(new Set(ids).size, ids.length); -}); - -test('a heavy-feed capture breaks its expectation when the tree flattens or the tree backend copes', () => { - const depth = { maxDepthAtLeast: 64 }; - assert.equal(expectationViolation(depth, { maxDepth: 64 }), undefined); - assert.match(expectationViolation(depth, { maxDepth: 63 })!, /below 64/); - assert.match(expectationViolation(depth, undefined)!, /unknown/); - - const degraded = { treeBackendDegraded: true }; - assert.equal( - expectationViolation(degraded, { backend: 'private-ax', state: 'recovered' }), - undefined, - ); - assert.equal(expectationViolation(degraded, { backend: 'tree', state: 'sparse' }), undefined); - assert.match( - expectationViolation(degraded, { backend: 'tree', state: 'healthy' })!, - /non-sparse/, - ); -}); diff --git a/scripts/perf/measurement.ts b/scripts/perf/measurement.ts index 1641361a3a..b06229527c 100644 --- a/scripts/perf/measurement.ts +++ b/scripts/perf/measurement.ts @@ -24,7 +24,7 @@ export function buildMeasurement(input: MeasurementInput): Measurement { ...input, wallClock: summarize(ok.map((s) => s.wallClockMs)), daemon: summarize(numbers(ok.map((s) => s.daemonMs))), - elementCount: summarize(numbers(ok.map((s) => s.elementCount ?? s.snapshot?.nodeCount))), + elementCount: summarize(numbers(ok.map((s) => s.snapshot?.nodeCount))), profile: aggregateProfiles(input.samples), failures, notes, diff --git a/scripts/perf/metadata.ts b/scripts/perf/metadata.ts index 6bb4e7d861..54ab9ecd1f 100644 --- a/scripts/perf/metadata.ts +++ b/scripts/perf/metadata.ts @@ -35,10 +35,23 @@ function simulator(udid: string): { name: string; runtime?: string } | undefined return undefined; } +function emulator(serial: string): { name: string; runtime?: string } | undefined { + const adb = (args: string[]) => capture('adb', ['-s', serial, ...args]); + const name = adb(['emu', 'avd', 'name'])?.split('\n')[0]?.trim(); + const release = adb(['shell', 'getprop', 'ro.build.version.release']); + const sdk = adb(['shell', 'getprop', 'ro.build.version.sdk']); + if (!name && !release) return undefined; + return { + name: name || serial, + runtime: release ? `Android ${release}${sdk ? ` (API ${sdk})` : ''}` : undefined, + }; +} + /** Records what the numbers depend on: source, toolchain, host, and device. */ -export function collectMetadata(cfg: PerfConfig, deviceName: string): RunMetadata { +export function collectMetadata(cfg: PerfConfig): RunMetadata { const env = process.env; - const sim = cfg.platform === 'ios' && cfg.udid ? simulator(cfg.udid) : undefined; + const selector = cfg.platform === 'ios' ? cfg.udid : cfg.serial; + const target = selector ? (cfg.platform === 'ios' ? simulator : emulator)(selector) : undefined; const cpus = os.cpus(); return { commit: env.GITHUB_SHA ?? capture('git', ['rev-parse', 'HEAD']) ?? 'unknown', @@ -59,11 +72,11 @@ export function collectMetadata(cfg: PerfConfig, deviceName: string): RunMetadat : undefined, node: process.version, device: { - name: sim?.name ?? deviceName, + name: target?.name ?? selector ?? 'unknown', udid: cfg.udid, serial: cfg.serial, - runtime: sim?.runtime, + runtime: target?.runtime, }, - fixture: cfg.platform === 'ios' ? { appId: cfg.appId, appPath: cfg.appPath } : undefined, + fixture: { appId: cfg.appId, appPath: cfg.appPath }, }; } diff --git a/scripts/perf/nightly-artifacts.test.ts b/scripts/perf/nightly-artifacts.test.ts index d302a99a94..ee2bbdc278 100644 --- a/scripts/perf/nightly-artifacts.test.ts +++ b/scripts/perf/nightly-artifacts.test.ts @@ -110,4 +110,12 @@ test('a rolling baseline accepts only runs that measured every target scenario', assert.equal(coversTarget(full, full), true); assert.equal(coversTarget(partial, full), false); assert.equal(coversTarget(full, partial), true); + const android = summaryFixture( + [ + { id: 'a/x', daemonMs: [1] }, + { id: 'b/y', daemonMs: [1] }, + ], + { platform: 'android' }, + ); + assert.equal(coversTarget(android, full), false); }); diff --git a/scripts/perf/nightly-artifacts.ts b/scripts/perf/nightly-artifacts.ts index 8f0525f077..cd22988ba5 100644 --- a/scripts/perf/nightly-artifacts.ts +++ b/scripts/perf/nightly-artifacts.ts @@ -3,10 +3,12 @@ import path from 'node:path'; import { runCmdSync } from '@agent-device/host-kit/command'; import { SUMMARY_FILE } from './report.ts'; import { parsePerfSummary } from './summary.ts'; -import type { PerfSummary } from './types.ts'; +import type { PerfSummary, Platform } from './types.ts'; -/** The stable name of the summary artifact that the nightly workflow uploads for every run. */ -export const SUMMARY_ARTIFACT = 'perf-ios-summary'; +/** The stable name of the summary artifact that the nightly workflow uploads for each platform. */ +export function summaryArtifact(platform: Platform): string { + return `perf-${platform}-summary`; +} export type WorkflowRun = { id: string; @@ -156,7 +158,8 @@ export function selectNightlyBaseline( * when the run has no such artifact (it failed before upload, or the artifact expired). */ export function fetchSummary(source: NightlySource, run: WorkflowRun): PerfSummary | undefined { - const dir = path.join(source.cacheDir, run.id); + // One run uploads a summary artifact per platform, so each artifact has its own directory. + const dir = path.join(source.cacheDir, run.id, source.artifact); const file = path.join(dir, SUMMARY_FILE); if (!fs.existsSync(file)) { fs.rmSync(dir, { recursive: true, force: true }); diff --git a/scripts/perf/platform-profiles.ts b/scripts/perf/platform-profiles.ts deleted file mode 100644 index 447d3b310f..0000000000 --- a/scripts/perf/platform-profiles.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { PerfConfig } from './config.ts'; - -// Local-convenience default for ad-hoc Android runs: an AVD name that `boot` can launch. CI -// passes --serial. The iOS suite drives the fixture app instead (see ios-suite.ts). -const DEFAULT_ANDROID_AVD = 'Pixel_9_Pro_XL_API_37'; - -export type ProfileSelectors = { - // A row on the Settings root that pushes a large sub-screen (big a11y tree). - deepScreen: string; - // The Settings search field (for press/focus; auto-picks a match). - searchField: string; - // A selector that uniquely targets the EDITABLE search field (for fill). - searchFieldEditable: string; - // A label reliably visible on the Settings root, for get/is (selector form). - anchorLabel: string; - // Plain text of the anchor, for wait text / find (not a selector). - anchorText: string; -}; - -export type ResolvedProfile = { - platform: 'android'; - deviceName: string; - serial?: string; - platformFlags: string[]; // --platform; applied to every call (only conflicts if it mismatches a locked session) - selectorFlags: string[]; // device selectors — ONLY on the session-establishing open / selectorless boot - appTarget: string; // `open` target for Settings - selectors: ProfileSelectors; -}; - -export function resolveProfile(cfg: PerfConfig): ResolvedProfile { - const avdName = cfg.device ?? DEFAULT_ANDROID_AVD; - const serialFlags = cfg.serial - ? ['--serial', cfg.serial, '--android-device-allowlist', cfg.serial] - : []; - return { - platform: 'android', - deviceName: cfg.serial ? `android (${cfg.serial})` : avdName, - serial: cfg.serial, - platformFlags: ['--platform', 'android'], - selectorFlags: ['--device', avdName, ...serialFlags], - appTarget: 'com.android.settings', - selectors: { - deepScreen: 'text="Network & internet"', - searchField: 'text="Search Settings"', - searchFieldEditable: 'editable', - anchorLabel: 'label="Network & internet"', - anchorText: 'Network & internet', - }, - }; -} diff --git a/scripts/perf/profile.test.ts b/scripts/perf/profile.test.ts index 91e16e4627..e4fd1ce678 100644 --- a/scripts/perf/profile.test.ts +++ b/scripts/perf/profile.test.ts @@ -5,6 +5,7 @@ import { profileRequestLog, readClientDiagnostics, readSettleFacts, + readSnapshotFacts, } from './profile.ts'; import type { Sample } from './types.ts'; @@ -110,3 +111,98 @@ test('reads the settle wait of a --settle response and nothing from other respon ); assert.equal(readSettleFacts({ data: { message: 'Tapped' } }), undefined); }); + +// Trimmed from Android heavy-feed request logs. +const androidCapture = (session: Record, oneShot = false) => + [ + line({ phase: 'android_snapshot_helper_session_capture', ...session }), + ...(oneShot + ? [ + line({ + level: 'debug', + phase: 'exec_command', + durationMs: 373, + data: { command: 'adb', event: 'settled' }, + }), + line({ phase: 'android_snapshot_helper_capture', durationMs: 373 }), + ] + : []), + line({ phase: 'snapshot_capture', durationMs: 380, data: { backend: 'android' } }), + ].join('\n'); + +test('names the Android helper transport that answered each capture', () => { + const session = profileRequestLog(androidCapture({ durationMs: 40 })); + assert.deepEqual(session.snapshotSources, { + 'android-helper/persistent-session': { count: 1, totalMs: 40, failures: 0 }, + }); + assert.deepEqual(session.fallbacks, []); + + // With the session off, the session timer settles cleanly and a one-shot capture answers. + const sessionOff = profileRequestLog(androidCapture({ durationMs: 0 }, true)); + assert.deepEqual(sessionOff.snapshotSources, { + 'android-helper/persistent-session': { count: 1, totalMs: 0, failures: 1 }, + 'android-helper/instrumentation': { count: 1, totalMs: 373, failures: 0 }, + }); + assert.deepEqual(sessionOff.spawns, { adb: { count: 1, totalMs: 373 } }); + assert.deepEqual( + aggregateProfiles([{ round: 0, ok: true, wallClockMs: 1, profile: sessionOff }])!.producers, + { + 'android-helper/instrumentation': 1, + }, + ); + + const sessionFailed = profileRequestLog( + [ + androidCapture({ level: 'error', durationMs: 5000, data: { error: 'socket closed' } }, true), + line({ + level: 'warn', + phase: 'android_snapshot_helper_session_fallback', + data: { reason: 'socket closed' }, + }), + ].join('\n'), + ); + assert.deepEqual(sessionFailed.snapshotSources['android-helper/persistent-session'], { + count: 1, + totalMs: 5000, + failures: 1, + }); + assert.deepEqual(sessionFailed.fallbacks, [ + 'android-helper/persistent-session: socket closed', + 'android_snapshot_helper_session_fallback: socket closed', + ]); +}); + +test('counts Android helper gestures as runner commands', () => { + const profile = profileRequestLog( + [ + line({ phase: 'android_touch_helper_gesture', durationMs: 210 }), + line({ + level: 'error', + phase: 'android_touch_helper_gesture', + durationMs: 30, + data: { error: 'busy' }, + }), + ].join('\n'), + ); + assert.deepEqual(profile.runnerCommands, { gesture: { count: 2, totalMs: 240 } }); + assert.deepEqual(profile.fallbacks, ['android_touch_helper_gesture: busy']); +}); + +test('reads the Android helper transport from a snapshot response', () => { + const facts = readSnapshotFacts({ + data: { + nodes: [{ depth: 0 }, { depth: 60 }], + snapshotQuality: { state: 'healthy', backend: 'android-helper' }, + androidSnapshot: { backend: 'android-helper', helperTransport: 'instrumentation' }, + }, + }); + assert.deepEqual(facts, { + backend: 'android-helper', + transport: 'instrumentation', + state: 'healthy', + reasonCode: undefined, + nodeCount: 2, + maxDepth: 60, + truncated: undefined, + }); +}); diff --git a/scripts/perf/profile.ts b/scripts/perf/profile.ts index a869d74c32..305e9d85a5 100644 --- a/scripts/perf/profile.ts +++ b/scripts/perf/profile.ts @@ -20,6 +20,13 @@ const RUNNER_COMMAND_PHASE = 'ios_runner_command_send'; const SPAWN_PHASE = 'exec_command'; const SNAPSHOT_SOURCE_PHASE = 'ios.snapshot-source.acquire'; const SNAPSHOT_RECOVERY_PHASE = 'ios_snapshot_source_recovery'; +// Android: the snapshot helper answers through its persistent session, or through a one-shot +// `am instrument` capture when the session is off or fails. Touch gestures run in the helper too. +const ANDROID_SESSION_CAPTURE_PHASE = 'android_snapshot_helper_session_capture'; +const ANDROID_ONE_SHOT_CAPTURE_PHASE = 'android_snapshot_helper_capture'; +const ANDROID_SESSION_PRODUCER = 'android-helper/persistent-session'; +const ANDROID_ONE_SHOT_PRODUCER = 'android-helper/instrumentation'; +const ANDROID_GESTURE_PHASE = 'android_touch_helper_gesture'; const STABILIZATION_PHASE = /settle|stabiliz/i; const FALLBACK_TEXT_LIMIT = 140; const CLIENT_DIAG_PREFIX = '[agent-device][diag] '; @@ -44,8 +51,10 @@ function eventDuration(event: DiagnosticEvent): number | undefined { /** * Reads one daemon request log (`--debug` ndjson). Process spawns count once, at their `settled` - * event. Fallbacks name every degraded path the request took: snapshot-source errors, truncated - * recovery, and warn- or error-level phases. + * event. Runner commands are XCTest runner commands on iOS and helper gestures on Android. Snapshot + * sources are the iOS accessibility producers and the Android helper transports. Fallbacks name + * every degraded path the request took: snapshot-source errors, truncated recovery, and warn- or + * error-level phases. */ export function profileRequestLog(ndjson: string): SampleProfile { const profile: SampleProfile = { @@ -58,6 +67,19 @@ export function profileRequestLog(ndjson: string): SampleProfile { fallbacks: [], }; const fallbacks = new Set(); + const addSource = (producer: string, event: DiagnosticEvent): boolean => { + const entry = (profile.snapshotSources[producer] ??= { count: 0, totalMs: 0, failures: 0 }); + entry.count += 1; + entry.totalMs += eventDuration(event) ?? 0; + const error = text(event.data?.error); + if (!error && event.level !== 'error') return true; + entry.failures += 1; + fallbacks.add(`${producer}: ${error ?? 'failed'}`); + return false; + }; + // A session capture that settles cleanly but is followed by a one-shot capture did not answer: + // the session is off, and the one-shot capture served the request. + let sessionAnswered = false; let firstTs: number | undefined; let lastTs: number | undefined; for (const line of ndjson.split('\n')) { @@ -83,15 +105,16 @@ export function profileRequestLog(ndjson: string): SampleProfile { } else if (phase === RUNNER_COMMAND_PHASE) { addTiming(profile.runnerCommands, text(data.command) ?? 'unknown', duration); } else if (phase === SNAPSHOT_SOURCE_PHASE) { - const producer = text(data.producer) ?? 'unknown'; - const entry = (profile.snapshotSources[producer] ??= { count: 0, totalMs: 0, failures: 0 }); - entry.count += 1; - entry.totalMs += duration ?? 0; - const error = text(data.error); - if (error || event.level === 'error') { - entry.failures += 1; - fallbacks.add(`${producer}: ${error ?? 'failed'}`); - } + addSource(text(data.producer) ?? 'unknown', event); + } else if (phase === ANDROID_SESSION_CAPTURE_PHASE) { + sessionAnswered = addSource(ANDROID_SESSION_PRODUCER, event); + } else if (phase === ANDROID_ONE_SHOT_CAPTURE_PHASE) { + if (sessionAnswered) profile.snapshotSources[ANDROID_SESSION_PRODUCER]!.failures += 1; + sessionAnswered = false; + addSource(ANDROID_ONE_SHOT_PRODUCER, event); + } else if (phase === ANDROID_GESTURE_PHASE) { + addTiming(profile.runnerCommands, 'gesture', duration); + if (event.level === 'error') fallbacks.add(`${phase}: ${text(data.error) ?? 'failed'}`); } else if (phase === SNAPSHOT_RECOVERY_PHASE && data.truncated === true) { fallbacks.add( `${text(data.producer) ?? 'unknown'}: truncated at ${String(data.acceptedLevels)} levels`, @@ -152,8 +175,10 @@ export function readSnapshotFacts(json: unknown): SnapshotFacts | undefined { const nodes = data.nodes as { depth?: unknown }[]; const quality = (data.snapshotQuality ?? {}) as Record; const depths = nodes.map((node) => (typeof node.depth === 'number' ? node.depth : 0)); + const android = (data.androidSnapshot ?? {}) as Record; return { backend: text(quality.backend), + transport: text(android.helperTransport), state: text(quality.state), reasonCode: text(quality.reasonCode), nodeCount: nodes.length, diff --git a/scripts/perf/run.ts b/scripts/perf/run.ts index cd712056ae..81e51cad95 100644 --- a/scripts/perf/run.ts +++ b/scripts/perf/run.ts @@ -1,17 +1,17 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { parseConfig, REPO_ROOT, usesSourceCli, type PerfConfig } from './config.ts'; -import { runScenario, setupIsolation, teardownIsolation } from './harness.ts'; -import { iosRunFinalizer, runIosSuite, type IosSuiteContext } from './ios-harness.ts'; +import { parseConfig, REPO_ROOT, type PerfConfig } from './config.ts'; +import { collectMetadata } from './metadata.ts'; +import { writeRunOutput } from './report.ts'; import { describeCoverage, FIXTURE_APP_EVENT_URL_TEMPLATE, - IOS_SUITE, - IOS_SUITE_EXCLUSIONS, -} from './ios-suite.ts'; -import { collectMetadata } from './metadata.ts'; -import { writeRunOutput } from './report.ts'; + SUITE_EXCLUSIONS, + suiteFor, + type SuiteScenario, +} from './suite.ts'; +import { runFinalizer, runSuite, type SuiteContext } from './suite-harness.ts'; import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary } from './types.ts'; function log(msg: string): void { @@ -29,73 +29,59 @@ function onInterrupt(cleanup: () => void): void { }); } -function selectScenarios(cfg: PerfConfig) { - if (!cfg.scenarios) return IOS_SUITE; - const unknown = cfg.scenarios.filter((id) => !IOS_SUITE.some((s) => s.id === id)); +function selectScenarios(cfg: PerfConfig): SuiteScenario[] { + const suite = suiteFor(cfg.platform); + if (!cfg.scenarios) return suite; + const unknown = cfg.scenarios.filter((id) => !suite.some((s) => s.id === id)); if (unknown.length > 0) { throw new Error( - `Unknown scenarios: ${unknown.join(', ')} (known: ${IOS_SUITE.map((s) => s.id).join(', ')})`, + `Unknown scenarios: ${unknown.join(', ')} (known: ${suite.map((s) => s.id).join(', ')})`, ); } - return IOS_SUITE.filter((s) => cfg.scenarios!.includes(s.id)); + return suite.filter((s) => cfg.scenarios!.includes(s.id)); } -async function runIos( +async function runPlatformSuite( cfg: PerfConfig, ): Promise<{ measurements: Measurement[]; coverage: PerfSummary['coverage'] }> { const scenarios = selectScenarios(cfg); - // The daemon inherits this from the first CLI call; it routes trigger-app-event to the fixture. + // The daemon inherits these from the first CLI call; they route trigger-app-event to the fixture. process.env.AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE ??= FIXTURE_APP_EVENT_URL_TEMPLATE; + process.env.AGENT_DEVICE_ANDROID_APP_EVENT_URL_TEMPLATE ??= FIXTURE_APP_EVENT_URL_TEMPLATE; const rawDir = path.join(cfg.outDir, 'raw'); - const ctx: IosSuiteContext = { + const ctx: SuiteContext = { + platform: cfg.platform, stateDir: fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-perf-')), - udid: cfg.udid!, + device: cfg.platform === 'ios' ? cfg.udid! : cfg.serial!, appId: cfg.appId, artifactsDir: path.join(rawDir, 'artifacts'), rawDir, }; fs.mkdirSync(ctx.artifactsDir, { recursive: true }); log(`state-dir: ${ctx.stateDir}`); - const finalize = iosRunFinalizer(ctx, cfg.keepArtifacts); + const finalize = runFinalizer(ctx, cfg.keepArtifacts); onInterrupt(finalize); try { - const measurements = await runIosSuite(ctx, cfg, scenarios); - return { measurements, coverage: describeCoverage(scenarios, IOS_SUITE_EXCLUSIONS) }; - } finally { - finalize(); - } -} - -function runAndroid(cfg: PerfConfig): { - measurements: Measurement[]; - coverage: PerfSummary['coverage']; -} { - const ctx = setupIsolation(cfg); - onInterrupt(() => teardownIsolation(ctx, cfg)); - try { - const measurements = runScenario(ctx, cfg); + const measurements = await runSuite(ctx, cfg, scenarios); return { measurements, - coverage: { measured: [...new Set(measurements.map((m) => m.command))].sort(), excluded: [] }, + coverage: describeCoverage(scenarios, SUITE_EXCLUSIONS[cfg.platform]), }; } finally { - teardownIsolation(ctx, cfg); + finalize(); } } async function main(): Promise { const cfg = parseConfig(process.argv.slice(2)); - // The iOS forced-backend probe loads the built SDK; the dist CLI needs the build as well. - if ( - (cfg.platform === 'ios' || !usesSourceCli()) && - !fs.existsSync(path.join(REPO_ROOT, 'dist', 'src')) - ) { + // The CLI and the iOS forced-backend probe run from the build. + if (!fs.existsSync(path.join(REPO_ROOT, 'dist', 'src'))) { throw new Error('dist/ is missing — run `pnpm build` first.'); } const startedAt = new Date().toISOString(); // Read before the run, so a checkout change during a long run cannot relabel it. - const metadata = collectMetadata(cfg, cfg.serial ?? cfg.device ?? 'unknown'); - const { measurements, coverage } = cfg.platform === 'ios' ? await runIos(cfg) : runAndroid(cfg); + const metadata = collectMetadata(cfg); + const { measurements, coverage } = await runPlatformSuite(cfg); const summary: PerfSummary = { schema: PERF_SUMMARY_SCHEMA, platform: cfg.platform, diff --git a/scripts/perf/scenario.ts b/scripts/perf/scenario.ts deleted file mode 100644 index 1e34cad240..0000000000 --- a/scripts/perf/scenario.ts +++ /dev/null @@ -1,148 +0,0 @@ -import path from 'node:path'; -import type { ResolvedProfile } from './platform-profiles.ts'; - -// A structured batch step: maps through the same command input as MCP and the Node client. -export type BatchStepSpec = { - command: string; - input: Record; -}; - -type ScenarioStepBase = { - label: string; - command: string; - // When set, the harness runs an untimed `open --relaunch` (reset to root, top of list) - // before timing this step. Used for steps whose precondition is a clean root, since - // earlier commands (find/is, search) leave the list scrolled or in a different surface. - freshRoot?: boolean; -}; - -// Discriminated on execMode so the invoker gets the right payload without `!`/`?? []`: -// standalone carries full CLI args; batch carries one structured batch step. -export type ScenarioStep = - | (ScenarioStepBase & { execMode: 'standalone'; args: string[] }) - | (ScenarioStepBase & { execMode: 'batch'; step: BatchStepSpec; isSnapshot?: boolean }); - -export type StepContext = { artifactsDir: string }; - -function std(label: string, command: string, args: string[]): ScenarioStep { - return { label, command, execMode: 'standalone', args }; -} - -function bat( - label: string, - command: string, - step: BatchStepSpec, - opts: { isSnapshot?: boolean; freshRoot?: boolean } = {}, -): ScenarioStep { - return { label, command, execMode: 'batch' as const, step, ...opts }; -} - -// One ordered pass over Android Settings. The harness repeats this N (+warmup) times; -// the leading `open --relaunch` resets the app to its root each round, so every -// round starts from a known state while commands run in their natural order. -export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): ScenarioStep[] { - const s = p.selectors; - const shot = path.join(ctx.artifactsDir, 'shot.png'); - const rec = path.join(ctx.artifactsDir, 'rec.mp4'); - const trace = path.join(ctx.artifactsDir, 'trace.log'); - - // Android reveals the editable search field only after a tap on the search card. - const textEntry: ScenarioStep[] = [ - bat( - 'press search field', - 'press', - { - command: 'press', - input: { target: { kind: 'selector', selector: s.searchField } }, - }, - { freshRoot: true }, - ), - bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), - bat('fill search', 'fill', { - command: 'fill', - input: { - target: { kind: 'selector', selector: s.searchFieldEditable }, - text: 'general', - }, - }), - bat('get editable text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.searchFieldEditable }, - }, - }), - ]; - - return [ - // --- reset to root via relaunch --- - std('open (relaunch → root)', 'open', ['open', p.appTarget, '--relaunch']), - - // --- reads on the root tree (snapshots first; anchor label is visible here) --- - bat( - 'snapshot -i (root)', - 'snapshot', - { command: 'snapshot', input: { interactiveOnly: true } }, - { isSnapshot: true }, - ), - bat('snapshot (root)', 'snapshot', { command: 'snapshot', input: {} }, { isSnapshot: true }), - - // --- navigate into a sub-screen from a fresh root (freshRoot resets scroll so the - // deep-screen row is in view), read it, then return --- - bat( - 'press → deep screen', - 'press', - { - command: 'press', - input: { target: { kind: 'selector', selector: s.deepScreen } }, - }, - { freshRoot: true }, - ), - bat('snapshot (deep)', 'snapshot', { command: 'snapshot', input: {} }, { isSnapshot: true }), - bat( - 'snapshot -i (deep)', - 'snapshot', - { command: 'snapshot', input: { interactiveOnly: true } }, - { isSnapshot: true }, - ), - bat('back', 'back', { command: 'back', input: {} }), - - // --- targeted reads against the visible anchor (freshRoot so the anchor is on screen) --- - bat( - 'wait text', - 'wait', - { command: 'wait', input: { text: s.anchorText, timeoutMs: 3000 } }, - { freshRoot: true }, - ), - bat('find', 'find', { command: 'find', input: { query: s.anchorText } }), - bat('get text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.anchorLabel }, - }, - }), - bat('is visible', 'is', { - command: 'is', - input: { predicate: 'visible', selector: s.anchorLabel }, - }), - - // --- text entry (platform-specific order; see textEntry above) then scroll results --- - ...textEntry, - bat('scroll down', 'scroll', { command: 'scroll', input: { direction: 'down' } }), - - // --- artifact-producing commands; record brackets the rest so the clip has >1s of - // footage (an instant start→stop makes simctl recordVideo fail to finalize) --- - std('record start', 'record', ['record', 'start', rec, '--hide-touches']), - bat('screenshot', 'screenshot', { command: 'screenshot', input: { path: shot } }), - bat('logs mark', 'logs', { - command: 'logs', - input: { action: 'mark', message: 'perf-mark' }, - }), - bat('logs clear', 'logs', { command: 'logs', input: { action: 'clear' } }), - std('trace start', 'trace', ['trace', 'start', trace]), - std('trace stop', 'trace', ['trace', 'stop']), - bat('perf frames', 'perf', { command: 'perf', input: { area: 'frames' } }), - std('record stop', 'record', ['record', 'stop']), - ]; -} diff --git a/scripts/perf/suite-harness.test.ts b/scripts/perf/suite-harness.test.ts new file mode 100644 index 0000000000..7bd871cb06 --- /dev/null +++ b/scripts/perf/suite-harness.test.ts @@ -0,0 +1,134 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { test, vi } from 'vitest'; +import type { PerfConfig } from './config.ts'; + +// Stands in for the CLI: records each teardown call, and whether the logs were already saved, +// then appends to the daemon log and fails like a teardown that went wrong. +const FAKE_CLI = ` +import fs from 'node:fs'; +const args = process.argv.slice(2); +const stateDir = args[args.indexOf('--state-dir') + 1]; +const command = args[0] === 'daemon' ? 'daemon stop' : args[0]; +const saved = fs.existsSync(process.env.PERF_FAKE_SAVED_LOG); +fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' saved=' + saved + '\\n'); +fs.appendFileSync(stateDir + '/daemon.log', command + '\\n'); +process.exit(1); +`; + +test('finalizing a run saves the logs before and after a failing teardown, once', async () => { + const root = mkdtempForTestSync('perf-finalize-'); + const fakeCli = path.join(root, 'fake-cli.mjs'); + fs.writeFileSync(fakeCli, FAKE_CLI); + const stateDir = path.join(root, 'state'); + const rawDir = path.join(root, 'out', 'raw'); + fs.mkdirSync(path.join(stateDir, 'sessions', 'perf'), { recursive: true }); + fs.writeFileSync(path.join(stateDir, 'daemon.log'), 'suite\n'); + fs.writeFileSync(path.join(stateDir, 'sessions', 'perf', 'runner.log'), 'runner\n'); + fs.writeFileSync(path.join(stateDir, 'daemon.json'), '{"token":"secret"}'); + const calls = path.join(root, 'calls.log'); + vi.stubEnv('AGENT_DEVICE_PERF_CLI', fakeCli); + vi.stubEnv('PERF_FAKE_CALLS', calls); + vi.stubEnv('PERF_FAKE_SAVED_LOG', path.join(rawDir, 'state', 'daemon.log')); + // The CLI command is read when the harness loads, so load it after the stub. + const { runFinalizer } = await import('./suite-harness.ts'); + + const finalize = runFinalizer( + { + platform: 'ios', + stateDir, + device: 'sim', + appId: 'app', + artifactsDir: path.join(rawDir, 'artifacts'), + rawDir, + }, + false, + ); + finalize(); + // The interrupt handler and the normal finish both call it; the second call does nothing. + finalize(); + + assert.equal(fs.readFileSync(calls, 'utf8'), 'close saved=true\ndaemon stop saved=true\n'); + const saved = path.join(rawDir, 'state'); + assert.equal( + fs.readFileSync(path.join(saved, 'daemon.log'), 'utf8'), + 'suite\nclose\ndaemon stop\n', + ); + assert.equal( + fs.readFileSync(path.join(saved, 'sessions', 'perf', 'runner.log'), 'utf8'), + 'runner\n', + ); + assert.equal(fs.existsSync(path.join(saved, 'daemon.json')), false); + assert.equal(fs.existsSync(stateDir), false); +}); + +// Stands in for the CLI and its daemon: records each call with the session variable it saw, and +// answers a snapshot with the transport that variable selects. +const FAKE_DAEMON_CLI = ` +import fs from 'node:fs'; +const args = process.argv.slice(2); +const command = args.slice(0, args.indexOf('--state-dir')).join(' '); +const session = process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION ?? 'default'; +fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' session=' + session + '\\n'); +const transport = session === '0' ? 'instrumentation' : 'persistent-session'; +const data = args[0] === 'snapshot' + ? { nodes: [{ depth: 0 }], androidSnapshot: { helperTransport: transport } } + : {}; +process.stdout.write(JSON.stringify({ success: true, data })); +`; + +test('a step with daemon variables runs against a restarted daemon and restores it after', async () => { + const root = mkdtempForTestSync('perf-daemon-env-'); + const fakeCli = path.join(root, 'fake-cli.mjs'); + fs.writeFileSync(fakeCli, FAKE_DAEMON_CLI); + const calls = path.join(root, 'calls.log'); + vi.stubEnv('AGENT_DEVICE_PERF_CLI', fakeCli); + vi.stubEnv('PERF_FAKE_CALLS', calls); + vi.resetModules(); + const { runSuite } = await import('./suite-harness.ts'); + + const [measurement] = await runSuite( + { + platform: 'android', + stateDir: path.join(root, 'state'), + device: 'emulator-5554', + appId: 'app', + artifactsDir: path.join(root, 'artifacts'), + rawDir: path.join(root, 'raw'), + }, + { warmup: 0, rounds: 1 } as PerfConfig, + [ + { + id: 'heavy-feed', + title: 'heavy feed', + setup: [], + steps: [ + { + id: 'one-shot', + command: 'snapshot', + args: ['snapshot', '--raw'], + daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' }, + expect: { transport: 'instrumentation' }, + }, + ], + }, + ], + ); + + assert.equal( + fs.readFileSync(calls, 'utf8'), + [ + 'open app --relaunch --serial emulator-5554 session=default', + 'daemon stop session=default', + 'open app --serial emulator-5554 session=0', + 'snapshot --raw --debug session=0', + 'daemon stop session=default', + 'open app --serial emulator-5554 session=default', + '', + ].join('\n'), + ); + assert.equal(measurement!.failures, 0); + assert.equal(measurement!.samples[0]!.snapshot?.transport, 'instrumentation'); +}); diff --git a/scripts/perf/ios-harness.ts b/scripts/perf/suite-harness.ts similarity index 77% rename from scripts/perf/ios-harness.ts rename to scripts/perf/suite-harness.ts index 16579f7506..91a742d092 100644 --- a/scripts/perf/ios-harness.ts +++ b/scripts/perf/suite-harness.ts @@ -7,7 +7,7 @@ import { import { invokeCli } from './cli.ts'; import type { PerfConfig } from './config.ts'; import { captureWithForcedBackend } from './forced-backend.ts'; -import { expectationViolation, type SuiteScenario, type SuiteStep } from './ios-suite.ts'; +import { expectationViolation, type PlatformStep, type SuiteScenario } from './suite.ts'; import { buildMeasurement } from './measurement.ts'; import { profileRequestLog, @@ -15,39 +15,51 @@ import { readSettleFacts, readSnapshotFacts, } from './profile.ts'; -import type { CliResult, Measurement, Sample } from './types.ts'; +import type { CliResult, Measurement, Platform, Sample } from './types.ts'; const SESSION = 'perf'; -export type IosSuiteContext = { +export type SuiteContext = { + platform: Platform; stateDir: string; - udid: string; + // The iOS simulator UDID or the Android emulator serial. + device: string; appId: string; artifactsDir: string; rawDir: string; // Set when a request outlived the client timeout. That request still holds the session, so // the suite restarts the daemon before the next scenario. daemonStuck?: boolean; + // Variables that the running daemon was started with, set by a step with `daemonEnv`. + daemonEnv?: Readonly>; }; function log(msg: string): void { process.stderr.write(`[perf] ${msg}\n`); } -function baseFlags(ctx: IosSuiteContext): string[] { - return ['--state-dir', ctx.stateDir, '--session', SESSION, '--platform', 'ios']; +function baseFlags(ctx: SuiteContext): string[] { + return ['--state-dir', ctx.stateDir, '--session', SESSION, '--platform', ctx.platform]; } -function run(ctx: IosSuiteContext, args: readonly string[]): CliResult { - return watch(ctx, invokeCli(expandStatic(args, ctx), baseFlags(ctx))); +function deviceFlags(ctx: SuiteContext): string[] { + return ctx.platform === 'ios' ? ['--udid', ctx.device] : ['--serial', ctx.device]; } -function watch(ctx: IosSuiteContext, result: CliResult): CliResult { +function cli(ctx: SuiteContext, args: string[]): CliResult { + return watch(ctx, invokeCli(args, baseFlags(ctx), { env: ctx.daemonEnv })); +} + +function run(ctx: SuiteContext, args: readonly string[]): CliResult { + return cli(ctx, expandStatic(args, ctx)); +} + +function watch(ctx: SuiteContext, result: CliResult): CliResult { if (daemonTimedOut(result)) ctx.daemonStuck = true; return result; } -function expandStatic(args: readonly string[], ctx: IosSuiteContext): string[] { +function expandStatic(args: readonly string[], ctx: SuiteContext): string[] { return args.map((arg) => arg === '{app}' ? ctx.appId @@ -64,7 +76,7 @@ type SnapshotNode = { const TARGET_TOKEN = /^\{(ref|point):(.+)\}$/; /** Expands `{ref:id}` and `{point:id}` from one untimed interactive snapshot. */ -function resolveTargets(args: string[], ctx: IosSuiteContext): string[] | { error: string } { +function resolveTargets(args: string[], ctx: SuiteContext): string[] | { error: string } { if (!args.some((arg) => TARGET_TOKEN.test(arg))) return args; const snapshot = run(ctx, ['snapshot', '-i']); const nodes = ((snapshot.json as { data?: { nodes?: SnapshotNode[] } })?.data?.nodes ?? @@ -92,7 +104,7 @@ function resolveTargets(args: string[], ctx: IosSuiteContext): string[] | { erro /** Runs untimed calls in order and describes the first failure. */ function untimed( - ctx: IosSuiteContext, + ctx: SuiteContext, calls: readonly (readonly string[])[] = [], ): string | undefined { for (const args of calls) { @@ -117,14 +129,14 @@ function errorOf(result: CliResult): Pick }; } -function readProfile(ctx: IosSuiteContext, requestId: string | undefined) { +function readProfile(ctx: SuiteContext, requestId: string | undefined) { if (!requestId) return undefined; const sessionDir = resolveSessionDir(path.join(ctx.stateDir, 'sessions'), SESSION); const logPath = resolveSessionRequestLogPath(sessionDir, requestId); return fs.existsSync(logPath) ? profileRequestLog(fs.readFileSync(logPath, 'utf8')) : undefined; } -function recordFailure(ctx: IosSuiteContext, id: string, round: number, detail: unknown): void { +function recordFailure(ctx: SuiteContext, id: string, round: number, detail: unknown): void { const dir = path.join(ctx.rawDir, 'failures'); fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync( @@ -133,15 +145,35 @@ function recordFailure(ctx: IosSuiteContext, id: string, round: number, detail: ); } +/** + * Restarts the daemon with `env` and reopens the session on the current screen. A daemon reads + * its host environment once, at start. + */ +function restartDaemon( + ctx: SuiteContext, + env: Readonly> | undefined, +): string | undefined { + const stop = invokeCli(['daemon', 'stop'], ['--state-dir', ctx.stateDir]); + if (!stop.ok) return `daemon stop: ${errorOf(stop).errorMessage}`; + ctx.daemonEnv = env; + return sessionError(openSession(ctx, { relaunch: false })); +} + async function runTimedStep( scenario: SuiteScenario, - step: SuiteStep, - ctx: IosSuiteContext, + step: PlatformStep, + ctx: SuiteContext, round: number, ): Promise { const id = `${scenario.id}/${step.id}`; - const before = untimed(ctx, step.before); + const restoreDaemon = (): void => { + // The rest of the scenario cannot run without its session; the next scenario reopens it. + if (step.daemonEnv && restartDaemon(ctx, undefined)) ctx.daemonStuck = true; + }; + const before = + (step.daemonEnv ? restartDaemon(ctx, step.daemonEnv) : undefined) ?? untimed(ctx, step.before); if (before) { + restoreDaemon(); log(` ✗ ${id} (before_failed: ${before})`); return { round, ok: false, wallClockMs: 0, errorCode: 'before_failed', errorMessage: before }; } @@ -150,7 +182,7 @@ async function runTimedStep( const { result: forced, requestId } = await captureWithForcedBackend({ stateDir: ctx.stateDir, session: SESSION, - udid: ctx.udid, + udid: ctx.device, backend: 'tree', }); const result = watch(ctx, forced); @@ -176,7 +208,7 @@ async function runTimedStep( errorMessage: args.error, }; } else { - const result = watch(ctx, invokeCli([...args, '--debug'], baseFlags(ctx))); + const result = cli(ctx, [...args, '--debug']); const diagnostics = readClientDiagnostics(result.stderr); sample = { round, @@ -201,6 +233,7 @@ async function runTimedStep( const violation = sample.ok ? expectationViolation(step.expect, sample.snapshot) : undefined; if (violation) Object.assign(sample, { ok: false, errorCode: 'expectation_failed', errorMessage: violation }); + restoreDaemon(); const after = untimed(ctx, step.after); if (after) log(` ! ${id} after: ${after}`); if (!sample.ok) log(` ✗ ${id} (${sample.errorCode}: ${sample.errorMessage})`); @@ -214,23 +247,26 @@ function sessionError(result: CliResult): string | undefined { return message; } -function stopStuckDaemon(ctx: IosSuiteContext): void { +function stopStuckDaemon(ctx: SuiteContext): void { log(' ! a request outlived its timeout; restarting the daemon'); invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); ctx.daemonStuck = false; + ctx.daemonEnv = undefined; } /** Opens the session with device selectors; later calls address it by session name alone. */ -function establishSession(ctx: IosSuiteContext): CliResult { +function openSession(ctx: SuiteContext, options: { relaunch: boolean }): CliResult { + const relaunch = options.relaunch ? ['--relaunch'] : []; + return cli(ctx, ['open', ctx.appId, ...relaunch, ...deviceFlags(ctx)]); +} + +function establishSession(ctx: SuiteContext): CliResult { if (ctx.daemonStuck) stopStuckDaemon(ctx); - return watch( - ctx, - invokeCli(['open', ctx.appId, '--relaunch', '--udid', ctx.udid], baseFlags(ctx)), - ); + return openSession(ctx, { relaunch: true }); } -export async function runIosSuite( - ctx: IosSuiteContext, +export async function runSuite( + ctx: SuiteContext, cfg: PerfConfig, scenarios: readonly SuiteScenario[], ): Promise { @@ -273,7 +309,7 @@ export async function runIosSuite( label: step.args.join(' '), command: step.command, args: [...step.args], - platform: 'ios', + platform: ctx.platform, execMode: step.via === 'sdk-forced-tree' ? 'sdk' : 'standalone', samples: samples.get(`${scenario.id}/${step.id}`) ?? [], warmupDropped: cfg.warmup, @@ -290,7 +326,7 @@ const TEARDOWN_TIMEOUT_MS = 60_000; * the logs again with the teardown, and remove the state directory unless it is kept. It runs once, * so the interrupt handler and the normal finish can both call it. */ -export function iosRunFinalizer(ctx: IosSuiteContext, keepArtifacts: boolean): () => void { +export function runFinalizer(ctx: SuiteContext, keepArtifacts: boolean): () => void { let finalized = false; const saveLogs = (): void => { try { @@ -315,7 +351,7 @@ export function iosRunFinalizer(ctx: IosSuiteContext, keepArtifacts: boolean): ( } /** Copies daemon and session logs into the raw artifact; the daemon token file stays behind. */ -function copyRawState(ctx: IosSuiteContext): void { +function copyRawState(ctx: SuiteContext): void { const target = path.join(ctx.rawDir, 'state'); fs.mkdirSync(target, { recursive: true }); const daemonLog = path.join(ctx.stateDir, 'daemon.log'); diff --git a/scripts/perf/suite.test.ts b/scripts/perf/suite.test.ts new file mode 100644 index 0000000000..742b162ab3 --- /dev/null +++ b/scripts/perf/suite.test.ts @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict'; +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import { test } from 'vitest'; +import { + describeCoverage, + expectationViolation, + SUITE, + SUITE_EXCLUSIONS, + suiteFor, + type SuiteScenario, + type SuiteStep, +} from './suite.ts'; +import type { Platform } from './types.ts'; + +const PLATFORMS: readonly Platform[] = ['ios', 'android']; + +test('on each platform, every public command is either measured or excluded with a reason', () => { + for (const platform of PLATFORMS) { + const { measured, excluded } = describeCoverage(suiteFor(platform), SUITE_EXCLUSIONS[platform]); + const classified = [...measured, ...excluded.map((e) => e.command)].sort(); + assert.deepEqual(classified, Object.values(PUBLIC_COMMANDS).sort(), platform); + for (const { reason } of excluded) assert.ok(reason.length > 0); + } +}); + +test('step ids are unique per platform, so comparisons key each step once', () => { + for (const platform of PLATFORMS) { + const ids = suiteFor(platform).flatMap((scenario) => + scenario.steps.map((s) => `${scenario.id}/${s.id}`), + ); + assert.equal(new Set(ids).size, ids.length, platform); + } +}); + +test('a platform suite drops the other platform steps and applies its own deltas', () => { + const shared = { id: 'swipe', command: PUBLIC_COMMANDS.swipe, args: ['swipe', '1'] }; + const suite: SuiteScenario[] = [ + { + id: 'scenario', + title: 'scenario', + setup: [], + steps: [ + { ...shared, on: { android: { args: ['swipe', '2'], before: [] } }, before: [['wait']] }, + { id: 'ios-only', command: PUBLIC_COMMANDS.actionButton, args: [], only: 'ios' }, + ], + }, + ]; + assert.deepEqual(suiteFor('ios', suite)[0]!.steps, [ + { ...shared, before: [['wait']] }, + { id: 'ios-only', command: PUBLIC_COMMANDS.actionButton, args: [] }, + ]); + assert.deepEqual(suiteFor('android', suite)[0]!.steps, [ + { ...shared, args: ['swipe', '2'], before: [] }, + ]); +}); + +test('every platform runs at least one step of every scenario', () => { + for (const scenario of SUITE) { + for (const platform of PLATFORMS) { + const resolved = suiteFor(platform).find((s) => s.id === scenario.id); + assert.ok(resolved && resolved.steps.length > 0, `${platform} ${scenario.id}`); + } + } +}); + +test('a heavy-feed capture breaks its expectation when the tree flattens or the tree backend copes', () => { + const depth = { maxDepthAtLeast: 64 }; + assert.equal(expectationViolation(depth, { maxDepth: 64 }), undefined); + assert.match(expectationViolation(depth, { maxDepth: 63 })!, /below 64/); + assert.match(expectationViolation(depth, undefined)!, /unknown/); + + const degraded = { treeBackendDegraded: true }; + assert.equal( + expectationViolation(degraded, { backend: 'private-ax', state: 'recovered' }), + undefined, + ); + assert.equal(expectationViolation(degraded, { backend: 'tree', state: 'sparse' }), undefined); + assert.match( + expectationViolation(degraded, { backend: 'tree', state: 'healthy' })!, + /non-sparse/, + ); +}); + +test('an Android capture breaks its expectation when another helper transport served it', () => { + const oneShot = { transport: 'instrumentation' }; + assert.equal(expectationViolation(oneShot, { transport: 'instrumentation' }), undefined); + assert.match( + expectationViolation(oneShot, { transport: 'persistent-session' })!, + /persistent-session transport served the capture, not instrumentation/, + ); + assert.match(expectationViolation(oneShot, {})!, /unknown transport/); +}); diff --git a/scripts/perf/ios-suite.ts b/scripts/perf/suite.ts similarity index 67% rename from scripts/perf/ios-suite.ts rename to scripts/perf/suite.ts index bde981db05..c3feeea3ef 100644 --- a/scripts/perf/ios-suite.ts +++ b/scripts/perf/suite.ts @@ -1,5 +1,5 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import type { SnapshotFacts } from './types.ts'; +import type { Platform, SnapshotFacts } from './types.ts'; /** Routes `trigger-app-event` into the fixture's automation screen. */ export const FIXTURE_APP_EVENT_URL_TEMPLATE = @@ -17,7 +17,7 @@ type Cli = readonly string[]; * or center point of the element with that identifier in an untimed `snapshot -i` taken just * before the step. `before` and `after` run untimed around the step in every round. */ -export type SuiteStep = { +export type PlatformStep = { id: string; command: PublicCommand; args: Cli; @@ -26,16 +26,32 @@ export type SuiteStep = { // `sdk-forced-tree` captures a raw snapshot through the daemon with the XCTest tree backend // pinned; the CLI has no flag for it. via?: 'cli' | 'sdk-forced-tree'; + // The step runs against a daemon started with these variables. The harness restarts the daemon + // and reopens the session on the current screen, untimed, before and after the step. + daemonEnv?: Readonly>; // A sample that succeeds but breaks the expectation fails with `expectation_failed`, so a // fixture that stops reproducing its shape shows up as a new failure. expect?: StepExpectation; }; +type StepDelta = Partial>; + +/** + * A step of the shared suite. It runs on every platform unless `only` names one. `on` replaces + * fields on one platform, for example screen coordinates or a platform-only wait. + */ +export type SuiteStep = PlatformStep & { + only?: Platform; + on?: Partial>; +}; + export type StepExpectation = { // Deepest raw node depth, counted from the application at 0. maxDepthAtLeast?: number; // The pinned XCTest tree backend did not serve a healthy capture. treeBackendDegraded?: boolean; + // The Android helper transport that served the capture. + transport?: string; }; /** Describes how a snapshot breaks the step expectation, or returns undefined. */ @@ -50,17 +66,33 @@ export function expectationViolation( if (expect.treeBackendDegraded && snapshot?.backend === 'tree' && snapshot.state !== 'sparse') { return 'the pinned tree backend served a non-sparse capture'; } + if (expect.transport !== undefined && snapshot?.transport !== expect.transport) { + return `the ${snapshot?.transport ?? 'unknown'} transport served the capture, not ${expect.transport}`; + } return undefined; } /** A scenario starts from `setup` (untimed, every round) and runs its steps in order. */ -export type SuiteScenario = { +export type SuiteScenario = { id: string; title: string; setup: readonly Cli[]; - steps: SuiteStep[]; + steps: TStep[]; }; +/** Resolves the shared suite for one platform: drops other platforms' steps, applies deltas. */ +export function suiteFor( + platform: Platform, + suite: readonly SuiteScenario[] = SUITE, +): SuiteScenario[] { + return suite.map(({ steps, ...scenario }) => ({ + ...scenario, + steps: steps + .filter((s) => s.only === undefined || s.only === platform) + .map(({ only: _only, on, ...shared }) => ({ ...shared, ...on?.[platform] })), + })); +} + // A cold relaunch on a hosted runner can take longer than 15 s before the first landmark. const LANDMARK_WAIT_MS = '60000'; const relaunch: Cli = ['open', '{app}', '--relaunch']; @@ -76,6 +108,7 @@ const toSettingsLab = (button: string, landmark: string): Cli[] => [ ['click', `id="${button}"`], ['wait', landmark, LANDMARK_WAIT_MS], ]; +const SWIPE_COUNT_FLAGS = ['--count', '2', '--pause-ms', '50', '--pattern', 'ping-pong'] as const; const step = ( id: string, command: PublicCommand, @@ -83,7 +116,7 @@ const step = ( extra: Omit = {}, ): SuiteStep => ({ id, command, args, ...extra }); -export const IOS_SUITE: readonly SuiteScenario[] = [ +export const SUITE: readonly SuiteScenario[] = [ { id: 'lifecycle', title: 'App lifecycle, device state, and inventory', @@ -99,9 +132,26 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('settings-appearance', C.settings, ['settings', 'appearance', 'light']), step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf']), step('clipboard-read', C.clipboard, ['clipboard', 'read']), - step('action-button', C.actionButton, ['action-button']), - step('prepare-runner', C.prepare, ['prepare', 'ios-runner']), - step('devices', C.devices, ['devices', '--platform', 'ios']), + step('action-button', C.actionButton, ['action-button'], { only: 'ios' }), + // `wake` is the one hardware key that leaves an awake device as it was. + step('hardware-button', C.hardwareButton, ['hardware-button', 'wake'], { only: 'android' }), + step( + 'push', + C.push, + [ + 'push', + '{app}', + '{"action":"com.callstack.agentdevicelab.PERF","extras":{"source":"perf"}}', + ], + { only: 'android' }, + ), + step('telephony-sms', C.telephony, ['telephony', 'sms', '5551234', 'agent-device perf'], { + only: 'android', + // The SMS notification can cover the app; reopen it for the next step. + after: [['open', '{app}']], + }), + step('prepare-runner', C.prepare, ['prepare', 'ios-runner'], { only: 'ios' }), + step('devices', C.devices, ['devices']), step('apps', C.apps, ['apps']), step('capabilities', C.capabilities, ['capabilities']), step('doctor', C.doctor, ['doctor']), @@ -118,6 +168,8 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ ['alert', 'accept'], ['wait', 'id="automation-title"', LANDMARK_WAIT_MS], ], + // Android opens the event URL without a prompt. + on: { android: { after: [['wait', 'id="automation-title"', LANDMARK_WAIT_MS]] } }, }, ), ], @@ -212,20 +264,13 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), step('scroll-top', C.scroll, ['scroll', 'top']), step('scroll-until', C.scroll, ['scroll', 'down', '--until', 'id="catalog-footer"']), - step('swipe', C.swipe, ['swipe', '200', '300', '200', '650']), - step('swipe-count', C.swipe, [ - 'swipe', - '200', - '650', - '200', - '450', - '--count', - '2', - '--pause-ms', - '50', - '--pattern', - 'ping-pong', - ]), + // Coordinates are iOS points and Android pixels; the Android values keep the same screen share. + step('swipe', C.swipe, ['swipe', '200', '300', '200', '650'], { + on: { android: { args: ['swipe', '540', '800', '540', '1750'] } }, + }), + step('swipe-count', C.swipe, ['swipe', '200', '650', '200', '450', ...SWIPE_COUNT_FLAGS], { + on: { android: { args: ['swipe', '540', '1750', '540', '1200', ...SWIPE_COUNT_FLAGS] } }, + }), ], }, { @@ -238,6 +283,8 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('type-suffix', C.type, ['type', '.test']), step('keyboard-dismiss', C.keyboard, ['keyboard', 'dismiss'], { before: [['wait', 'label="Done" visible=true', '10000']], + // Android emulators type through the headless test IME, which has no Done key. + on: { android: { before: [] } }, }), step('press-checkbox', C.press, ['press', 'id="checkbox-agree"'], { before: [['scroll', 'down', '--until', 'id="checkbox-agree"']], @@ -300,12 +347,16 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ }, { id: 'heavy-feed', - title: 'Heavy feed: Bluesky-depth tree (≥65 native levels) and virtualized list', + title: 'Heavy feed: Bluesky-depth tree and virtualized list', setup: toSettingsLab('open-heavy-feed', 'id="heavy-feed-title"'), steps: [ step('snapshot-i', C.snapshot, ['snapshot', '-i']), step('snapshot', C.snapshot, ['snapshot']), - step('snapshot-raw', C.snapshot, ['snapshot', '--raw'], { expect: { maxDepthAtLeast: 64 } }), + // The fixture renders more than 65 native levels on iOS and 60 on Android. + step('snapshot-raw', C.snapshot, ['snapshot', '--raw'], { + expect: { maxDepthAtLeast: 64 }, + on: { android: { expect: { maxDepthAtLeast: 56 } } }, + }), step('press-like-selector', C.press, ['press', 'id="heavy-post-0-like"']), step('press-like-settle', C.press, ['press', 'id="heavy-post-1-like"', '--settle']), step('press-like-ref', C.press, ['press', '{ref:heavy-post-0-like}']), @@ -318,14 +369,25 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('scroll-down', C.scroll, ['scroll', 'down']), step('snapshot-i-scrolled', C.snapshot, ['snapshot', '-i']), step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), - step('swipe', C.swipe, ['swipe', '200', '650', '200', '300']), + step('swipe', C.swipe, ['swipe', '200', '650', '200', '300'], { + on: { android: { args: ['swipe', '540', '1750', '540', '800'] } }, + }), // The pinned XCTest capture outlives its watchdog by about a second, and the next runner // command refuses until it ends; the untimed wait keeps that cost out of the next step. step('snapshot-raw-forced-tree', C.snapshot, ['snapshot', '--raw'], { + only: 'ios', via: 'sdk-forced-tree', expect: { treeBackendDegraded: true }, after: [['wait', '2000']], }), + // Android has one snapshot engine. Its other path is the one-shot `am instrument` capture that + // serves a request when the helper session is off or fails; only the daemon environment + // turns the session off. + step('snapshot-raw-one-shot', C.snapshot, ['snapshot', '--raw'], { + only: 'android', + daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' }, + expect: { maxDepthAtLeast: 56, transport: 'instrumentation' }, + }), step('back', C.back, ['back']), ], }, @@ -337,32 +399,60 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ }, ]; -/** - * Catalog commands the iOS suite does not time, with the reason. Every public command is either - * measured by a step above or listed here; `ios-suite.test.ts` holds that partition. - */ -export const IOS_SUITE_EXCLUSIONS: Partial> = { - [C.boot]: - 'Simulator power cycle: CoreSimulator boot time dominates and it ends the session under test.', - [C.shutdown]: 'Simulator power cycle: it ends the session and the runner under test.', - [C.install]: - 'Bundle provisioning: simctl copy time dominates; the job installs the fixture once.', - [C.reinstall]: - 'Bundle provisioning: simctl copy time dominates and it resets the app under test.', - [C.uninstall]: 'Bundle provisioning: it removes the app under test.', - [C.installFromSource]: - 'Bundle provisioning from a source build; the job installs a cached Release fixture.', +const COMPOSITE_RUNNERS: Partial> = { [C.replay]: 'Composite runner: a replay is the sum of the measured commands it runs.', [C.test]: 'Composite runner: a test suite is the sum of the measured commands it runs.', - [C.push]: - 'The fixture holds no notification authorization, so simctl refuses the payload; iOS push coverage is contract-only.', - [C.audio]: 'Host audio capture needs a macOS audio permission that hosted runners do not grant.', - [C.hover]: 'The iOS runtime refuses hover; it is a web pointer contract.', - [C.viewport]: 'The iOS runtime refuses viewport; it resizes web viewports only.', - [C.tvRemote]: 'tvOS only.', - [C.fold]: 'iPhone Duo only; the suite device has one panel.', - [C.telephony]: 'The iOS simulator runtime refuses simulated calls and SMS.', - [C.hardwareButton]: 'The iOS simulator runtime refuses hardware-button presses.', +}; + +/** + * Catalog commands that the suite does not time on each platform, with the reason. On each + * platform, every public command is either measured by a step above or listed here; + * `suite.test.ts` holds that partition. + */ +export const SUITE_EXCLUSIONS: Record>> = { + ios: { + ...COMPOSITE_RUNNERS, + [C.boot]: + 'Simulator power cycle: CoreSimulator boot time dominates and it ends the session under test.', + [C.shutdown]: 'Simulator power cycle: it ends the session and the runner under test.', + [C.install]: + 'Bundle provisioning: simctl copy time dominates; the job installs the fixture once.', + [C.reinstall]: + 'Bundle provisioning: simctl copy time dominates and it resets the app under test.', + [C.uninstall]: 'Bundle provisioning: it removes the app under test.', + [C.installFromSource]: + 'Bundle provisioning from a source build; the job installs a cached Release fixture.', + [C.push]: + 'The fixture holds no notification authorization, so simctl refuses the payload; iOS push coverage is contract-only.', + [C.audio]: + 'Host audio capture needs a macOS audio permission that hosted runners do not grant.', + [C.hover]: 'The iOS runtime refuses hover; it is a web pointer contract.', + [C.viewport]: 'The iOS runtime refuses viewport; it resizes web viewports only.', + [C.tvRemote]: 'tvOS only.', + [C.fold]: 'iPhone Duo only; the suite device has one panel.', + [C.telephony]: 'The iOS simulator runtime refuses simulated calls and SMS.', + [C.hardwareButton]: 'The iOS simulator runtime refuses hardware-button presses.', + }, + android: { + ...COMPOSITE_RUNNERS, + [C.boot]: + 'Emulator power cycle: emulator boot time dominates and it ends the session under test.', + [C.shutdown]: 'Emulator power cycle: it ends the session and the helper under test.', + [C.install]: 'APK provisioning: adb install time dominates; the job installs the fixture once.', + [C.reinstall]: 'APK provisioning: adb install time dominates and it resets the app under test.', + [C.uninstall]: 'APK provisioning: it removes the app under test.', + [C.installFromSource]: + 'APK provisioning from a source build; the job installs a cached Release fixture.', + [C.prepare]: + 'prepare has only the ios-runner action; the snapshot helper installs on first use.', + [C.actionButton]: 'Android has no key event for the Action button.', + [C.audio]: + 'Host audio capture uses macOS ScreenCaptureKit; the Linux emulator runner has none.', + [C.hover]: 'The Android runtime refuses hover; it is a web pointer contract.', + [C.viewport]: 'The Android runtime refuses viewport; it resizes web viewports only.', + [C.tvRemote]: 'Android TV only.', + [C.fold]: 'Foldable emulators only; the suite emulator is a pixel_7 with one display.', + }, }; /** Partitions the measured and excluded catalog commands for the report. */ diff --git a/scripts/perf/summary-fixture.ts b/scripts/perf/summary-fixture.ts index b86ad31ef8..1892e3e09b 100644 --- a/scripts/perf/summary-fixture.ts +++ b/scripts/perf/summary-fixture.ts @@ -1,5 +1,11 @@ import { buildMeasurement } from './measurement.ts'; -import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary, type Sample } from './types.ts'; +import { + PERF_SUMMARY_SCHEMA, + type Measurement, + type PerfSummary, + type Platform, + type Sample, +} from './types.ts'; export type StepFixture = { id: string; @@ -37,11 +43,11 @@ function measurement(step: StepFixture): Measurement { /** A schema-valid summary whose steps carry the given timings. */ export function summaryFixture( steps: StepFixture[], - overrides: { startedAt?: string; xcode?: string; runId?: string } = {}, + overrides: { startedAt?: string; xcode?: string; runId?: string; platform?: Platform } = {}, ): PerfSummary { return { schema: PERF_SUMMARY_SCHEMA, - platform: 'ios', + platform: overrides.platform ?? 'ios', startedAt: overrides.startedAt ?? '2026-10-20T03:00:00.000Z', finishedAt: '2026-10-20T03:30:00.000Z', config: { diff --git a/scripts/perf/types.ts b/scripts/perf/types.ts index a3cc091be5..b523e273be 100644 --- a/scripts/perf/types.ts +++ b/scripts/perf/types.ts @@ -2,7 +2,7 @@ export type Platform = 'ios' | 'android'; -export type ExecMode = 'batch' | 'standalone' | 'sdk'; +export type ExecMode = 'standalone' | 'sdk'; export type CliResult = { exitCode: number; @@ -44,6 +44,8 @@ export type SettleFacts = { waitedMs: number; captures?: number; settled?: boole export type SnapshotFacts = { backend?: string; + // The Android helper transport: `persistent-session` or `instrumentation`. + transport?: string; state?: string; reasonCode?: string; nodeCount?: number; @@ -54,10 +56,9 @@ export type SnapshotFacts = { export type Sample = { round: number; wallClockMs: number; - // Daemon request round trip: the client's `daemon_request` timer for CLI calls, the batch step - // duration for batch calls, or the request-log span for SDK calls. + // Daemon request round trip: the client's `daemon_request` timer for CLI calls, or the + // request-log span for SDK calls. daemonMs?: number; - elementCount?: number; requestId?: string; snapshot?: SnapshotFacts; settle?: SettleFacts; From 4f18a1a4f19581830e9bc30fb0304ec3b176372a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 09:22:52 +0200 Subject: [PATCH 06/15] fix(perf): keep Android clipboard and restore failures out of timings The API 35 and 36 emulator images ship no shell clipboard service, so clipboard is an Android exclusion. A daemon restart that reopens the session clears the stuck flag, and one that cannot reopen it skips the rest of the scenario as restore_failed. The one-shot step warms the restarted daemon's helper install check untimed. latest and date references resolve to the newest run with the platform's artifact. The tour-only --n alias is gone. Co-Authored-By: Claude Opus 5.5 --- scripts/perf/README.md | 3 +- scripts/perf/compare-cli.ts | 13 ++++--- scripts/perf/config.test.ts | 4 +-- scripts/perf/config.ts | 1 - scripts/perf/nightly-artifacts.test.ts | 5 +++ scripts/perf/nightly-artifacts.ts | 17 ++++++--- scripts/perf/suite-harness.test.ts | 48 +++++++++++++++++++------- scripts/perf/suite-harness.ts | 34 +++++++++++------- scripts/perf/suite.ts | 10 ++++-- 9 files changed, 94 insertions(+), 41 deletions(-) diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 66f4a04810..8cdfcaa4b4 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -55,6 +55,7 @@ Some shared steps use other screen coordinates or waits on Android. The `on.andr | --- | --- | --- | | `action-button` | lifecycle | Not timed: Android has no key event for the Action button. | | `prepare` | lifecycle (`prepare ios-runner`) | Not timed: `prepare` has only the `ios-runner` action. | +| `clipboard` | lifecycle ×2 | Not timed: the API 35 and 36 emulator images have no shell clipboard service. | | `hardware-button` | Not timed: the simulator refuses hardware-button presses. | lifecycle (`hardware-button wake`) | | `push` | Not timed: the fixture has no notification authorization. | lifecycle (adb broadcast) | | `telephony` | Not timed: the simulator refuses calls and SMS. | lifecycle (`telephony sms`) | @@ -95,7 +96,7 @@ All times are milliseconds. Statistics use only the successful samples of a step | `producer` | The number of samples that each snapshot producer answered: `simulator-ax-bridge` and `apple-runner` on iOS; `android-helper/persistent-session` and `android-helper/instrumentation` (one-shot `am instrument`) on Android. | | backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected` or `android-helper/healthy`. | | fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. | -| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth, or that the forced path did not serve the capture. | +| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `restore_failed` means that the daemon restart after a forced-path step could not reopen the session. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth, or that the forced path did not serve the capture. | ## Nightly results diff --git a/scripts/perf/compare-cli.ts b/scripts/perf/compare-cli.ts index 8b130c457d..a638c97ce9 100644 --- a/scripts/perf/compare-cli.ts +++ b/scripts/perf/compare-cli.ts @@ -134,14 +134,17 @@ function main(): void { const load = (ref: string): Loaded => { if (fs.existsSync(ref)) return { summary: readLocalSummary(ref), label: path.basename(path.resolve(ref)) }; - const run = resolveRunRef(ref, allRuns()); + const summaries = new Map(); + const run = resolveRunRef(ref, allRuns(), (candidate) => { + const summary = fetchSummary(options.source, candidate); + if (summary) summaries.set(candidate.id, summary); + return summary !== undefined; + }); if (!run) throw new Error( - `No completed ${options.source.workflow} run on ${options.source.branch} matches ${ref}.`, + `No completed ${options.source.workflow} run on ${options.source.branch} matches ${ref} with a ${options.source.artifact} artifact.`, ); - const summary = fetchSummary(options.source, run); - if (!summary) throw new Error(`Run ${run.id} has no ${options.source.artifact} artifact.`); - return { summary, label: runLabel(run), run }; + return { summary: summaries.get(run.id)!, label: runLabel(run), run }; }; const loadRuns = (candidates: WorkflowRun[]): Loaded[] => candidates.flatMap((run) => { diff --git a/scripts/perf/config.test.ts b/scripts/perf/config.test.ts index 18408db309..4e14d554bc 100644 --- a/scripts/perf/config.test.ts +++ b/scripts/perf/config.test.ts @@ -6,8 +6,8 @@ test('a run measures at least three rounds, so every step can get a timing verdi assert.equal(parseConfig(['--udid', 'sim', '--rounds', '3']).rounds, 3); assert.throws(() => parseConfig(['--udid', 'sim', '--rounds', '2']), /--rounds must be .* >= 3/); assert.throws( - () => parseConfig(['--platform', 'android', '--serial', 'emulator-5554', '--n', '1']), - /--n must be .* >= 3/, + () => parseConfig(['--platform', 'android', '--serial', 'emulator-5554', '--rounds', '1']), + /--rounds must be .* >= 3/, ); }); diff --git a/scripts/perf/config.ts b/scripts/perf/config.ts index ca5812e3a2..7c2bf3de3e 100644 --- a/scripts/perf/config.ts +++ b/scripts/perf/config.ts @@ -66,7 +66,6 @@ export function parseConfig(argv: string[]): PerfConfig { cfg.platform = v; break; } - case '--n': case '--rounds': cfg.rounds = readIntValue(argv, i++, a, MIN_MEASURED_ROUNDS); break; diff --git a/scripts/perf/nightly-artifacts.test.ts b/scripts/perf/nightly-artifacts.test.ts index ee2bbdc278..2b45790051 100644 --- a/scripts/perf/nightly-artifacts.test.ts +++ b/scripts/perf/nightly-artifacts.test.ts @@ -30,6 +30,11 @@ test('resolves latest, run ids, and dates to the newest matching run', () => { assert.equal(resolveRunRef('2026-10-19', RUNS)?.id, '103'); assert.equal(resolveRunRef('2026-10-18', RUNS), undefined); assert.throws(() => resolveRunRef('yesterday', RUNS), /Unknown run reference/); + // A run without the platform artifact gives way to the newest run that has it. + const withArtifact = (run: WorkflowRun) => run.id !== '104' && run.id !== '103'; + assert.equal(resolveRunRef('latest', RUNS, withArtifact)?.id, '102'); + assert.equal(resolveRunRef('2026-10-19', RUNS, withArtifact)?.id, '102'); + assert.equal(resolveRunRef('104', RUNS, withArtifact), undefined); }); const baselineIds = ( diff --git a/scripts/perf/nightly-artifacts.ts b/scripts/perf/nightly-artifacts.ts index cd22988ba5..7daaf517f9 100644 --- a/scripts/perf/nightly-artifacts.ts +++ b/scripts/perf/nightly-artifacts.ts @@ -81,13 +81,20 @@ function sortNewestFirst(runs: WorkflowRun[]): WorkflowRun[] { /** * Resolves one run reference: `latest`, a numeric run id, or a `YYYY-MM-DD` date (the newest run - * created that UTC day). Returns undefined when no listed run matches. + * created that UTC day). Only runs that `accept` takes count, so a run that skipped a platform + * does not hide an older run of that platform. Returns undefined when no listed run matches. */ -export function resolveRunRef(ref: string, runs: WorkflowRun[]): WorkflowRun | undefined { +export function resolveRunRef( + ref: string, + runs: WorkflowRun[], + accept: (run: WorkflowRun) => boolean = () => true, +): WorkflowRun | undefined { const newestFirst = sortNewestFirst(runs); - if (ref === 'latest') return newestFirst[0]; - if (/^\d+$/.test(ref)) return newestFirst.find((run) => run.id === ref); - if (DATE_REF.test(ref)) return newestFirst.find((run) => run.createdAt.slice(0, 10) === ref); + const first = (match: (run: WorkflowRun) => boolean) => + newestFirst.find((run) => match(run) && accept(run)); + if (ref === 'latest') return first(() => true); + if (/^\d+$/.test(ref)) return first((run) => run.id === ref); + if (DATE_REF.test(ref)) return first((run) => run.createdAt.slice(0, 10) === ref); throw new Error( `Unknown run reference ${JSON.stringify(ref)}: use latest, a run id, a YYYY-MM-DD date, or a path.`, ); diff --git a/scripts/perf/suite-harness.test.ts b/scripts/perf/suite-harness.test.ts index 7bd871cb06..fb3b33c5d5 100644 --- a/scripts/perf/suite-harness.test.ts +++ b/scripts/perf/suite-harness.test.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { test, vi } from 'vitest'; import type { PerfConfig } from './config.ts'; +import type { PlatformStep } from './suite.ts'; // Stands in for the CLI: records each teardown call, and whether the logs were already saved, // then appends to the daemon log and fails like a teardown that went wrong. @@ -72,6 +73,10 @@ const args = process.argv.slice(2); const command = args.slice(0, args.indexOf('--state-dir')).join(' '); const session = process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION ?? 'default'; fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' session=' + session + '\\n'); +if (process.env.PERF_FAKE_FAIL_RESTORE && command === 'open app --serial emulator-5554' && session === 'default') { + process.stdout.write(JSON.stringify({ success: false, error: { code: 'DEVICE_IN_USE', message: 'claimed' } })); + process.exit(1); +} const transport = session === '0' ? 'instrumentation' : 'persistent-session'; const data = args[0] === 'snapshot' ? { nodes: [{ depth: 0 }], androidSnapshot: { helperTransport: transport } } @@ -79,7 +84,15 @@ const data = args[0] === 'snapshot' process.stdout.write(JSON.stringify({ success: true, data })); `; -test('a step with daemon variables runs against a restarted daemon and restores it after', async () => { +const ONE_SHOT: PlatformStep = { + id: 'one-shot', + command: 'snapshot', + args: ['snapshot', '--raw'], + daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' }, + expect: { transport: 'instrumentation' }, +}; + +async function runFakeDaemonSuite(steps: PlatformStep[]) { const root = mkdtempForTestSync('perf-daemon-env-'); const fakeCli = path.join(root, 'fake-cli.mjs'); fs.writeFileSync(fakeCli, FAKE_DAEMON_CLI); @@ -88,8 +101,7 @@ test('a step with daemon variables runs against a restarted daemon and restores vi.stubEnv('PERF_FAKE_CALLS', calls); vi.resetModules(); const { runSuite } = await import('./suite-harness.ts'); - - const [measurement] = await runSuite( + const measurements = await runSuite( { platform: 'android', stateDir: path.join(root, 'state'), @@ -104,21 +116,21 @@ test('a step with daemon variables runs against a restarted daemon and restores id: 'heavy-feed', title: 'heavy feed', setup: [], - steps: [ - { - id: 'one-shot', - command: 'snapshot', - args: ['snapshot', '--raw'], - daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' }, - expect: { transport: 'instrumentation' }, - }, - ], + steps, }, ], ); + return { measurements, calls: fs.readFileSync(calls, 'utf8') }; +} + +test('a step with daemon variables runs against a restarted daemon and restores it after', async () => { + const { + measurements: [measurement], + calls, + } = await runFakeDaemonSuite([ONE_SHOT]); assert.equal( - fs.readFileSync(calls, 'utf8'), + calls, [ 'open app --relaunch --serial emulator-5554 session=default', 'daemon stop session=default', @@ -132,3 +144,13 @@ test('a step with daemon variables runs against a restarted daemon and restores assert.equal(measurement!.failures, 0); assert.equal(measurement!.samples[0]!.snapshot?.transport, 'instrumentation'); }); + +test('a failed daemon restore skips the rest of the scenario as restore_failed', async () => { + vi.stubEnv('PERF_FAKE_FAIL_RESTORE', '1'); + const { measurements } = await runFakeDaemonSuite([ + ONE_SHOT, + { id: 'back', command: 'back', args: ['back'] }, + ]); + assert.equal(measurements[0]!.failures, 0); + assert.equal(measurements[1]!.samples[0]!.errorCode, 'restore_failed'); +}); diff --git a/scripts/perf/suite-harness.ts b/scripts/perf/suite-harness.ts index 91a742d092..43fca20d76 100644 --- a/scripts/perf/suite-harness.ts +++ b/scripts/perf/suite-harness.ts @@ -32,6 +32,9 @@ export type SuiteContext = { daemonStuck?: boolean; // Variables that the running daemon was started with, set by a step with `daemonEnv`. daemonEnv?: Readonly>; + // Set when the daemon restart after a `daemonEnv` step could not reopen the session. The rest + // of the scenario is skipped, and the next scenario reopens the session. + restoreFailed?: string; }; function log(msg: string): void { @@ -156,7 +159,10 @@ function restartDaemon( const stop = invokeCli(['daemon', 'stop'], ['--state-dir', ctx.stateDir]); if (!stop.ok) return `daemon stop: ${errorOf(stop).errorMessage}`; ctx.daemonEnv = env; - return sessionError(openSession(ctx, { relaunch: false })); + const failure = sessionError(openSession(ctx, { relaunch: false })); + // A fresh daemon holds no request from before the restart. + if (!failure) ctx.daemonStuck = false; + return failure; } async function runTimedStep( @@ -167,8 +173,7 @@ async function runTimedStep( ): Promise { const id = `${scenario.id}/${step.id}`; const restoreDaemon = (): void => { - // The rest of the scenario cannot run without its session; the next scenario reopens it. - if (step.daemonEnv && restartDaemon(ctx, undefined)) ctx.daemonStuck = true; + if (step.daemonEnv) ctx.restoreFailed = restartDaemon(ctx, undefined); }; const before = (step.daemonEnv ? restartDaemon(ctx, step.daemonEnv) : undefined) ?? untimed(ctx, step.before); @@ -252,6 +257,7 @@ function stopStuckDaemon(ctx: SuiteContext): void { invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); ctx.daemonStuck = false; ctx.daemonEnv = undefined; + ctx.restoreFailed = undefined; } /** Opens the session with device selectors; later calls address it by session name alone. */ @@ -261,7 +267,7 @@ function openSession(ctx: SuiteContext, options: { relaunch: boolean }): CliResu } function establishSession(ctx: SuiteContext): CliResult { - if (ctx.daemonStuck) stopStuckDaemon(ctx); + if (ctx.daemonStuck || ctx.restoreFailed) stopStuckDaemon(ctx); return openSession(ctx, { relaunch: true }); } @@ -277,8 +283,10 @@ export async function runSuite( log(`round ${round + 1}/${totalRounds}${measured ? '' : ' (warmup, dropped)'}`); let sessionFailure = sessionError(establishSession(ctx)); for (const scenario of scenarios) { - // A request that outlived its timeout still holds the session; reopen before the next scenario. - if (ctx.daemonStuck) sessionFailure = sessionError(establishSession(ctx)); + // A request that outlived its timeout still holds the session, and a failed daemon restore + // lost it; reopen before the next scenario. + if (ctx.daemonStuck || ctx.restoreFailed) + sessionFailure = sessionError(establishSession(ctx)); log(` scenario ${scenario.id}`); const setupFailure = sessionFailure ?? untimed(ctx, scenario.setup); if (setupFailure) log(` ✗ setup failed: ${setupFailure}`); @@ -288,12 +296,14 @@ export async function runSuite( const id = `${scenario.id}/${step.id}`; const skipped = setupFailure ? { errorCode: 'setup_failed', errorMessage: setupFailure } - : ctx.daemonStuck - ? { - errorCode: 'daemon_stuck', - errorMessage: 'an earlier request still holds the session', - } - : undefined; + : ctx.restoreFailed + ? { errorCode: 'restore_failed', errorMessage: ctx.restoreFailed } + : ctx.daemonStuck + ? { + errorCode: 'daemon_stuck', + errorMessage: 'an earlier request still holds the session', + } + : undefined; const sample: Sample = skipped ? { round, ok: false, wallClockMs: 0, ...skipped } : await runTimedStep(scenario, step, ctx, round); diff --git a/scripts/perf/suite.ts b/scripts/perf/suite.ts index c3feeea3ef..42fe76aaa0 100644 --- a/scripts/perf/suite.ts +++ b/scripts/perf/suite.ts @@ -130,8 +130,10 @@ export const SUITE: readonly SuiteScenario[] = [ step('orientation-landscape', C.orientation, ['orientation', 'landscape-left']), step('orientation-portrait', C.orientation, ['orientation', 'portrait']), step('settings-appearance', C.settings, ['settings', 'appearance', 'light']), - step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf']), - step('clipboard-read', C.clipboard, ['clipboard', 'read']), + step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf'], { + only: 'ios', + }), + step('clipboard-read', C.clipboard, ['clipboard', 'read'], { only: 'ios' }), step('action-button', C.actionButton, ['action-button'], { only: 'ios' }), // `wake` is the one hardware key that leaves an awake device as it was. step('hardware-button', C.hardwareButton, ['hardware-button', 'wake'], { only: 'android' }), @@ -386,6 +388,8 @@ export const SUITE: readonly SuiteScenario[] = [ step('snapshot-raw-one-shot', C.snapshot, ['snapshot', '--raw'], { only: 'android', daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' }, + // The restarted daemon verifies the helper install on its first capture; keep that out. + before: [['snapshot', '-i']], expect: { maxDepthAtLeast: 56, transport: 'instrumentation' }, }), step('back', C.back, ['back']), @@ -446,6 +450,8 @@ export const SUITE_EXCLUSIONS: Record Date: Wed, 21 Oct 2026 21:42:25 +0200 Subject: [PATCH 07/15] feat(perf): time every iOS command against the fixture app pnpm perf --platform ios now drives the test app through seven scenarios that time each catalog command the iOS simulator serves, including the heavy feed and a raw capture with the XCTest tree backend pinned. The suite records a reason for every command it does not time, and a unit test keeps that partition complete. Each sample keeps the client daemon round trip and a phase profile from its --debug request log: runner commands, host process spawns, snapshot sources, retries, stabilization, and fallbacks. A run writes summary.json, report.md, and the raw daemon and session logs. The iOS Settings tour is removed; the Android tour stays as it was. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/perf-nightly.yml | 42 +++- scripts/perf/README.md | 51 ++++ scripts/perf/config.ts | 27 +- scripts/perf/forced-backend.ts | 90 +++++++ scripts/perf/harness.ts | 48 ++-- scripts/perf/ios-harness.ts | 294 ++++++++++++++++++++++ scripts/perf/ios-suite.test.ts | 39 +++ scripts/perf/ios-suite.ts | 380 +++++++++++++++++++++++++++++ scripts/perf/measurement.ts | 32 +++ scripts/perf/metadata.ts | 69 ++++++ scripts/perf/platform-profiles.ts | 36 +-- scripts/perf/profile.test.ts | 112 +++++++++ scripts/perf/profile.ts | 228 +++++++++++++++++ scripts/perf/report.ts | 151 ++++++++---- scripts/perf/run.ts | 159 ++++++++---- scripts/perf/scenario.ts | 118 +++------ scripts/perf/stats.test.ts | 28 +++ scripts/perf/stats.ts | 35 ++- scripts/perf/types.ts | 104 ++++++-- vitest.config.ts | 2 + 20 files changed, 1763 insertions(+), 282 deletions(-) create mode 100644 scripts/perf/README.md create mode 100644 scripts/perf/forced-backend.ts create mode 100644 scripts/perf/ios-harness.ts create mode 100644 scripts/perf/ios-suite.test.ts create mode 100644 scripts/perf/ios-suite.ts create mode 100644 scripts/perf/measurement.ts create mode 100644 scripts/perf/metadata.ts create mode 100644 scripts/perf/profile.test.ts create mode 100644 scripts/perf/profile.ts create mode 100644 scripts/perf/stats.test.ts diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index 5814855ead..1d6c4938a5 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -12,24 +12,24 @@ on: rounds: description: 'Measured rounds per command (samples)' required: false - default: '5' + default: '3' permissions: contents: read + actions: read concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: - AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts' - PERF_ROUNDS: ${{ github.event.inputs.rounds || '5' }} + PERF_ROUNDS: ${{ github.event.inputs.rounds || '3' }} jobs: perf-ios: name: iOS Command Perf runs-on: macos-26 - timeout-minutes: 80 + timeout-minutes: 100 env: IOS_RUNTIME_VERSION: '26.2' AGENT_DEVICE_XCUITEST_ARCHS: 'arm64' @@ -42,6 +42,15 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + with: + cache-dependency-path: | + pnpm-lock.yaml + examples/test-app/pnpm-lock.yaml + + - name: Build the shared CLI + uses: ./.github/actions/run-gate + with: + gate: build - name: Restore and build iOS XCTest runner uses: ./.github/actions/setup-apple-runner-build @@ -54,22 +63,35 @@ jobs: xcuitest-destination: generic/platform=iOS Simulator - name: Boot iOS test simulator + id: ios-simulator uses: ./.github/actions/boot-ios-test-simulator with: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro + - name: Install the fixture app + id: fixture-app + uses: ./.github/actions/setup-fixture-app + - name: Preflight iOS runner through public CLI + env: + UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} run: | + # The keyboard must stay expanded for keyboard dismiss to find its Done key. + xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "$UDID" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" pnpm clean:daemon - - name: Run iOS command perf benchmark + - name: Run iOS command perf suite + env: + UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + APP_ID: ${{ steps.fixture-app.outputs.app-id }} + APP_PATH: ${{ steps.fixture-app.outputs.app-path }} run: | - pnpm perf --platform ios \ - --device "iPhone 17 Pro" \ - --n "$PERF_ROUNDS" --warmup 1 \ + pnpm perf --platform ios --udid "$UDID" \ + --app-id "$APP_ID" --app-path "$APP_PATH" \ + --rounds "$PERF_ROUNDS" --warmup 1 \ --out-dir "$GITHUB_WORKSPACE/perf-results" - name: Upload iOS perf report @@ -84,6 +106,8 @@ jobs: name: Android Command Perf runs-on: ubuntu-latest timeout-minutes: 80 + env: + AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts' steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/scripts/perf/README.md b/scripts/perf/README.md new file mode 100644 index 0000000000..42d61d4301 --- /dev/null +++ b/scripts/perf/README.md @@ -0,0 +1,51 @@ +# Command performance suite + +`pnpm perf --platform ios` times every interaction command against the fixture app (`examples/test-app`). +The steps are in `ios-suite.ts`. Each step belongs to one scenario. +Each scenario starts from an untimed setup in every round. +The heavy-feed scenario uses a screen whose accessibility tree is deeper than 65 levels. +The suite records a reason for each catalog command that it does not time. `ios-suite.test.ts` keeps that list complete. +`pnpm perf --platform android` runs the older report-only Settings tour. + +## Run locally + +1. Run `pnpm build`. The suite drives the built CLI and the built SDK. +2. Use your own booted simulator. Install a current Release fixture on it. For the cached CI build and the JavaScript repack, see `examples/test-app/README.md`. +3. Set `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH` to a path in your worktree. +4. Run the suite: + +```sh +pnpm perf --platform ios --udid --rounds 3 --warmup 1 --out-dir .tmp/perf/local +``` + +`--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id. +The suite uses its own temporary state directory and daemon. It stops them at the end. + +## Output + +- `summary.json`: the machine-readable run (`agent-device-perf-summary/v1`). It holds the metadata, the coverage, and every sample. +- `report.md`: one table per scenario, the backends and fallbacks, the failures, and the coverage. +- `raw/state/`: the daemon log, and for each session `runner.log`, `events.ndjson`, and the `--debug` request log of each request. `requestId` in a sample names its request log. +- `raw/failures/`: the JSON response and stderr of each failed sample. + +## Metrics + +All times are milliseconds. Statistics use only the successful samples of a step. + +| Metric | Meaning | +| --- | --- | +| `wall` | The CLI process time, from spawn to exit. | +| `daemon` | The daemon request round trip that the client measures (`daemon_request`). For the pinned-backend capture, the span of its request log. Comparisons use this metric. | +| `mean ± sd`, `cv` | The mean, the sample standard deviation, and the standard deviation divided by the mean. | +| `median`, `p95`, `min`, `max` | The median (the mean of the two middle values for an even count), the nearest-rank 95th percentile, and the extremes. | +| `runner` | The median sum of XCTest runner command times in one request. | +| `spawns` | The median count and time of host processes that one request started. | +| `snapshot source` | The median time of host accessibility-bridge acquisitions in one request. | +| `settle wait` | The median `settle.waitedMs` that a `--settle` response reports. | +| `stabilization` | The median time of post-gesture stabilization phases in one request. | +| `retries` | The failed attempts of retried operations in all successful samples. | +| `top phases` | The five phases with the largest median time per request. | +| `producer` | The number of samples that each snapshot producer answered (`simulator-ax-bridge`, `apple-runner`). | +| backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected`. | +| fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. | +| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth. | diff --git a/scripts/perf/config.ts b/scripts/perf/config.ts index 10c3e84630..1db472fa88 100644 --- a/scripts/perf/config.ts +++ b/scripts/perf/config.ts @@ -13,11 +13,16 @@ export type PerfConfig = { warmup: number; // leading rounds dropped from stats keepArtifacts: boolean; // keep temp state dir + leave device booted outDir: string; - udid?: string; // iOS device override (UDID) - device?: string; // device override by name (e.g. "iPhone 17 Pro"); preferred over udid + udid?: string; // iOS simulator UDID (required for iOS) + device?: string; // Android AVD name for boot serial?: string; // Android device override + appId: string; // iOS fixture bundle id + appPath?: string; // iOS fixture bundle path, recorded in the run metadata + scenarios?: string[]; // iOS scenario ids; all when omitted }; +export const DEFAULT_IOS_APP_ID = 'com.callstack.agentdevicelab'; + // How to invoke the CLI. Defaults to the built dist binary (bin/agent-device.mjs). // Set AGENT_DEVICE_PERF_CLI to run from source instead, e.g. on CI: // AGENT_DEVICE_PERF_CLI="--experimental-strip-types src/bin.ts" @@ -50,10 +55,11 @@ function readIntValue(argv: string[], i: number, flag: string, min: number): num export function parseConfig(argv: string[]): PerfConfig { const cfg: PerfConfig = { platform: 'ios', - rounds: 5, + rounds: 3, warmup: 1, keepArtifacts: false, - outDir: DEFAULT_OUT_DIR, + outDir: path.join(DEFAULT_OUT_DIR, new Date().toISOString().replaceAll(/[:.]/g, '-')), + appId: DEFAULT_IOS_APP_ID, }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; @@ -86,9 +92,22 @@ export function parseConfig(argv: string[]): PerfConfig { case '--serial': cfg.serial = readValue(argv, i++, a); break; + case '--app-id': + cfg.appId = readValue(argv, i++, a); + break; + case '--app-path': + cfg.appPath = path.resolve(readValue(argv, i++, a)); + break; + case '--scenarios': + cfg.scenarios = readValue(argv, i++, a) + .split(',') + .map((id) => id.trim()) + .filter(Boolean); + break; default: throw new Error(`Unknown flag: ${a}`); } } + if (cfg.platform === 'ios' && !cfg.udid) throw new Error('--udid is required for --platform ios'); return cfg; } diff --git a/scripts/perf/forced-backend.ts b/scripts/perf/forced-backend.ts new file mode 100644 index 0000000000..387b22fea2 --- /dev/null +++ b/scripts/perf/forced-backend.ts @@ -0,0 +1,90 @@ +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { pathToFileURL } from 'node:url'; +import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { createRequestId } from '@agent-device/host-kit/diagnostics'; +import { captureHostEnvironment } from '@agent-device/host-kit/process'; +import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot'; +import { readDaemonInfo } from '../../src/daemon-client/daemon-client-metadata.ts'; +import { sendRequest } from '../../src/daemon-client/daemon-client-transport.ts'; +import { REPO_ROOT } from './config.ts'; +import type { CliResult } from './types.ts'; + +type AgentDeviceSdk = typeof import('../../src/sdk/index.ts'); + +const FORCED_CAPTURE_TIMEOUT_MS = 90_000; + +/** + * Captures one raw snapshot with the internal backend pin that the CLI does not expose. The + * transport talks to the daemon that the built CLI already started and never starts or replaces + * it, so the measured daemon stays the daemon under test. + */ +export async function captureWithForcedBackend(options: { + stateDir: string; + session: string; + udid: string; + backend: SnapshotPreferredBackend; +}): Promise<{ result: CliResult; requestId: string }> { + const requestId = createRequestId(); + const sdk = (await import( + pathToFileURL(path.join(REPO_ROOT, 'dist/src/index.js')).href + )) as AgentDeviceSdk; + const client = sdk.createAgentDeviceClient( + { session: options.session, stateDir: options.stateDir }, + { transport: forcedBackendTransport(options.backend, requestId) }, + ); + const t0 = performance.now(); + try { + const snapshot = await client.capture.snapshot({ + raw: true, + platform: 'ios', + udid: options.udid, + timeoutMs: FORCED_CAPTURE_TIMEOUT_MS, + }); + const json = { success: true, data: snapshot }; + const wallClockMs = performance.now() - t0; + return { + result: { exitCode: 0, wallClockMs, stdout: '', stderr: '', json, ok: true }, + requestId, + }; + } catch (error) { + const normalized = sdk.normalizeAgentDeviceError(error); + const json = { success: false, error: normalized }; + const wallClockMs = performance.now() - t0; + return { + result: { exitCode: 1, wallClockMs, stdout: '', stderr: normalized.message, json, ok: false }, + requestId, + }; + } +} + +function forcedBackendTransport( + backend: SnapshotPreferredBackend, + requestId: string, +): AgentDeviceDaemonTransport { + const host = captureHostEnvironment(process); + return async (request, context) => { + const stateDir = request.flags?.stateDir; + if (request.session === undefined || typeof stateDir !== 'string') { + throw new Error('A forced-backend capture requires an explicit session and state dir.'); + } + const paths = resolveDaemonPaths(stateDir, { env: process.env }); + const info = readDaemonInfo(paths.infoPath); + if (!info) throw new Error(`No daemon is running for state dir ${stateDir}.`); + return await sendRequest( + info, + { + ...request, + session: request.session, + token: info.token, + flags: { ...request.flags, snapshotPreferredBackend: backend }, + meta: { ...request.meta, debug: true, requestId }, + }, + 'auto', + paths, + FORCED_CAPTURE_TIMEOUT_MS, + { host, signal: context?.signal, timeoutRecovery: 'none' }, + ); + }; +} diff --git a/scripts/perf/harness.ts b/scripts/perf/harness.ts index 49f0bb6f93..09063440a7 100644 --- a/scripts/perf/harness.ts +++ b/scripts/perf/harness.ts @@ -10,8 +10,8 @@ import { } from './cli.ts'; import type { PerfConfig } from './config.ts'; import { resolveProfile, type ResolvedProfile } from './platform-profiles.ts'; +import { buildMeasurement } from './measurement.ts'; import { buildSettingsTour, type ScenarioStep } from './scenario.ts'; -import { summarize } from './stats.ts'; import type { CliResult, Measurement, Sample } from './types.ts'; export type IsolationContext = { @@ -32,7 +32,7 @@ export function setupIsolation(cfg: PerfConfig): IsolationContext { fs.mkdirSync(artifactsDir, { recursive: true }); const baseFlags = ['--state-dir', stateDir, '--session', 'perf', ...profile.platformFlags]; log(`state-dir: ${stateDir}`); - log(`device: ${profile.deviceName} (${profile.udid ?? profile.serial})`); + log(`device: ${profile.deviceName} (${profile.serial ?? 'default'})`); return { stateDir, artifactsDir, baseFlags, profile }; } @@ -72,8 +72,8 @@ function toSample(r: CliResult, round: number): Sample { return sample; } -// The first interaction after open/relaunch pays the iOS XCUITest runner startup (~10s+ cold) -// and a per-relaunch first-AX-query settle cost. Run an untimed throwaway interaction so that +// The first interaction after open/relaunch pays the Android helper startup and a per-relaunch +// first-query settle cost. Run an untimed throwaway interaction so that // cost is never attributed to a measured command. function warmRunner(ctx: IsolationContext): void { invokeCli(['snapshot', '-i'], ctx.baseFlags); @@ -91,39 +91,31 @@ function runStep(step: ScenarioStep, ctx: IsolationContext, round: number): Samp : invokeBatchStep(step.step, ctx.baseFlags); const sample = toSample(r, round); if (step.execMode === 'batch') { - sample.daemonDurationMs = readBatchStepDurationMs(r); + sample.daemonMs = readBatchStepDurationMs(r); if (step.isSnapshot) sample.elementCount = countElements(r); } return sample; } -function buildMeasurement( +const TOUR_SCENARIO = 'settings-tour'; + +function tourMeasurement( step: Pick, platform: ResolvedProfile['platform'], samples: Sample[], warmupDropped: number, ): Measurement { - const ok = samples.filter((s) => s.ok); - const failures = samples.length - ok.length; - const notes: string[] = []; - if (failures > 0) { - const codes = [...new Set(samples.filter((s) => !s.ok).map((s) => s.errorCode))].join(', '); - notes.push(`${failures}/${samples.length} samples failed: ${codes}`); - } - const num = (xs: (number | undefined)[]) => xs.filter((n): n is number => typeof n === 'number'); - return { - command: step.command, + return buildMeasurement({ + id: `${TOUR_SCENARIO}/${step.label}`, + scenario: TOUR_SCENARIO, label: step.label, + command: step.command, + args: [], platform, execMode: step.execMode, samples, warmupDropped, - wallClock: summarize(ok.map((s) => s.wallClockMs)), - daemonDuration: summarize(num(ok.map((s) => s.daemonDurationMs))), - elementCount: summarize(num(ok.map((s) => s.elementCount))), - failures, - notes, - }; + }); } function setupMeasurementFailed(measurement: Measurement): boolean { @@ -137,7 +129,7 @@ function bootOnce(ctx: IsolationContext): Measurement { const bootFlags = ['--state-dir', ctx.stateDir, ...ctx.profile.platformFlags]; const r = invokeCli(['boot', ...ctx.profile.selectorFlags], bootFlags); const sample = toSample(r, 0); - return buildMeasurement( + return tourMeasurement( { command: 'boot', label: 'boot device', execMode: 'standalone' }, ctx.profile.platform, [sample], @@ -152,7 +144,7 @@ function establishSession(ctx: IsolationContext): Measurement { log('establishing session (open with device selectors)'); const r = invokeCli(['open', ctx.profile.appTarget, ...ctx.profile.selectorFlags], ctx.baseFlags); const sample = toSample(r, 0); - return buildMeasurement( + return tourMeasurement( { command: 'open', label: 'open (establish + cold)', execMode: 'standalone' }, ctx.profile.platform, [sample], @@ -176,10 +168,8 @@ export function runScenario(ctx: IsolationContext, cfg: PerfConfig): Measurement // Android accessibility dumps time out while the UI is animating; disable animations // up front (untimed) so snapshot/get/is/fill can read an idle hierarchy. - if (ctx.profile.platform === 'android') { - log('disabling animations (android)'); - invokeCli(['settings', 'animations', 'off'], ctx.baseFlags); - } + log('disabling animations'); + invokeCli(['settings', 'animations', 'off'], ctx.baseFlags); const totalRounds = cfg.warmup + cfg.rounds; for (let round = 0; round < totalRounds; round++) { @@ -197,7 +187,7 @@ export function runScenario(ctx: IsolationContext, cfg: PerfConfig): Measurement } const tourMeasurements = steps.map((step) => - buildMeasurement(step, ctx.profile.platform, acc.get(step.label)!, cfg.warmup), + tourMeasurement(step, ctx.profile.platform, acc.get(step.label)!, cfg.warmup), ); return [boot, establish, ...tourMeasurements]; } diff --git a/scripts/perf/ios-harness.ts b/scripts/perf/ios-harness.ts new file mode 100644 index 0000000000..bcd2c3f4a6 --- /dev/null +++ b/scripts/perf/ios-harness.ts @@ -0,0 +1,294 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + resolveSessionDir, + resolveSessionRequestLogPath, +} from '@agent-device/daemon-contracts/session-artifact-paths'; +import { invokeCli } from './cli.ts'; +import type { PerfConfig } from './config.ts'; +import { captureWithForcedBackend } from './forced-backend.ts'; +import { expectationViolation, type SuiteScenario, type SuiteStep } from './ios-suite.ts'; +import { buildMeasurement } from './measurement.ts'; +import { + profileRequestLog, + readClientDiagnostics, + readSettleFacts, + readSnapshotFacts, +} from './profile.ts'; +import type { CliResult, Measurement, Sample } from './types.ts'; + +const SESSION = 'perf'; + +export type IosSuiteContext = { + stateDir: string; + udid: string; + appId: string; + artifactsDir: string; + rawDir: string; + // Set when a request outlived the client timeout. That request still holds the session, so + // the suite restarts the daemon before the next scenario. + daemonStuck?: boolean; +}; + +function log(msg: string): void { + process.stderr.write(`[perf] ${msg}\n`); +} + +function baseFlags(ctx: IosSuiteContext): string[] { + return ['--state-dir', ctx.stateDir, '--session', SESSION, '--platform', 'ios']; +} + +function run(ctx: IosSuiteContext, args: readonly string[]): CliResult { + return watch(ctx, invokeCli(expandStatic(args, ctx), baseFlags(ctx))); +} + +function watch(ctx: IosSuiteContext, result: CliResult): CliResult { + if (daemonTimedOut(result)) ctx.daemonStuck = true; + return result; +} + +function expandStatic(args: readonly string[], ctx: IosSuiteContext): string[] { + return args.map((arg) => + arg === '{app}' + ? ctx.appId + : arg.replace(/^\{out:(.+)\}$/, (_, file: string) => path.join(ctx.artifactsDir, file)), + ); +} + +type SnapshotNode = { + ref?: string; + identifier?: string; + rect?: { x: number; y: number; width: number; height: number }; +}; + +const TARGET_TOKEN = /^\{(ref|point):(.+)\}$/; + +/** Expands `{ref:id}` and `{point:id}` from one untimed interactive snapshot. */ +function resolveTargets(args: string[], ctx: IosSuiteContext): string[] | { error: string } { + if (!args.some((arg) => TARGET_TOKEN.test(arg))) return args; + const snapshot = run(ctx, ['snapshot', '-i']); + const nodes = ((snapshot.json as { data?: { nodes?: SnapshotNode[] } })?.data?.nodes ?? + []) as SnapshotNode[]; + const expanded: string[] = []; + for (const arg of args) { + const match = TARGET_TOKEN.exec(arg); + if (!match) { + expanded.push(arg); + continue; + } + const node = nodes.find((candidate) => candidate.identifier === match[2]); + if (!node?.ref || !node.rect) return { error: `target ${match[2]} is not in snapshot -i` }; + if (match[1] === 'ref') { + expanded.push(`@${node.ref}`); + } else { + expanded.push( + String(Math.round(node.rect.x + node.rect.width / 2)), + String(Math.round(node.rect.y + node.rect.height / 2)), + ); + } + } + return expanded; +} + +/** Runs untimed calls in order and describes the first failure. */ +function untimed( + ctx: IosSuiteContext, + calls: readonly (readonly string[])[] = [], +): string | undefined { + for (const args of calls) { + const result = run(ctx, args); + if (!result.ok) return `${args.join(' ')}: ${errorOf(result).errorMessage}`; + } + return undefined; +} + +/** True when the client gave up on a daemon request that is still running. */ +function daemonTimedOut(result: CliResult): boolean { + const details = (result.json as { error?: { details?: { reason?: unknown } } } | undefined)?.error + ?.details; + return details?.reason === 'daemon_transport_timeout'; +} + +function errorOf(result: CliResult): Pick { + const error = (result.json as { error?: { code?: string; message?: string } } | undefined)?.error; + return { + errorCode: error?.code ?? `exit:${result.exitCode}`, + errorMessage: (error?.message ?? result.stderr.trim().split('\n').pop() ?? '').slice(0, 300), + }; +} + +function readProfile(ctx: IosSuiteContext, requestId: string | undefined) { + if (!requestId) return undefined; + const sessionDir = resolveSessionDir(path.join(ctx.stateDir, 'sessions'), SESSION); + const logPath = resolveSessionRequestLogPath(sessionDir, requestId); + return fs.existsSync(logPath) ? profileRequestLog(fs.readFileSync(logPath, 'utf8')) : undefined; +} + +function recordFailure(ctx: IosSuiteContext, id: string, round: number, detail: unknown): void { + const dir = path.join(ctx.rawDir, 'failures'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync( + path.join(dir, `${id.replace('/', '__')}.round-${round}.json`), + JSON.stringify(detail, null, 2), + ); +} + +async function runTimedStep( + scenario: SuiteScenario, + step: SuiteStep, + ctx: IosSuiteContext, + round: number, +): Promise { + const id = `${scenario.id}/${step.id}`; + const before = untimed(ctx, step.before); + if (before) { + log(` ✗ ${id} (before_failed: ${before})`); + return { round, ok: false, wallClockMs: 0, errorCode: 'before_failed', errorMessage: before }; + } + let sample: Sample; + if (step.via === 'sdk-forced-tree') { + const { result: forced, requestId } = await captureWithForcedBackend({ + stateDir: ctx.stateDir, + session: SESSION, + udid: ctx.udid, + backend: 'tree', + }); + const result = watch(ctx, forced); + const profile = readProfile(ctx, requestId); + sample = { + round, + ok: result.ok, + wallClockMs: result.wallClockMs, + daemonMs: profile?.requestSpanMs, + requestId, + snapshot: readSnapshotFacts(result.json), + profile, + }; + if (!result.ok) Object.assign(sample, errorOf(result)); + } else { + const args = resolveTargets(expandStatic(step.args, ctx), ctx); + if ('error' in args) { + sample = { + round, + ok: false, + wallClockMs: 0, + errorCode: 'target_unresolved', + errorMessage: args.error, + }; + } else { + const result = watch(ctx, invokeCli([...args, '--debug'], baseFlags(ctx))); + const diagnostics = readClientDiagnostics(result.stderr); + sample = { + round, + ok: result.ok, + wallClockMs: result.wallClockMs, + daemonMs: diagnostics.daemonMs, + requestId: diagnostics.requestId, + snapshot: step.command === 'snapshot' ? readSnapshotFacts(result.json) : undefined, + settle: readSettleFacts(result.json), + profile: readProfile(ctx, diagnostics.requestId), + }; + if (!result.ok) { + Object.assign(sample, errorOf(result)); + recordFailure(ctx, id, round, { + args, + json: result.json, + stderrTail: result.stderr.slice(-4000), + }); + } + } + } + const violation = sample.ok ? expectationViolation(step.expect, sample.snapshot) : undefined; + if (violation) + Object.assign(sample, { ok: false, errorCode: 'expectation_failed', errorMessage: violation }); + const after = untimed(ctx, step.after); + if (after) log(` ! ${id} after: ${after}`); + if (!sample.ok) log(` ✗ ${id} (${sample.errorCode}: ${sample.errorMessage})`); + return sample; +} + +function sessionError(result: CliResult): string | undefined { + if (result.ok) return undefined; + const message = `open: ${errorOf(result).errorMessage}`; + log(` ✗ session failed: ${message}`); + return message; +} + +function stopStuckDaemon(ctx: IosSuiteContext): void { + log(' ! a request outlived its timeout; restarting the daemon'); + invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); + ctx.daemonStuck = false; +} + +/** Opens the session with device selectors; later calls address it by session name alone. */ +function establishSession(ctx: IosSuiteContext): CliResult { + if (ctx.daemonStuck) stopStuckDaemon(ctx); + return watch( + ctx, + invokeCli(['open', ctx.appId, '--relaunch', '--udid', ctx.udid], baseFlags(ctx)), + ); +} + +export async function runIosSuite( + ctx: IosSuiteContext, + cfg: PerfConfig, + scenarios: readonly SuiteScenario[], +): Promise { + const samples = new Map(); + const totalRounds = cfg.warmup + cfg.rounds; + for (let round = 0; round < totalRounds; round++) { + const measured = round >= cfg.warmup; + log(`round ${round + 1}/${totalRounds}${measured ? '' : ' (warmup, dropped)'}`); + let sessionFailure = sessionError(establishSession(ctx)); + for (const scenario of scenarios) { + // A request that outlived its timeout still holds the session; reopen before the next scenario. + if (ctx.daemonStuck) sessionFailure = sessionError(establishSession(ctx)); + log(` scenario ${scenario.id}`); + const setupFailure = sessionFailure ?? untimed(ctx, scenario.setup); + if (setupFailure) log(` ✗ setup failed: ${setupFailure}`); + for (const step of scenario.steps) { + // CLI calls block the event loop; yield so an interrupt can stop the run between steps. + await new Promise((resolve) => setImmediate(resolve)); + const id = `${scenario.id}/${step.id}`; + const skipped = setupFailure + ? { errorCode: 'setup_failed', errorMessage: setupFailure } + : ctx.daemonStuck + ? { + errorCode: 'daemon_stuck', + errorMessage: 'an earlier request still holds the session', + } + : undefined; + const sample: Sample = skipped + ? { round, ok: false, wallClockMs: 0, ...skipped } + : await runTimedStep(scenario, step, ctx, round); + if (measured) samples.set(id, [...(samples.get(id) ?? []), sample]); + } + } + } + return scenarios.flatMap((scenario) => + scenario.steps.map((step) => + buildMeasurement({ + id: `${scenario.id}/${step.id}`, + scenario: scenario.id, + label: step.args.join(' '), + command: step.command, + args: [...step.args], + platform: 'ios', + execMode: step.via === 'sdk-forced-tree' ? 'sdk' : 'standalone', + samples: samples.get(`${scenario.id}/${step.id}`) ?? [], + warmupDropped: cfg.warmup, + }), + ), + ); +} + +/** Copies daemon and session logs into the raw artifact; the daemon token file stays behind. */ +export function copyRawState(ctx: IosSuiteContext): void { + const target = path.join(ctx.rawDir, 'state'); + fs.mkdirSync(target, { recursive: true }); + const daemonLog = path.join(ctx.stateDir, 'daemon.log'); + if (fs.existsSync(daemonLog)) fs.copyFileSync(daemonLog, path.join(target, 'daemon.log')); + const sessions = path.join(ctx.stateDir, 'sessions'); + if (fs.existsSync(sessions)) + fs.cpSync(sessions, path.join(target, 'sessions'), { recursive: true }); +} diff --git a/scripts/perf/ios-suite.test.ts b/scripts/perf/ios-suite.test.ts new file mode 100644 index 0000000000..cc92840b1e --- /dev/null +++ b/scripts/perf/ios-suite.test.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import { test } from 'vitest'; +import { + describeCoverage, + expectationViolation, + IOS_SUITE, + IOS_SUITE_EXCLUSIONS, +} from './ios-suite.ts'; + +test('every public command is either measured by the iOS suite or excluded with a reason', () => { + const { measured, excluded } = describeCoverage(IOS_SUITE, IOS_SUITE_EXCLUSIONS); + const classified = [...measured, ...excluded.map((e) => e.command)].sort(); + assert.deepEqual(classified, Object.values(PUBLIC_COMMANDS).sort()); + for (const { reason } of excluded) assert.ok(reason.length > 0); +}); + +test('step ids are unique, so comparisons key each step once', () => { + const ids = IOS_SUITE.flatMap((scenario) => scenario.steps.map((s) => `${scenario.id}/${s.id}`)); + assert.equal(new Set(ids).size, ids.length); +}); + +test('a heavy-feed capture breaks its expectation when the tree flattens or the tree backend copes', () => { + const depth = { maxDepthAtLeast: 64 }; + assert.equal(expectationViolation(depth, { maxDepth: 64 }), undefined); + assert.match(expectationViolation(depth, { maxDepth: 63 })!, /below 64/); + assert.match(expectationViolation(depth, undefined)!, /unknown/); + + const degraded = { treeBackendDegraded: true }; + assert.equal( + expectationViolation(degraded, { backend: 'private-ax', state: 'recovered' }), + undefined, + ); + assert.equal(expectationViolation(degraded, { backend: 'tree', state: 'sparse' }), undefined); + assert.match( + expectationViolation(degraded, { backend: 'tree', state: 'healthy' })!, + /non-sparse/, + ); +}); diff --git a/scripts/perf/ios-suite.ts b/scripts/perf/ios-suite.ts new file mode 100644 index 0000000000..bde981db05 --- /dev/null +++ b/scripts/perf/ios-suite.ts @@ -0,0 +1,380 @@ +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { SnapshotFacts } from './types.ts'; + +/** Routes `trigger-app-event` into the fixture's automation screen. */ +export const FIXTURE_APP_EVENT_URL_TEMPLATE = + 'agent-device-test-app:///automation?event={event}&payload={payload}'; + +export type PublicCommand = (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS]; + +const C = PUBLIC_COMMANDS; + +type Cli = readonly string[]; + +/** + * One timed command. `args` tokens expand before each run: `{app}` is the fixture bundle id, + * `{out:}` a path in the run's artifact directory, and `{ref:}` / `{point:}` the ref + * or center point of the element with that identifier in an untimed `snapshot -i` taken just + * before the step. `before` and `after` run untimed around the step in every round. + */ +export type SuiteStep = { + id: string; + command: PublicCommand; + args: Cli; + before?: readonly Cli[]; + after?: readonly Cli[]; + // `sdk-forced-tree` captures a raw snapshot through the daemon with the XCTest tree backend + // pinned; the CLI has no flag for it. + via?: 'cli' | 'sdk-forced-tree'; + // A sample that succeeds but breaks the expectation fails with `expectation_failed`, so a + // fixture that stops reproducing its shape shows up as a new failure. + expect?: StepExpectation; +}; + +export type StepExpectation = { + // Deepest raw node depth, counted from the application at 0. + maxDepthAtLeast?: number; + // The pinned XCTest tree backend did not serve a healthy capture. + treeBackendDegraded?: boolean; +}; + +/** Describes how a snapshot breaks the step expectation, or returns undefined. */ +export function expectationViolation( + expect: StepExpectation | undefined, + snapshot: SnapshotFacts | undefined, +): string | undefined { + if (!expect) return undefined; + if (expect.maxDepthAtLeast !== undefined && (snapshot?.maxDepth ?? -1) < expect.maxDepthAtLeast) { + return `max depth ${snapshot?.maxDepth ?? 'unknown'} is below ${expect.maxDepthAtLeast}`; + } + if (expect.treeBackendDegraded && snapshot?.backend === 'tree' && snapshot.state !== 'sparse') { + return 'the pinned tree backend served a non-sparse capture'; + } + return undefined; +} + +/** A scenario starts from `setup` (untimed, every round) and runs its steps in order. */ +export type SuiteScenario = { + id: string; + title: string; + setup: readonly Cli[]; + steps: SuiteStep[]; +}; + +// A cold relaunch on a hosted runner can take longer than 15 s before the first landmark. +const LANDMARK_WAIT_MS = '60000'; +const relaunch: Cli = ['open', '{app}', '--relaunch']; +const toTab = (label: string, landmark: string): Cli[] => [ + relaunch, + ['wait', `label="${label}"`, LANDMARK_WAIT_MS], + ['click', `label="${label}"`], + ['wait', landmark, LANDMARK_WAIT_MS], +]; +const toSettingsLab = (button: string, landmark: string): Cli[] => [ + ...toTab('Settings', 'id="settings-title"'), + ['scroll', 'down', '--until', `id="${button}"`], + ['click', `id="${button}"`], + ['wait', landmark, LANDMARK_WAIT_MS], +]; +const step = ( + id: string, + command: PublicCommand, + args: Cli, + extra: Omit = {}, +): SuiteStep => ({ id, command, args, ...extra }); + +export const IOS_SUITE: readonly SuiteScenario[] = [ + { + id: 'lifecycle', + title: 'App lifecycle, device state, and inventory', + setup: [relaunch, ['wait', 'text', 'Agent Device Tester', LANDMARK_WAIT_MS]], + steps: [ + step('open-relaunch', C.open, relaunch), + step('appstate', C.appState, ['appstate']), + step('home', C.home, ['home']), + step('open-resume', C.open, ['open', '{app}']), + step('app-switcher', C.appSwitcher, ['app-switcher'], { after: [['open', '{app}']] }), + step('orientation-landscape', C.orientation, ['orientation', 'landscape-left']), + step('orientation-portrait', C.orientation, ['orientation', 'portrait']), + step('settings-appearance', C.settings, ['settings', 'appearance', 'light']), + step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf']), + step('clipboard-read', C.clipboard, ['clipboard', 'read']), + step('action-button', C.actionButton, ['action-button']), + step('prepare-runner', C.prepare, ['prepare', 'ios-runner']), + step('devices', C.devices, ['devices', '--platform', 'ios']), + step('apps', C.apps, ['apps']), + step('capabilities', C.capabilities, ['capabilities']), + step('doctor', C.doctor, ['doctor']), + // Some runtimes ask "Open in Agent Device Tester?" for the event URL, and the prompt outlives a + // relaunch. The answer opens the URL seconds later, so wait for its route before the next + // relaunch. The untimed calls stop at alert get when no prompt is up. + step( + 'trigger-app-event', + C.triggerAppEvent, + ['trigger-app-event', 'fixture.ready', '{"source":"perf"}'], + { + after: [ + ['alert', 'get'], + ['alert', 'accept'], + ['wait', 'id="automation-title"', LANDMARK_WAIT_MS], + ], + }, + ), + ], + }, + { + id: 'home-reads', + title: 'Reads on the home screen (light tree)', + setup: [relaunch, ['wait', 'text', 'Agent Device Tester', LANDMARK_WAIT_MS]], + steps: [ + step('snapshot-i', C.snapshot, ['snapshot', '-i']), + step('snapshot', C.snapshot, ['snapshot']), + step('snapshot-raw', C.snapshot, ['snapshot', '--raw']), + step('snapshot-diff', C.snapshot, ['snapshot', '--diff']), + step('diff-snapshot', C.diff, ['diff', 'snapshot']), + step('wait-text', C.wait, ['wait', 'text', 'Agent Device Tester']), + step('wait-selector', C.wait, ['wait', 'id="gesture-canary-ready"']), + step('wait-absent', C.wait, ['wait', 'absent', 'id="heavy-feed-title"']), + step('wait-stable', C.wait, ['wait', 'stable', '300', '5000']), + step('is-visible', C.is, ['is', 'visible', 'id="gesture-canary-ready"']), + step('is-exists', C.is, ['is', 'exists', 'text="Gesture lab"']), + step('get-text', C.get, ['get', 'text', 'id="gesture-fling-status"']), + step('get-attrs', C.get, ['get', 'attrs', 'id="drag-source"']), + step('find-text', C.find, ['find', 'text', 'Gesture lab', 'exists']), + step('batch-reads', C.batch, [ + 'batch', + '--steps', + JSON.stringify([ + { + command: 'get', + input: { + format: 'text', + target: { kind: 'selector', selector: 'id="gesture-fling-status"' }, + }, + }, + { command: 'is', input: { predicate: 'visible', selector: 'id="gesture-canary-ready"' } }, + ]), + ]), + step('gesture-pan', C.gesture, [ + 'gesture', + 'pan', + '{point:gesture-target}', + '40', + '0', + '300', + ]), + step('gesture-pinch', C.gesture, ['gesture', 'pinch', '1.4', '{point:gesture-target}']), + step('screenshot', C.screenshot, ['screenshot', '{out:home.png}']), + step('screenshot-overlay', C.screenshot, [ + 'screenshot', + '{out:home-refs.png}', + '--overlay-refs', + ]), + step('events', C.events, ['events']), + step('logs-mark', C.logs, ['logs', 'mark', 'agent-device perf']), + step('perf-memory', C.perf, ['perf', 'memory', 'sample']), + step('react-native', C.reactNative, ['react-native', 'dismiss-overlay']), + step('artifacts', C.artifacts, ['artifacts']), + ], + }, + { + id: 'catalog', + title: 'Catalog list: press forms, text entry, and scrolling', + setup: toTab('Catalog', 'id="catalog-title"'), + steps: [ + step('press-selector', C.press, ['press', 'id="category-produce"']), + step('press-selector-settle', C.press, ['press', 'id="category-bakery"', '--settle']), + step('press-ref', C.press, ['press', '{ref:category-pantry}']), + step('press-ref-settle', C.press, ['press', '{ref:category-produce}', '--settle']), + step('press-point', C.press, ['press', '{point:category-bakery}']), + step('press-point-settle', C.press, ['press', '{point:category-pantry}', '--settle']), + step('click-selector', C.click, ['click', 'id="category-all"']), + step('click-ref', C.click, ['click', '{ref:category-bakery}']), + step('click-point', C.click, ['click', '{point:category-pantry}']), + step('click-settle', C.click, ['click', 'id="category-all"', '--settle']), + step('press-count', C.press, [ + 'press', + '{point:category-produce}', + '--count', + '2', + '--interval-ms', + '50', + ]), + step('fill', C.fill, ['fill', 'id="catalog-search"', 'tart']), + step('fill-settle', C.fill, ['fill', 'id="catalog-search"', 'kit', '--settle']), + step('focus', C.focus, ['focus', '{point:catalog-search}']), + step('type', C.type, ['type', ' box']), + step('keyboard-return', C.keyboard, ['keyboard', 'return']), + step('fill-clear', C.fill, ['fill', 'id="catalog-search"', ''], { + after: [['keyboard', 'return']], + }), + step('scroll-down', C.scroll, ['scroll', 'down']), + step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), + step('scroll-top', C.scroll, ['scroll', 'top']), + step('scroll-until', C.scroll, ['scroll', 'down', '--until', 'id="catalog-footer"']), + step('swipe', C.swipe, ['swipe', '200', '300', '200', '650']), + step('swipe-count', C.swipe, [ + 'swipe', + '200', + '650', + '200', + '450', + '--count', + '2', + '--pause-ms', + '50', + '--pattern', + 'ping-pong', + ]), + ], + }, + { + id: 'form', + title: 'Form fields, keyboard, and submit', + setup: toTab('Form', 'id="form-title"'), + steps: [ + step('fill-name', C.fill, ['fill', 'id="field-name"', 'Ada Lovelace']), + step('fill-email', C.fill, ['fill', 'id="field-email"', 'ada@example']), + step('type-suffix', C.type, ['type', '.test']), + step('keyboard-dismiss', C.keyboard, ['keyboard', 'dismiss'], { + before: [['wait', 'label="Done" visible=true', '10000']], + }), + step('press-checkbox', C.press, ['press', 'id="checkbox-agree"'], { + before: [['scroll', 'down', '--until', 'id="checkbox-agree"']], + }), + step('press-submit-settle', C.press, ['press', 'id="submit-order"', '--settle'], { + before: [['scroll', 'down', '--until', 'id="submit-order"']], + }), + ], + }, + { + id: 'automation', + title: 'Long press, native alerts, recording, tracing, and back navigation', + setup: toSettingsLab('open-automation-lab', 'id="automation-title"'), + steps: [ + step('record-start', C.record, ['record', 'start', '{out:automation.mp4}', '--hide-touches']), + step('record-stop', C.record, ['record', 'stop'], { + before: [ + ['scroll', 'down', '--until', 'id="automation-press"'], + ['press', 'id="automation-press"'], + ['wait', '1200'], + ], + }), + step('trace-start', C.trace, ['trace', 'start', '{out:automation.trace}']), + step('trace-stop', C.trace, ['trace', 'stop', '{out:automation.trace}'], { + before: [['press', 'id="automation-press"']], + }), + step('longpress', C.longPress, ['longpress', 'id="automation-longpress"', '800'], { + before: [['scroll', 'down', '--until', 'id="automation-longpress"']], + }), + step('longpress-settle', C.longPress, [ + 'longpress', + 'id="automation-longpress"', + '800', + '--settle', + ]), + step('alert-open', C.press, ['press', 'id="automation-open-alert"'], { + before: [['scroll', 'down', '--until', 'id="automation-open-alert"']], + }), + step('alert-wait', C.alert, ['alert', 'wait', '10000']), + step('alert-get', C.alert, ['alert', 'get']), + step('alert-accept', C.alert, ['alert', 'accept']), + step('alert-dismiss', C.alert, ['alert', 'dismiss'], { + before: [ + ['press', 'id="automation-open-alert"'], + ['alert', 'wait', '10000'], + ], + }), + step('logs-start', C.logs, ['logs', 'start']), + // A stop issued 400 ms after start has hung its request in local runs; let the stream run. + step('logs-stop', C.logs, ['logs', 'stop'], { before: [['wait', '1000']] }), + step('network-dump', C.network, ['network', 'dump', '10']), + step('back', C.back, ['back']), + step('back-settle', C.back, ['back', '--settle'], { + before: [ + ['click', 'id="open-automation-lab"'], + ['wait', 'id="automation-title"', LANDMARK_WAIT_MS], + ], + }), + ], + }, + { + id: 'heavy-feed', + title: 'Heavy feed: Bluesky-depth tree (≥65 native levels) and virtualized list', + setup: toSettingsLab('open-heavy-feed', 'id="heavy-feed-title"'), + steps: [ + step('snapshot-i', C.snapshot, ['snapshot', '-i']), + step('snapshot', C.snapshot, ['snapshot']), + step('snapshot-raw', C.snapshot, ['snapshot', '--raw'], { expect: { maxDepthAtLeast: 64 } }), + step('press-like-selector', C.press, ['press', 'id="heavy-post-0-like"']), + step('press-like-settle', C.press, ['press', 'id="heavy-post-1-like"', '--settle']), + step('press-like-ref', C.press, ['press', '{ref:heavy-post-0-like}']), + step('press-like-point', C.press, ['press', '{point:heavy-post-0-like}']), + step('wait-text', C.wait, ['wait', 'text', 'Liked posts: 2']), + step('is-visible', C.is, ['is', 'visible', 'id="heavy-post-1-like"']), + step('get-text', C.get, ['get', 'text', 'id="heavy-feed-last-action"']), + step('find-text', C.find, ['find', 'text', 'Post 1:', 'exists']), + step('longpress-post', C.longPress, ['longpress', 'id="heavy-post-1"', '800']), + step('scroll-down', C.scroll, ['scroll', 'down']), + step('snapshot-i-scrolled', C.snapshot, ['snapshot', '-i']), + step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), + step('swipe', C.swipe, ['swipe', '200', '650', '200', '300']), + // The pinned XCTest capture outlives its watchdog by about a second, and the next runner + // command refuses until it ends; the untimed wait keeps that cost out of the next step. + step('snapshot-raw-forced-tree', C.snapshot, ['snapshot', '--raw'], { + via: 'sdk-forced-tree', + expect: { treeBackendDegraded: true }, + after: [['wait', '2000']], + }), + step('back', C.back, ['back']), + ], + }, + { + id: 'session', + title: 'Session teardown', + setup: [], + steps: [step('close', C.close, ['close'])], + }, +]; + +/** + * Catalog commands the iOS suite does not time, with the reason. Every public command is either + * measured by a step above or listed here; `ios-suite.test.ts` holds that partition. + */ +export const IOS_SUITE_EXCLUSIONS: Partial> = { + [C.boot]: + 'Simulator power cycle: CoreSimulator boot time dominates and it ends the session under test.', + [C.shutdown]: 'Simulator power cycle: it ends the session and the runner under test.', + [C.install]: + 'Bundle provisioning: simctl copy time dominates; the job installs the fixture once.', + [C.reinstall]: + 'Bundle provisioning: simctl copy time dominates and it resets the app under test.', + [C.uninstall]: 'Bundle provisioning: it removes the app under test.', + [C.installFromSource]: + 'Bundle provisioning from a source build; the job installs a cached Release fixture.', + [C.replay]: 'Composite runner: a replay is the sum of the measured commands it runs.', + [C.test]: 'Composite runner: a test suite is the sum of the measured commands it runs.', + [C.push]: + 'The fixture holds no notification authorization, so simctl refuses the payload; iOS push coverage is contract-only.', + [C.audio]: 'Host audio capture needs a macOS audio permission that hosted runners do not grant.', + [C.hover]: 'The iOS runtime refuses hover; it is a web pointer contract.', + [C.viewport]: 'The iOS runtime refuses viewport; it resizes web viewports only.', + [C.tvRemote]: 'tvOS only.', + [C.fold]: 'iPhone Duo only; the suite device has one panel.', + [C.telephony]: 'The iOS simulator runtime refuses simulated calls and SMS.', + [C.hardwareButton]: 'The iOS simulator runtime refuses hardware-button presses.', +}; + +/** Partitions the measured and excluded catalog commands for the report. */ +export function describeCoverage( + scenarios: readonly SuiteScenario[], + exclusions: Partial>, +): { measured: string[]; excluded: { command: string; reason: string }[] } { + const measured = [...new Set(scenarios.flatMap((s) => s.steps.map((step) => step.command)))]; + return { + measured: measured.sort(), + excluded: Object.entries(exclusions) + .map(([command, reason]) => ({ command, reason: reason! })) + .sort((a, b) => a.command.localeCompare(b.command)), + }; +} diff --git a/scripts/perf/measurement.ts b/scripts/perf/measurement.ts new file mode 100644 index 0000000000..1641361a3a --- /dev/null +++ b/scripts/perf/measurement.ts @@ -0,0 +1,32 @@ +import { aggregateProfiles } from './profile.ts'; +import { summarize } from './stats.ts'; +import type { Measurement } from './types.ts'; + +type MeasurementInput = Omit< + Measurement, + 'wallClock' | 'daemon' | 'elementCount' | 'profile' | 'failures' | 'notes' +>; + +function numbers(values: (number | undefined)[]): number[] { + return values.filter((n): n is number => typeof n === 'number'); +} + +/** Summarizes the successful samples; failed samples count only toward `failures`. */ +export function buildMeasurement(input: MeasurementInput): Measurement { + const ok = input.samples.filter((s) => s.ok); + const failures = input.samples.length - ok.length; + const notes: string[] = []; + if (failures > 0) { + const codes = [...new Set(input.samples.filter((s) => !s.ok).map((s) => s.errorCode))]; + notes.push(`${failures}/${input.samples.length} failed: ${codes.join(', ')}`); + } + return { + ...input, + wallClock: summarize(ok.map((s) => s.wallClockMs)), + daemon: summarize(numbers(ok.map((s) => s.daemonMs))), + elementCount: summarize(numbers(ok.map((s) => s.elementCount ?? s.snapshot?.nodeCount))), + profile: aggregateProfiles(input.samples), + failures, + notes, + }; +} diff --git a/scripts/perf/metadata.ts b/scripts/perf/metadata.ts new file mode 100644 index 0000000000..6bb4e7d861 --- /dev/null +++ b/scripts/perf/metadata.ts @@ -0,0 +1,69 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { REPO_ROOT, type PerfConfig } from './config.ts'; +import type { RunMetadata } from './types.ts'; + +function capture(cmd: string, args: string[]): string | undefined { + try { + const result = runCmdSync(cmd, args, { cwd: REPO_ROOT, allowFailure: true }); + return result.exitCode === 0 ? result.stdout.trim() : undefined; + } catch { + return undefined; + } +} + +function readVersion(): string { + try { + const pkg = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')); + return typeof pkg.version === 'string' ? pkg.version : 'unknown'; + } catch { + return 'unknown'; + } +} + +function simulator(udid: string): { name: string; runtime?: string } | undefined { + const raw = capture('xcrun', ['simctl', 'list', 'devices', '--json']); + if (!raw) return undefined; + const devices = (JSON.parse(raw) as { devices: Record }) + .devices; + for (const [runtime, list] of Object.entries(devices)) { + const device = list.find((entry) => entry.udid === udid); + if (device) return { name: device.name, runtime: runtime.split('.').pop() }; + } + return undefined; +} + +/** Records what the numbers depend on: source, toolchain, host, and device. */ +export function collectMetadata(cfg: PerfConfig, deviceName: string): RunMetadata { + const env = process.env; + const sim = cfg.platform === 'ios' && cfg.udid ? simulator(cfg.udid) : undefined; + const cpus = os.cpus(); + return { + commit: env.GITHUB_SHA ?? capture('git', ['rev-parse', 'HEAD']) ?? 'unknown', + ref: env.GITHUB_REF_NAME ?? capture('git', ['rev-parse', '--abbrev-ref', 'HEAD']), + runId: env.GITHUB_RUN_ID, + runAttempt: env.GITHUB_RUN_ATTEMPT, + agentDeviceVersion: readVersion(), + host: { + os: `${os.type()} ${os.release()}`, + arch: os.arch(), + cpu: cpus[0]?.model ?? 'unknown', + cores: cpus.length, + memoryGb: Math.round(os.totalmem() / 1024 ** 3), + }, + xcode: + cfg.platform === 'ios' + ? capture('xcodebuild', ['-version'])?.replaceAll('\n', ' ') + : undefined, + node: process.version, + device: { + name: sim?.name ?? deviceName, + udid: cfg.udid, + serial: cfg.serial, + runtime: sim?.runtime, + }, + fixture: cfg.platform === 'ios' ? { appId: cfg.appId, appPath: cfg.appPath } : undefined, + }; +} diff --git a/scripts/perf/platform-profiles.ts b/scripts/perf/platform-profiles.ts index c082cdab06..447d3b310f 100644 --- a/scripts/perf/platform-profiles.ts +++ b/scripts/perf/platform-profiles.ts @@ -1,10 +1,7 @@ import type { PerfConfig } from './config.ts'; -import type { Platform } from './types.ts'; -// Local-convenience defaults for ad-hoc runs; CI always overrides them (--device / --serial). -// The iOS UDID is a specific local "iPhone 17" sim; the Android default is an AVD name that -// `boot` can launch. Pass --udid/--device/--serial to target your own device. -const DEFAULT_IOS_UDID = 'D74E0B66-57EB-4EC1-92DC-DA0A30581FE7'; +// Local-convenience default for ad-hoc Android runs: an AVD name that `boot` can launch. CI +// passes --serial. The iOS suite drives the fixture app instead (see ios-suite.ts). const DEFAULT_ANDROID_AVD = 'Pixel_9_Pro_XL_API_37'; export type ProfileSelectors = { @@ -14,10 +11,6 @@ export type ProfileSelectors = { searchField: string; // A selector that uniquely targets the EDITABLE search field (for fill). searchFieldEditable: string; - // iOS exposes an editable search field at the Settings root (fill works without focusing - // first; focusing then filling can hang). Android only reveals the editable after tapping - // the search card, so it must press the search entry before fill/type. - searchEditableAtRoot: boolean; // A label reliably visible on the Settings root, for get/is (selector form). anchorLabel: string; // Plain text of the anchor, for wait text / find (not a selector). @@ -25,9 +18,8 @@ export type ProfileSelectors = { }; export type ResolvedProfile = { - platform: Platform; + platform: 'android'; deviceName: string; - udid?: string; serial?: string; platformFlags: string[]; // --platform; applied to every call (only conflicts if it mismatches a locked session) selectorFlags: string[]; // device selectors — ONLY on the session-establishing open / selectorless boot @@ -36,27 +28,6 @@ export type ResolvedProfile = { }; export function resolveProfile(cfg: PerfConfig): ResolvedProfile { - if (cfg.platform === 'ios') { - // Prefer targeting by device name (CI boots a named simulator); fall back to a UDID. - const useName = cfg.device !== undefined; - const udid = useName ? undefined : (cfg.udid ?? DEFAULT_IOS_UDID); - return { - platform: 'ios', - deviceName: cfg.device ?? 'iPhone 17', - udid, - platformFlags: ['--platform', 'ios'], - selectorFlags: useName ? ['--device', cfg.device!] : ['--udid', udid!], - appTarget: 'settings', - selectors: { - deepScreen: 'label="General"', - searchField: 'label="Search"', - searchFieldEditable: 'label="Search" editable', - searchEditableAtRoot: true, - anchorLabel: 'label="General"', - anchorText: 'General', - }, - }; - } const avdName = cfg.device ?? DEFAULT_ANDROID_AVD; const serialFlags = cfg.serial ? ['--serial', cfg.serial, '--android-device-allowlist', cfg.serial] @@ -72,7 +43,6 @@ export function resolveProfile(cfg: PerfConfig): ResolvedProfile { deepScreen: 'text="Network & internet"', searchField: 'text="Search Settings"', searchFieldEditable: 'editable', - searchEditableAtRoot: false, anchorLabel: 'label="Network & internet"', anchorText: 'Network & internet', }, diff --git a/scripts/perf/profile.test.ts b/scripts/perf/profile.test.ts new file mode 100644 index 0000000000..91e16e4627 --- /dev/null +++ b/scripts/perf/profile.test.ts @@ -0,0 +1,112 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + aggregateProfiles, + profileRequestLog, + readClientDiagnostics, + readSettleFacts, +} from './profile.ts'; +import type { Sample } from './types.ts'; + +const line = (event: Record) => JSON.stringify({ level: 'info', ...event }); + +// Trimmed from a heavy-feed `press` request log. +const PRESS_LOG = [ + line({ ts: '2026-10-21T19:00:00.000Z', phase: 'request_start', data: { command: 'press' } }), + line({ + phase: 'exec_command', + durationMs: 0.02, + data: { command: 'xcrun', event: 'spawn_start' }, + }), + line({ phase: 'exec_command', durationMs: 9, data: { command: 'xcrun', event: 'spawn' } }), + line({ phase: 'exec_command', durationMs: 120, data: { command: 'xcrun', event: 'exit' } }), + line({ phase: 'exec_command', durationMs: 121, data: { command: 'xcrun', event: 'settled' } }), + line({ + level: 'error', + phase: 'ios.snapshot-source.acquire', + durationMs: 80, + data: { producer: 'simulator-ax-bridge', error: 'unsupported: foreground-owner-unverified' }, + }), + line({ + phase: 'ios.snapshot-source.acquire', + durationMs: 160, + data: { producer: 'simulator-ax-bridge' }, + }), + line({ + phase: 'ios_snapshot_source_recovery', + data: { producer: 'simulator-ax-bridge', truncated: true, acceptedLevels: 32 }, + }), + line({ phase: 'retry', data: { event: 'attempt_failed', attempt: 1 } }), + line({ phase: 'ios_runner_command_send', durationMs: 900, data: { command: 'tap' } }), + line({ phase: 'post_gesture_snapshot_stabilized', data: { attempts: 2, durationMs: 300 } }), + line({ + ts: '2026-10-21T19:00:01.250Z', + level: 'warn', + phase: 'ios_runner_session_invalidated', + data: { reason: 'stale' }, + }), + 'not json', +].join('\n'); + +test('profiles runner, spawn, snapshot-source, retry, and settle phases of one request', () => { + const profile = profileRequestLog(PRESS_LOG); + assert.deepEqual(profile.spawns, { xcrun: { count: 1, totalMs: 121 } }); + assert.deepEqual(profile.runnerCommands, { tap: { count: 1, totalMs: 900 } }); + assert.deepEqual(profile.snapshotSources, { + 'simulator-ax-bridge': { count: 2, totalMs: 240, failures: 1 }, + }); + assert.equal(profile.retries, 1); + assert.equal(profile.requestSpanMs, 1250); + assert.deepEqual(profile.stabilization, { count: 1, totalMs: 300 }); + assert.deepEqual(profile.fallbacks, [ + 'ios_runner_session_invalidated: stale', + 'simulator-ax-bridge: truncated at 32 levels', + 'simulator-ax-bridge: unsupported: foreground-owner-unverified', + ]); +}); + +test('counts a producer only for samples where it answered', () => { + const failedOnly = profileRequestLog( + line({ + level: 'error', + phase: 'ios.snapshot-source.acquire', + durationMs: 5, + data: { producer: 'simulator-ax-bridge', error: 'cancelled' }, + }), + ); + const runnerSnapshot = profileRequestLog( + line({ phase: 'ios_runner_command_send', durationMs: 4000, data: { command: 'snapshot' } }), + ); + const samples: Sample[] = [ + { round: 0, ok: true, wallClockMs: 1, profile: failedOnly }, + { + round: 1, + ok: true, + wallClockMs: 1, + profile: runnerSnapshot, + snapshot: { backend: 'private-ax', state: 'recovered', reasonCode: 'ax-rejected' }, + }, + { round: 2, ok: false, wallClockMs: 1, profile: profileRequestLog(PRESS_LOG) }, + ]; + const aggregate = aggregateProfiles(samples)!; + assert.deepEqual(aggregate.producers, { 'apple-runner': 1 }); + assert.deepEqual(aggregate.backends, { 'private-ax/recovered/ax-rejected': 1 }); +}); + +test('reads the client daemon timer from --debug stderr', () => { + const stderr = [ + '[agent-device][diag] {"phase":"daemon_startup","durationMs":30,"requestId":"r1"}', + 'noise', + '[agent-device][diag] {"phase":"daemon_request","durationMs":812,"requestId":"r1"}', + ].join('\n'); + assert.deepEqual(readClientDiagnostics(stderr), { daemonMs: 812, requestId: 'r1' }); + assert.deepEqual(readClientDiagnostics('[agent-device][diag] {"phase":"daemon_req'), {}); +}); + +test('reads the settle wait of a --settle response and nothing from other responses', () => { + assert.deepEqual( + readSettleFacts({ data: { settle: { settled: true, waitedMs: 1665, captures: 2 } } }), + { waitedMs: 1665, captures: 2, settled: true }, + ); + assert.equal(readSettleFacts({ data: { message: 'Tapped' } }), undefined); +}); diff --git a/scripts/perf/profile.ts b/scripts/perf/profile.ts new file mode 100644 index 0000000000..a869d74c32 --- /dev/null +++ b/scripts/perf/profile.ts @@ -0,0 +1,228 @@ +import { median } from './stats.ts'; +import type { + MeasurementProfile, + Sample, + SampleProfile, + SettleFacts, + SnapshotFacts, + Timing, +} from './types.ts'; + +type DiagnosticEvent = { + ts?: string; + level?: string; + phase?: string; + durationMs?: number; + data?: Record; +}; + +const RUNNER_COMMAND_PHASE = 'ios_runner_command_send'; +const SPAWN_PHASE = 'exec_command'; +const SNAPSHOT_SOURCE_PHASE = 'ios.snapshot-source.acquire'; +const SNAPSHOT_RECOVERY_PHASE = 'ios_snapshot_source_recovery'; +const STABILIZATION_PHASE = /settle|stabiliz/i; +const FALLBACK_TEXT_LIMIT = 140; +const CLIENT_DIAG_PREFIX = '[agent-device][diag] '; + +function addTiming(map: Record, key: string, durationMs: number | undefined): void { + const entry = (map[key] ??= { count: 0, totalMs: 0 }); + entry.count += 1; + entry.totalMs += durationMs ?? 0; +} + +function text(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 + ? value.slice(0, FALLBACK_TEXT_LIMIT) + : undefined; +} + +function eventDuration(event: DiagnosticEvent): number | undefined { + if (typeof event.durationMs === 'number') return event.durationMs; + const nested = event.data?.durationMs; + return typeof nested === 'number' ? nested : undefined; +} + +/** + * Reads one daemon request log (`--debug` ndjson). Process spawns count once, at their `settled` + * event. Fallbacks name every degraded path the request took: snapshot-source errors, truncated + * recovery, and warn- or error-level phases. + */ +export function profileRequestLog(ndjson: string): SampleProfile { + const profile: SampleProfile = { + phases: {}, + runnerCommands: {}, + spawns: {}, + snapshotSources: {}, + retries: 0, + stabilization: { count: 0, totalMs: 0 }, + fallbacks: [], + }; + const fallbacks = new Set(); + let firstTs: number | undefined; + let lastTs: number | undefined; + for (const line of ndjson.split('\n')) { + if (!line.trim()) continue; + let event: DiagnosticEvent; + try { + event = JSON.parse(line) as DiagnosticEvent; + } catch { + continue; + } + const phase = event.phase; + if (!phase) continue; + const ts = typeof event.ts === 'string' ? Date.parse(event.ts) : Number.NaN; + if (Number.isFinite(ts)) { + firstTs ??= ts; + lastTs = ts; + } + const data = event.data ?? {}; + const duration = eventDuration(event); + if (phase === SPAWN_PHASE) { + if (data.event !== 'settled') continue; + addTiming(profile.spawns, text(data.command) ?? 'unknown', duration); + } else if (phase === RUNNER_COMMAND_PHASE) { + addTiming(profile.runnerCommands, text(data.command) ?? 'unknown', duration); + } else if (phase === SNAPSHOT_SOURCE_PHASE) { + const producer = text(data.producer) ?? 'unknown'; + const entry = (profile.snapshotSources[producer] ??= { count: 0, totalMs: 0, failures: 0 }); + entry.count += 1; + entry.totalMs += duration ?? 0; + const error = text(data.error); + if (error || event.level === 'error') { + entry.failures += 1; + fallbacks.add(`${producer}: ${error ?? 'failed'}`); + } + } else if (phase === SNAPSHOT_RECOVERY_PHASE && data.truncated === true) { + fallbacks.add( + `${text(data.producer) ?? 'unknown'}: truncated at ${String(data.acceptedLevels)} levels`, + ); + } else if (phase === 'retry' && data.event === 'attempt_failed') { + profile.retries += 1; + } else if (event.level === 'warn' || event.level === 'error') { + fallbacks.add(`${phase}${text(data.reason) ? `: ${text(data.reason)}` : ''}`); + } + if (STABILIZATION_PHASE.test(phase)) { + profile.stabilization.count += 1; + profile.stabilization.totalMs += duration ?? 0; + } + addTiming(profile.phases, phase, duration); + } + profile.fallbacks = [...fallbacks].sort(); + if (firstTs !== undefined && lastTs !== undefined) profile.requestSpanMs = lastTs - firstTs; + return profile; +} + +/** Reads the client's `daemon_request` timer and request id from `--debug` stderr. */ +export function readClientDiagnostics(stderr: string): { daemonMs?: number; requestId?: string } { + for (const line of stderr.split('\n')) { + const start = line.indexOf(CLIENT_DIAG_PREFIX); + if (start < 0) continue; + try { + const event = JSON.parse(line.slice(start + CLIENT_DIAG_PREFIX.length)) as { + phase?: string; + durationMs?: number; + requestId?: string; + }; + if (event.phase === 'daemon_request') { + return { daemonMs: event.durationMs, requestId: event.requestId }; + } + } catch { + // A truncated diagnostic line carries no timer. + } + } + return {}; +} + +/** Reads the settle outcome of a `--settle` response. */ +export function readSettleFacts(json: unknown): SettleFacts | undefined { + const settle = (json as { data?: { settle?: Record } } | undefined)?.data + ?.settle; + if (!settle || typeof settle.waitedMs !== 'number') return undefined; + return { + waitedMs: settle.waitedMs, + captures: typeof settle.captures === 'number' ? settle.captures : undefined, + settled: typeof settle.settled === 'boolean' ? settle.settled : undefined, + }; +} + +/** Reads snapshot quality facts from a `--json` snapshot response. */ +export function readSnapshotFacts(json: unknown): SnapshotFacts | undefined { + const data = (json as { data?: Record } | undefined)?.data; + if (!data || !Array.isArray(data.nodes)) return undefined; + const nodes = data.nodes as { depth?: unknown }[]; + const quality = (data.snapshotQuality ?? {}) as Record; + const depths = nodes.map((node) => (typeof node.depth === 'number' ? node.depth : 0)); + return { + backend: text(quality.backend), + state: text(quality.state), + reasonCode: text(quality.reasonCode), + nodeCount: nodes.length, + maxDepth: depths.length > 0 ? Math.max(...depths) : undefined, + truncated: typeof data.truncated === 'boolean' ? data.truncated : undefined, + }; +} + +function medianOrNull(values: number[]): number | null { + return values.length === 0 ? null : median(values); +} + +function sumTimings(map: Record): Timing { + return Object.values(map).reduce( + (sum, t) => ({ count: sum.count + t.count, totalMs: sum.totalMs + t.totalMs }), + { count: 0, totalMs: 0 }, + ); +} + +function tally(counts: Record, keys: Iterable): void { + for (const key of new Set(keys)) counts[key] = (counts[key] ?? 0) + 1; +} + +/** + * Summarizes the successful samples' profiles. Medians describe a typical request; producers, + * backends, and fallbacks count how many samples took each path. + */ +export function aggregateProfiles(samples: Sample[]): MeasurementProfile | null { + const profiled = samples.filter((s) => s.ok && s.profile); + if (profiled.length === 0) return null; + const profiles = profiled.map((s) => s.profile!); + const producers: Record = {}; + const backends: Record = {}; + const fallbacks: Record = {}; + for (const sample of profiled) { + const profile = sample.profile!; + tally( + producers, + Object.entries(profile.snapshotSources) + .filter(([, source]) => source.count > source.failures) + .map(([producer]) => producer), + ); + if (profile.runnerCommands.snapshot) tally(producers, ['apple-runner']); + if (sample.snapshot?.backend) { + const { backend, state, reasonCode } = sample.snapshot; + tally(backends, [[backend, state, reasonCode].filter(Boolean).join('/')]); + } + tally(fallbacks, profile.fallbacks); + } + const phaseNames = new Set(profiles.flatMap((p) => Object.keys(p.phases))); + const topPhases = [...phaseNames] + .map((phase) => ({ + phase, + medianMs: median(profiles.map((p) => p.phases[phase]?.totalMs ?? 0)), + })) + .filter((entry) => entry.medianMs > 0) + .sort((a, b) => b.medianMs - a.medianMs) + .slice(0, 5); + return { + runnerMs: medianOrNull(profiles.map((p) => sumTimings(p.runnerCommands).totalMs)), + spawnCount: medianOrNull(profiles.map((p) => sumTimings(p.spawns).count)), + spawnMs: medianOrNull(profiles.map((p) => sumTimings(p.spawns).totalMs)), + snapshotSourceMs: medianOrNull(profiles.map((p) => sumTimings(p.snapshotSources).totalMs)), + stabilizationMs: medianOrNull(profiles.map((p) => p.stabilization.totalMs)), + settleWaitMs: medianOrNull(profiled.flatMap((s) => (s.settle ? [s.settle.waitedMs] : []))), + retries: profiles.reduce((sum, p) => sum + p.retries, 0), + producers, + backends, + fallbacks, + topPhases, + }; +} diff --git a/scripts/perf/report.ts b/scripts/perf/report.ts index 536639eadb..480978638d 100644 --- a/scripts/perf/report.ts +++ b/scripts/perf/report.ts @@ -1,73 +1,134 @@ import fs from 'node:fs'; import path from 'node:path'; -import type { Measurement, RunResult, Stat } from './types.ts'; +import type { Measurement, PerfSummary, Stat } from './types.ts'; -function ms(n: number | undefined): string { +export const SUMMARY_FILE = 'summary.json'; +export const REPORT_FILE = 'report.md'; + +function ms(n: number | null | undefined): string { return typeof n === 'number' && Number.isFinite(n) ? n.toFixed(0) : '–'; } +function pct(n: number | undefined): string { + return typeof n === 'number' && Number.isFinite(n) ? `${(n * 100).toFixed(0)}%` : '–'; +} + +function keys(counts: Record | undefined): string { + const entries = Object.entries(counts ?? {}); + return entries.length === 0 ? '–' : entries.map(([key, n]) => `${key}×${n}`).join(', '); +} + function wallCells(s: Stat | null): string { - if (!s) return '– | – | – | –'; - return `${ms(s.min)} | ${ms(s.median)} | ${ms(s.p95)} | ${ms(s.max)}`; + if (!s) return '– | – | – | – | – | –'; + return `${ms(s.mean)} ± ${ms(s.stddev)} | ${ms(s.median)} | ${ms(s.p95)} | ${ms(s.min)} | ${ms(s.max)} | ${pct(s.cv)}`; } -function stampName(platform: string, startedAt: string): string { - return `perf-${platform}-${startedAt.replaceAll(/[:.]/g, '-')}`; +function cell(text: string): string { + return text.replaceAll('|', String.raw`\|`); } -function measurementRow(m: Measurement): string { - const daemon = m.daemonDuration ? ms(m.daemonDuration.median) : '–'; - const elements = m.elementCount ? ms(m.elementCount.median) : '–'; - const n = m.wallClock?.n ?? 0; - return `| ${m.label} | ${m.command} | ${m.execMode} | ${n} | ${wallCells(m.wallClock)} | ${daemon} | ${elements} | ${m.notes.join('; ')} |`; +function timingRow(m: Measurement): string { + return [ + m.id.slice(m.scenario.length + 1), + `\`${cell(m.label)}\``, + String(m.samples.length), + wallCells(m.wallClock), + ms(m.daemon?.median), + String(m.failures), + ].join(' | '); } -function toMarkdown(run: RunResult): string { - const lines: string[] = []; - lines.push(`# agent-device command perf — ${run.platform}`); - lines.push(''); - lines.push(`- **Device**: ${run.device.name} (${run.device.udid ?? run.device.serial ?? '?'})`); - lines.push(`- **agent-device**: ${run.agentDeviceVersion}`); - lines.push(`- **Rounds**: ${run.config.rounds} (warmup ${run.config.warmup} dropped)`); - lines.push(`- **Started**: ${run.startedAt}`); - lines.push(`- **Finished**: ${run.finishedAt}`); - lines.push(''); - lines.push('All times in milliseconds. `wall-clock` includes process spawn + socket overhead;'); - lines.push( - '`daemon` is the batch step round-trip (spawn overhead ≈ wall-median − daemon-median).', - ); - lines.push('`elements` = node count in the snapshot payload (tree-size proxy).'); - lines.push('An untimed warmup interaction runs after each open/relaunch, so measured commands'); - lines.push('do not pay the one-time iOS-runner startup or post-relaunch first-AX-query cost.'); - lines.push(''); - lines.push( - '| command | cli | mode | n | wall min | wall median | wall p95 | wall max | daemon median | elements | notes |', - ); - lines.push('|---|---|---|---|---|---|---|---|---|---|---|'); - for (const m of run.measurements) lines.push(measurementRow(m)); - lines.push(''); +function profileRow(m: Measurement): string { + const p = m.profile; + return [ + m.id.slice(m.scenario.length + 1), + ms(p?.runnerMs), + p?.spawnCount == null ? '–' : `${p.spawnCount} / ${ms(p.spawnMs)}`, + ms(p?.snapshotSourceMs), + ms(p?.settleWaitMs), + ms(p?.stabilizationMs), + String(p?.retries ?? '–'), + cell(keys(p?.producers)), + cell(p?.topPhases.map((t) => `${t.phase} ${ms(t.medianMs)}`).join(', ') || '–'), + ].join(' | '); +} - const failed = run.measurements.filter((m) => m.failures > 0); - if (failed.length > 0) { - lines.push('## Failures'); +/** Renders the human report for one run. */ +export function renderReport(summary: PerfSummary): string { + const meta = summary.metadata; + const lines: string[] = [ + `# agent-device perf — ${summary.platform}`, + '', + `- **Commit**: ${meta.commit.slice(0, 12)}${meta.ref ? ` (${meta.ref})` : ''}${meta.runId ? ` · run ${meta.runId}` : ''}`, + `- **Device**: ${meta.device.name}${meta.device.runtime ? ` · ${meta.device.runtime}` : ''}`, + `- **Toolchain**: ${meta.xcode ?? 'no Xcode'} · Node ${meta.node} · agent-device ${meta.agentDeviceVersion}`, + `- **Host**: ${meta.host.os} · ${meta.host.cpu} · ${meta.host.cores} cores · ${meta.host.memoryGb} GB`, + `- **Rounds**: ${summary.config.rounds} measured after ${summary.config.warmup} warmup · ${summary.startedAt} → ${summary.finishedAt}`, + '', + 'Times are milliseconds. See scripts/perf/README.md for each metric. The second table of each', + 'scenario holds per-request medians from the daemon `--debug` request logs.', + '', + ]; + const scenarios = [...new Set(summary.measurements.map((m) => m.scenario))]; + for (const scenario of scenarios) { + const measurements = summary.measurements.filter((m) => m.scenario === scenario); + lines.push( + `## ${scenario}`, + '', + '| step | command | n | wall mean ± sd | median | p95 | min | max | cv | daemon | fail |', + '|---|---|---|---|---|---|---|---|---|---|---|', + ...measurements.map((m) => `| ${timingRow(m)} |`), + '', + '| step | runner | spawns | snapshot source | settle wait | stabilization | retries | producer | top phases |', + '|---|---|---|---|---|---|---|---|---|', + ...measurements.map((m) => `| ${profileRow(m)} |`), + '', + ); + } + const degraded = summary.measurements.filter( + (m) => + Object.keys(m.profile?.fallbacks ?? {}).length > 0 || + Object.keys(m.profile?.backends ?? {}).length > 0, + ); + if (degraded.length > 0) { + lines.push('## Backends and fallbacks', ''); + for (const m of degraded) { + lines.push( + `- **${m.id}** — backends ${keys(m.profile?.backends)}; fallbacks ${keys(m.profile?.fallbacks)}`, + ); + } lines.push(''); + } + const failed = summary.measurements.filter((m) => m.failures > 0); + if (failed.length > 0) { + lines.push('## Failures', ''); for (const m of failed) { const sample = m.samples.find((s) => !s.ok); lines.push( - `- **${m.label}** — ${m.notes.join('; ')}${sample?.errorMessage ? ` — ${sample.errorMessage}` : ''}`, + `- **${m.id}** — ${m.notes.join('; ')}${sample?.errorMessage ? ` — ${sample.errorMessage}` : ''}`, ); } lines.push(''); } + lines.push( + '## Coverage', + '', + `Measured commands (${summary.coverage.measured.length}): ${summary.coverage.measured.join(', ')}.`, + '', + ...summary.coverage.excluded.map((e) => `- \`${e.command}\` not measured: ${e.reason}`), + '', + ); return lines.join('\n'); } -export function writeReports(run: RunResult, outDir: string): { jsonPath: string; mdPath: string } { +export function writeRunOutput( + summary: PerfSummary, + outDir: string, +): { jsonPath: string; mdPath: string } { fs.mkdirSync(outDir, { recursive: true }); - const base = stampName(run.platform, run.startedAt); - const jsonPath = path.join(outDir, `${base}.json`); - const mdPath = path.join(outDir, `${base}.md`); - fs.writeFileSync(jsonPath, JSON.stringify(run, null, 2)); - fs.writeFileSync(mdPath, toMarkdown(run)); + const jsonPath = path.join(outDir, SUMMARY_FILE); + const mdPath = path.join(outDir, REPORT_FILE); + fs.writeFileSync(jsonPath, `${JSON.stringify(summary, null, 2)}\n`); + fs.writeFileSync(mdPath, renderReport(summary)); return { jsonPath, mdPath }; } diff --git a/scripts/perf/run.ts b/scripts/perf/run.ts index f1f390cc4e..51cff7c18d 100644 --- a/scripts/perf/run.ts +++ b/scripts/perf/run.ts @@ -1,72 +1,129 @@ import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; -import { parseConfig, REPO_ROOT, usesSourceCli } from './config.ts'; +import { invokeCli } from './cli.ts'; +import { parseConfig, REPO_ROOT, usesSourceCli, type PerfConfig } from './config.ts'; +import { runScenario, setupIsolation, teardownIsolation } from './harness.ts'; +import { copyRawState, runIosSuite, type IosSuiteContext } from './ios-harness.ts'; import { - runScenario, - setupIsolation, - teardownIsolation, - type IsolationContext, -} from './harness.ts'; -import { writeReports } from './report.ts'; -import type { RunResult } from './types.ts'; + describeCoverage, + FIXTURE_APP_EVENT_URL_TEMPLATE, + IOS_SUITE, + IOS_SUITE_EXCLUSIONS, +} from './ios-suite.ts'; +import { collectMetadata } from './metadata.ts'; +import { writeRunOutput } from './report.ts'; +import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary } from './types.ts'; -function readVersion(): string { - try { - const pkg = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')); - return typeof pkg.version === 'string' ? pkg.version : 'unknown'; - } catch { - return 'unknown'; - } +function log(msg: string): void { + process.stderr.write(`[perf] ${msg}\n`); } -function main(): void { - const cfg = parseConfig(process.argv.slice(2)); - // The dist binary needs a build; running from source (AGENT_DEVICE_PERF_CLI) does not. - if (!usesSourceCli() && !fs.existsSync(path.join(REPO_ROOT, 'dist', 'src'))) { - process.stderr.write('[perf] dist/ is missing — run `pnpm build` first.\n'); - process.exit(1); - } - - const startedAt = new Date().toISOString(); - let ctx: IsolationContext | null = null; - let exitCode = 0; - - const cleanup = (): void => { - if (ctx) { - teardownIsolation(ctx, cfg); - ctx = null; - } - }; - process.on('SIGINT', () => { +function onInterrupt(cleanup: () => void): void { + process.once('SIGINT', () => { cleanup(); process.exit(130); }); - process.on('SIGTERM', () => { + process.once('SIGTERM', () => { cleanup(); process.exit(143); }); +} + +function selectScenarios(cfg: PerfConfig) { + if (!cfg.scenarios) return IOS_SUITE; + const unknown = cfg.scenarios.filter((id) => !IOS_SUITE.some((s) => s.id === id)); + if (unknown.length > 0) { + throw new Error( + `Unknown scenarios: ${unknown.join(', ')} (known: ${IOS_SUITE.map((s) => s.id).join(', ')})`, + ); + } + return IOS_SUITE.filter((s) => cfg.scenarios!.includes(s.id)); +} +async function runIos( + cfg: PerfConfig, +): Promise<{ measurements: Measurement[]; coverage: PerfSummary['coverage'] }> { + const scenarios = selectScenarios(cfg); + // The daemon inherits this from the first CLI call; it routes trigger-app-event to the fixture. + process.env.AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE ??= FIXTURE_APP_EVENT_URL_TEMPLATE; + const rawDir = path.join(cfg.outDir, 'raw'); + const ctx: IosSuiteContext = { + stateDir: fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-perf-')), + udid: cfg.udid!, + appId: cfg.appId, + artifactsDir: path.join(rawDir, 'artifacts'), + rawDir, + }; + fs.mkdirSync(ctx.artifactsDir, { recursive: true }); + log(`state-dir: ${ctx.stateDir}`); + const stop = (): void => { + invokeCli(['close'], ['--state-dir', ctx.stateDir, '--session', 'perf']); + invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); + }; + onInterrupt(stop); + try { + const measurements = await runIosSuite(ctx, cfg, scenarios); + return { measurements, coverage: describeCoverage(scenarios, IOS_SUITE_EXCLUSIONS) }; + } finally { + stop(); + copyRawState(ctx); + if (!cfg.keepArtifacts) fs.rmSync(ctx.stateDir, { recursive: true, force: true }); + } +} + +function runAndroid(cfg: PerfConfig): { + measurements: Measurement[]; + coverage: PerfSummary['coverage']; +} { + const ctx = setupIsolation(cfg); + onInterrupt(() => teardownIsolation(ctx, cfg)); try { - ctx = setupIsolation(cfg); const measurements = runScenario(ctx, cfg); - const run: RunResult = { - startedAt, - finishedAt: new Date().toISOString(), - platform: cfg.platform, - device: { udid: ctx.profile.udid, serial: ctx.profile.serial, name: ctx.profile.deviceName }, - config: { rounds: cfg.rounds, warmup: cfg.warmup, keepArtifacts: cfg.keepArtifacts }, - agentDeviceVersion: readVersion(), + return { measurements, + coverage: { measured: [...new Set(measurements.map((m) => m.command))].sort(), excluded: [] }, }; - const { jsonPath, mdPath } = writeReports(run, cfg.outDir); - process.stderr.write(`\n[perf] report: ${mdPath}\n[perf] json: ${jsonPath}\n`); - } catch (error) { - process.stderr.write(`[perf] error: ${(error as Error).stack ?? String(error)}\n`); - exitCode = 1; } finally { - cleanup(); + teardownIsolation(ctx, cfg); } - process.exit(exitCode); } -main(); +async function main(): Promise { + const cfg = parseConfig(process.argv.slice(2)); + // The iOS forced-backend probe loads the built SDK; the dist CLI needs the build as well. + if ( + (cfg.platform === 'ios' || !usesSourceCli()) && + !fs.existsSync(path.join(REPO_ROOT, 'dist', 'src')) + ) { + throw new Error('dist/ is missing — run `pnpm build` first.'); + } + const startedAt = new Date().toISOString(); + // Read before the run, so a checkout change during a long run cannot relabel it. + const metadata = collectMetadata(cfg, cfg.serial ?? cfg.device ?? 'unknown'); + const { measurements, coverage } = cfg.platform === 'ios' ? await runIos(cfg) : runAndroid(cfg); + const summary: PerfSummary = { + schema: PERF_SUMMARY_SCHEMA, + platform: cfg.platform, + startedAt, + finishedAt: new Date().toISOString(), + config: { + rounds: cfg.rounds, + warmup: cfg.warmup, + scenarios: [...new Set(measurements.map((m) => m.scenario))], + }, + metadata, + coverage, + measurements, + }; + const { jsonPath, mdPath } = writeRunOutput(summary, cfg.outDir); + log(`report: ${mdPath}`); + log(`summary: ${jsonPath}`); +} + +main() + .then(() => process.exit(0)) + .catch((error: unknown) => { + process.stderr.write(`[perf] error: ${(error as Error).stack ?? String(error)}\n`); + process.exit(1); + }); diff --git a/scripts/perf/scenario.ts b/scripts/perf/scenario.ts index d99ddd2bba..1e34cad240 100644 --- a/scripts/perf/scenario.ts +++ b/scripts/perf/scenario.ts @@ -37,7 +37,7 @@ function bat( return { label, command, execMode: 'batch' as const, step, ...opts }; } -// One ordered pass over Settings. The harness repeats this N (+warmup) times; +// One ordered pass over Android Settings. The harness repeats this N (+warmup) times; // the leading `open --relaunch` resets the app to its root each round, so every // round starts from a known state while commands run in their natural order. export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): ScenarioStep[] { @@ -46,92 +46,33 @@ export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): Scenari const rec = path.join(ctx.artifactsDir, 'rec.mp4'); const trace = path.join(ctx.artifactsDir, 'trace.log'); - // Text entry differs per platform: iOS fills the root search field directly (focusing it - // first can hang); Android must open the search screen before an editable field exists. - const textEntry: ScenarioStep[] = p.selectors.searchEditableAtRoot - ? [ - // iOS: editable search field exists at root; fill it directly (freshRoot resets scroll). - bat( - 'fill search', - 'fill', - { - command: 'fill', - input: { - target: { kind: 'selector', selector: s.searchFieldEditable }, - text: 'general', - }, - }, - { freshRoot: true }, - ), - bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), - bat('get editable text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.searchFieldEditable }, - }, - }), - bat('keyboard return', 'keyboard', { command: 'keyboard', input: { action: 'return' } }), - ] - : [ - // Android: tap the search entry first to reveal the editable, then type/fill it. - bat( - 'press search field', - 'press', - { - command: 'press', - input: { target: { kind: 'selector', selector: s.searchField } }, - }, - { freshRoot: true }, - ), - bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), - bat('fill search', 'fill', { - command: 'fill', - input: { - target: { kind: 'selector', selector: s.searchFieldEditable }, - text: 'general', - }, - }), - bat('get editable text', 'get', { - command: 'get', - input: { - format: 'text', - target: { kind: 'selector', selector: s.searchFieldEditable }, - }, - }), - ]; - - // These iOS-only repeated gesture forms fuse into `sequence` runner requests: - // press --count > 1 and swipe --count > 1 both batch their steps into one command. - const iosRunnerSeries: ScenarioStep[] = - p.platform === 'ios' - ? [ - bat( - 'press series (sequence)', - 'press', - { - command: 'press', - input: { target: { kind: 'point', x: 200, y: 95 }, count: 2, intervalMs: 50 }, - }, - { freshRoot: true }, - ), - bat( - 'swipe series (sequence)', - 'swipe', - { - command: 'swipe', - input: { - from: { x: 200, y: 650 }, - to: { x: 200, y: 450 }, - count: 2, - pauseMs: 50, - pattern: 'ping-pong', - }, - }, - { freshRoot: true }, - ), - ] - : []; + // Android reveals the editable search field only after a tap on the search card. + const textEntry: ScenarioStep[] = [ + bat( + 'press search field', + 'press', + { + command: 'press', + input: { target: { kind: 'selector', selector: s.searchField } }, + }, + { freshRoot: true }, + ), + bat('type', 'type', { command: 'type', input: { text: 'wifi' } }), + bat('fill search', 'fill', { + command: 'fill', + input: { + target: { kind: 'selector', selector: s.searchFieldEditable }, + text: 'general', + }, + }), + bat('get editable text', 'get', { + command: 'get', + input: { + format: 'text', + target: { kind: 'selector', selector: s.searchFieldEditable }, + }, + }), + ]; return [ // --- reset to root via relaunch --- @@ -166,9 +107,6 @@ export function buildSettingsTour(p: ResolvedProfile, ctx: StepContext): Scenari ), bat('back', 'back', { command: 'back', input: {} }), - // --- iOS runner series commands surfaced by PR #643 --- - ...iosRunnerSeries, - // --- targeted reads against the visible anchor (freshRoot so the anchor is on screen) --- bat( 'wait text', diff --git a/scripts/perf/stats.test.ts b/scripts/perf/stats.test.ts new file mode 100644 index 0000000000..088c7e1806 --- /dev/null +++ b/scripts/perf/stats.test.ts @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { summarize } from './stats.ts'; + +test('summarize reports sample spread, the middle median, and nearest-rank p95', () => { + const stat = summarize([40, 10, 30, 20, Number.NaN])!; + assert.equal(stat.n, 4); + assert.equal(stat.mean, 25); + // Sample (n − 1) deviation: sqrt((15² + 5² + 5² + 15²) / 3). + assert.ok(Math.abs(stat.stddev - Math.sqrt(500 / 3)) < 1e-9); + assert.ok(Math.abs(stat.cv - Math.sqrt(500 / 3) / 25) < 1e-9); + assert.deepEqual([stat.min, stat.median, stat.p95, stat.max], [10, 25, 40, 40]); +}); + +test('summarize gives a single sample zero spread and no samples no summary', () => { + assert.deepEqual(summarize([7]), { + n: 1, + mean: 7, + stddev: 0, + cv: 0, + median: 7, + min: 7, + max: 7, + p95: 7, + }); + assert.equal(summarize([]), null); + assert.equal(summarize([0, 0])?.cv, 0); +}); diff --git a/scripts/perf/stats.ts b/scripts/perf/stats.ts index 14b846cac2..f972ec4e37 100644 --- a/scripts/perf/stats.ts +++ b/scripts/perf/stats.ts @@ -1,22 +1,43 @@ import type { Stat } from './types.ts'; -// Nearest-rank percentile over a copy of the values. +// Nearest-rank percentile over sorted values. function percentile(sorted: number[], p: number): number { - if (sorted.length === 0) return Number.NaN; const rank = Math.ceil((p / 100) * sorted.length); const idx = Math.min(sorted.length - 1, Math.max(0, rank - 1)); - return sorted[idx]; + return sorted[idx]!; } +/** + * Describes one sample set. `stddev` is the sample standard deviation (n − 1), so it is 0 for a + * single value; `cv` is stddev / mean and is 0 when the mean is 0. + */ export function summarize(values: number[]): Stat | null { const clean = values.filter((v) => Number.isFinite(v)); if (clean.length === 0) return null; const sorted = [...clean].sort((a, b) => a - b); + const n = sorted.length; + const mean = sorted.reduce((sum, v) => sum + v, 0) / n; + const variance = n > 1 ? sorted.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (n - 1) : 0; + const stddev = Math.sqrt(variance); return { - n: sorted.length, - min: sorted[0], - median: percentile(sorted, 50), + n, + mean, + stddev, + cv: mean === 0 ? 0 : stddev / mean, + median: middle(sorted), + min: sorted[0]!, + max: sorted.at(-1)!, p95: percentile(sorted, 95), - max: sorted.at(-1), }; } + +// The median of sorted values; an even count averages the two middle values. +function middle(sorted: number[]): number { + const half = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[half]! : (sorted[half - 1]! + sorted[half]!) / 2; +} + +export function median(values: number[]): number { + const sorted = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b); + return sorted.length === 0 ? Number.NaN : middle(sorted); +} diff --git a/scripts/perf/types.ts b/scripts/perf/types.ts index 175481a889..a3cc091be5 100644 --- a/scripts/perf/types.ts +++ b/scripts/perf/types.ts @@ -1,8 +1,8 @@ -// Shared data shapes for the e2e perf benchmark harness. +// Shared data shapes for the perf suite and its night-over-night comparison. export type Platform = 'ios' | 'android'; -export type ExecMode = 'batch' | 'standalone'; +export type ExecMode = 'batch' | 'standalone' | 'sdk'; export type CliResult = { exitCode: number; @@ -13,38 +13,114 @@ export type CliResult = { ok: boolean; // exit 0 AND (json.ok !== false) }; +export type Stat = { + n: number; + mean: number; + stddev: number; + cv: number; + median: number; + min: number; + max: number; + p95: number; +}; + +export type Timing = { count: number; totalMs: number }; + +/** Phase breakdown of one request, read from its daemon `--debug` request log. */ +export type SampleProfile = { + phases: Record; + runnerCommands: Record; + spawns: Record; + snapshotSources: Record; + retries: number; + stabilization: Timing; + fallbacks: string[]; + // Daemon-side request time: from `request_start` to the last event of the request log. + requestSpanMs?: number; +}; + +/** The settle outcome that a `--settle` response reports. */ +export type SettleFacts = { waitedMs: number; captures?: number; settled?: boolean }; + +export type SnapshotFacts = { + backend?: string; + state?: string; + reasonCode?: string; + nodeCount?: number; + maxDepth?: number; + truncated?: boolean; +}; + export type Sample = { round: number; wallClockMs: number; - daemonDurationMs?: number; // from batch results[0].durationMs (batch mode only) - elementCount?: number; // for snapshot rows: parsed @eN count, a tree-size proxy + // Daemon request round trip: the client's `daemon_request` timer for CLI calls, the batch step + // duration for batch calls, or the request-log span for SDK calls. + daemonMs?: number; + elementCount?: number; + requestId?: string; + snapshot?: SnapshotFacts; + settle?: SettleFacts; + profile?: SampleProfile; ok: boolean; errorCode?: string; errorMessage?: string; }; -export type Stat = { n: number; min: number; median: number; p95: number; max: number }; +export type MeasurementProfile = { + runnerMs: number | null; + spawnCount: number | null; + spawnMs: number | null; + snapshotSourceMs: number | null; + stabilizationMs: number | null; + settleWaitMs: number | null; + retries: number; + producers: Record; + backends: Record; + fallbacks: Record; + topPhases: { phase: string; medianMs: number }[]; +}; export type Measurement = { - command: string; + id: string; // stable `/` comparison key + scenario: string; label: string; + command: string; // catalog command name + args: string[]; platform: Platform; execMode: ExecMode; - samples: Sample[]; // kept samples only (warmup rounds dropped) + samples: Sample[]; // measured rounds only (warmup rounds dropped) warmupDropped: number; wallClock: Stat | null; - daemonDuration: Stat | null; // null for standalone or when no ok samples - elementCount: Stat | null; // null unless snapshot row + daemon: Stat | null; + elementCount: Stat | null; + profile: MeasurementProfile | null; failures: number; notes: string[]; }; -export type RunResult = { +export type RunMetadata = { + commit: string; + ref?: string; + runId?: string; + runAttempt?: string; + agentDeviceVersion: string; + host: { os: string; arch: string; cpu: string; cores: number; memoryGb: number }; + xcode?: string; + node: string; + device: { name: string; udid?: string; serial?: string; runtime?: string }; + fixture?: { appId: string; appPath?: string }; +}; + +export const PERF_SUMMARY_SCHEMA = 'agent-device-perf-summary/v1'; + +export type PerfSummary = { + schema: typeof PERF_SUMMARY_SCHEMA; + platform: Platform; startedAt: string; finishedAt: string; - platform: Platform; - device: { udid?: string; serial?: string; name: string }; - config: { rounds: number; warmup: number; keepArtifacts: boolean }; - agentDeviceVersion: string; + config: { rounds: number; warmup: number; scenarios: string[] }; + metadata: RunMetadata; + coverage: { measured: string[]; excluded: { command: string; reason: string }[] }; measurements: Measurement[]; }; diff --git a/vitest.config.ts b/vitest.config.ts index b5a54e80f7..5da1a3c156 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -196,6 +196,8 @@ export default defineConfig({ 'scripts/__tests__/release-version.test.ts', 'scripts/__tests__/release-plan.test.ts', 'scripts/ios-snapshot-benchmark/*.test.ts', + // The perf suite's pure logic, without a device. + 'scripts/perf/*.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts', 'test/ci/upload-artifact-hidden-paths.test.ts', From 34ec510f409d94d9a3bc706cc7a5a34ba7385853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 04:52:54 +0200 Subject: [PATCH 08/15] feat(perf): compare nightly iOS runs and run the suite every night (#3886) pnpm perf:compare judges one run against earlier runs: by default the seven nights before it, or a chosen run, date, range, or local output. A step's baseline is the median of its nightly medians. A change is flagged only when it is larger than 20%, three times the night-to-night scaled MAD, twice the run's own cv/sqrt(n), and 50 ms; new failures and fixed steps are flagged too. perf-nightly runs the iOS suite at 02:40 UTC again (#1781 A3). Each run uploads perf-ios-summary (90 days) and perf-ios-raw (30 days), and writes the comparison to the job summary with warnings for regressions and new failures. Android stays report-only behind a dispatch input. Co-authored-by: Claude Opus 5.5 --- .github/workflows/perf-nightly.yml | 101 +++++-- package.json | 1 + scripts/perf/README.md | 36 +++ scripts/perf/compare-cli.ts | 196 ++++++++++++++ scripts/perf/compare.test.ts | 104 +++++++ scripts/perf/compare.ts | 270 +++++++++++++++++++ scripts/perf/nightly-artifacts.test.ts | 70 +++++ scripts/perf/nightly-artifacts.ts | 159 +++++++++++ scripts/perf/stats.test.ts | 7 +- scripts/perf/stats.ts | 6 + scripts/perf/summary-fixture.ts | 64 +++++ scripts/perf/summary.test.ts | 32 +++ scripts/perf/summary.ts | 63 +++++ test/repo-guards/npm-package-scripts.test.ts | 2 +- 14 files changed, 1088 insertions(+), 23 deletions(-) create mode 100644 scripts/perf/compare-cli.ts create mode 100644 scripts/perf/compare.test.ts create mode 100644 scripts/perf/compare.ts create mode 100644 scripts/perf/nightly-artifacts.test.ts create mode 100644 scripts/perf/nightly-artifacts.ts create mode 100644 scripts/perf/summary-fixture.ts create mode 100644 scripts/perf/summary.test.ts create mode 100644 scripts/perf/summary.ts diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index 1d6c4938a5..243cd08c17 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -1,29 +1,57 @@ name: Perf Nightly -# Parked to workflow_dispatch by #1781 A3: it only writes a report and compares nothing, so it -# structurally cannot catch a regression, and iOS wall-clock medians swing up to +122% -# night-to-night at n=5 with no consumer reading the report. `pnpm perf` / scripts/perf are -# unaffected — still runnable on demand. Un-park once a comparison step with multi-night -# smoothing (someone reading it) exists, or a perf arc needs nightly data. +# Nightly iOS command performance against the fixture app, compared night over night (#1781 A3). +# - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts +# failing. PR lanes check behavior, not time. +# - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians +# swing up to +122% between single nights at n=5, so a step is judged against the median of up to +# seven earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. +# - Cost: one macos-26 job of about 45 minutes a night; summaries are kept 90 days, raw logs 30. +# - Delete or park when: no flagged change has been acted on for 60 nights. +# Android stays report-only and runs only on request. on: + schedule: + - cron: '40 2 * * *' workflow_dispatch: inputs: rounds: - description: 'Measured rounds per command (samples)' + description: 'Measured rounds per step (samples)' required: false default: '3' + warmup: + description: 'Warmup rounds dropped before measuring' + required: false + default: '1' + scenarios: + description: 'Comma-separated iOS scenario ids (empty runs all)' + required: false + default: '' + baseline-nights: + description: 'Earlier runs in the rolling baseline' + required: false + default: '7' + baseline-branch: + description: 'Branch whose runs form the baseline' + required: false + default: 'main' + android: + description: 'Also run the report-only Android Settings tour' + type: boolean + required: false + default: true permissions: contents: read actions: read concurrency: - group: ci-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: perf-nightly-${{ github.ref }} + cancel-in-progress: false env: PERF_ROUNDS: ${{ github.event.inputs.rounds || '3' }} + PERF_WARMUP: ${{ github.event.inputs.warmup || '1' }} jobs: perf-ios: @@ -52,6 +80,13 @@ jobs: with: gate: build + - name: Boot iOS test simulator + id: ios-simulator + uses: ./.github/actions/boot-ios-test-simulator + with: + runtime-version: ${{ env.IOS_RUNTIME_VERSION }} + preferred-device-name: iPhone 17 Pro + - name: Restore and build iOS XCTest runner uses: ./.github/actions/setup-apple-runner-build with: @@ -62,13 +97,6 @@ jobs: xcuitest-platform: ios xcuitest-destination: generic/platform=iOS Simulator - - name: Boot iOS test simulator - id: ios-simulator - uses: ./.github/actions/boot-ios-test-simulator - with: - runtime-version: ${{ env.IOS_RUNTIME_VERSION }} - preferred-device-name: iPhone 17 Pro - - name: Install the fixture app id: fixture-app uses: ./.github/actions/setup-fixture-app @@ -88,22 +116,53 @@ jobs: UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} APP_ID: ${{ steps.fixture-app.outputs.app-id }} APP_PATH: ${{ steps.fixture-app.outputs.app-path }} + SCENARIOS: ${{ github.event.inputs.scenarios }} run: | pnpm perf --platform ios --udid "$UDID" \ --app-id "$APP_ID" --app-path "$APP_PATH" \ - --rounds "$PERF_ROUNDS" --warmup 1 \ + --rounds "$PERF_ROUNDS" --warmup "$PERF_WARMUP" \ + ${SCENARIOS:+--scenarios "$SCENARIOS"} \ --out-dir "$GITHUB_WORKSPACE/perf-results" - - name: Upload iOS perf report - if: always() + - name: Compare with earlier nights + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} + env: + GH_TOKEN: ${{ github.token }} + BASELINE_BRANCH: ${{ github.event.inputs.baseline-branch || 'main' }} + BASELINE_NIGHTS: ${{ github.event.inputs.baseline-nights || '7' }} + run: | + pnpm perf:compare --to perf-results \ + --branch "$BASELINE_BRANCH" \ + --nights "$BASELINE_NIGHTS" \ + --exclude-run "$GITHUB_RUN_ID" \ + --out perf-results --github + echo "Full report: the perf-ios-summary artifact holds report.md and summary.json." >> "$GITHUB_STEP_SUMMARY" + + - name: Upload iOS perf summary + if: ${{ !cancelled() && hashFiles('perf-results/summary.json') != '' }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: perf-ios - path: perf-results/ + name: perf-ios-summary + path: | + perf-results/summary.json + perf-results/report.md + perf-results/comparison.json + perf-results/comparison.md + retention-days: 90 + if-no-files-found: error + + - name: Upload iOS perf raw logs + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: perf-ios-raw + path: perf-results/raw/ + retention-days: 30 if-no-files-found: warn perf-android: name: Android Command Perf + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.android == 'true' }} runs-on: ubuntu-latest timeout-minutes: 80 env: @@ -134,7 +193,7 @@ jobs: adb -s emulator-5554 shell settings put global window_animation_scale 0 || true; adb -s emulator-5554 shell settings put global transition_animation_scale 0 || true; adb -s emulator-5554 shell settings put global animator_duration_scale 0 || true; - pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup 1 --out-dir "$GITHUB_WORKSPACE/perf-results" + pnpm perf --platform android --serial emulator-5554 --n "$PERF_ROUNDS" --warmup "$PERF_WARMUP" --out-dir "$GITHUB_WORKSPACE/perf-results" - name: Upload Android perf report if: always() diff --git a/package.json b/package.json index 07a4584c2b..b19f477964 100644 --- a/package.json +++ b/package.json @@ -131,6 +131,7 @@ "test:ios-snapshot-differential": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift test --package-path apple/snapshot-presentation && node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/ios-snapshot-differential.test.ts", "size": "node scripts/size-report.mjs", "perf": "node --experimental-strip-types scripts/perf/run.ts", + "perf:compare": "node --experimental-strip-types scripts/perf/compare-cli.ts", "bench:ios-snapshot": "node --experimental-strip-types scripts/ios-snapshot-benchmark/run.ts", "bench:ios-snapshot:deep-button": "node --experimental-strip-types scripts/ios-snapshot-benchmark/deep-button.ts", "bench:ios-snapshot:evidence": "node --experimental-strip-types scripts/ios-snapshot-benchmark/evidence.ts", diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 42d61d4301..4f088d0aea 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -49,3 +49,39 @@ All times are milliseconds. Statistics use only the successful samples of a step | backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected`. | | fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. | | `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth. | + +## Nightly results + +`.github/workflows/perf-nightly.yml` runs the iOS suite every night on `main`. +Each run uploads `perf-ios-summary` (summary, report, comparison; 90 days) and `perf-ios-raw` (logs; 30 days). +The job compares the run with up to seven earlier runs and writes the result to the job summary. + +To fetch and compare runs on your machine, use `gh` with access to the repository: + +```sh +pnpm perf:compare # latest night against the 7 nights before it +pnpm perf:compare --to 2026-10-28 # that night against the 7 nights before it +pnpm perf:compare --to latest --from 2026-10-21 # latest against one night +pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare +pnpm perf:compare --to .tmp/perf/local # a local run against recent nights +``` + +A reference is `latest`, a run id, a `YYYY-MM-DD` date, or a local output path. +Downloads stay in `.tmp/perf-nightly//`. +Do not judge a local run against CI nights. The comparison warns when the device, Xcode, or CPU differs. + +## Comparison rule + +The baseline of a step is the median of its nightly medians. +The noise of a step is the scaled median absolute deviation of those medians. +A step is a `regression` or an `improvement` only when its change is more than all of these: + +- 20% of the baseline. +- Three times the night-to-night noise. +- Two times the coefficient of variation of the run divided by √n. +- 50 ms. + +The rolling baseline skips runs that did not run every scenario of the judged run, such as a partial dispatch. +Fewer than three baseline nights give `insufficient-baseline`. +`new-failures` means that the step failed and no baseline night failed. `fixed` means the reverse. +The thresholds are in `compare.ts`. diff --git a/scripts/perf/compare-cli.ts b/scripts/perf/compare-cli.ts new file mode 100644 index 0000000000..0336beca96 --- /dev/null +++ b/scripts/perf/compare-cli.ts @@ -0,0 +1,196 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { compareRuns, renderComparison, type Comparison } from './compare.ts'; +import { REPO_ROOT } from './config.ts'; +import { + baselineCandidates, + coversTarget, + fetchSummary, + listRuns, + readLocalSummary, + resolveRunRef, + selectRange, + SUMMARY_ARTIFACT, + type NightlySource, + type WorkflowRun, +} from './nightly-artifacts.ts'; +import type { PerfSummary } from './types.ts'; + +const USAGE = `Usage: pnpm perf:compare [--to ] [--from |..] [--nights ] [options] + +Compares one perf run with earlier runs and prints a Markdown verdict per step. + +A is latest, a workflow run id, a YYYY-MM-DD date (the newest run that UTC day), or a local +path to a summary.json or a \`pnpm perf\` output directory. + + --to Run to judge (default: latest). + --from Baseline run, or an inclusive range ... Without it the baseline is + the --nights runs before --to. + --nights Rolling baseline size (default: 7). + --branch Branch whose nightly runs are listed (default: main). + --repo Repository (default: $GITHUB_REPOSITORY or callstack/agent-device). + --workflow Workflow file (default: perf-nightly.yml). + --cache-dir Download cache (default: .tmp/perf-nightly). + --exclude-run Never use this run as baseline (repeatable). + --out Also write comparison.json and comparison.md there. + --github Append the Markdown to $GITHUB_STEP_SUMMARY and annotate flagged steps.`; + +type Options = { + to: string; + from?: string; + nights: number; + out?: string; + github: boolean; + exclude: Set; + source: NightlySource; +}; + +function parse(argv: string[]): Options { + const options: Options = { + to: 'latest', + nights: 7, + github: false, + exclude: new Set(), + source: { + repo: process.env.GITHUB_REPOSITORY ?? 'callstack/agent-device', + workflow: 'perf-nightly.yml', + branch: 'main', + artifact: SUMMARY_ARTIFACT, + cacheDir: path.join(REPO_ROOT, '.tmp', 'perf-nightly'), + }, + }; + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]!; + const value = (): string => { + const next = argv[++i]; + if (next === undefined) throw new Error(`Missing value for ${flag}`); + return next; + }; + switch (flag) { + case '--to': + options.to = value(); + break; + case '--from': + options.from = value(); + break; + case '--nights': { + const n = Number(value()); + if (!Number.isInteger(n) || n < 1) throw new Error('--nights must be a positive integer'); + options.nights = n; + break; + } + case '--branch': + options.source.branch = value(); + break; + case '--repo': + options.source.repo = value(); + break; + case '--workflow': + options.source.workflow = value(); + break; + case '--cache-dir': + options.source.cacheDir = path.resolve(value()); + break; + case '--exclude-run': + options.exclude.add(value()); + break; + case '--out': + options.out = path.resolve(value()); + break; + case '--github': + options.github = true; + break; + case '--help': + case '-h': + process.stdout.write(`${USAGE}\n`); + process.exit(0); + break; + default: + throw new Error(`Unknown flag: ${flag}\n\n${USAGE}`); + } + } + return options; +} + +type Loaded = { summary: PerfSummary; label: string; run?: WorkflowRun }; + +function main(): void { + const options = parse(process.argv.slice(2)); + let runs: WorkflowRun[] | undefined; + const allRuns = (): WorkflowRun[] => (runs ??= listRuns(options.source)); + const runLabel = (run: WorkflowRun) => `run ${run.id} (${run.createdAt.slice(0, 10)})`; + const load = (ref: string): Loaded => { + if (fs.existsSync(ref)) + return { summary: readLocalSummary(ref), label: path.basename(path.resolve(ref)) }; + const run = resolveRunRef(ref, allRuns()); + if (!run) + throw new Error( + `No completed ${options.source.workflow} run on ${options.source.branch} matches ${ref}.`, + ); + const summary = fetchSummary(options.source, run); + if (!summary) throw new Error(`Run ${run.id} has no ${options.source.artifact} artifact.`); + return { summary, label: runLabel(run), run }; + }; + const loadRuns = ( + candidates: WorkflowRun[], + limit: number, + accept: (summary: PerfSummary) => boolean = () => true, + ): Loaded[] => { + const loaded: Loaded[] = []; + for (const run of candidates) { + if (loaded.length >= limit) break; + const summary = fetchSummary(options.source, run); + if (summary && accept(summary)) loaded.push({ summary, label: runLabel(run), run }); + } + return loaded; + }; + + const target = load(options.to); + const exclude = new Set([...options.exclude, ...(target.run ? [target.run.id] : [])]); + let baseline: Loaded[]; + if (options.from?.includes('..')) { + const [fromRef, toRef] = options.from.split('..'); + const range = selectRange(allRuns(), fromRef!, toRef!).filter((run) => !exclude.has(run.id)); + baseline = loadRuns(range, range.length); + } else if (options.from) { + baseline = [load(options.from)]; + } else { + const before = target.run?.createdAt ?? target.summary.startedAt; + baseline = loadRuns(baselineCandidates(allRuns(), before, exclude), options.nights, (summary) => + coversTarget(summary, target.summary), + ); + } + + const comparison = compareRuns(target, baseline); + const markdown = renderComparison(comparison); + process.stdout.write(`${markdown}\n`); + if (options.out) writeComparison(comparison, markdown, options.out); + if (options.github) publishToGitHub(comparison, markdown); +} + +function writeComparison(comparison: Comparison, markdown: string, dir: string): void { + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'comparison.json'), `${JSON.stringify(comparison, null, 2)}\n`); + fs.writeFileSync(path.join(dir, 'comparison.md'), markdown); +} + +function publishToGitHub(comparison: Comparison, markdown: string): void { + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (summaryPath) fs.appendFileSync(summaryPath, `${markdown}\n`); + for (const row of comparison.rows) { + if (row.verdict === 'regression' || row.verdict === 'new-failures') { + const change = + row.verdict === 'regression' + ? `median ${row.target?.median.toFixed(0)} ms vs ${row.baseline?.median.toFixed(0)} ms baseline` + : `${row.target?.failures}/${row.target?.samples} samples failed`; + process.stdout.write(`::warning title=perf ${row.verdict}::${row.id}: ${change}\n`); + } + } +} + +try { + main(); +} catch (error) { + process.stderr.write(`[perf:compare] ${(error as Error).message}\n`); + process.exit(1); +} diff --git a/scripts/perf/compare.test.ts b/scripts/perf/compare.test.ts new file mode 100644 index 0000000000..d7046d1860 --- /dev/null +++ b/scripts/perf/compare.test.ts @@ -0,0 +1,104 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { compareRuns, type ComparisonRow } from './compare.ts'; +import { summaryFixture, type StepFixture } from './summary-fixture.ts'; + +const STEP = 'heavy-feed/snapshot'; + +function nights(...steps: StepFixture[][]) { + return steps.map((night, index) => ({ summary: summaryFixture(night), label: `night ${index}` })); +} + +function row(target: StepFixture[], baseline: StepFixture[][], id = STEP): ComparisonRow { + const comparison = compareRuns( + { summary: summaryFixture(target), label: 'tonight' }, + nights(...baseline), + ); + return comparison.rows.find((r) => r.id === id)!; +} + +const flat = (ms: number, extra: Partial = {}): StepFixture[] => [ + { id: STEP, daemonMs: [ms, ms, ms], ...extra }, +]; + +test('flags a slowdown that exceeds a quiet baseline and the relative floor', () => { + const result = row( + [{ id: STEP, daemonMs: [1450, 1500, 1550] }], + [flat(1000), flat(1010), flat(990)], + ); + assert.equal(result.verdict, 'regression'); + assert.equal(result.metric, 'daemon'); + assert.equal(result.baseline?.median, 1000); + assert.equal(result.deltaMs, 500); + assert.equal(result.threshold, 0.2); +}); + +test('does not flag the same slowdown when nightly medians already swing that much', () => { + const result = row( + [{ id: STEP, daemonMs: [1450, 1500, 1550] }], + [flat(500), flat(1000), flat(1500), flat(700), flat(1300)], + ); + assert.equal(result.verdict, 'unchanged'); + // 3 × scaled MAD: deviations from 1000 are 0, 300, 300, 500, 500. + assert.ok(Math.abs(result.threshold! - (3 * 1.4826 * 300) / 1000) < 1e-9); +}); + +test('does not flag a large relative change below the absolute floor', () => { + assert.equal(row(flat(60), [flat(40), flat(40), flat(40)]).verdict, 'unchanged'); +}); + +test('flags a speedup beyond noise as an improvement', () => { + assert.equal(row(flat(500), [flat(1000), flat(1010), flat(990)]).verdict, 'improvement'); +}); + +test('withholds a timing verdict until enough nights exist', () => { + const result = row(flat(1500), [flat(1000), flat(1000)]); + assert.equal(result.verdict, 'insufficient-baseline'); + assert.equal(result.deltaRelative, 0.5); +}); + +test('reports failures that no baseline night had, and failures that went away', () => { + const clean = [flat(1000), flat(1000), flat(1000)]; + assert.equal(row(flat(1000, { failures: 1 }), clean).verdict, 'new-failures'); + const failing = [ + flat(1000, { failures: 1 }), + flat(1000, { failures: 2 }), + flat(1000, { failures: 1 }), + ]; + assert.equal(row(flat(1000), failing).verdict, 'fixed'); + // A step that already failed on some baseline night is judged on timing instead. + const flaky = [flat(1000, { failures: 1 }), flat(1000), flat(1000)]; + assert.equal(row(flat(1000, { failures: 1 }), flaky).verdict, 'unchanged'); +}); + +test('falls back to wall time when any run lacks the daemon timer', () => { + const result = row(flat(1000), [ + flat(1000), + [{ id: STEP, wallMs: [1300, 1300, 1300] }], + flat(1000), + ]); + assert.equal(result.metric, 'wall'); +}); + +test('marks steps that only one side measured', () => { + const comparison = compareRuns( + { summary: summaryFixture([{ id: 'a/new', daemonMs: [1] }]), label: 't' }, + [{ summary: summaryFixture([{ id: 'a/old', daemonMs: [1] }]), label: 'b' }], + ); + assert.deepEqual( + comparison.rows.map((r) => [r.id, r.verdict]), + [ + ['a/new', 'new'], + ['a/old', 'missing'], + ], + ); +}); + +test('warns when the baseline ran on another toolchain', () => { + const comparison = compareRuns( + { summary: summaryFixture(flat(1000), { xcode: 'Xcode 26.3' }), label: 't' }, + [{ summary: summaryFixture(flat(1000)), label: 'b' }], + ); + assert.equal(comparison.warnings.length, 1); + assert.match(comparison.warnings[0]!, /xcode/); +}); diff --git a/scripts/perf/compare.ts b/scripts/perf/compare.ts new file mode 100644 index 0000000000..11dae30520 --- /dev/null +++ b/scripts/perf/compare.ts @@ -0,0 +1,270 @@ +import { median, scaledMad } from './stats.ts'; +import type { Measurement, PerfSummary, Stat } from './types.ts'; + +export type ComparisonThresholds = { + // A change must exceed every one of these to be flagged. + minRelative: number; // fraction of the baseline median + minAbsoluteMs: number; + spreadFactor: number; // × night-to-night relative spread (scaled MAD of nightly medians) + withinFactor: number; // × tonight's cv / √n (standard error of tonight's median) + minBaselineNights: number; // fewer nights give `insufficient-baseline` +}; + +const DEFAULT_THRESHOLDS: ComparisonThresholds = { + minRelative: 0.2, + minAbsoluteMs: 50, + spreadFactor: 3, + withinFactor: 2, + minBaselineNights: 3, +}; + +export type Verdict = + | 'regression' + | 'improvement' + | 'unchanged' + | 'new-failures' + | 'fixed' + | 'failing' + | 'insufficient-baseline' + | 'new' + | 'missing'; + +/** Verdicts that need a reader's attention, in report order. */ +const FLAGGED_VERDICTS: readonly Verdict[] = ['new-failures', 'regression', 'fixed', 'improvement']; + +export type ComparisonRow = { + id: string; + scenario: string; + command: string; + verdict: Verdict; + metric: 'daemon' | 'wall'; + target: { median: number; n: number; cv: number; failures: number; samples: number } | null; + baseline: { median: number; nights: number; spread: number; failingNights: number } | null; + deltaMs: number | null; + deltaRelative: number | null; + threshold: number | null; +}; + +export type RunRef = { + label: string; + runId?: string; + commit: string; + startedAt: string; + device: string; + xcode?: string; + rounds: number; +}; + +const COMPARISON_SCHEMA = 'agent-device-perf-comparison/v1'; + +export type Comparison = { + schema: typeof COMPARISON_SCHEMA; + target: RunRef; + baseline: RunRef[]; + thresholds: ComparisonThresholds; + counts: Partial>; + warnings: string[]; + rows: ComparisonRow[]; +}; + +function describeRun(summary: PerfSummary, label: string): RunRef { + return { + label, + runId: summary.metadata.runId, + commit: summary.metadata.commit, + startedAt: summary.startedAt, + device: `${summary.metadata.device.name}${summary.metadata.device.runtime ? ` ${summary.metadata.device.runtime}` : ''}`, + xcode: summary.metadata.xcode, + rounds: summary.config.rounds, + }; +} + +function metricStat(m: Measurement, metric: 'daemon' | 'wall'): Stat | null { + return metric === 'daemon' ? m.daemon : m.wallClock; +} + +function compareStep( + id: string, + target: Measurement | undefined, + history: Measurement[], + thresholds: ComparisonThresholds, +): ComparisonRow { + const any = target ?? history[0]!; + const base = { id, scenario: any.scenario, command: any.command }; + const metric: 'daemon' | 'wall' = [target, ...history].every( + (m) => !m || !m.wallClock || m.daemon, + ) + ? 'daemon' + : 'wall'; + const empty = { deltaMs: null, deltaRelative: null, threshold: null }; + const okNights = history + .map((m) => metricStat(m, metric)) + .filter((s): s is Stat => s !== null) + .map((s) => s.median); + const failingNights = history.filter((m) => m.failures > 0).length; + const baselineMedian = okNights.length > 0 ? median(okNights) : null; + const baseline = + history.length === 0 + ? null + : { + median: baselineMedian ?? Number.NaN, + nights: history.length, + spread: baselineMedian ? scaledMad(okNights) / baselineMedian : 0, + failingNights, + }; + if (!target) return { ...base, verdict: 'missing', metric, target: null, baseline, ...empty }; + const stat = metricStat(target, metric); + const targetFacts = stat + ? { + median: stat.median, + n: stat.n, + cv: stat.cv, + failures: target.failures, + samples: target.samples.length, + } + : { + median: Number.NaN, + n: 0, + cv: 0, + failures: target.failures, + samples: target.samples.length, + }; + const row = { ...base, metric, target: targetFacts, baseline }; + if (!baseline) return { ...row, verdict: 'new', ...empty }; + if (target.failures > 0 && failingNights === 0) + return { ...row, verdict: 'new-failures', ...empty }; + if (target.failures === 0 && failingNights === history.length) + return { ...row, verdict: 'fixed', ...empty }; + if (!stat || baselineMedian === null) return { ...row, verdict: 'failing', ...empty }; + + const deltaMs = stat.median - baselineMedian; + const deltaRelative = baselineMedian === 0 ? 0 : deltaMs / baselineMedian; + const threshold = Math.max( + thresholds.minRelative, + thresholds.spreadFactor * baseline.spread, + (thresholds.withinFactor * stat.cv) / Math.sqrt(stat.n), + ); + const measured = { deltaMs, deltaRelative, threshold }; + if (okNights.length < thresholds.minBaselineNights) { + return { ...row, verdict: 'insufficient-baseline', ...measured }; + } + const beyondNoise = + Math.abs(deltaRelative) > threshold && Math.abs(deltaMs) > thresholds.minAbsoluteMs; + const verdict: Verdict = !beyondNoise ? 'unchanged' : deltaMs > 0 ? 'regression' : 'improvement'; + return { ...row, verdict, ...measured }; +} + +function environmentWarnings(target: PerfSummary, baseline: PerfSummary[]): string[] { + const facts = (s: PerfSummary) => ({ + device: `${s.metadata.device.name} ${s.metadata.device.runtime ?? ''}`.trim(), + xcode: s.metadata.xcode ?? 'none', + cpu: s.metadata.host.cpu, + }); + const mine = facts(target); + const warnings: string[] = []; + for (const key of ['device', 'xcode', 'cpu'] as const) { + const differing = baseline.filter((s) => facts(s)[key] !== mine[key]).length; + if (differing > 0) { + warnings.push( + `${differing}/${baseline.length} baseline runs used a different ${key} than tonight (${mine[key]}); their timings are not like for like.`, + ); + } + } + return warnings; +} + +/** + * Compares one run against a rolling baseline of earlier runs. The baseline value of a step is the + * median of its nightly medians, and its noise is the scaled MAD of those medians. A step is flagged + * only when its change exceeds the relative floor, the night-to-night spread, tonight's own + * sampling error, and an absolute floor together. + */ +export function compareRuns( + target: { summary: PerfSummary; label: string }, + baseline: { summary: PerfSummary; label: string }[], + thresholds: ComparisonThresholds = DEFAULT_THRESHOLDS, +): Comparison { + const targetById = new Map(target.summary.measurements.map((m) => [m.id, m])); + const ids = [ + ...target.summary.measurements.map((m) => m.id), + ...baseline.flatMap((b) => b.summary.measurements.map((m) => m.id)), + ]; + const rows = [...new Set(ids)].map((id) => + compareStep( + id, + targetById.get(id), + baseline.flatMap((b) => b.summary.measurements.filter((m) => m.id === id)), + thresholds, + ), + ); + const counts: Partial> = {}; + for (const row of rows) counts[row.verdict] = (counts[row.verdict] ?? 0) + 1; + return { + schema: COMPARISON_SCHEMA, + target: describeRun(target.summary, target.label), + baseline: baseline.map((b) => describeRun(b.summary, b.label)), + thresholds, + counts, + warnings: environmentWarnings( + target.summary, + baseline.map((b) => b.summary), + ), + rows, + }; +} + +function ms(n: number | null | undefined): string { + return typeof n === 'number' && Number.isFinite(n) ? n.toFixed(0) : '–'; +} + +function pct(n: number | null | undefined, signed = false): string { + if (typeof n !== 'number' || !Number.isFinite(n)) return '–'; + const value = `${(n * 100).toFixed(0)}%`; + return signed && n > 0 ? `+${value}` : value; +} + +function tableRow(row: ComparisonRow): string { + const failures = row.target && row.target.failures > 0 ? ` (${row.target.failures} failed)` : ''; + return `| ${row.verdict} | \`${row.id}\` | ${row.metric} | ${ms(row.target?.median)}${failures} | ${ms(row.baseline?.median)} | ${ms(row.deltaMs)} | ${pct(row.deltaRelative, true)} | ${pct(row.threshold)} | ${row.baseline?.nights ?? 0} |`; +} + +const TABLE_HEAD = [ + '| verdict | step | metric | tonight median | baseline median | Δ ms | Δ | threshold | nights |', + '|---|---|---|---|---|---|---|---|---|', +]; + +/** Renders the comparison for the job summary and the artifact. */ +export function renderComparison(comparison: Comparison): string { + const t = comparison.thresholds; + const flagged = comparison.rows + .filter((row) => FLAGGED_VERDICTS.includes(row.verdict)) + .sort((a, b) => FLAGGED_VERDICTS.indexOf(a.verdict) - FLAGGED_VERDICTS.indexOf(b.verdict)); + const counts = Object.entries(comparison.counts) + .map(([verdict, n]) => `${n} ${verdict}`) + .join(', '); + const lines = [ + `## agent-device perf comparison`, + '', + `- **Tonight**: ${comparison.target.label} · ${comparison.target.commit.slice(0, 12)} · ${comparison.target.startedAt} · ${comparison.target.device}`, + `- **Baseline**: ${comparison.baseline.length === 0 ? 'none' : comparison.baseline.map((b) => `${b.label} (${b.commit.slice(0, 7)})`).join(', ')}`, + `- **Rule**: flag a step when its median moves by more than max(${pct(t.minRelative)}, ${t.spreadFactor}× night-to-night spread, ${t.withinFactor}× tonight cv/√n) and more than ${t.minAbsoluteMs} ms against the median of the baseline nights; at least ${t.minBaselineNights} nights are needed for a verdict.`, + `- **Steps**: ${counts || 'none'}`, + '', + ]; + for (const warning of comparison.warnings) lines.push(`> ⚠️ ${warning}`, ''); + if (flagged.length > 0) { + lines.push(...TABLE_HEAD, ...flagged.map(tableRow), ''); + } else { + lines.push('No step moved beyond its noise threshold.', ''); + } + lines.push( + '
All steps', + '', + ...TABLE_HEAD, + ...comparison.rows.map(tableRow), + '', + '
', + '', + ); + return lines.join('\n'); +} diff --git a/scripts/perf/nightly-artifacts.test.ts b/scripts/perf/nightly-artifacts.test.ts new file mode 100644 index 0000000000..74671a936d --- /dev/null +++ b/scripts/perf/nightly-artifacts.test.ts @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + baselineCandidates, + coversTarget, + resolveRunRef, + selectRange, + type WorkflowRun, +} from './nightly-artifacts.ts'; +import { summaryFixture } from './summary-fixture.ts'; + +const run = (id: string, createdAt: string): WorkflowRun => ({ + id, + createdAt, + headSha: `sha-${id}`, + event: 'schedule', +}); + +// Deliberately unsorted, with two runs on 2026-10-19. +const RUNS = [ + run('101', '2026-10-17T03:00:00Z'), + run('104', '2026-10-20T03:00:00Z'), + run('102', '2026-10-19T03:00:00Z'), + run('103', '2026-10-19T15:30:00Z'), +]; + +test('resolves latest, run ids, and dates to the newest matching run', () => { + assert.equal(resolveRunRef('latest', RUNS)?.id, '104'); + assert.equal(resolveRunRef('102', RUNS)?.id, '102'); + assert.equal(resolveRunRef('2026-10-19', RUNS)?.id, '103'); + assert.equal(resolveRunRef('2026-10-18', RUNS), undefined); + assert.throws(() => resolveRunRef('yesterday', RUNS), /Unknown run reference/); +}); + +test('baseline candidates are earlier runs, newest first, without excluded runs', () => { + const candidates = baselineCandidates(RUNS, '2026-10-20T03:00:00Z', new Set(['103'])); + assert.deepEqual( + candidates.map((r) => r.id), + ['102', '101'], + ); +}); + +test('a range includes both ends whichever order they are given in', () => { + const ids = (from: string, to: string) => selectRange(RUNS, from, to).map((r) => r.id); + assert.deepEqual(ids('101', '102'), ['102', '101']); + assert.deepEqual(ids('102', '101'), ['102', '101']); +}); + +test('a date end of a range covers its whole UTC day', () => { + // 2026-10-19 has a 03:00 run and a later 15:30 run; both belong to the range. + assert.deepEqual( + selectRange(RUNS, '2026-10-17', '2026-10-19').map((r) => r.id), + ['103', '102', '101'], + ); + assert.deepEqual( + selectRange(RUNS, '2026-10-19', '104').map((r) => r.id), + ['104', '103', '102'], + ); +}); + +test('a rolling baseline accepts only runs that measured every target scenario', () => { + const full = summaryFixture([ + { id: 'a/x', daemonMs: [1] }, + { id: 'b/y', daemonMs: [1] }, + ]); + const partial = summaryFixture([{ id: 'b/y', daemonMs: [1] }]); + assert.equal(coversTarget(full, full), true); + assert.equal(coversTarget(partial, full), false); + assert.equal(coversTarget(full, partial), true); +}); diff --git a/scripts/perf/nightly-artifacts.ts b/scripts/perf/nightly-artifacts.ts new file mode 100644 index 0000000000..08b00b7797 --- /dev/null +++ b/scripts/perf/nightly-artifacts.ts @@ -0,0 +1,159 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { SUMMARY_FILE } from './report.ts'; +import { parsePerfSummary } from './summary.ts'; +import type { PerfSummary } from './types.ts'; + +/** The stable name of the summary artifact that the nightly workflow uploads for every run. */ +export const SUMMARY_ARTIFACT = 'perf-ios-summary'; + +export type WorkflowRun = { + id: string; + createdAt: string; + headSha: string; + event: string; +}; + +export type NightlySource = { + repo: string; + workflow: string; + branch: string; + artifact: string; + cacheDir: string; +}; + +const DATE_REF = /^\d{4}-\d{2}-\d{2}$/; + +function gh(args: string[]): { ok: boolean; stdout: string; stderr: string } { + const result = runCmdSync('gh', args, { allowFailure: true, maxBuffer: 64 * 1024 * 1024 }); + return { ok: result.exitCode === 0, stdout: result.stdout, stderr: result.stderr }; +} + +/** Lists the workflow's completed runs on one branch, newest first. */ +export function listRuns(source: NightlySource, limit = 100): WorkflowRun[] { + const result = gh([ + 'run', + 'list', + '--repo', + source.repo, + '--workflow', + source.workflow, + '--branch', + source.branch, + '--status', + 'completed', + '--limit', + String(limit), + '--json', + 'databaseId,createdAt,headSha,event', + ]); + if (!result.ok) throw new Error(`gh run list failed: ${result.stderr.trim()}`); + const rows = JSON.parse(result.stdout) as { + databaseId: number; + createdAt: string; + headSha: string; + event: string; + }[]; + return sortNewestFirst( + rows.map((row) => ({ + id: String(row.databaseId), + createdAt: row.createdAt, + headSha: row.headSha, + event: row.event, + })), + ); +} + +function sortNewestFirst(runs: WorkflowRun[]): WorkflowRun[] { + return [...runs].sort((a, b) => b.createdAt.localeCompare(a.createdAt)); +} + +/** + * Resolves one run reference: `latest`, a numeric run id, or a `YYYY-MM-DD` date (the newest run + * created that UTC day). Returns undefined when no listed run matches. + */ +export function resolveRunRef(ref: string, runs: WorkflowRun[]): WorkflowRun | undefined { + const newestFirst = sortNewestFirst(runs); + if (ref === 'latest') return newestFirst[0]; + if (/^\d+$/.test(ref)) return newestFirst.find((run) => run.id === ref); + if (DATE_REF.test(ref)) return newestFirst.find((run) => run.createdAt.slice(0, 10) === ref); + throw new Error( + `Unknown run reference ${JSON.stringify(ref)}: use latest, a run id, a YYYY-MM-DD date, or a path.`, + ); +} + +/** + * Runs between two references inclusive, newest first. A date end covers its whole UTC day, so a + * dispatch later that day does not hide the night's scheduled run. + */ +export function selectRange(runs: WorkflowRun[], fromRef: string, toRef: string): WorkflowRun[] { + const bounds = (ref: string): [string, string] | undefined => { + if (DATE_REF.test(ref)) return [`${ref}T00:00:00Z`, `${ref}T23:59:59.999Z`]; + const run = resolveRunRef(ref, runs); + return run && [run.createdAt, run.createdAt]; + }; + const from = bounds(fromRef); + const to = bounds(toRef); + if (!from || !to) throw new Error(`No runs match the range ${fromRef}..${toRef}.`); + const low = from[0] < to[0] ? from[0] : to[0]; + const high = from[1] > to[1] ? from[1] : to[1]; + return sortNewestFirst(runs).filter((run) => run.createdAt >= low && run.createdAt <= high); +} + +/** + * A rolling-baseline run must have run every scenario of the target on the same platform, so a + * partial dispatch does not take the place of a full night. + */ +export function coversTarget(candidate: PerfSummary, target: PerfSummary): boolean { + return ( + candidate.platform === target.platform && + target.config.scenarios.every((scenario) => candidate.config.scenarios.includes(scenario)) + ); +} + +/** Baseline candidates: runs created before the target, newest first, never the target itself. */ +export function baselineCandidates( + runs: WorkflowRun[], + before: string, + exclude: ReadonlySet, +): WorkflowRun[] { + return sortNewestFirst(runs).filter((run) => run.createdAt < before && !exclude.has(run.id)); +} + +/** + * Returns a run's summary, downloading the summary artifact once into the cache. Returns undefined + * when the run has no such artifact (it failed before upload, or the artifact expired). + */ +export function fetchSummary(source: NightlySource, run: WorkflowRun): PerfSummary | undefined { + const dir = path.join(source.cacheDir, run.id); + const file = path.join(dir, SUMMARY_FILE); + if (!fs.existsSync(file)) { + fs.rmSync(dir, { recursive: true, force: true }); + const result = gh([ + 'run', + 'download', + run.id, + '--repo', + source.repo, + '--name', + source.artifact, + '--dir', + dir, + ]); + if (!result.ok || !fs.existsSync(file)) { + process.stderr.write( + `[perf] run ${run.id}: no ${source.artifact} artifact (${result.stderr.trim()})\n`, + ); + fs.rmSync(dir, { recursive: true, force: true }); + return undefined; + } + } + return parsePerfSummary(JSON.parse(fs.readFileSync(file, 'utf8')), file); +} + +/** Reads a local run: a `summary.json` path or a run output directory. */ +export function readLocalSummary(location: string): PerfSummary { + const file = fs.statSync(location).isDirectory() ? path.join(location, SUMMARY_FILE) : location; + return parsePerfSummary(JSON.parse(fs.readFileSync(file, 'utf8')), file); +} diff --git a/scripts/perf/stats.test.ts b/scripts/perf/stats.test.ts index 088c7e1806..9b5d61ac46 100644 --- a/scripts/perf/stats.test.ts +++ b/scripts/perf/stats.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { summarize } from './stats.ts'; +import { scaledMad, summarize } from './stats.ts'; test('summarize reports sample spread, the middle median, and nearest-rank p95', () => { const stat = summarize([40, 10, 30, 20, Number.NaN])!; @@ -26,3 +26,8 @@ test('summarize gives a single sample zero spread and no samples no summary', () assert.equal(summarize([]), null); assert.equal(summarize([0, 0])?.cv, 0); }); + +test('scaledMad ignores one outlying night', () => { + // Deviations from the median 101 are 49, 1, 1, 3; their median is 2. + assert.ok(Math.abs(scaledMad([150, 100, 98, 102]) - 1.4826 * 2) < 1e-9); +}); diff --git a/scripts/perf/stats.ts b/scripts/perf/stats.ts index f972ec4e37..e4941f4608 100644 --- a/scripts/perf/stats.ts +++ b/scripts/perf/stats.ts @@ -41,3 +41,9 @@ export function median(values: number[]): number { const sorted = values.filter((v) => Number.isFinite(v)).sort((a, b) => a - b); return sorted.length === 0 ? Number.NaN : middle(sorted); } + +/** Median absolute deviation scaled to estimate a normal standard deviation. */ +export function scaledMad(values: number[]): number { + const center = median(values); + return 1.4826 * median(values.map((v) => Math.abs(v - center))); +} diff --git a/scripts/perf/summary-fixture.ts b/scripts/perf/summary-fixture.ts new file mode 100644 index 0000000000..b86ad31ef8 --- /dev/null +++ b/scripts/perf/summary-fixture.ts @@ -0,0 +1,64 @@ +import { buildMeasurement } from './measurement.ts'; +import { PERF_SUMMARY_SCHEMA, type Measurement, type PerfSummary, type Sample } from './types.ts'; + +export type StepFixture = { + id: string; + // Daemon round trips of the successful samples; wall adds a fixed CLI overhead. + daemonMs?: number[]; + wallMs?: number[]; + failures?: number; +}; + +function measurement(step: StepFixture): Measurement { + const timings = step.daemonMs ?? step.wallMs ?? []; + const samples: Sample[] = timings.map((ms, round) => ({ + round, + ok: true, + wallClockMs: step.daemonMs ? ms + 300 : ms, + daemonMs: step.daemonMs ? ms : undefined, + })); + for (let i = 0; i < (step.failures ?? 0); i++) { + samples.push({ round: samples.length, ok: false, wallClockMs: 0, errorCode: 'COMMAND_FAILED' }); + } + const [scenario] = step.id.split('/'); + return buildMeasurement({ + id: step.id, + scenario: scenario!, + label: step.id, + command: 'snapshot', + args: [], + platform: 'ios', + execMode: 'standalone', + samples, + warmupDropped: 1, + }); +} + +/** A schema-valid summary whose steps carry the given timings. */ +export function summaryFixture( + steps: StepFixture[], + overrides: { startedAt?: string; xcode?: string; runId?: string } = {}, +): PerfSummary { + return { + schema: PERF_SUMMARY_SCHEMA, + platform: 'ios', + startedAt: overrides.startedAt ?? '2026-10-20T03:00:00.000Z', + finishedAt: '2026-10-20T03:30:00.000Z', + config: { + rounds: 5, + warmup: 1, + scenarios: [...new Set(steps.map((s) => s.id.split('/')[0]!))], + }, + metadata: { + commit: '0123456789abcdef', + runId: overrides.runId, + agentDeviceVersion: '0.0.0', + host: { os: 'Darwin', arch: 'arm64', cpu: 'Apple M1 (Virtual)', cores: 3, memoryGb: 7 }, + xcode: overrides.xcode ?? 'Xcode 26.2', + node: 'v24.0.0', + device: { name: 'iPhone 17 Pro', runtime: 'iOS-26-2' }, + }, + coverage: { measured: ['snapshot'], excluded: [] }, + measurements: steps.map(measurement), + }; +} diff --git a/scripts/perf/summary.test.ts b/scripts/perf/summary.test.ts new file mode 100644 index 0000000000..421f234f17 --- /dev/null +++ b/scripts/perf/summary.test.ts @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { parsePerfSummary } from './summary.ts'; +import { summaryFixture } from './summary-fixture.ts'; + +const valid = () => + JSON.parse(JSON.stringify(summaryFixture([{ id: 's/step', daemonMs: [10, 20] }]))) as Record< + string, + unknown + >; + +test('accepts a summary that carries every field the comparison reads', () => { + const summary = parsePerfSummary(valid(), 'summary.json'); + assert.equal(summary.measurements[0]?.daemon?.median, 15); +}); + +test('rejects another schema version and names the field', () => { + assert.throws( + () => parsePerfSummary({ ...valid(), schema: 'agent-device-perf-summary/v0' }, 'a.json'), + /a\.json: schema must be "agent-device-perf-summary\/v1"/, + ); +}); + +test('rejects a measurement whose timing summary lost a statistic', () => { + const summary = valid(); + const [measurement] = summary.measurements as { daemon: Record }[]; + delete measurement!.daemon.cv; + assert.throws( + () => parsePerfSummary(summary, 'b.json'), + /b\.json: measurements\[0\]\.daemon\.cv must be a finite number/, + ); +}); diff --git a/scripts/perf/summary.ts b/scripts/perf/summary.ts new file mode 100644 index 0000000000..2feff211aa --- /dev/null +++ b/scripts/perf/summary.ts @@ -0,0 +1,63 @@ +import { PERF_SUMMARY_SCHEMA, type PerfSummary } from './types.ts'; + +type Json = Record; + +const STAT_FIELDS = ['n', 'mean', 'stddev', 'cv', 'median', 'min', 'max', 'p95'] as const; + +function fail(source: string, at: string, expected: string): never { + throw new Error(`${source}: ${at} must be ${expected}`); +} + +function object(value: unknown, source: string, at: string): Json { + if (value === null || typeof value !== 'object' || Array.isArray(value)) + fail(source, at, 'an object'); + return value as Json; +} + +function string(value: unknown, source: string, at: string): string { + if (typeof value !== 'string') fail(source, at, 'a string'); + return value; +} + +function number(value: unknown, source: string, at: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) fail(source, at, 'a finite number'); + return value; +} + +function stat(value: unknown, source: string, at: string): void { + if (value === null) return; + const s = object(value, source, at); + for (const field of STAT_FIELDS) number(s[field], source, `${at}.${field}`); +} + +/** + * Validates a `summary.json` read from disk or a downloaded artifact. Only the fields that the + * comparison and report read are checked; a run from another schema version is rejected. + */ +export function parsePerfSummary(value: unknown, source: string): PerfSummary { + const root = object(value, source, 'summary'); + if (root.schema !== PERF_SUMMARY_SCHEMA) fail(source, 'schema', `"${PERF_SUMMARY_SCHEMA}"`); + if (root.platform !== 'ios' && root.platform !== 'android') + fail(source, 'platform', 'ios or android'); + string(root.startedAt, source, 'startedAt'); + const config = object(root.config, source, 'config'); + number(config.rounds, source, 'config.rounds'); + const metadata = object(root.metadata, source, 'metadata'); + string(metadata.commit, source, 'metadata.commit'); + string(object(metadata.host, source, 'metadata.host').cpu, source, 'metadata.host.cpu'); + string(object(metadata.device, source, 'metadata.device').name, source, 'metadata.device.name'); + if (!Array.isArray(config.scenarios)) fail(source, 'config.scenarios', 'an array'); + if (!Array.isArray(root.measurements)) fail(source, 'measurements', 'an array'); + root.measurements.forEach((entry, index) => { + const at = `measurements[${index}]`; + const m = object(entry, source, at); + string(m.id, source, `${at}.id`); + string(m.scenario, source, `${at}.scenario`); + string(m.command, source, `${at}.command`); + number(m.failures, source, `${at}.failures`); + if (!Array.isArray(m.samples)) fail(source, `${at}.samples`, 'an array'); + stat(m.wallClock, source, `${at}.wallClock`); + stat(m.daemon, source, `${at}.daemon`); + }); + return value as PerfSummary; +} diff --git a/test/repo-guards/npm-package-scripts.test.ts b/test/repo-guards/npm-package-scripts.test.ts index 7f35bea71e..9b12030361 100644 --- a/test/repo-guards/npm-package-scripts.test.ts +++ b/test/repo-guards/npm-package-scripts.test.ts @@ -36,7 +36,7 @@ test('perf uses one platform-selectable entry point', () => { assert.equal(script('perf'), 'node --experimental-strip-types scripts/perf/run.ts'); assert.equal(packageJson.scripts['perf:ios'], undefined); assert.equal(packageJson.scripts['perf:android'], undefined); - assert.doesNotMatch(perfNightlyWorkflow, /pnpm perf:/); + assert.doesNotMatch(perfNightlyWorkflow, /pnpm perf:(ios|android)\b/); assert.match(perfNightlyWorkflow, /pnpm perf --platform ios/); assert.match(perfNightlyWorkflow, /pnpm perf --platform android/); }); From fed2a4cc9dbb9fe6c6048549604c0d7b7c1c76df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 05:53:53 +0200 Subject: [PATCH 09/15] fix(perf): judge nights, samples, and failures soundly; keep logs on interrupt - Rolling baselines take one scheduled run per UTC night (the newest usable one) and page through workflow runs until enough nights are found. Explicit --to/--from references still accept any trigger. - A run measures at least three rounds. A timing verdict needs three successful samples tonight and on each counted baseline night. - Failures get their own verdict. more-failures flags tonight failing more than every baseline night, in count and share; a faster median with worse failures is not an improvement. - The interrupt handler and the normal finish share one finalizer: save logs, stop session and daemon within a deadline, save again. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/perf-nightly.yml | 4 +- scripts/perf/README.md | 30 +++-- scripts/perf/cli.ts | 7 +- scripts/perf/compare-cli.ts | 54 ++++---- scripts/perf/compare.test.ts | 46 +++++-- scripts/perf/compare.ts | 167 ++++++++++++++++--------- scripts/perf/config.test.ts | 9 ++ scripts/perf/config.ts | 7 +- scripts/perf/ios-harness.test.ts | 57 +++++++++ scripts/perf/ios-harness.ts | 34 ++++- scripts/perf/nightly-artifacts.test.ts | 61 +++++++-- scripts/perf/nightly-artifacts.ts | 110 ++++++++++------ scripts/perf/run.ts | 14 +-- 13 files changed, 435 insertions(+), 165 deletions(-) create mode 100644 scripts/perf/config.test.ts create mode 100644 scripts/perf/ios-harness.test.ts diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index 243cd08c17..f4d7cd4863 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -16,7 +16,7 @@ on: workflow_dispatch: inputs: rounds: - description: 'Measured rounds per step (samples)' + description: 'Measured rounds per step (samples, at least 3)' required: false default: '3' warmup: @@ -28,7 +28,7 @@ on: required: false default: '' baseline-nights: - description: 'Earlier runs in the rolling baseline' + description: 'Earlier scheduled nights in the rolling baseline' required: false default: '7' baseline-branch: diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 4f088d0aea..0715aa7061 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -18,8 +18,9 @@ The suite records a reason for each catalog command that it does not time. `ios- pnpm perf --platform ios --udid --rounds 3 --warmup 1 --out-dir .tmp/perf/local ``` -`--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id. +`--rounds` is at least 3. `--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id. The suite uses its own temporary state directory and daemon. It stops them at the end. +At the end, or on SIGINT or SIGTERM, the suite saves the logs to `raw/state/`, stops the session and daemon within a deadline, and saves the logs again. ## Output @@ -54,19 +55,22 @@ All times are milliseconds. Statistics use only the successful samples of a step `.github/workflows/perf-nightly.yml` runs the iOS suite every night on `main`. Each run uploads `perf-ios-summary` (summary, report, comparison; 90 days) and `perf-ios-raw` (logs; 30 days). -The job compares the run with up to seven earlier runs and writes the result to the job summary. +The job compares the run with up to seven earlier nights and writes the result to the job summary. +A night is the newest scheduled run of one UTC day that ran every scenario of the judged run. +Manual dispatches are never part of the rolling baseline. To fetch and compare runs on your machine, use `gh` with access to the repository: ```sh pnpm perf:compare # latest night against the 7 nights before it pnpm perf:compare --to 2026-10-28 # that night against the 7 nights before it -pnpm perf:compare --to latest --from 2026-10-21 # latest against one night -pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare +pnpm perf:compare --to latest --from 2026-10-21 # latest against the newest run of that day +pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare # every run in the range pnpm perf:compare --to .tmp/perf/local # a local run against recent nights ``` A reference is `latest`, a run id, a `YYYY-MM-DD` date, or a local output path. +An explicit `--to` or `--from` reference accepts runs of any trigger, so it can compare dispatches and local runs. Downloads stay in `.tmp/perf-nightly//`. Do not judge a local run against CI nights. The comparison warns when the device, Xcode, or CPU differs. @@ -81,7 +85,19 @@ A step is a `regression` or an `improvement` only when its change is more than a - Two times the coefficient of variation of the run divided by √n. - 50 ms. -The rolling baseline skips runs that did not run every scenario of the judged run, such as a partial dispatch. -Fewer than three baseline nights give `insufficient-baseline`. -`new-failures` means that the step failed and no baseline night failed. `fixed` means the reverse. +Each step gets a timing verdict and a failure verdict. +A timing verdict uses only successful samples: + +- `insufficient-samples`: the judged run has fewer than three successful samples of the step. +- `insufficient-baseline`: fewer than three baseline nights have three successful samples of the step. Nights with fewer do not count. + +The failure verdict compares failed samples with the worst baseline night: + +- `new-failures`: the step failed and no baseline night failed. +- `more-failures`: the step failed more often than on every baseline night, both in count and in share of its samples. +- `fixed`: the step did not fail and every baseline night failed. + +When failures are new or more, a faster median is `insufficient-samples`, not `improvement`. +The samples that still pass do not show that the step got faster. +Regressions and worse failures are workflow warnings. The job stays green. The thresholds are in `compare.ts`. diff --git a/scripts/perf/cli.ts b/scripts/perf/cli.ts index 57ee22c56a..cada28867d 100644 --- a/scripts/perf/cli.ts +++ b/scripts/perf/cli.ts @@ -33,7 +33,11 @@ function jsonOk(json: unknown): boolean { // Invoke the built CLI once. `args` includes the command + positionals + dash-flags; // `baseFlags` carries the isolation + device flags shared by every call. -export function invokeCli(args: string[], baseFlags: string[]): CliResult { +export function invokeCli( + args: string[], + baseFlags: string[], + options: { timeoutMs?: number } = {}, +): CliResult { const full = [...CLI_ARGV, ...args, ...baseFlags, '--json']; const t0 = performance.now(); let stdout = ''; @@ -46,6 +50,7 @@ export function invokeCli(args: string[], baseFlags: string[]): CliResult { cwd: REPO_ROOT, maxBuffer: MAX_BUFFER, allowFailure: true, + timeoutMs: options.timeoutMs, }); stdout = r.stdout; stderr = r.stderr; diff --git a/scripts/perf/compare-cli.ts b/scripts/perf/compare-cli.ts index 0336beca96..3b3593acfe 100644 --- a/scripts/perf/compare-cli.ts +++ b/scripts/perf/compare-cli.ts @@ -3,12 +3,12 @@ import path from 'node:path'; import { compareRuns, renderComparison, type Comparison } from './compare.ts'; import { REPO_ROOT } from './config.ts'; import { - baselineCandidates, coversTarget, fetchSummary, listRuns, readLocalSummary, resolveRunRef, + selectNightlyBaseline, selectRange, SUMMARY_ARTIFACT, type NightlySource, @@ -24,9 +24,9 @@ A is latest, a workflow run id, a YYYY-MM-DD date (the newest run that UTC path to a summary.json or a \`pnpm perf\` output directory. --to Run to judge (default: latest). - --from Baseline run, or an inclusive range ... Without it the baseline is - the --nights runs before --to. - --nights Rolling baseline size (default: 7). + --from Baseline run, or an inclusive range .. of runs of any trigger. + Without it the baseline is the --nights scheduled nights before --to. + --nights Rolling baseline size in UTC nights, one scheduled run each (default: 7). --branch Branch whose nightly runs are listed (default: main). --repo Repository (default: $GITHUB_REPOSITORY or callstack/agent-device). --workflow Workflow file (default: perf-nightly.yml). @@ -117,7 +117,7 @@ type Loaded = { summary: PerfSummary; label: string; run?: WorkflowRun }; function main(): void { const options = parse(process.argv.slice(2)); let runs: WorkflowRun[] | undefined; - const allRuns = (): WorkflowRun[] => (runs ??= listRuns(options.source)); + const allRuns = (): WorkflowRun[] => (runs ??= [...listRuns(options.source)]); const runLabel = (run: WorkflowRun) => `run ${run.id} (${run.createdAt.slice(0, 10)})`; const load = (ref: string): Loaded => { if (fs.existsSync(ref)) @@ -131,19 +131,11 @@ function main(): void { if (!summary) throw new Error(`Run ${run.id} has no ${options.source.artifact} artifact.`); return { summary, label: runLabel(run), run }; }; - const loadRuns = ( - candidates: WorkflowRun[], - limit: number, - accept: (summary: PerfSummary) => boolean = () => true, - ): Loaded[] => { - const loaded: Loaded[] = []; - for (const run of candidates) { - if (loaded.length >= limit) break; + const loadRuns = (candidates: WorkflowRun[]): Loaded[] => + candidates.flatMap((run) => { const summary = fetchSummary(options.source, run); - if (summary && accept(summary)) loaded.push({ summary, label: runLabel(run), run }); - } - return loaded; - }; + return summary ? [{ summary, label: runLabel(run), run }] : []; + }); const target = load(options.to); const exclude = new Set([...options.exclude, ...(target.run ? [target.run.id] : [])]); @@ -151,14 +143,19 @@ function main(): void { if (options.from?.includes('..')) { const [fromRef, toRef] = options.from.split('..'); const range = selectRange(allRuns(), fromRef!, toRef!).filter((run) => !exclude.has(run.id)); - baseline = loadRuns(range, range.length); + baseline = loadRuns(range); } else if (options.from) { baseline = [load(options.from)]; } else { const before = target.run?.createdAt ?? target.summary.startedAt; - baseline = loadRuns(baselineCandidates(allRuns(), before, exclude), options.nights, (summary) => - coversTarget(summary, target.summary), - ); + baseline = selectNightlyBaseline( + runs ?? listRuns(options.source), + { before, exclude, nights: options.nights }, + (run) => { + const summary = fetchSummary(options.source, run); + return summary && coversTarget(summary, target.summary) ? summary : undefined; + }, + ).map(({ run, value }) => ({ summary: value, label: runLabel(run), run })); } const comparison = compareRuns(target, baseline); @@ -178,12 +175,15 @@ function publishToGitHub(comparison: Comparison, markdown: string): void { const summaryPath = process.env.GITHUB_STEP_SUMMARY; if (summaryPath) fs.appendFileSync(summaryPath, `${markdown}\n`); for (const row of comparison.rows) { - if (row.verdict === 'regression' || row.verdict === 'new-failures') { - const change = - row.verdict === 'regression' - ? `median ${row.target?.median.toFixed(0)} ms vs ${row.baseline?.median.toFixed(0)} ms baseline` - : `${row.target?.failures}/${row.target?.samples} samples failed`; - process.stdout.write(`::warning title=perf ${row.verdict}::${row.id}: ${change}\n`); + if (row.timing === 'regression') { + process.stdout.write( + `::warning title=perf regression::${row.id}: median ${row.target?.median.toFixed(0)} ms vs ${row.baseline?.median.toFixed(0)} ms baseline\n`, + ); + } + if (row.failures === 'new-failures' || row.failures === 'more-failures') { + process.stdout.write( + `::warning title=perf ${row.failures}::${row.id}: ${row.target?.failures}/${row.target?.samples} samples failed\n`, + ); } } } diff --git a/scripts/perf/compare.test.ts b/scripts/perf/compare.test.ts index d7046d1860..51fb75d55c 100644 --- a/scripts/perf/compare.test.ts +++ b/scripts/perf/compare.test.ts @@ -26,7 +26,7 @@ test('flags a slowdown that exceeds a quiet baseline and the relative floor', () [{ id: STEP, daemonMs: [1450, 1500, 1550] }], [flat(1000), flat(1010), flat(990)], ); - assert.equal(result.verdict, 'regression'); + assert.equal(result.timing, 'regression'); assert.equal(result.metric, 'daemon'); assert.equal(result.baseline?.median, 1000); assert.equal(result.deltaMs, 500); @@ -38,37 +38,63 @@ test('does not flag the same slowdown when nightly medians already swing that mu [{ id: STEP, daemonMs: [1450, 1500, 1550] }], [flat(500), flat(1000), flat(1500), flat(700), flat(1300)], ); - assert.equal(result.verdict, 'unchanged'); + assert.equal(result.timing, 'unchanged'); // 3 × scaled MAD: deviations from 1000 are 0, 300, 300, 500, 500. assert.ok(Math.abs(result.threshold! - (3 * 1.4826 * 300) / 1000) < 1e-9); }); test('does not flag a large relative change below the absolute floor', () => { - assert.equal(row(flat(60), [flat(40), flat(40), flat(40)]).verdict, 'unchanged'); + assert.equal(row(flat(60), [flat(40), flat(40), flat(40)]).timing, 'unchanged'); }); test('flags a speedup beyond noise as an improvement', () => { - assert.equal(row(flat(500), [flat(1000), flat(1010), flat(990)]).verdict, 'improvement'); + assert.equal(row(flat(500), [flat(1000), flat(1010), flat(990)]).timing, 'improvement'); +}); + +test('withholds a timing verdict when tonight or a baseline night has fewer than three successes', () => { + const one = (ms: number): StepFixture[] => [{ id: STEP, daemonMs: [ms] }]; + const fromOneSample = row(one(1500), [one(1000), one(1000), one(1000)]); + assert.equal(fromOneSample.timing, 'insufficient-samples'); + assert.equal(fromOneSample.baseline?.timedNights, 0); + // Three successes tonight still need three nights that each had three successes. + const thinNights = row(flat(1500), [flat(1000), flat(1000), one(1000)]); + assert.equal(thinNights.timing, 'insufficient-baseline'); + assert.equal(thinNights.baseline?.timedNights, 2); + assert.equal(thinNights.deltaMs, 500); }); test('withholds a timing verdict until enough nights exist', () => { const result = row(flat(1500), [flat(1000), flat(1000)]); - assert.equal(result.verdict, 'insufficient-baseline'); + assert.equal(result.timing, 'insufficient-baseline'); assert.equal(result.deltaRelative, 0.5); }); test('reports failures that no baseline night had, and failures that went away', () => { const clean = [flat(1000), flat(1000), flat(1000)]; - assert.equal(row(flat(1000, { failures: 1 }), clean).verdict, 'new-failures'); + assert.equal(row(flat(1000, { failures: 1 }), clean).failures, 'new-failures'); const failing = [ flat(1000, { failures: 1 }), flat(1000, { failures: 2 }), flat(1000, { failures: 1 }), ]; - assert.equal(row(flat(1000), failing).verdict, 'fixed'); - // A step that already failed on some baseline night is judged on timing instead. + assert.equal(row(flat(1000), failing).failures, 'fixed'); + // As many failures as the worst baseline night is not a change; timing is judged as usual. + const flaky = [flat(1000, { failures: 1 }), flat(1000), flat(1000)]; + const steady = row(flat(1000, { failures: 1 }), flaky); + assert.equal(steady.failures, 'unchanged'); + assert.equal(steady.timing, 'unchanged'); +}); + +test('reports worse failures over a flaky baseline and never as an improvement', () => { const flaky = [flat(1000, { failures: 1 }), flat(1000), flat(1000)]; - assert.equal(row(flat(1000, { failures: 1 }), flaky).verdict, 'unchanged'); + const mostlyFailing = row([{ id: STEP, daemonMs: [500], failures: 2 }], flaky); + assert.equal(mostlyFailing.failures, 'more-failures'); + assert.equal(mostlyFailing.timing, 'insufficient-samples'); + // Enough successes, but they are the subset that still passes, so the speedup is withheld. + const faster = row(flat(500, { failures: 2 }), flaky); + assert.equal(faster.failures, 'more-failures'); + assert.equal(faster.timing, 'insufficient-samples'); + assert.equal(faster.deltaMs, -500); }); test('falls back to wall time when any run lacks the daemon timer', () => { @@ -86,7 +112,7 @@ test('marks steps that only one side measured', () => { [{ summary: summaryFixture([{ id: 'a/old', daemonMs: [1] }]), label: 'b' }], ); assert.deepEqual( - comparison.rows.map((r) => [r.id, r.verdict]), + comparison.rows.map((r) => [r.id, r.timing]), [ ['a/new', 'new'], ['a/old', 'missing'], diff --git a/scripts/perf/compare.ts b/scripts/perf/compare.ts index 11dae30520..22b1f28a61 100644 --- a/scripts/perf/compare.ts +++ b/scripts/perf/compare.ts @@ -1,3 +1,4 @@ +import { MIN_MEASURED_ROUNDS } from './config.ts'; import { median, scaledMad } from './stats.ts'; import type { Measurement, PerfSummary, Stat } from './types.ts'; @@ -7,7 +8,8 @@ export type ComparisonThresholds = { minAbsoluteMs: number; spreadFactor: number; // × night-to-night relative spread (scaled MAD of nightly medians) withinFactor: number; // × tonight's cv / √n (standard error of tonight's median) - minBaselineNights: number; // fewer nights give `insufficient-baseline` + minBaselineNights: number; // fewer timed nights give `insufficient-baseline` + minSamples: number; // successful samples that tonight and each timed night need }; const DEFAULT_THRESHOLDS: ComparisonThresholds = { @@ -16,30 +18,47 @@ const DEFAULT_THRESHOLDS: ComparisonThresholds = { spreadFactor: 3, withinFactor: 2, minBaselineNights: 3, + minSamples: MIN_MEASURED_ROUNDS, }; -export type Verdict = +export type TimingVerdict = | 'regression' | 'improvement' | 'unchanged' - | 'new-failures' - | 'fixed' - | 'failing' + | 'insufficient-samples' | 'insufficient-baseline' | 'new' | 'missing'; +export type FailureVerdict = 'new-failures' | 'more-failures' | 'fixed' | 'unchanged'; + +type Flagged = Exclude; + /** Verdicts that need a reader's attention, in report order. */ -const FLAGGED_VERDICTS: readonly Verdict[] = ['new-failures', 'regression', 'fixed', 'improvement']; +const FLAGGED_VERDICTS: readonly Flagged[] = [ + 'new-failures', + 'more-failures', + 'regression', + 'fixed', + 'improvement', +]; export type ComparisonRow = { id: string; scenario: string; command: string; - verdict: Verdict; + timing: TimingVerdict; + // Null when only one side measured the step. + failures: FailureVerdict | null; metric: 'daemon' | 'wall'; target: { median: number; n: number; cv: number; failures: number; samples: number } | null; - baseline: { median: number; nights: number; spread: number; failingNights: number } | null; + baseline: { + median: number; + nights: number; + timedNights: number; + spread: number; + failingNights: number; + } | null; deltaMs: number | null; deltaRelative: number | null; threshold: number | null; @@ -62,7 +81,10 @@ export type Comparison = { target: RunRef; baseline: RunRef[]; thresholds: ComparisonThresholds; - counts: Partial>; + counts: { + timing: Partial>; + failures: Partial>; + }; warnings: string[]; rows: ComparisonRow[]; }; @@ -83,6 +105,24 @@ function metricStat(m: Measurement, metric: 'daemon' | 'wall'): Stat | null { return metric === 'daemon' ? m.daemon : m.wallClock; } +function failureRate(m: Measurement): number { + return m.samples.length === 0 ? 0 : m.failures / m.samples.length; +} + +/** + * Failures worsen when tonight has more failed samples than every baseline night, both as a count + * and as a share of samples, so a different round count cannot fake a change. + */ +function judgeFailures(target: Measurement, history: Measurement[]): FailureVerdict { + const failingNights = history.filter((m) => m.failures > 0).length; + const worstCount = Math.max(...history.map((m) => m.failures)); + const worstRate = Math.max(...history.map(failureRate)); + if (target.failures > worstCount && failureRate(target) > worstRate) + return failingNights === 0 ? 'new-failures' : 'more-failures'; + if (target.failures === 0 && failingNights === history.length) return 'fixed'; + return 'unchanged'; +} + function compareStep( id: string, target: Measurement | undefined, @@ -97,45 +137,39 @@ function compareStep( ? 'daemon' : 'wall'; const empty = { deltaMs: null, deltaRelative: null, threshold: null }; - const okNights = history + const timed = (s: Stat | null): s is Stat => s !== null && s.n >= thresholds.minSamples; + const timedNights = history .map((m) => metricStat(m, metric)) - .filter((s): s is Stat => s !== null) + .filter(timed) .map((s) => s.median); - const failingNights = history.filter((m) => m.failures > 0).length; - const baselineMedian = okNights.length > 0 ? median(okNights) : null; + const baselineMedian = timedNights.length > 0 ? median(timedNights) : null; const baseline = history.length === 0 ? null : { median: baselineMedian ?? Number.NaN, nights: history.length, - spread: baselineMedian ? scaledMad(okNights) / baselineMedian : 0, - failingNights, + timedNights: timedNights.length, + spread: baselineMedian ? scaledMad(timedNights) / baselineMedian : 0, + failingNights: history.filter((m) => m.failures > 0).length, }; - if (!target) return { ...base, verdict: 'missing', metric, target: null, baseline, ...empty }; + if (!target) + return { ...base, timing: 'missing', failures: null, metric, target: null, baseline, ...empty }; const stat = metricStat(target, metric); - const targetFacts = stat - ? { - median: stat.median, - n: stat.n, - cv: stat.cv, - failures: target.failures, - samples: target.samples.length, - } - : { - median: Number.NaN, - n: 0, - cv: 0, - failures: target.failures, - samples: target.samples.length, - }; + const targetFacts = { + median: stat?.median ?? Number.NaN, + n: stat?.n ?? 0, + cv: stat?.cv ?? 0, + failures: target.failures, + samples: target.samples.length, + }; const row = { ...base, metric, target: targetFacts, baseline }; - if (!baseline) return { ...row, verdict: 'new', ...empty }; - if (target.failures > 0 && failingNights === 0) - return { ...row, verdict: 'new-failures', ...empty }; - if (target.failures === 0 && failingNights === history.length) - return { ...row, verdict: 'fixed', ...empty }; - if (!stat || baselineMedian === null) return { ...row, verdict: 'failing', ...empty }; + if (!baseline) return { ...row, timing: 'new', failures: null, ...empty }; + const failures = judgeFailures(target, history); + if (!stat || baselineMedian === null) { + const timing = timed(stat) ? 'insufficient-baseline' : 'insufficient-samples'; + return { ...row, timing, failures, ...empty }; + } const deltaMs = stat.median - baselineMedian; const deltaRelative = baselineMedian === 0 ? 0 : deltaMs / baselineMedian; @@ -144,14 +178,18 @@ function compareStep( thresholds.spreadFactor * baseline.spread, (thresholds.withinFactor * stat.cv) / Math.sqrt(stat.n), ); - const measured = { deltaMs, deltaRelative, threshold }; - if (okNights.length < thresholds.minBaselineNights) { - return { ...row, verdict: 'insufficient-baseline', ...measured }; + const measured = { ...row, failures, deltaMs, deltaRelative, threshold }; + if (!timed(stat)) return { ...measured, timing: 'insufficient-samples' }; + if (timedNights.length < thresholds.minBaselineNights) { + return { ...measured, timing: 'insufficient-baseline' }; } const beyondNoise = Math.abs(deltaRelative) > threshold && Math.abs(deltaMs) > thresholds.minAbsoluteMs; - const verdict: Verdict = !beyondNoise ? 'unchanged' : deltaMs > 0 ? 'regression' : 'improvement'; - return { ...row, verdict, ...measured }; + if (!beyondNoise) return { ...measured, timing: 'unchanged' }; + if (deltaMs > 0) return { ...measured, timing: 'regression' }; + // The samples that still succeed while failures worsen do not show that the step got faster. + const worsened = failures === 'new-failures' || failures === 'more-failures'; + return { ...measured, timing: worsened ? 'insufficient-samples' : 'improvement' }; } function environmentWarnings(target: PerfSummary, baseline: PerfSummary[]): string[] { @@ -177,7 +215,8 @@ function environmentWarnings(target: PerfSummary, baseline: PerfSummary[]): stri * Compares one run against a rolling baseline of earlier runs. The baseline value of a step is the * median of its nightly medians, and its noise is the scaled MAD of those medians. A step is flagged * only when its change exceeds the relative floor, the night-to-night spread, tonight's own - * sampling error, and an absolute floor together. + * sampling error, and an absolute floor together. Tonight and every night that counts need + * `minSamples` successful samples. Failures are judged apart from timing. */ export function compareRuns( target: { summary: PerfSummary; label: string }, @@ -197,8 +236,11 @@ export function compareRuns( thresholds, ), ); - const counts: Partial> = {}; - for (const row of rows) counts[row.verdict] = (counts[row.verdict] ?? 0) + 1; + const counts: Comparison['counts'] = { timing: {}, failures: {} }; + for (const row of rows) { + counts.timing[row.timing] = (counts.timing[row.timing] ?? 0) + 1; + if (row.failures) counts.failures[row.failures] = (counts.failures[row.failures] ?? 0) + 1; + } return { schema: COMPARISON_SCHEMA, target: describeRun(target.summary, target.label), @@ -223,32 +265,45 @@ function pct(n: number | null | undefined, signed = false): string { return signed && n > 0 ? `+${value}` : value; } +function flagRank(row: ComparisonRow): number { + const ranks = [row.failures, row.timing].map((v) => FLAGGED_VERDICTS.indexOf(v as Flagged)); + const flagged = ranks.filter((rank) => rank >= 0); + return flagged.length === 0 ? -1 : Math.min(...flagged); +} + function tableRow(row: ComparisonRow): string { - const failures = row.target && row.target.failures > 0 ? ` (${row.target.failures} failed)` : ''; - return `| ${row.verdict} | \`${row.id}\` | ${row.metric} | ${ms(row.target?.median)}${failures} | ${ms(row.baseline?.median)} | ${ms(row.deltaMs)} | ${pct(row.deltaRelative, true)} | ${pct(row.threshold)} | ${row.baseline?.nights ?? 0} |`; + const failures = row.target ? `${row.target.failures}/${row.target.samples}` : '–'; + const nights = row.baseline ? `${row.baseline.timedNights}/${row.baseline.nights}` : '0'; + return `| ${row.timing} | ${row.failures ?? '–'} | \`${row.id}\` | ${row.metric} | ${ms(row.target?.median)} | ${ms(row.baseline?.median)} | ${ms(row.deltaMs)} | ${pct(row.deltaRelative, true)} | ${pct(row.threshold)} | ${failures} | ${nights} |`; } const TABLE_HEAD = [ - '| verdict | step | metric | tonight median | baseline median | Δ ms | Δ | threshold | nights |', - '|---|---|---|---|---|---|---|---|---|', + '| timing | failures | step | metric | tonight median | baseline median | Δ ms | Δ | threshold | failed tonight | timed nights |', + '|---|---|---|---|---|---|---|---|---|---|---|', ]; +function describeCounts(counts: Partial>): string { + return ( + Object.entries(counts) + .map(([verdict, n]) => `${n} ${verdict}`) + .join(', ') || 'none' + ); +} + /** Renders the comparison for the job summary and the artifact. */ export function renderComparison(comparison: Comparison): string { const t = comparison.thresholds; const flagged = comparison.rows - .filter((row) => FLAGGED_VERDICTS.includes(row.verdict)) - .sort((a, b) => FLAGGED_VERDICTS.indexOf(a.verdict) - FLAGGED_VERDICTS.indexOf(b.verdict)); - const counts = Object.entries(comparison.counts) - .map(([verdict, n]) => `${n} ${verdict}`) - .join(', '); + .filter((row) => flagRank(row) >= 0) + .sort((a, b) => flagRank(a) - flagRank(b)); const lines = [ `## agent-device perf comparison`, '', `- **Tonight**: ${comparison.target.label} · ${comparison.target.commit.slice(0, 12)} · ${comparison.target.startedAt} · ${comparison.target.device}`, `- **Baseline**: ${comparison.baseline.length === 0 ? 'none' : comparison.baseline.map((b) => `${b.label} (${b.commit.slice(0, 7)})`).join(', ')}`, - `- **Rule**: flag a step when its median moves by more than max(${pct(t.minRelative)}, ${t.spreadFactor}× night-to-night spread, ${t.withinFactor}× tonight cv/√n) and more than ${t.minAbsoluteMs} ms against the median of the baseline nights; at least ${t.minBaselineNights} nights are needed for a verdict.`, - `- **Steps**: ${counts || 'none'}`, + `- **Rule**: flag a step when its median moves by more than max(${pct(t.minRelative)}, ${t.spreadFactor}× night-to-night spread, ${t.withinFactor}× tonight cv/√n) and more than ${t.minAbsoluteMs} ms against the median of the baseline nights; a timing verdict needs ${t.minBaselineNights} nights, and tonight and each of those nights need ${t.minSamples} successful samples. Failures are flagged when tonight has more failed samples than every baseline night, in count and in share.`, + `- **Timing**: ${describeCounts(comparison.counts.timing)}`, + `- **Failures**: ${describeCounts(comparison.counts.failures)}`, '', ]; for (const warning of comparison.warnings) lines.push(`> ⚠️ ${warning}`, ''); diff --git a/scripts/perf/config.test.ts b/scripts/perf/config.test.ts new file mode 100644 index 0000000000..d8bb3b363a --- /dev/null +++ b/scripts/perf/config.test.ts @@ -0,0 +1,9 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { parseConfig } from './config.ts'; + +test('a run measures at least three rounds, so every step can get a timing verdict', () => { + assert.equal(parseConfig(['--udid', 'sim', '--rounds', '3']).rounds, 3); + assert.throws(() => parseConfig(['--udid', 'sim', '--rounds', '2']), /--rounds must be .* >= 3/); + assert.throws(() => parseConfig(['--platform', 'android', '--n', '1']), /--n must be .* >= 3/); +}); diff --git a/scripts/perf/config.ts b/scripts/perf/config.ts index 1db472fa88..32623a8ebf 100644 --- a/scripts/perf/config.ts +++ b/scripts/perf/config.ts @@ -23,6 +23,9 @@ export type PerfConfig = { export const DEFAULT_IOS_APP_ID = 'com.callstack.agentdevicelab'; +/** The fewest measured rounds that a run accepts; a comparison needs as many successful samples. */ +export const MIN_MEASURED_ROUNDS = 3; + // How to invoke the CLI. Defaults to the built dist binary (bin/agent-device.mjs). // Set AGENT_DEVICE_PERF_CLI to run from source instead, e.g. on CI: // AGENT_DEVICE_PERF_CLI="--experimental-strip-types src/bin.ts" @@ -55,7 +58,7 @@ function readIntValue(argv: string[], i: number, flag: string, min: number): num export function parseConfig(argv: string[]): PerfConfig { const cfg: PerfConfig = { platform: 'ios', - rounds: 3, + rounds: MIN_MEASURED_ROUNDS, warmup: 1, keepArtifacts: false, outDir: path.join(DEFAULT_OUT_DIR, new Date().toISOString().replaceAll(/[:.]/g, '-')), @@ -72,7 +75,7 @@ export function parseConfig(argv: string[]): PerfConfig { } case '--n': case '--rounds': - cfg.rounds = readIntValue(argv, i++, a, 1); + cfg.rounds = readIntValue(argv, i++, a, MIN_MEASURED_ROUNDS); break; case '--warmup': cfg.warmup = readIntValue(argv, i++, a, 0); diff --git a/scripts/perf/ios-harness.test.ts b/scripts/perf/ios-harness.test.ts new file mode 100644 index 0000000000..e8e88f5129 --- /dev/null +++ b/scripts/perf/ios-harness.test.ts @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { test, vi } from 'vitest'; + +// Stands in for the CLI: records each teardown call, and whether the logs were already saved, +// then appends to the daemon log and fails like a teardown that went wrong. +const FAKE_CLI = ` +import fs from 'node:fs'; +const args = process.argv.slice(2); +const stateDir = args[args.indexOf('--state-dir') + 1]; +const command = args[0] === 'daemon' ? 'daemon stop' : args[0]; +const saved = fs.existsSync(process.env.PERF_FAKE_SAVED_LOG); +fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' saved=' + saved + '\\n'); +fs.appendFileSync(stateDir + '/daemon.log', command + '\\n'); +process.exit(1); +`; + +test('finalizing a run saves the logs before and after a failing teardown, once', async () => { + const root = mkdtempForTestSync('perf-finalize-'); + const fakeCli = path.join(root, 'fake-cli.mjs'); + fs.writeFileSync(fakeCli, FAKE_CLI); + const stateDir = path.join(root, 'state'); + const rawDir = path.join(root, 'out', 'raw'); + fs.mkdirSync(path.join(stateDir, 'sessions', 'perf'), { recursive: true }); + fs.writeFileSync(path.join(stateDir, 'daemon.log'), 'suite\n'); + fs.writeFileSync(path.join(stateDir, 'sessions', 'perf', 'runner.log'), 'runner\n'); + fs.writeFileSync(path.join(stateDir, 'daemon.json'), '{"token":"secret"}'); + const calls = path.join(root, 'calls.log'); + vi.stubEnv('AGENT_DEVICE_PERF_CLI', fakeCli); + vi.stubEnv('PERF_FAKE_CALLS', calls); + vi.stubEnv('PERF_FAKE_SAVED_LOG', path.join(rawDir, 'state', 'daemon.log')); + // The CLI command is read when the harness loads, so load it after the stub. + const { iosRunFinalizer } = await import('./ios-harness.ts'); + + const finalize = iosRunFinalizer( + { stateDir, udid: 'sim', appId: 'app', artifactsDir: path.join(rawDir, 'artifacts'), rawDir }, + false, + ); + finalize(); + // The interrupt handler and the normal finish both call it; the second call does nothing. + finalize(); + + assert.equal(fs.readFileSync(calls, 'utf8'), 'close saved=true\ndaemon stop saved=true\n'); + const saved = path.join(rawDir, 'state'); + assert.equal( + fs.readFileSync(path.join(saved, 'daemon.log'), 'utf8'), + 'suite\nclose\ndaemon stop\n', + ); + assert.equal( + fs.readFileSync(path.join(saved, 'sessions', 'perf', 'runner.log'), 'utf8'), + 'runner\n', + ); + assert.equal(fs.existsSync(path.join(saved, 'daemon.json')), false); + assert.equal(fs.existsSync(stateDir), false); +}); diff --git a/scripts/perf/ios-harness.ts b/scripts/perf/ios-harness.ts index bcd2c3f4a6..16579f7506 100644 --- a/scripts/perf/ios-harness.ts +++ b/scripts/perf/ios-harness.ts @@ -282,8 +282,40 @@ export async function runIosSuite( ); } +// A stalled close or daemon stop must not keep the run, and its logs, from finishing. +const TEARDOWN_TIMEOUT_MS = 60_000; + +/** + * Returns the end of a run: save the logs, stop the session and the daemon within a deadline, save + * the logs again with the teardown, and remove the state directory unless it is kept. It runs once, + * so the interrupt handler and the normal finish can both call it. + */ +export function iosRunFinalizer(ctx: IosSuiteContext, keepArtifacts: boolean): () => void { + let finalized = false; + const saveLogs = (): void => { + try { + copyRawState(ctx); + } catch (error) { + log(`finalize: saving logs failed: ${(error as Error).message}`); + } + }; + const teardown = (args: string[], flags: string[]): void => { + const result = invokeCli(args, flags, { timeoutMs: TEARDOWN_TIMEOUT_MS }); + if (!result.ok) log(`finalize: ${args.join(' ')} failed: ${result.stderr.trim().slice(-400)}`); + }; + return () => { + if (finalized) return; + finalized = true; + saveLogs(); + teardown(['close'], ['--state-dir', ctx.stateDir, '--session', SESSION]); + teardown(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); + saveLogs(); + if (!keepArtifacts) fs.rmSync(ctx.stateDir, { recursive: true, force: true }); + }; +} + /** Copies daemon and session logs into the raw artifact; the daemon token file stays behind. */ -export function copyRawState(ctx: IosSuiteContext): void { +function copyRawState(ctx: IosSuiteContext): void { const target = path.join(ctx.rawDir, 'state'); fs.mkdirSync(target, { recursive: true }); const daemonLog = path.join(ctx.stateDir, 'daemon.log'); diff --git a/scripts/perf/nightly-artifacts.test.ts b/scripts/perf/nightly-artifacts.test.ts index 74671a936d..d302a99a94 100644 --- a/scripts/perf/nightly-artifacts.test.ts +++ b/scripts/perf/nightly-artifacts.test.ts @@ -1,19 +1,19 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { - baselineCandidates, coversTarget, resolveRunRef, + selectNightlyBaseline, selectRange, type WorkflowRun, } from './nightly-artifacts.ts'; import { summaryFixture } from './summary-fixture.ts'; -const run = (id: string, createdAt: string): WorkflowRun => ({ +const run = (id: string, createdAt: string, event = 'schedule'): WorkflowRun => ({ id, createdAt, headSha: `sha-${id}`, - event: 'schedule', + event, }); // Deliberately unsorted, with two runs on 2026-10-19. @@ -32,12 +32,55 @@ test('resolves latest, run ids, and dates to the newest matching run', () => { assert.throws(() => resolveRunRef('yesterday', RUNS), /Unknown run reference/); }); -test('baseline candidates are earlier runs, newest first, without excluded runs', () => { - const candidates = baselineCandidates(RUNS, '2026-10-20T03:00:00Z', new Set(['103'])); - assert.deepEqual( - candidates.map((r) => r.id), - ['102', '101'], - ); +const baselineIds = ( + runs: Iterable, + options: { nights?: number; exclude?: string[]; reject?: string[] } = {}, +) => + selectNightlyBaseline( + runs, + { + before: '2026-10-20T03:00:00Z', + exclude: new Set(options.exclude), + nights: options.nights ?? 3, + }, + (r) => (options.reject?.includes(r.id) ? undefined : r.id), + ).map(({ value }) => value); + +test('a rolling baseline takes one scheduled run per UTC night and skips dispatches', () => { + const newestFirst = [ + run('20', '2026-10-19T16:00:00Z', 'workflow_dispatch'), + run('19', '2026-10-19T15:00:00Z', 'workflow_dispatch'), + run('18', '2026-10-19T14:00:00Z', 'workflow_dispatch'), + run('17', '2026-10-19T02:50:00Z'), + run('16', '2026-10-19T02:40:00Z'), + run('15', '2026-10-18T02:40:00Z'), + run('14', '2026-10-17T02:40:00Z'), + ]; + // Three dispatches in one afternoon are not three nights; 2026-10-19 counts its newest run once. + assert.deepEqual(baselineIds(newestFirst), ['17', '15', '14']); +}); + +test('a night without a usable summary gives way to an older scheduled run of that night', () => { + const newestFirst = [ + run('17', '2026-10-19T02:50:00Z'), + run('16', '2026-10-19T02:40:00Z'), + run('15', '2026-10-18T02:40:00Z'), + run('14', '2026-10-17T02:40:00Z'), + ]; + assert.deepEqual(baselineIds(newestFirst, { reject: ['17'], exclude: ['15'] }), ['16', '14']); +}); + +test('a rolling baseline reads past 100 runs and stops once it has enough nights', () => { + function* history(): Generator { + for (let i = 0; i < 150; i++) { + yield run(`d${i}`, '2026-10-19T12:00:00Z', 'workflow_dispatch'); + } + yield run('n3', '2026-10-18T02:40:00Z'); + yield run('n2', '2026-10-17T02:40:00Z'); + yield run('n1', '2026-10-16T02:40:00Z'); + throw new Error('read past the last needed night'); + } + assert.deepEqual(baselineIds(history()), ['n3', 'n2', 'n1']); }); test('a range includes both ends whichever order they are given in', () => { diff --git a/scripts/perf/nightly-artifacts.ts b/scripts/perf/nightly-artifacts.ts index 08b00b7797..8f0525f077 100644 --- a/scripts/perf/nightly-artifacts.ts +++ b/scripts/perf/nightly-artifacts.ts @@ -30,39 +30,47 @@ function gh(args: string[]): { ok: boolean; stdout: string; stderr: string } { return { ok: result.exitCode === 0, stdout: result.stdout, stderr: result.stderr }; } -/** Lists the workflow's completed runs on one branch, newest first. */ -export function listRuns(source: NightlySource, limit = 100): WorkflowRun[] { - const result = gh([ - 'run', - 'list', - '--repo', - source.repo, - '--workflow', - source.workflow, - '--branch', - source.branch, - '--status', - 'completed', - '--limit', - String(limit), - '--json', - 'databaseId,createdAt,headSha,event', - ]); - if (!result.ok) throw new Error(`gh run list failed: ${result.stderr.trim()}`); - const rows = JSON.parse(result.stdout) as { - databaseId: number; - createdAt: string; - headSha: string; - event: string; - }[]; - return sortNewestFirst( - rows.map((row) => ({ - id: String(row.databaseId), - createdAt: row.createdAt, - headSha: row.headSha, - event: row.event, - })), - ); +const RUNS_PER_PAGE = 100; + +/** + * Yields the workflow's completed runs on one branch, newest first, one API page at a time. A + * caller that stops early fetches only the pages that it read. + */ +export function* listRuns(source: NightlySource): Generator { + const seen = new Set(); + for (let page = 1; ; page++) { + const result = gh([ + 'api', + '--method', + 'GET', + `repos/${source.repo}/actions/workflows/${source.workflow}/runs`, + '-f', + `branch=${source.branch}`, + '-f', + 'status=completed', + '-f', + `per_page=${RUNS_PER_PAGE}`, + '-f', + `page=${page}`, + '--jq', + '[.workflow_runs[] | {id, created_at, head_sha, event}]', + ]); + if (!result.ok) throw new Error(`gh api workflow runs failed: ${result.stderr.trim()}`); + const rows = JSON.parse(result.stdout) as { + id: number; + created_at: string; + head_sha: string; + event: string; + }[]; + for (const row of rows) { + const id = String(row.id); + // A run created during paging moves later pages by one place. + if (seen.has(id)) continue; + seen.add(id); + yield { id, createdAt: row.created_at, headSha: row.head_sha, event: row.event }; + } + if (rows.length < RUNS_PER_PAGE) return; + } } function sortNewestFirst(runs: WorkflowRun[]): WorkflowRun[] { @@ -112,13 +120,35 @@ export function coversTarget(candidate: PerfSummary, target: PerfSummary): boole ); } -/** Baseline candidates: runs created before the target, newest first, never the target itself. */ -export function baselineCandidates( - runs: WorkflowRun[], - before: string, - exclude: ReadonlySet, -): WorkflowRun[] { - return sortNewestFirst(runs).filter((run) => run.createdAt < before && !exclude.has(run.id)); +/** + * Selects the rolling baseline: one scheduled run for each UTC night before the target, newest night + * first. A night counts the newest of its scheduled runs that `load` accepts, so manual dispatches + * cannot fill the baseline with measurements from one afternoon. `runs` is newest first and is read + * only until `nights` nights are found. + */ +export function selectNightlyBaseline( + runs: Iterable, + options: { before: string; exclude: ReadonlySet; nights: number }, + load: (run: WorkflowRun) => T | undefined, +): { run: WorkflowRun; value: T }[] { + const selected: { run: WorkflowRun; value: T }[] = []; + const nights = new Set(); + for (const run of runs) { + const night = run.createdAt.slice(0, 10); + if ( + run.event !== 'schedule' || + run.createdAt >= options.before || + options.exclude.has(run.id) || + nights.has(night) + ) + continue; + const value = load(run); + if (value === undefined) continue; + nights.add(night); + selected.push({ run, value }); + if (selected.length >= options.nights) break; + } + return selected; } /** diff --git a/scripts/perf/run.ts b/scripts/perf/run.ts index 51cff7c18d..cd712056ae 100644 --- a/scripts/perf/run.ts +++ b/scripts/perf/run.ts @@ -1,10 +1,9 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { invokeCli } from './cli.ts'; import { parseConfig, REPO_ROOT, usesSourceCli, type PerfConfig } from './config.ts'; import { runScenario, setupIsolation, teardownIsolation } from './harness.ts'; -import { copyRawState, runIosSuite, type IosSuiteContext } from './ios-harness.ts'; +import { iosRunFinalizer, runIosSuite, type IosSuiteContext } from './ios-harness.ts'; import { describeCoverage, FIXTURE_APP_EVENT_URL_TEMPLATE, @@ -57,18 +56,13 @@ async function runIos( }; fs.mkdirSync(ctx.artifactsDir, { recursive: true }); log(`state-dir: ${ctx.stateDir}`); - const stop = (): void => { - invokeCli(['close'], ['--state-dir', ctx.stateDir, '--session', 'perf']); - invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); - }; - onInterrupt(stop); + const finalize = iosRunFinalizer(ctx, cfg.keepArtifacts); + onInterrupt(finalize); try { const measurements = await runIosSuite(ctx, cfg, scenarios); return { measurements, coverage: describeCoverage(scenarios, IOS_SUITE_EXCLUSIONS) }; } finally { - stop(); - copyRawState(ctx); - if (!cfg.keepArtifacts) fs.rmSync(ctx.stateDir, { recursive: true, force: true }); + finalize(); } } From 8958314a1624d0b28238c22dcd89920c6736d304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 10:42:45 +0200 Subject: [PATCH 10/15] fix(perf): pin the tree backend on a capture it serves, and share host readers - The forced-tree step ran a raw capture. The runner honors a backend pin only on the regular plan, so the raw plan recovered through private AX and the expectation passed a degraded row. The step is now a regular pinned capture on the home screen, and its expectation requires the pinned backend with the runner's `requested-backend` code. - Perf metadata reads the revision, toolchain, and simulator through the readers of the iOS snapshot benchmark, moved to scripts/lib/bench-host.ts. - host-kit and the daemon client export the phase names that the profiler reads. - summary-fixture.ts becomes summary.fixtures.ts; the nightly header matches 3 rounds. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/perf-nightly.yml | 6 +- packages/host-kit/src/command.ts | 1 + packages/host-kit/src/diagnostics.ts | 1 + packages/host-kit/src/internal/diagnostics.ts | 5 +- packages/host-kit/src/internal/exec-trace.ts | 5 +- packages/host-kit/src/internal/retry.ts | 5 +- packages/host-kit/src/retry.ts | 1 + .../benchmark-config.ts | 2 +- scripts/ios-snapshot-benchmark/run.test.ts | 2 +- scripts/ios-snapshot-benchmark/run.ts | 2 +- scripts/ios-snapshot-benchmark/types.ts | 34 +---------- .../host.test.ts => lib/bench-host.test.ts} | 2 +- .../host.ts => lib/bench-host.ts} | 33 ++++++++++- scripts/perf/README.md | 4 +- scripts/perf/compare.test.ts | 2 +- scripts/perf/forced-backend.ts | 3 +- scripts/perf/ios-suite.test.ts | 26 +++++--- scripts/perf/ios-suite.ts | 35 +++++++---- scripts/perf/metadata.ts | 59 ++++++------------- scripts/perf/nightly-artifacts.test.ts | 2 +- scripts/perf/profile.test.ts | 24 +++++--- scripts/perf/profile.ts | 31 ++++++---- scripts/perf/report.ts | 4 +- ...summary-fixture.ts => summary.fixtures.ts} | 1 + scripts/perf/summary.test.ts | 2 +- scripts/perf/types.ts | 2 + src/daemon-client/daemon-client.ts | 5 +- 27 files changed, 163 insertions(+), 136 deletions(-) rename scripts/{ios-snapshot-benchmark/host.test.ts => lib/bench-host.test.ts} (93%) rename scripts/{ios-snapshot-benchmark/host.ts => lib/bench-host.ts} (88%) rename scripts/perf/{summary-fixture.ts => summary.fixtures.ts} (99%) diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml index f4d7cd4863..d6bfb51342 100644 --- a/.github/workflows/perf-nightly.yml +++ b/.github/workflows/perf-nightly.yml @@ -4,8 +4,10 @@ name: Perf Nightly # - Catches: a command whose daemon time moves beyond the noise of earlier nights, or that starts # failing. PR lanes check behavior, not time. # - Wanted for: tuning runner, snapshot, and settle paths while watching the effect. iOS medians -# swing up to +122% between single nights at n=5, so a step is judged against the median of up to -# seven earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. +# swing up to +122% between single nights, so a step is judged against the median of up to seven +# earlier nights and must clear their spread, its own sampling error, 20%, and 50 ms. A night +# runs 3 rounds, the minimum for a timing verdict, so one failed sample withholds that step's +# timing verdict. # - Cost: one macos-26 job of about 45 minutes a night; summaries are kept 90 days, raw logs 30. # - Delete or park when: no flagged change has been acted on for 60 nights. # Android stays report-only and runs only on request. diff --git a/packages/host-kit/src/command.ts b/packages/host-kit/src/command.ts index 73b8aeb4f3..c57e0157bd 100644 --- a/packages/host-kit/src/command.ts +++ b/packages/host-kit/src/command.ts @@ -26,3 +26,4 @@ export { withCommandHost, withoutCommandExecutorOverride, } from './internal/exec.ts'; +export { EXEC_COMMAND_PHASE } from './internal/exec-trace.ts'; diff --git a/packages/host-kit/src/diagnostics.ts b/packages/host-kit/src/diagnostics.ts index def5222700..3f69c72b4d 100644 --- a/packages/host-kit/src/diagnostics.ts +++ b/packages/host-kit/src/diagnostics.ts @@ -1,6 +1,7 @@ export { countDiagnosticEventsByPhase, createRequestId, + DIAGNOSTIC_STDERR_PREFIX, type DiagnosticEventInput, emitDiagnostic, flushDiagnosticsToSessionFile, diff --git a/packages/host-kit/src/internal/diagnostics.ts b/packages/host-kit/src/internal/diagnostics.ts index 8d26f95e5f..9855bec828 100644 --- a/packages/host-kit/src/internal/diagnostics.ts +++ b/packages/host-kit/src/internal/diagnostics.ts @@ -53,6 +53,9 @@ type DiagnosticsScope = DiagnosticsScopeOptions & { sortedSensitiveValues?: string[]; }; +/** Prefixes each diagnostic event that a `--debug` client without a log file writes to stderr. */ +export const DIAGNOSTIC_STDERR_PREFIX = '[agent-device][diag] '; + const diagnosticsStorage = new AsyncLocalStorage(); export function createRequestId(): string { @@ -164,7 +167,7 @@ export function emitDiagnostic(event: DiagnosticEventInput): void { appendDiagnosticLine(scope, scope.traceLogPath, fileLine); } if (scope.debug && !scope.logPath && !scope.traceLogPath) { - process.stderr.write(`[agent-device][diag] ${fileLine}`); + process.stderr.write(`${DIAGNOSTIC_STDERR_PREFIX}${fileLine}`); } } catch { // Best-effort diagnostics should not break request flow. diff --git a/packages/host-kit/src/internal/exec-trace.ts b/packages/host-kit/src/internal/exec-trace.ts index 3a0ca05526..9b1f516476 100644 --- a/packages/host-kit/src/internal/exec-trace.ts +++ b/packages/host-kit/src/internal/exec-trace.ts @@ -9,6 +9,9 @@ import { updateDiagnosticsScope, } from './diagnostics.ts'; +/** The diagnostic phase of each host process lifecycle event. */ +export const EXEC_COMMAND_PHASE = 'exec_command'; + export type ExecSettlementReason = | 'success' | 'exit_failure' @@ -52,7 +55,7 @@ export function createExecTraceContext( pid = data.pid ?? pid; emitDiagnostic({ level: 'debug', - phase: 'exec_command', + phase: EXEC_COMMAND_PHASE, durationMs: performance.now() - started, data: { execId, command, event, pid, timeoutMs, ...data }, }); diff --git a/packages/host-kit/src/internal/retry.ts b/packages/host-kit/src/internal/retry.ts index c636d985b8..82f50606c3 100644 --- a/packages/host-kit/src/internal/retry.ts +++ b/packages/host-kit/src/internal/retry.ts @@ -9,6 +9,9 @@ type RetryPolicy = { shouldRetry?: (error: unknown, attempt: number) => boolean; }; +/** The diagnostic phase of each retry attempt and outcome. */ +export const RETRY_PHASE = 'retry'; + /** * The read side of a deadline. Holders are typed against this rather than the class so a caller * can hand one a substitute clock; {@link Deadline} is the production implementation. @@ -281,7 +284,7 @@ export function sleep(ms: number, signal?: AbortSignal): Promise { function publishRetryEvent(event: RetryTelemetryEvent): void { emitDiagnostic({ level: event.event === 'attempt_failed' || event.event === 'exhausted' ? 'warn' : 'debug', - phase: 'retry', + phase: RETRY_PHASE, data: { ...event, }, diff --git a/packages/host-kit/src/retry.ts b/packages/host-kit/src/retry.ts index bfc36cfe26..83e9eb6946 100644 --- a/packages/host-kit/src/retry.ts +++ b/packages/host-kit/src/retry.ts @@ -5,4 +5,5 @@ export { pollUntilSettled, retryWithPolicy, sleep, + RETRY_PHASE, } from './internal/retry.ts'; diff --git a/scripts/ios-snapshot-benchmark/benchmark-config.ts b/scripts/ios-snapshot-benchmark/benchmark-config.ts index 777beb2c35..4293114545 100644 --- a/scripts/ios-snapshot-benchmark/benchmark-config.ts +++ b/scripts/ios-snapshot-benchmark/benchmark-config.ts @@ -1,6 +1,6 @@ import path from 'node:path'; import { parseLocalStates, parseRtt, parseSampleCount, parseScreenIds } from './definitions.ts'; -import { resolveRepoRoot } from './host.ts'; +import { resolveRepoRoot } from '../lib/bench-host.ts'; import { assertBenchmarkOwner, assertOwnedDerivedPath, diff --git a/scripts/ios-snapshot-benchmark/run.test.ts b/scripts/ios-snapshot-benchmark/run.test.ts index 4639d11b35..d17a60e820 100644 --- a/scripts/ios-snapshot-benchmark/run.test.ts +++ b/scripts/ios-snapshot-benchmark/run.test.ts @@ -4,7 +4,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; import { mkdtempForTest } from '@agent-device/test-kit'; -import { resolveRepoRoot } from './host.ts'; +import { resolveRepoRoot } from '../lib/bench-host.ts'; test('CLI refuses an unmarked existing root before any descendant can be cleared', async () => { const repoRoot = resolveRepoRoot(); diff --git a/scripts/ios-snapshot-benchmark/run.ts b/scripts/ios-snapshot-benchmark/run.ts index e312da6980..89eff16655 100644 --- a/scripts/ios-snapshot-benchmark/run.ts +++ b/scripts/ios-snapshot-benchmark/run.ts @@ -10,7 +10,7 @@ import { classifyFailure, formatCliFailure, openFixture, type CliContext } from import { BenchmarkControlError, runDeepButtonControls } from './deep-control.ts'; import { CONTRACT, screenFixture } from './definitions.ts'; import { deepButtonFixtureEvidence } from './deep-button.ts'; -import { readGitRevision, readHostIdentity, readTarget, readToolchain } from './host.ts'; +import { readGitRevision, readHostIdentity, readTarget, readToolchain } from '../lib/bench-host.ts'; import { BenchmarkCellAdmissionError, BenchmarkContentionError, diff --git a/scripts/ios-snapshot-benchmark/types.ts b/scripts/ios-snapshot-benchmark/types.ts index e807e819fe..0ebd2d6504 100644 --- a/scripts/ios-snapshot-benchmark/types.ts +++ b/scripts/ios-snapshot-benchmark/types.ts @@ -1,3 +1,5 @@ +import type { GitRevision, HostIdentity, Target, Toolchain } from '../lib/bench-host.ts'; + export const BENCHMARK_SCHEMA_VERSION = 'ios-snapshot-convergence.v1' as const; export const ISSUE_ID = '#2189'; export const PARENT_ISSUE_ID = '#2188'; @@ -95,38 +97,6 @@ export type Measurement = { failureCategories: Partial>; }; -export type Toolchain = { - node: string; - pnpm: string; - xcode: string; - simctl: string; - os: string; - arch: string; -}; - -export type HostIdentity = { - model: string; - modelIdentifier: string; - cpu: string; - cpuCores: number; -}; - -export type GitRevision = { - commit: string; - branch: string; - dirty: boolean; -}; - -export type Target = { - platform: 'ios'; - kind: 'simulator'; - udid: string; - name: string; - runtime: string; - appId: string; - appPath?: string; -}; - export type PackageSize = { status: 'measured' | 'not-run'; revision: string; diff --git a/scripts/ios-snapshot-benchmark/host.test.ts b/scripts/lib/bench-host.test.ts similarity index 93% rename from scripts/ios-snapshot-benchmark/host.test.ts rename to scripts/lib/bench-host.test.ts index b2d1124d16..1a9cdc75b4 100644 --- a/scripts/ios-snapshot-benchmark/host.test.ts +++ b/scripts/lib/bench-host.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { parseHardwareProfile } from './host.ts'; +import { parseHardwareProfile } from './bench-host.ts'; test('parses the Mac model and CPU identity from system_profiler output', () => { assert.deepEqual( diff --git a/scripts/ios-snapshot-benchmark/host.ts b/scripts/lib/bench-host.ts similarity index 88% rename from scripts/ios-snapshot-benchmark/host.ts rename to scripts/lib/bench-host.ts index 7df8217649..03a7b29c14 100644 --- a/scripts/ios-snapshot-benchmark/host.ts +++ b/scripts/lib/bench-host.ts @@ -1,7 +1,38 @@ import { execFileSync } from 'node:child_process'; import os from 'node:os'; import path from 'node:path'; -import type { GitRevision, HostIdentity, Target, Toolchain } from './types.ts'; + +export type Toolchain = { + node: string; + pnpm: string; + xcode: string; + simctl: string; + os: string; + arch: string; +}; + +export type HostIdentity = { + model: string; + modelIdentifier: string; + cpu: string; + cpuCores: number; +}; + +export type GitRevision = { + commit: string; + branch: string; + dirty: boolean; +}; + +export type Target = { + platform: 'ios'; + kind: 'simulator'; + udid: string; + name: string; + runtime: string; + appId: string; + appPath?: string; +}; export function readToolchain(): Toolchain { return { diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 0715aa7061..b53c2e1725 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -36,7 +36,7 @@ All times are milliseconds. Statistics use only the successful samples of a step | Metric | Meaning | | --- | --- | | `wall` | The CLI process time, from spawn to exit. | -| `daemon` | The daemon request round trip that the client measures (`daemon_request`). For the pinned-backend capture, the span of its request log. Comparisons use this metric. | +| `daemon` | The daemon request round trip that the client measures (`daemon_request`). For the pinned-backend capture (`home-reads/snapshot-forced-tree`, marked SDK in the report), the span of its request log. Comparisons use this metric. | | `mean ± sd`, `cv` | The mean, the sample standard deviation, and the standard deviation divided by the mean. | | `median`, `p95`, `min`, `max` | The median (the mean of the two middle values for an even count), the nearest-rank 95th percentile, and the extremes. | | `runner` | The median sum of XCTest runner command times in one request. | @@ -49,7 +49,7 @@ All times are milliseconds. Statistics use only the successful samples of a step | `producer` | The number of samples that each snapshot producer answered (`simulator-ax-bridge`, `apple-runner`). | | backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected`. | | fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. | -| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth. | +| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth, or that a backend other than the pinned one served the pinned capture. | ## Nightly results diff --git a/scripts/perf/compare.test.ts b/scripts/perf/compare.test.ts index 51fb75d55c..79aec9562d 100644 --- a/scripts/perf/compare.test.ts +++ b/scripts/perf/compare.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { compareRuns, type ComparisonRow } from './compare.ts'; -import { summaryFixture, type StepFixture } from './summary-fixture.ts'; +import { summaryFixture, type StepFixture } from './summary.fixtures.ts'; const STEP = 'heavy-feed/snapshot'; diff --git a/scripts/perf/forced-backend.ts b/scripts/perf/forced-backend.ts index 387b22fea2..b72619b3e3 100644 --- a/scripts/perf/forced-backend.ts +++ b/scripts/perf/forced-backend.ts @@ -16,7 +16,7 @@ type AgentDeviceSdk = typeof import('../../src/sdk/index.ts'); const FORCED_CAPTURE_TIMEOUT_MS = 90_000; /** - * Captures one raw snapshot with the internal backend pin that the CLI does not expose. The + * Captures one snapshot with the internal backend pin that the CLI does not expose. The * transport talks to the daemon that the built CLI already started and never starts or replaces * it, so the measured daemon stays the daemon under test. */ @@ -37,7 +37,6 @@ export async function captureWithForcedBackend(options: { const t0 = performance.now(); try { const snapshot = await client.capture.snapshot({ - raw: true, platform: 'ios', udid: options.udid, timeoutMs: FORCED_CAPTURE_TIMEOUT_MS, diff --git a/scripts/perf/ios-suite.test.ts b/scripts/perf/ios-suite.test.ts index cc92840b1e..e9d58ece1f 100644 --- a/scripts/perf/ios-suite.test.ts +++ b/scripts/perf/ios-suite.test.ts @@ -20,20 +20,30 @@ test('step ids are unique, so comparisons key each step once', () => { assert.equal(new Set(ids).size, ids.length); }); -test('a heavy-feed capture breaks its expectation when the tree flattens or the tree backend copes', () => { +test('a heavy-feed capture breaks its expectation when the tree flattens', () => { const depth = { maxDepthAtLeast: 64 }; assert.equal(expectationViolation(depth, { maxDepth: 64 }), undefined); assert.match(expectationViolation(depth, { maxDepth: 63 })!, /below 64/); assert.match(expectationViolation(depth, undefined)!, /unknown/); +}); - const degraded = { treeBackendDegraded: true }; - assert.equal( - expectationViolation(degraded, { backend: 'private-ax', state: 'recovered' }), - undefined, +test('a pinned capture breaks its expectation unless the pinned backend served it', () => { + const pinned = { pinnedBackend: 'tree' } as const; + const served = { backend: 'tree', state: 'healthy', reasonCode: 'requested-backend' }; + assert.equal(expectationViolation(pinned, served), undefined); + // The raw plan ignores the pin and recovers through private AX on the heavy feed. + assert.match( + expectationViolation(pinned, { + backend: 'private-ax', + state: 'recovered', + reasonCode: 'ax-rejected', + })!, + /private-ax backend \(ax-rejected\)/, ); - assert.equal(expectationViolation(degraded, { backend: 'tree', state: 'sparse' }), undefined); + // An unpinned capture can still be tree-first on a light screen. assert.match( - expectationViolation(degraded, { backend: 'tree', state: 'healthy' })!, - /non-sparse/, + expectationViolation(pinned, { backend: 'tree', state: 'healthy' })!, + /no reason code/, ); + assert.match(expectationViolation(pinned, undefined)!, /unknown backend/); }); diff --git a/scripts/perf/ios-suite.ts b/scripts/perf/ios-suite.ts index bde981db05..26607307c2 100644 --- a/scripts/perf/ios-suite.ts +++ b/scripts/perf/ios-suite.ts @@ -1,4 +1,8 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { + SnapshotPreferredBackend, + SnapshotQualityVerdict, +} from '@agent-device/kernel/snapshot'; import type { SnapshotFacts } from './types.ts'; /** Routes `trigger-app-event` into the fixture's automation screen. */ @@ -23,8 +27,8 @@ export type SuiteStep = { args: Cli; before?: readonly Cli[]; after?: readonly Cli[]; - // `sdk-forced-tree` captures a raw snapshot through the daemon with the XCTest tree backend - // pinned; the CLI has no flag for it. + // `sdk-forced-tree` captures a snapshot through the daemon with the XCTest tree backend pinned; + // the CLI has no flag for it. via?: 'cli' | 'sdk-forced-tree'; // A sample that succeeds but breaks the expectation fails with `expectation_failed`, so a // fixture that stops reproducing its shape shows up as a new failure. @@ -34,10 +38,13 @@ export type SuiteStep = { export type StepExpectation = { // Deepest raw node depth, counted from the application at 0. maxDepthAtLeast?: number; - // The pinned XCTest tree backend did not serve a healthy capture. - treeBackendDegraded?: boolean; + // The pinned backend that must serve the capture. The runner marks a pinned capture with the + // reason code `requested-backend`; any other backend or code means the pin was not honored. + pinnedBackend?: SnapshotPreferredBackend; }; +const PINNED_REASON_CODE: SnapshotQualityVerdict['reasonCode'] = 'requested-backend'; + /** Describes how a snapshot breaks the step expectation, or returns undefined. */ export function expectationViolation( expect: StepExpectation | undefined, @@ -47,8 +54,11 @@ export function expectationViolation( if (expect.maxDepthAtLeast !== undefined && (snapshot?.maxDepth ?? -1) < expect.maxDepthAtLeast) { return `max depth ${snapshot?.maxDepth ?? 'unknown'} is below ${expect.maxDepthAtLeast}`; } - if (expect.treeBackendDegraded && snapshot?.backend === 'tree' && snapshot.state !== 'sparse') { - return 'the pinned tree backend served a non-sparse capture'; + if ( + expect.pinnedBackend !== undefined && + (snapshot?.backend !== expect.pinnedBackend || snapshot.reasonCode !== PINNED_REASON_CODE) + ) { + return `the ${snapshot?.backend ?? 'unknown'} backend (${snapshot?.reasonCode ?? 'no reason code'}) served the capture, not the pinned ${expect.pinnedBackend}`; } return undefined; } @@ -130,6 +140,12 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('snapshot-i', C.snapshot, ['snapshot', '-i']), step('snapshot', C.snapshot, ['snapshot']), step('snapshot-raw', C.snapshot, ['snapshot', '--raw']), + // Only the regular capture plan honors the pin; the raw plan runs its own backend ladder. The + // XCTest tree rejects the heavy feed, so the pinned capture runs on this light screen. + step('snapshot-forced-tree', C.snapshot, ['snapshot'], { + via: 'sdk-forced-tree', + expect: { pinnedBackend: 'tree' }, + }), step('snapshot-diff', C.snapshot, ['snapshot', '--diff']), step('diff-snapshot', C.diff, ['diff', 'snapshot']), step('wait-text', C.wait, ['wait', 'text', 'Agent Device Tester']), @@ -319,13 +335,6 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ step('snapshot-i-scrolled', C.snapshot, ['snapshot', '-i']), step('scroll-down-settle', C.scroll, ['scroll', 'down', '--settle']), step('swipe', C.swipe, ['swipe', '200', '650', '200', '300']), - // The pinned XCTest capture outlives its watchdog by about a second, and the next runner - // command refuses until it ends; the untimed wait keeps that cost out of the next step. - step('snapshot-raw-forced-tree', C.snapshot, ['snapshot', '--raw'], { - via: 'sdk-forced-tree', - expect: { treeBackendDegraded: true }, - after: [['wait', '2000']], - }), step('back', C.back, ['back']), ], }, diff --git a/scripts/perf/metadata.ts b/scripts/perf/metadata.ts index 6bb4e7d861..47f25b9098 100644 --- a/scripts/perf/metadata.ts +++ b/scripts/perf/metadata.ts @@ -1,19 +1,10 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { runCmdSync } from '@agent-device/host-kit/command'; +import { readGitRevision, readTarget, readToolchain } from '../lib/bench-host.ts'; import { REPO_ROOT, type PerfConfig } from './config.ts'; import type { RunMetadata } from './types.ts'; -function capture(cmd: string, args: string[]): string | undefined { - try { - const result = runCmdSync(cmd, args, { cwd: REPO_ROOT, allowFailure: true }); - return result.exitCode === 0 ? result.stdout.trim() : undefined; - } catch { - return undefined; - } -} - function readVersion(): string { try { const pkg = JSON.parse(fs.readFileSync(path.join(REPO_ROOT, 'package.json'), 'utf8')); @@ -23,47 +14,31 @@ function readVersion(): string { } } -function simulator(udid: string): { name: string; runtime?: string } | undefined { - const raw = capture('xcrun', ['simctl', 'list', 'devices', '--json']); - if (!raw) return undefined; - const devices = (JSON.parse(raw) as { devices: Record }) - .devices; - for (const [runtime, list] of Object.entries(devices)) { - const device = list.find((entry) => entry.udid === udid); - if (device) return { name: device.name, runtime: runtime.split('.').pop() }; - } - return undefined; -} - /** Records what the numbers depend on: source, toolchain, host, and device. */ export function collectMetadata(cfg: PerfConfig, deviceName: string): RunMetadata { - const env = process.env; - const sim = cfg.platform === 'ios' && cfg.udid ? simulator(cfg.udid) : undefined; + const revision = readGitRevision(REPO_ROOT); + const toolchain = readToolchain(); + const target = cfg.platform === 'ios' ? readTarget(cfg.udid!, cfg.appId, cfg.appPath) : undefined; const cpus = os.cpus(); return { - commit: env.GITHUB_SHA ?? capture('git', ['rev-parse', 'HEAD']) ?? 'unknown', - ref: env.GITHUB_REF_NAME ?? capture('git', ['rev-parse', '--abbrev-ref', 'HEAD']), - runId: env.GITHUB_RUN_ID, - runAttempt: env.GITHUB_RUN_ATTEMPT, + commit: revision.commit, + ref: revision.branch, + dirty: revision.dirty, + runId: process.env.GITHUB_RUN_ID, + runAttempt: process.env.GITHUB_RUN_ATTEMPT, agentDeviceVersion: readVersion(), host: { - os: `${os.type()} ${os.release()}`, - arch: os.arch(), + os: toolchain.os, + arch: toolchain.arch, cpu: cpus[0]?.model ?? 'unknown', cores: cpus.length, memoryGb: Math.round(os.totalmem() / 1024 ** 3), }, - xcode: - cfg.platform === 'ios' - ? capture('xcodebuild', ['-version'])?.replaceAll('\n', ' ') - : undefined, - node: process.version, - device: { - name: sim?.name ?? deviceName, - udid: cfg.udid, - serial: cfg.serial, - runtime: sim?.runtime, - }, - fixture: cfg.platform === 'ios' ? { appId: cfg.appId, appPath: cfg.appPath } : undefined, + xcode: target ? toolchain.xcode.replaceAll('\n', ' ') : undefined, + node: toolchain.node, + device: target + ? { name: target.name, udid: target.udid, runtime: target.runtime.split('.').pop() } + : { name: deviceName, serial: cfg.serial }, + fixture: target ? { appId: target.appId, appPath: target.appPath } : undefined, }; } diff --git a/scripts/perf/nightly-artifacts.test.ts b/scripts/perf/nightly-artifacts.test.ts index d302a99a94..34a1aac17a 100644 --- a/scripts/perf/nightly-artifacts.test.ts +++ b/scripts/perf/nightly-artifacts.test.ts @@ -7,7 +7,7 @@ import { selectRange, type WorkflowRun, } from './nightly-artifacts.ts'; -import { summaryFixture } from './summary-fixture.ts'; +import { summaryFixture } from './summary.fixtures.ts'; const run = (id: string, createdAt: string, event = 'schedule'): WorkflowRun => ({ id, diff --git a/scripts/perf/profile.test.ts b/scripts/perf/profile.test.ts index 91e16e4627..77f0a03a43 100644 --- a/scripts/perf/profile.test.ts +++ b/scripts/perf/profile.test.ts @@ -1,5 +1,9 @@ import assert from 'node:assert/strict'; +import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; +import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; +import { RETRY_PHASE } from '@agent-device/host-kit/retry'; import { test } from 'vitest'; +import { DAEMON_REQUEST_PHASE } from '../../src/daemon-client/daemon-client.ts'; import { aggregateProfiles, profileRequestLog, @@ -14,13 +18,17 @@ const line = (event: Record) => JSON.stringify({ level: 'info', const PRESS_LOG = [ line({ ts: '2026-10-21T19:00:00.000Z', phase: 'request_start', data: { command: 'press' } }), line({ - phase: 'exec_command', + phase: EXEC_COMMAND_PHASE, durationMs: 0.02, data: { command: 'xcrun', event: 'spawn_start' }, }), - line({ phase: 'exec_command', durationMs: 9, data: { command: 'xcrun', event: 'spawn' } }), - line({ phase: 'exec_command', durationMs: 120, data: { command: 'xcrun', event: 'exit' } }), - line({ phase: 'exec_command', durationMs: 121, data: { command: 'xcrun', event: 'settled' } }), + line({ phase: EXEC_COMMAND_PHASE, durationMs: 9, data: { command: 'xcrun', event: 'spawn' } }), + line({ phase: EXEC_COMMAND_PHASE, durationMs: 120, data: { command: 'xcrun', event: 'exit' } }), + line({ + phase: EXEC_COMMAND_PHASE, + durationMs: 121, + data: { command: 'xcrun', event: 'settled' }, + }), line({ level: 'error', phase: 'ios.snapshot-source.acquire', @@ -36,7 +44,7 @@ const PRESS_LOG = [ phase: 'ios_snapshot_source_recovery', data: { producer: 'simulator-ax-bridge', truncated: true, acceptedLevels: 32 }, }), - line({ phase: 'retry', data: { event: 'attempt_failed', attempt: 1 } }), + line({ phase: RETRY_PHASE, data: { event: 'attempt_failed', attempt: 1 } }), line({ phase: 'ios_runner_command_send', durationMs: 900, data: { command: 'tap' } }), line({ phase: 'post_gesture_snapshot_stabilized', data: { attempts: 2, durationMs: 300 } }), line({ @@ -95,12 +103,12 @@ test('counts a producer only for samples where it answered', () => { test('reads the client daemon timer from --debug stderr', () => { const stderr = [ - '[agent-device][diag] {"phase":"daemon_startup","durationMs":30,"requestId":"r1"}', + `${DIAGNOSTIC_STDERR_PREFIX}{"phase":"daemon_startup","durationMs":30,"requestId":"r1"}`, 'noise', - '[agent-device][diag] {"phase":"daemon_request","durationMs":812,"requestId":"r1"}', + `${DIAGNOSTIC_STDERR_PREFIX}{"phase":"${DAEMON_REQUEST_PHASE}","durationMs":812,"requestId":"r1"}`, ].join('\n'); assert.deepEqual(readClientDiagnostics(stderr), { daemonMs: 812, requestId: 'r1' }); - assert.deepEqual(readClientDiagnostics('[agent-device][diag] {"phase":"daemon_req'), {}); + assert.deepEqual(readClientDiagnostics(`${DIAGNOSTIC_STDERR_PREFIX}{"phase":"daemon_req`), {}); }); test('reads the settle wait of a --settle response and nothing from other responses', () => { diff --git a/scripts/perf/profile.ts b/scripts/perf/profile.ts index a869d74c32..b67314ea3f 100644 --- a/scripts/perf/profile.ts +++ b/scripts/perf/profile.ts @@ -1,3 +1,7 @@ +import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; +import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; +import { RETRY_PHASE } from '@agent-device/host-kit/retry'; +import { DAEMON_REQUEST_PHASE } from '../../src/daemon-client/daemon-client.ts'; import { median } from './stats.ts'; import type { MeasurementProfile, @@ -8,6 +12,12 @@ import type { Timing, } from './types.ts'; +// The platform-apple façades export only what production code consumes, so its phase names stay +// literal here. +const RUNNER_COMMAND_SEND_PHASE = 'ios_runner_command_send'; +const SNAPSHOT_SOURCE_ACQUIRE_PHASE = 'ios.snapshot-source.acquire'; +const SNAPSHOT_SOURCE_RECOVERY_PHASE = 'ios_snapshot_source_recovery'; + type DiagnosticEvent = { ts?: string; level?: string; @@ -16,13 +26,8 @@ type DiagnosticEvent = { data?: Record; }; -const RUNNER_COMMAND_PHASE = 'ios_runner_command_send'; -const SPAWN_PHASE = 'exec_command'; -const SNAPSHOT_SOURCE_PHASE = 'ios.snapshot-source.acquire'; -const SNAPSHOT_RECOVERY_PHASE = 'ios_snapshot_source_recovery'; const STABILIZATION_PHASE = /settle|stabiliz/i; const FALLBACK_TEXT_LIMIT = 140; -const CLIENT_DIAG_PREFIX = '[agent-device][diag] '; function addTiming(map: Record, key: string, durationMs: number | undefined): void { const entry = (map[key] ??= { count: 0, totalMs: 0 }); @@ -77,12 +82,12 @@ export function profileRequestLog(ndjson: string): SampleProfile { } const data = event.data ?? {}; const duration = eventDuration(event); - if (phase === SPAWN_PHASE) { + if (phase === EXEC_COMMAND_PHASE) { if (data.event !== 'settled') continue; addTiming(profile.spawns, text(data.command) ?? 'unknown', duration); - } else if (phase === RUNNER_COMMAND_PHASE) { + } else if (phase === RUNNER_COMMAND_SEND_PHASE) { addTiming(profile.runnerCommands, text(data.command) ?? 'unknown', duration); - } else if (phase === SNAPSHOT_SOURCE_PHASE) { + } else if (phase === SNAPSHOT_SOURCE_ACQUIRE_PHASE) { const producer = text(data.producer) ?? 'unknown'; const entry = (profile.snapshotSources[producer] ??= { count: 0, totalMs: 0, failures: 0 }); entry.count += 1; @@ -92,11 +97,11 @@ export function profileRequestLog(ndjson: string): SampleProfile { entry.failures += 1; fallbacks.add(`${producer}: ${error ?? 'failed'}`); } - } else if (phase === SNAPSHOT_RECOVERY_PHASE && data.truncated === true) { + } else if (phase === SNAPSHOT_SOURCE_RECOVERY_PHASE && data.truncated === true) { fallbacks.add( `${text(data.producer) ?? 'unknown'}: truncated at ${String(data.acceptedLevels)} levels`, ); - } else if (phase === 'retry' && data.event === 'attempt_failed') { + } else if (phase === RETRY_PHASE && data.event === 'attempt_failed') { profile.retries += 1; } else if (event.level === 'warn' || event.level === 'error') { fallbacks.add(`${phase}${text(data.reason) ? `: ${text(data.reason)}` : ''}`); @@ -115,15 +120,15 @@ export function profileRequestLog(ndjson: string): SampleProfile { /** Reads the client's `daemon_request` timer and request id from `--debug` stderr. */ export function readClientDiagnostics(stderr: string): { daemonMs?: number; requestId?: string } { for (const line of stderr.split('\n')) { - const start = line.indexOf(CLIENT_DIAG_PREFIX); + const start = line.indexOf(DIAGNOSTIC_STDERR_PREFIX); if (start < 0) continue; try { - const event = JSON.parse(line.slice(start + CLIENT_DIAG_PREFIX.length)) as { + const event = JSON.parse(line.slice(start + DIAGNOSTIC_STDERR_PREFIX.length)) as { phase?: string; durationMs?: number; requestId?: string; }; - if (event.phase === 'daemon_request') { + if (event.phase === DAEMON_REQUEST_PHASE) { return { daemonMs: event.durationMs, requestId: event.requestId }; } } catch { diff --git a/scripts/perf/report.ts b/scripts/perf/report.ts index 480978638d..8ff0e69b35 100644 --- a/scripts/perf/report.ts +++ b/scripts/perf/report.ts @@ -30,7 +30,7 @@ function cell(text: string): string { function timingRow(m: Measurement): string { return [ m.id.slice(m.scenario.length + 1), - `\`${cell(m.label)}\``, + `\`${cell(m.label)}\`${m.execMode === 'sdk' ? ' (SDK, in process; daemon = log span)' : ''}`, String(m.samples.length), wallCells(m.wallClock), ms(m.daemon?.median), @@ -59,7 +59,7 @@ export function renderReport(summary: PerfSummary): string { const lines: string[] = [ `# agent-device perf — ${summary.platform}`, '', - `- **Commit**: ${meta.commit.slice(0, 12)}${meta.ref ? ` (${meta.ref})` : ''}${meta.runId ? ` · run ${meta.runId}` : ''}`, + `- **Commit**: ${meta.commit.slice(0, 12)}${meta.dirty ? ' (dirty)' : ''}${meta.ref ? ` (${meta.ref})` : ''}${meta.runId ? ` · run ${meta.runId}` : ''}`, `- **Device**: ${meta.device.name}${meta.device.runtime ? ` · ${meta.device.runtime}` : ''}`, `- **Toolchain**: ${meta.xcode ?? 'no Xcode'} · Node ${meta.node} · agent-device ${meta.agentDeviceVersion}`, `- **Host**: ${meta.host.os} · ${meta.host.cpu} · ${meta.host.cores} cores · ${meta.host.memoryGb} GB`, diff --git a/scripts/perf/summary-fixture.ts b/scripts/perf/summary.fixtures.ts similarity index 99% rename from scripts/perf/summary-fixture.ts rename to scripts/perf/summary.fixtures.ts index b86ad31ef8..5ad12534e6 100644 --- a/scripts/perf/summary-fixture.ts +++ b/scripts/perf/summary.fixtures.ts @@ -51,6 +51,7 @@ export function summaryFixture( }, metadata: { commit: '0123456789abcdef', + dirty: false, runId: overrides.runId, agentDeviceVersion: '0.0.0', host: { os: 'Darwin', arch: 'arm64', cpu: 'Apple M1 (Virtual)', cores: 3, memoryGb: 7 }, diff --git a/scripts/perf/summary.test.ts b/scripts/perf/summary.test.ts index 421f234f17..70d199ef9e 100644 --- a/scripts/perf/summary.test.ts +++ b/scripts/perf/summary.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { parsePerfSummary } from './summary.ts'; -import { summaryFixture } from './summary-fixture.ts'; +import { summaryFixture } from './summary.fixtures.ts'; const valid = () => JSON.parse(JSON.stringify(summaryFixture([{ id: 's/step', daemonMs: [10, 20] }]))) as Record< diff --git a/scripts/perf/types.ts b/scripts/perf/types.ts index a3cc091be5..cfb14ffd2f 100644 --- a/scripts/perf/types.ts +++ b/scripts/perf/types.ts @@ -102,6 +102,8 @@ export type Measurement = { export type RunMetadata = { commit: string; ref?: string; + // Uncommitted changes in the checkout at the start of the run. + dirty: boolean; runId?: string; runAttempt?: string; agentDeviceVersion: string; diff --git a/src/daemon-client/daemon-client.ts b/src/daemon-client/daemon-client.ts index ddbe6e9611..bdd823b1f9 100644 --- a/src/daemon-client/daemon-client.ts +++ b/src/daemon-client/daemon-client.ts @@ -23,6 +23,9 @@ import { createRequestGuard, sendRequest } from './daemon-client-transport.ts'; import { isRemoteDaemon, type DaemonInfo } from './daemon-client-metadata.ts'; import { leaseScopeFromRequest } from '@agent-device/contracts/lease-scope'; +/** The client diagnostic phase that times one daemon request round trip. */ +export const DAEMON_REQUEST_PHASE = 'daemon_request'; + export type DaemonRequest = DaemonWireRequest; export type DaemonResponse = SharedDaemonResponse; type DaemonTransportOptions = AgentDeviceDaemonTransportContext & { @@ -138,7 +141,7 @@ async function sendToDaemonOnHost( settings, async () => { const response = await withDiagnosticTimer( - 'daemon_request', + DAEMON_REQUEST_PHASE, async () => await sendRequest( info, From 70faef463ce4bd560be72371359fe5741dc43eb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 10:48:36 +0200 Subject: [PATCH 11/15] fix(perf): read Android phase names from their emitters, and document the pinned capture - platform-android exports the helper capture and gesture phase names through its mechanics entry; the profiler and its test import them. - The README's snapshot-path section and parity table name the iOS pinned capture on home-reads and the reason code it must carry. Co-Authored-By: Claude Opus 5.5 --- packages/platform-android/src/mechanics.ts | 8 +++++++- packages/platform-android/src/snapshot.ts | 10 ++++++++-- packages/platform-android/src/touch-helper.ts | 4 +++- scripts/perf/README.md | 6 +++--- scripts/perf/metadata.ts | 3 +-- scripts/perf/profile.test.ts | 15 ++++++++++----- scripts/perf/profile.ts | 14 ++++++++------ 7 files changed, 40 insertions(+), 20 deletions(-) diff --git a/packages/platform-android/src/mechanics.ts b/packages/platform-android/src/mechanics.ts index 993c017204..f4098ec4a6 100644 --- a/packages/platform-android/src/mechanics.ts +++ b/packages/platform-android/src/mechanics.ts @@ -308,7 +308,12 @@ export { resetAndroidSnapshotHelperRuntime, retireAndroidSnapshotHelperAfterContentFailure, } from './snapshot-helper-runtime.ts'; -export { snapshotAndroid, type AndroidSnapshotOptions } from './snapshot.ts'; +export { + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, + snapshotAndroid, + type AndroidSnapshotOptions, +} from './snapshot.ts'; export { screenshotAndroid, type AndroidScreenshotOptions } from './screenshot.ts'; export { ANDROID_SNAPSHOT_PRESENTATION_DEFAULT_DEADLINE_MS, @@ -337,6 +342,7 @@ export { ensureAndroidSdkPathConfigured } from './sdk.ts'; export { fillAndroid, typeAndroid } from './text-input.ts'; export { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts'; export { + ANDROID_TOUCH_HELPER_GESTURE_PHASE, ANDROID_TOUCH_PLAN_PROTOCOL, executeAndroidTouchHelperPlan, normalizeAndroidTouchHelperGestureRequest, diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index ddbf8f47ec..4705384cad 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -102,6 +102,12 @@ const HELPER_CONTENT_RECAPTURE_DELAY_MS = 250; * a busy screen run a snapshot past the daemon request envelope. */ const HELPER_CONTENT_RECAPTURE_WINDOW_MS = 10_000; +/** The diagnostic phase of a capture through the snapshot helper's persistent session. */ +export const ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE = + 'android_snapshot_helper_session_capture'; +/** The diagnostic phase of a one-shot `am instrument` snapshot helper capture. */ +export const ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE = 'android_snapshot_helper_capture'; + export type AndroidSnapshotOptions = SnapshotOptions & { idleWait?: AndroidSnapshotHelperCaptureOptions['idleWait']; appBundleId?: string; @@ -465,7 +471,7 @@ async function captureAndroidUiHierarchyFromHelper(params: { captureOptions.idleWait = params.idleWait; try { const sessionCapture = await withDiagnosticTimer( - 'android_snapshot_helper_session_capture', + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, async () => await captureAndroidSnapshotWithHelperSession(captureOptions), { packageName: artifact.manifest.packageName, @@ -487,7 +493,7 @@ async function captureAndroidUiHierarchyFromHelper(params: { await resetAndroidSnapshotHelperRuntime(adb, artifact.manifest.packageName); } return await withDiagnosticTimer( - 'android_snapshot_helper_capture', + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, async () => await captureAndroidSnapshotWithHelper(captureOptions), { packageName: artifact.manifest.packageName, diff --git a/packages/platform-android/src/touch-helper.ts b/packages/platform-android/src/touch-helper.ts index 3ad1c7b616..196d0e136a 100644 --- a/packages/platform-android/src/touch-helper.ts +++ b/packages/platform-android/src/touch-helper.ts @@ -33,6 +33,8 @@ import { } from './snapshot-helper-types.ts'; export const ANDROID_TOUCH_PLAN_PROTOCOL = 'android-touch-plan-v1'; +/** The diagnostic phase of one gesture that the snapshot helper injects. */ +export const ANDROID_TOUCH_HELPER_GESTURE_PHASE = 'android_touch_helper_gesture'; const HELPER_GESTURE_TIMEOUT_MS = 45_000; const HELPER_GESTURE_TIMEOUT_OVERHEAD_MS = 15_000; const HELPER_VIEWPORT_TIMEOUT_MS = 45_000; @@ -67,7 +69,7 @@ export async function executeAndroidTouchHelperPlan( JSON.stringify({ protocol: ANDROID_TOUCH_PLAN_PROTOCOL, ...request }), ).toString('base64'); const output = await withDiagnosticTimer( - 'android_touch_helper_gesture', + ANDROID_TOUCH_HELPER_GESTURE_PHASE, async () => { // A live snapshot session already owns UiAutomation; injecting through it avoids the // instrumentation stop/restart that previously taxed every gesture. Gesture session errors diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 2abda4e587..b31b0a21f2 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -41,9 +41,9 @@ At the end, or on SIGINT or SIGTERM, the suite saves the logs to `raw/state/`, s ## Snapshot paths -The heavy-feed scenario times the normal snapshot path and one forced path on each platform. +Each platform times the normal snapshot path and one forced path. -- iOS: the simulator accessibility bridge answers normal captures. `snapshot-raw-forced-tree` pins the XCTest tree backend through the SDK, because the CLI has no flag for it. On this screen, the tree backend must degrade to `sparse`. +- iOS: the simulator accessibility bridge answers normal captures. `home-reads/snapshot-forced-tree` pins the XCTest tree backend through the SDK, because the CLI has no flag for it. Only a regular capture honors the pin; a raw capture runs its own backend ladder. The response must name the `tree` backend with the `requested-backend` reason code. The XCTest tree rejects the heavy feed, so the step runs on the home screen. - Android: the snapshot helper APK is the only snapshot engine (ADR 0002). Its persistent session answers normal captures. `snapshot-raw-one-shot` restarts the daemon with `AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION=0`, so a one-shot `am instrument` capture answers. The harness restarts the daemon without the variable after the step. Both raw captures must reach depth 56. The forced capture must report the `instrumentation` transport. ## Platform parity @@ -59,7 +59,7 @@ Some shared steps use other screen coordinates or waits on Android. The `on.andr | `hardware-button` | Not timed: the simulator refuses hardware-button presses. | lifecycle (`hardware-button wake`) | | `push` | Not timed: the fixture has no notification authorization. | lifecycle (adb broadcast) | | `telephony` | Not timed: the simulator refuses calls and SMS. | lifecycle (`telephony sms`) | -| `snapshot` | heavy-feed: `snapshot-raw-forced-tree` | heavy-feed: `snapshot-raw-one-shot` | +| `snapshot` | home-reads: `snapshot-forced-tree` | heavy-feed: `snapshot-raw-one-shot` | | `audio` | Not timed: hosted runners grant no macOS audio permission. | Not timed: the Linux runner has no ScreenCaptureKit. | | `fold` | Not timed: iPhone Duo only. | Not timed: foldable emulators only. | | `tv-remote` | Not timed: tvOS only. | Not timed: Android TV only. | diff --git a/scripts/perf/metadata.ts b/scripts/perf/metadata.ts index d2c616241a..ab6719fa47 100644 --- a/scripts/perf/metadata.ts +++ b/scripts/perf/metadata.ts @@ -39,8 +39,7 @@ function emulator(serial: string): RunMetadata['device'] { export function collectMetadata(cfg: PerfConfig): RunMetadata { const revision = readGitRevision(REPO_ROOT); const toolchain = readToolchain(); - const target = - cfg.platform === 'ios' ? readTarget(cfg.udid!, cfg.appId, cfg.appPath) : undefined; + const target = cfg.platform === 'ios' ? readTarget(cfg.udid!, cfg.appId, cfg.appPath) : undefined; const cpus = os.cpus(); return { commit: revision.commit, diff --git a/scripts/perf/profile.test.ts b/scripts/perf/profile.test.ts index 3158a65948..f2b4835480 100644 --- a/scripts/perf/profile.test.ts +++ b/scripts/perf/profile.test.ts @@ -2,6 +2,11 @@ import assert from 'node:assert/strict'; import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; import { RETRY_PHASE } from '@agent-device/host-kit/retry'; +import { + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, + ANDROID_TOUCH_HELPER_GESTURE_PHASE, +} from '@agent-device/platform-android/mechanics'; import { test } from 'vitest'; import { DAEMON_REQUEST_PHASE } from '../../src/daemon-client/daemon-client.ts'; import { @@ -123,7 +128,7 @@ test('reads the settle wait of a --settle response and nothing from other respon // Trimmed from Android heavy-feed request logs. const androidCapture = (session: Record, oneShot = false) => [ - line({ phase: 'android_snapshot_helper_session_capture', ...session }), + line({ phase: ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, ...session }), ...(oneShot ? [ line({ @@ -132,7 +137,7 @@ const androidCapture = (session: Record, oneShot = false) => durationMs: 373, data: { command: 'adb', event: 'settled' }, }), - line({ phase: 'android_snapshot_helper_capture', durationMs: 373 }), + line({ phase: ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, durationMs: 373 }), ] : []), line({ phase: 'snapshot_capture', durationMs: 380, data: { backend: 'android' } }), @@ -183,17 +188,17 @@ test('names the Android helper transport that answered each capture', () => { test('counts Android helper gestures as runner commands', () => { const profile = profileRequestLog( [ - line({ phase: 'android_touch_helper_gesture', durationMs: 210 }), + line({ phase: ANDROID_TOUCH_HELPER_GESTURE_PHASE, durationMs: 210 }), line({ level: 'error', - phase: 'android_touch_helper_gesture', + phase: ANDROID_TOUCH_HELPER_GESTURE_PHASE, durationMs: 30, data: { error: 'busy' }, }), ].join('\n'), ); assert.deepEqual(profile.runnerCommands, { gesture: { count: 2, totalMs: 240 } }); - assert.deepEqual(profile.fallbacks, ['android_touch_helper_gesture: busy']); + assert.deepEqual(profile.fallbacks, [`${ANDROID_TOUCH_HELPER_GESTURE_PHASE}: busy`]); }); test('reads the Android helper transport from a snapshot response', () => { diff --git a/scripts/perf/profile.ts b/scripts/perf/profile.ts index f048aa2f6f..61aa839c51 100644 --- a/scripts/perf/profile.ts +++ b/scripts/perf/profile.ts @@ -1,6 +1,11 @@ import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; import { RETRY_PHASE } from '@agent-device/host-kit/retry'; +import { + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, + ANDROID_TOUCH_HELPER_GESTURE_PHASE, +} from '@agent-device/platform-android/mechanics'; import { DAEMON_REQUEST_PHASE } from '../../src/daemon-client/daemon-client.ts'; import { median } from './stats.ts'; import type { @@ -28,11 +33,8 @@ type DiagnosticEvent = { // Android: the snapshot helper answers through its persistent session, or through a one-shot // `am instrument` capture when the session is off or fails. Touch gestures run in the helper too. -const ANDROID_SESSION_CAPTURE_PHASE = 'android_snapshot_helper_session_capture'; -const ANDROID_ONE_SHOT_CAPTURE_PHASE = 'android_snapshot_helper_capture'; const ANDROID_SESSION_PRODUCER = 'android-helper/persistent-session'; const ANDROID_ONE_SHOT_PRODUCER = 'android-helper/instrumentation'; -const ANDROID_GESTURE_PHASE = 'android_touch_helper_gesture'; const STABILIZATION_PHASE = /settle|stabiliz/i; const FALLBACK_TEXT_LIMIT = 140; @@ -111,13 +113,13 @@ export function profileRequestLog(ndjson: string): SampleProfile { addTiming(profile.runnerCommands, text(data.command) ?? 'unknown', duration); } else if (phase === SNAPSHOT_SOURCE_ACQUIRE_PHASE) { addSource(text(data.producer) ?? 'unknown', event); - } else if (phase === ANDROID_SESSION_CAPTURE_PHASE) { + } else if (phase === ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE) { sessionAnswered = addSource(ANDROID_SESSION_PRODUCER, event); - } else if (phase === ANDROID_ONE_SHOT_CAPTURE_PHASE) { + } else if (phase === ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE) { if (sessionAnswered) profile.snapshotSources[ANDROID_SESSION_PRODUCER]!.failures += 1; sessionAnswered = false; addSource(ANDROID_ONE_SHOT_PRODUCER, event); - } else if (phase === ANDROID_GESTURE_PHASE) { + } else if (phase === ANDROID_TOUCH_HELPER_GESTURE_PHASE) { addTiming(profile.runnerCommands, 'gesture', duration); if (event.level === 'error') fallbacks.add(`${phase}: ${text(data.error) ?? 'failed'}`); } else if (phase === SNAPSHOT_SOURCE_RECOVERY_PHASE && data.truncated === true) { From 181d5260bb6752adcf69e237f1b50fc041ae5c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 11:20:24 +0200 Subject: [PATCH 12/15] fix(perf): name the failed restore when restarting the daemon, and fix the local compare example Co-Authored-By: Claude Opus 5.5 --- scripts/perf/README.md | 3 ++- scripts/perf/suite-harness.ts | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/perf/README.md b/scripts/perf/README.md index b31b0a21f2..5ea3867642 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -115,7 +115,8 @@ pnpm perf:compare --platform android # the same for Android pnpm perf:compare --to 2026-10-28 # that night against the 7 nights before it pnpm perf:compare --to latest --from 2026-10-21 # latest against the newest run of that day pnpm perf:compare --to latest --from 2026-10-21..2026-10-25 --out .tmp/perf/compare # every run in the range -pnpm perf:compare --to .tmp/perf/local # a local run against recent nights +pnpm perf:compare --to .tmp/perf/ios # a local run against recent nights +pnpm perf:compare --platform android --to .tmp/perf/android ``` A reference is `latest`, a run id, a `YYYY-MM-DD` date, or a local output path. diff --git a/scripts/perf/suite-harness.ts b/scripts/perf/suite-harness.ts index 43fca20d76..6cf0be7521 100644 --- a/scripts/perf/suite-harness.ts +++ b/scripts/perf/suite-harness.ts @@ -253,7 +253,11 @@ function sessionError(result: CliResult): string | undefined { } function stopStuckDaemon(ctx: SuiteContext): void { - log(' ! a request outlived its timeout; restarting the daemon'); + log( + ctx.restoreFailed + ? ` ! the daemon restore failed (${ctx.restoreFailed}); restarting the daemon` + : ' ! a request outlived its timeout; restarting the daemon', + ); invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]); ctx.daemonStuck = false; ctx.daemonEnv = undefined; From 9e9ccc3addd00222dd705e9ae8e0a34c58c106ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 12:01:32 +0200 Subject: [PATCH 13/15] fix(perf): keep the daemon request phase literal Only scripts read it, so a root export has no production consumer and fails the fallow audit. Co-Authored-By: Claude Opus 5.5 --- scripts/perf/profile.test.ts | 3 +-- scripts/perf/profile.ts | 6 +++--- src/daemon-client/daemon-client.ts | 5 +---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/perf/profile.test.ts b/scripts/perf/profile.test.ts index 77f0a03a43..9e9b9e61c1 100644 --- a/scripts/perf/profile.test.ts +++ b/scripts/perf/profile.test.ts @@ -3,7 +3,6 @@ import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; import { RETRY_PHASE } from '@agent-device/host-kit/retry'; import { test } from 'vitest'; -import { DAEMON_REQUEST_PHASE } from '../../src/daemon-client/daemon-client.ts'; import { aggregateProfiles, profileRequestLog, @@ -105,7 +104,7 @@ test('reads the client daemon timer from --debug stderr', () => { const stderr = [ `${DIAGNOSTIC_STDERR_PREFIX}{"phase":"daemon_startup","durationMs":30,"requestId":"r1"}`, 'noise', - `${DIAGNOSTIC_STDERR_PREFIX}{"phase":"${DAEMON_REQUEST_PHASE}","durationMs":812,"requestId":"r1"}`, + `${DIAGNOSTIC_STDERR_PREFIX}{"phase":"daemon_request","durationMs":812,"requestId":"r1"}`, ].join('\n'); assert.deepEqual(readClientDiagnostics(stderr), { daemonMs: 812, requestId: 'r1' }); assert.deepEqual(readClientDiagnostics(`${DIAGNOSTIC_STDERR_PREFIX}{"phase":"daemon_req`), {}); diff --git a/scripts/perf/profile.ts b/scripts/perf/profile.ts index b67314ea3f..96a4a188f6 100644 --- a/scripts/perf/profile.ts +++ b/scripts/perf/profile.ts @@ -1,7 +1,6 @@ import { EXEC_COMMAND_PHASE } from '@agent-device/host-kit/command'; import { DIAGNOSTIC_STDERR_PREFIX } from '@agent-device/host-kit/diagnostics'; import { RETRY_PHASE } from '@agent-device/host-kit/retry'; -import { DAEMON_REQUEST_PHASE } from '../../src/daemon-client/daemon-client.ts'; import { median } from './stats.ts'; import type { MeasurementProfile, @@ -12,11 +11,12 @@ import type { Timing, } from './types.ts'; -// The platform-apple façades export only what production code consumes, so its phase names stay -// literal here. +// The platform-apple façades and the root client export only what production code consumes, so +// these phase names stay literal here. const RUNNER_COMMAND_SEND_PHASE = 'ios_runner_command_send'; const SNAPSHOT_SOURCE_ACQUIRE_PHASE = 'ios.snapshot-source.acquire'; const SNAPSHOT_SOURCE_RECOVERY_PHASE = 'ios_snapshot_source_recovery'; +const DAEMON_REQUEST_PHASE = 'daemon_request'; type DiagnosticEvent = { ts?: string; diff --git a/src/daemon-client/daemon-client.ts b/src/daemon-client/daemon-client.ts index bdd823b1f9..ddbe6e9611 100644 --- a/src/daemon-client/daemon-client.ts +++ b/src/daemon-client/daemon-client.ts @@ -23,9 +23,6 @@ import { createRequestGuard, sendRequest } from './daemon-client-transport.ts'; import { isRemoteDaemon, type DaemonInfo } from './daemon-client-metadata.ts'; import { leaseScopeFromRequest } from '@agent-device/contracts/lease-scope'; -/** The client diagnostic phase that times one daemon request round trip. */ -export const DAEMON_REQUEST_PHASE = 'daemon_request'; - export type DaemonRequest = DaemonWireRequest; export type DaemonResponse = SharedDaemonResponse; type DaemonTransportOptions = AgentDeviceDaemonTransportContext & { @@ -141,7 +138,7 @@ async function sendToDaemonOnHost( settings, async () => { const response = await withDiagnosticTimer( - DAEMON_REQUEST_PHASE, + 'daemon_request', async () => await sendRequest( info, From aed5aa7e94f574003f9951c97aa24e8d3cc3e4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 12:33:08 +0200 Subject: [PATCH 14/15] fix(perf): stop the log stream right after it starts #3892 fixed the iOS simulator logs stop hang, so the untimed wait before logs-stop is no longer needed. Co-Authored-By: Claude Opus 5.5 --- scripts/perf/ios-suite.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/perf/ios-suite.ts b/scripts/perf/ios-suite.ts index 26607307c2..efc7795de1 100644 --- a/scripts/perf/ios-suite.ts +++ b/scripts/perf/ios-suite.ts @@ -302,8 +302,7 @@ export const IOS_SUITE: readonly SuiteScenario[] = [ ], }), step('logs-start', C.logs, ['logs', 'start']), - // A stop issued 400 ms after start has hung its request in local runs; let the stream run. - step('logs-stop', C.logs, ['logs', 'stop'], { before: [['wait', '1000']] }), + step('logs-stop', C.logs, ['logs', 'stop']), step('network-dump', C.network, ['network', 'dump', '10']), step('back', C.back, ['back']), step('back-settle', C.back, ['back', '--settle'], { From 1fddbe9d0f32b1451b41057904fa708ebc7b0611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 22 Oct 2026 16:07:47 +0200 Subject: [PATCH 15/15] refactor(android): record the answering helper transport on its capture timer The snapshot helper's capture timers now name the transport they drove, and the session timer records whether the session answered. profile.ts reads those fields per event instead of inferring a silent session from event order, and no longer re-spells the producer names platform-android owns. Co-Authored-By: Claude Opus 5.5 --- .../snapshot-helper-capture-timers.test.ts | 74 +++++++++++++++++++ packages/platform-android/src/snapshot.ts | 24 ++++-- scripts/perf/profile.test.ts | 70 +++++++++++++----- scripts/perf/profile.ts | 31 ++++---- 4 files changed, 158 insertions(+), 41 deletions(-) create mode 100644 packages/platform-android/src/__tests__/snapshot-helper-capture-timers.test.ts diff --git a/packages/platform-android/src/__tests__/snapshot-helper-capture-timers.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-capture-timers.test.ts new file mode 100644 index 0000000000..7f314f6b7b --- /dev/null +++ b/packages/platform-android/src/__tests__/snapshot-helper-capture-timers.test.ts @@ -0,0 +1,74 @@ +import { afterEach, beforeEach, test } from 'vitest'; +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { withDiagnosticsScope } from '@agent-device/host-kit/diagnostics'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { + ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE as ONE_SHOT_PHASE, + ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE as SESSION_PHASE, + snapshotAndroid, +} from '../snapshot.ts'; +import { resetAndroidSnapshotHelperInstallCache } from '../snapshot-helper-install.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; +import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './test-utils/android-snapshot-helper.ts'; +import { createPersistentSnapshotHelperProvider } from './snapshot-helper-session.fixtures.ts'; +import { mkdtempForTest } from './test-utils/tmp-dir.ts'; + +const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, +}; + +beforeEach(async () => { + await resetAndroidSnapshotHelperSessions(); + resetAndroidSnapshotHelperInstallCache(); +}); + +afterEach(async () => { + await resetAndroidSnapshotHelperSessions(); +}); + +async function captureTimers(sessionResponseMode?: 'malformed') { + const provider = createPersistentSnapshotHelperProvider({ + calls: [], + spawnArgs: [], + processes: [], + sessionResponseMode, + oneShotXml: '', + }); + const tmpDir = await mkdtempForTest('agent-device-android-capture-timers-'); + const logPath = path.join(tmpDir, 'diag.ndjson'); + try { + await withDiagnosticsScope( + { debug: true, logPath, session: 'android-test', requestId: 'req-1', command: 'snapshot' }, + async () => + await snapshotAndroid(device, { + helperAdb: provider, + helperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT, + }), + ); + return (await fs.readFile(logPath, 'utf8')) + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as { phase: string; data?: Record }) + .filter(({ phase }) => phase === SESSION_PHASE || phase === ONE_SHOT_PHASE) + .map(({ phase, data }) => ({ phase, transport: data?.transport, answered: data?.answered })); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +} + +test('each helper capture timer records its transport and whether the session answered', async () => { + assert.deepEqual(await captureTimers(), [ + { phase: SESSION_PHASE, transport: 'persistent-session', answered: true }, + ]); + await resetAndroidSnapshotHelperSessions(); + assert.deepEqual(await captureTimers('malformed'), [ + { phase: SESSION_PHASE, transport: 'persistent-session', answered: false }, + { phase: ONE_SHOT_PHASE, transport: 'instrumentation', answered: undefined }, + ]); +}); diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index 4705384cad..31f9e3d75e 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -36,6 +36,7 @@ import { ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, type AndroidHelperSessionScope, type AndroidSnapshotHelperCaptureOptions, + type AndroidSnapshotHelperTransport, } from './snapshot-helper-types.ts'; import { captureAndroidSnapshotWithHelper, @@ -102,10 +103,17 @@ const HELPER_CONTENT_RECAPTURE_DELAY_MS = 250; * a busy screen run a snapshot past the daemon request envelope. */ const HELPER_CONTENT_RECAPTURE_WINDOW_MS = 10_000; -/** The diagnostic phase of a capture through the snapshot helper's persistent session. */ +/** + * The diagnostic phase of a capture through the snapshot helper's persistent session. Its + * `data.transport` names the session transport, and `data.answered` is false when the session + * returned no capture (off, backed off, failed to start, or retired) and a one-shot capture follows. + */ export const ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE = 'android_snapshot_helper_session_capture'; -/** The diagnostic phase of a one-shot `am instrument` snapshot helper capture. */ +/** + * The diagnostic phase of a one-shot `am instrument` snapshot helper capture. Its `data.transport` + * names the instrumentation transport. + */ export const ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE = 'android_snapshot_helper_capture'; export type AndroidSnapshotOptions = SnapshotOptions & { @@ -469,15 +477,18 @@ async function captureAndroidUiHierarchyFromHelper(params: { signal, }); captureOptions.idleWait = params.idleWait; + let sessionCapture: AndroidSnapshotHelperOutput | undefined; try { - const sessionCapture = await withDiagnosticTimer( + sessionCapture = await withDiagnosticTimer( ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, - async () => await captureAndroidSnapshotWithHelperSession(captureOptions), - { + async () => (sessionCapture = await captureAndroidSnapshotWithHelperSession(captureOptions)), + () => ({ packageName: artifact.manifest.packageName, version: artifact.manifest.version, timeoutMs: ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, - }, + transport: 'persistent-session' satisfies AndroidSnapshotHelperTransport, + answered: sessionCapture !== undefined, + }), ); if (sessionCapture) return sessionCapture; } catch (error) { @@ -500,6 +511,7 @@ async function captureAndroidUiHierarchyFromHelper(params: { version: artifact.manifest.version, timeoutMs: ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, commandTimeoutMs: ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, + transport: 'instrumentation' satisfies AndroidSnapshotHelperTransport, }, ); } diff --git a/scripts/perf/profile.test.ts b/scripts/perf/profile.test.ts index 1573df6805..e3d3582739 100644 --- a/scripts/perf/profile.test.ts +++ b/scripts/perf/profile.test.ts @@ -6,6 +6,7 @@ import { ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, ANDROID_TOUCH_HELPER_GESTURE_PHASE, + type AndroidSnapshotHelperTransport, } from '@agent-device/platform-android/mechanics'; import { test } from 'vitest'; import { @@ -125,32 +126,46 @@ test('reads the settle wait of a --settle response and nothing from other respon }); // Trimmed from Android heavy-feed request logs. -const androidCapture = (session: Record, oneShot = false) => +const sessionTimer = (answered: boolean, event: Record = {}) => + line({ + phase: ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, + ...event, + data: { + transport: 'persistent-session' satisfies AndroidSnapshotHelperTransport, + answered, + ...(event.data as Record | undefined), + }, + }); +const oneShotTimer = [ + line({ + level: 'debug', + phase: 'exec_command', + durationMs: 373, + data: { command: 'adb', event: 'settled' }, + }), + line({ + phase: ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, + durationMs: 373, + data: { transport: 'instrumentation' satisfies AndroidSnapshotHelperTransport }, + }), +]; +const androidCapture = (...timers: string[]) => [ - line({ phase: ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE, ...session }), - ...(oneShot - ? [ - line({ - level: 'debug', - phase: 'exec_command', - durationMs: 373, - data: { command: 'adb', event: 'settled' }, - }), - line({ phase: ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE, durationMs: 373 }), - ] - : []), + ...timers, line({ phase: 'snapshot_capture', durationMs: 380, data: { backend: 'android' } }), ].join('\n'); test('names the Android helper transport that answered each capture', () => { - const session = profileRequestLog(androidCapture({ durationMs: 40 })); + const session = profileRequestLog(androidCapture(sessionTimer(true, { durationMs: 40 }))); assert.deepEqual(session.snapshotSources, { 'android-helper/persistent-session': { count: 1, totalMs: 40, failures: 0 }, }); assert.deepEqual(session.fallbacks, []); - // With the session off, the session timer settles cleanly and a one-shot capture answers. - const sessionOff = profileRequestLog(androidCapture({ durationMs: 0 }, true)); + // With the session off, the session timer settles unanswered and a one-shot capture answers. + const sessionOff = profileRequestLog( + androidCapture(sessionTimer(false, { durationMs: 0 }), ...oneShotTimer), + ); assert.deepEqual(sessionOff.snapshotSources, { 'android-helper/persistent-session': { count: 1, totalMs: 0, failures: 1 }, 'android-helper/instrumentation': { count: 1, totalMs: 373, failures: 0 }, @@ -165,7 +180,10 @@ test('names the Android helper transport that answered each capture', () => { const sessionFailed = profileRequestLog( [ - androidCapture({ level: 'error', durationMs: 5000, data: { error: 'socket closed' } }, true), + androidCapture( + sessionTimer(false, { level: 'error', durationMs: 5000, data: { error: 'socket closed' } }), + ...oneShotTimer, + ), line({ level: 'warn', phase: 'android_snapshot_helper_session_fallback', @@ -184,6 +202,24 @@ test('names the Android helper transport that answered each capture', () => { ]); }); +test('reads whether the Android session answered from its own timer, not from event order', () => { + // A content re-capture can follow a session capture that answered. The answer still counts. + const recaptured = profileRequestLog( + androidCapture(sessionTimer(true, { durationMs: 40 }), ...oneShotTimer), + ); + assert.deepEqual(recaptured.snapshotSources['android-helper/persistent-session'], { + count: 1, + totalMs: 40, + failures: 0, + }); + + // A request that ends before any one-shot capture still records the unanswered session. + const unanswered = profileRequestLog(androidCapture(sessionTimer(false, { durationMs: 12 }))); + assert.deepEqual(unanswered.snapshotSources, { + 'android-helper/persistent-session': { count: 1, totalMs: 12, failures: 1 }, + }); +}); + test('counts Android helper gestures as runner commands', () => { const profile = profileRequestLog( [ diff --git a/scripts/perf/profile.ts b/scripts/perf/profile.ts index 19038c70aa..1124eb077c 100644 --- a/scripts/perf/profile.ts +++ b/scripts/perf/profile.ts @@ -31,10 +31,6 @@ type DiagnosticEvent = { data?: Record; }; -// Android: the snapshot helper answers through its persistent session, or through a one-shot -// `am instrument` capture when the session is off or fails. Touch gestures run in the helper too. -const ANDROID_SESSION_PRODUCER = 'android-helper/persistent-session'; -const ANDROID_ONE_SHOT_PRODUCER = 'android-helper/instrumentation'; const STABILIZATION_PHASE = /settle|stabiliz/i; const FALLBACK_TEXT_LIMIT = 140; @@ -74,19 +70,15 @@ export function profileRequestLog(ndjson: string): SampleProfile { fallbacks: [], }; const fallbacks = new Set(); - const addSource = (producer: string, event: DiagnosticEvent): boolean => { + const addSource = (producer: string, event: DiagnosticEvent, answered = true): void => { const entry = (profile.snapshotSources[producer] ??= { count: 0, totalMs: 0, failures: 0 }); entry.count += 1; entry.totalMs += eventDuration(event) ?? 0; const error = text(event.data?.error); - if (!error && event.level !== 'error') return true; - entry.failures += 1; - fallbacks.add(`${producer}: ${error ?? 'failed'}`); - return false; + const failed = error !== undefined || event.level === 'error'; + if (failed || !answered) entry.failures += 1; + if (failed) fallbacks.add(`${producer}: ${error ?? 'failed'}`); }; - // A session capture that settles cleanly but is followed by a one-shot capture did not answer: - // the session is off, and the one-shot capture served the request. - let sessionAnswered = false; let firstTs: number | undefined; let lastTs: number | undefined; for (const line of ndjson.split('\n')) { @@ -113,12 +105,15 @@ export function profileRequestLog(ndjson: string): SampleProfile { addTiming(profile.runnerCommands, text(data.command) ?? 'unknown', duration); } else if (phase === SNAPSHOT_SOURCE_ACQUIRE_PHASE) { addSource(text(data.producer) ?? 'unknown', event); - } else if (phase === ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE) { - sessionAnswered = addSource(ANDROID_SESSION_PRODUCER, event); - } else if (phase === ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE) { - if (sessionAnswered) profile.snapshotSources[ANDROID_SESSION_PRODUCER]!.failures += 1; - sessionAnswered = false; - addSource(ANDROID_ONE_SHOT_PRODUCER, event); + } else if ( + phase === ANDROID_SNAPSHOT_HELPER_SESSION_CAPTURE_PHASE || + phase === ANDROID_SNAPSHOT_HELPER_CAPTURE_PHASE + ) { + addSource( + `android-helper/${text(data.transport) ?? 'unknown'}`, + event, + data.answered !== false, + ); } else if (phase === ANDROID_TOUCH_HELPER_GESTURE_PHASE) { addTiming(profile.runnerCommands, 'gesture', duration); if (event.level === 'error') fallbacks.add(`${phase}: ${text(data.error) ?? 'failed'}`);