diff --git a/.fallowrc.json b/.fallowrc.json index baf47664c8..93fcfda95c 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -29,6 +29,8 @@ "scripts/patch-xcuitest-runner-icon.ts", "scripts/write-xcuitest-cache-metadata.ts", "scripts/xcuitest-build-settings.ts", + // Shell-invoked executable: scripts/replays/android-manual.sh. + "scripts/replays/android-search-diagnostic.ts", "packages/capture-kit/src/ios-snapshot-engine/replay.ts", // #1596 regression fixture: runs as a real `node --experimental-strip-types` // subprocess (test/integration/daemon-replace-exit-flush.test.ts), so diff --git a/.github/workflows/replays-manual.yml b/.github/workflows/replays-manual.yml index 59c76e1d2c..92d77f7503 100644 --- a/.github/workflows/replays-manual.yml +++ b/.github/workflows/replays-manual.yml @@ -9,13 +9,23 @@ name: Replay Manual on: workflow_dispatch: + inputs: + readiness_wait: + description: 'Bounded #3077 comparison; normal preserves the ordinary full manual lane' + type: choice + options: [normal, enabled, disabled] + default: normal + android_diagnostic: + description: 'Android-only search diagnostic after comparison timing; use enabled or disabled' + type: boolean + default: false permissions: contents: read actions: read concurrency: - group: ci-${{ github.workflow }}-${{ github.ref }} + group: ci-${{ github.workflow }}-${{ github.ref }}-${{ inputs.readiness_wait != 'normal' && github.run_id || 'normal' }} cancel-in-progress: true jobs: @@ -32,6 +42,12 @@ jobs: - name: Setup toolchain uses: ./.github/actions/setup-node-pnpm + - name: Prepare readiness comparison source + if: inputs.readiness_wait != 'normal' + env: + READINESS_MODE: ${{ inputs.readiness_wait }} + run: node --experimental-strip-types scripts/replays/readiness-experiment.ts prepare "$READINESS_MODE" "$GITHUB_SHA" + - name: Setup Android replay host id: android-replay-host uses: ./.github/actions/setup-android-replay-host @@ -39,6 +55,7 @@ jobs: package-helpers: 'true' - name: Restore Android fixture APK + if: inputs.readiness_wait == 'normal' id: fixture-app uses: ./.github/actions/setup-fixture-app with: @@ -58,25 +75,7 @@ jobs: profile: pixel_7 target: google_apis_playstore emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics - script: | - # android-emulator-runner executes this with /usr/bin/sh, which is dash on - # ubuntu runners and rejects `-o pipefail`. The serial assignment below is - # validated by the `test -n` guard instead. - set -eu - ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" - test -n "$ANDROID_SERIAL" - BOOT_FINISHED_AT="$(date +%s)" - pnpm gate build - pnpm clean:daemon - adb -s "$ANDROID_SERIAL" install -r "${{ steps.fixture-app.outputs.apk-path }}" - node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag-android.ad --platform android --serial "$ANDROID_SERIAL" --session nightly-android-target-drag --json - pnpm clean:daemon - FULL_STARTED_AT="$(date +%s)" - AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=full AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts - FULL_FINISHED_AT="$(date +%s)" - echo "Android setup before emulator: $((BOOT_FINISHED_AT - ${{ steps.android-setup.outputs.seconds }}))s" - echo "Android full scenario wall time: $((FULL_FINISHED_AT - FULL_STARTED_AT))s" - pnpm gate replay-android --retries 2 --artifacts-dir test/artifacts/replays-android --report-junit test/artifacts/replays-android.junit.xml + script: sh scripts/replays/android-manual.sh "${{ inputs.readiness_wait }}" "${{ steps.fixture-app.outputs.apk-path }}" "${{ steps.fixture-app.outputs.app-id }}" "${{ steps.android-setup.outputs.seconds }}" "${{ inputs.android_diagnostic }}" - name: Upload Android artifacts if: always() @@ -86,6 +85,7 @@ jobs: agent-state-dir: ${{ env.AGENT_DEVICE_STATE_DIR }} manual-ios: + if: ${{ !inputs.android_diagnostic }} name: iOS Replay Suite runs-on: macos-26 timeout-minutes: 80 @@ -107,6 +107,12 @@ jobs: pnpm-lock.yaml examples/test-app/pnpm-lock.yaml + - name: Prepare readiness comparison source + if: inputs.readiness_wait != 'normal' + env: + READINESS_MODE: ${{ inputs.readiness_wait }} + run: node --experimental-strip-types scripts/replays/readiness-experiment.ts prepare "$READINESS_MODE" "$GITHUB_SHA" + - name: Restore and build iOS XCTest runner uses: ./.github/actions/setup-apple-runner-build with: @@ -117,6 +123,12 @@ jobs: xcuitest-platform: ios xcuitest-destination: generic/platform=iOS Simulator + - name: Build readiness comparison runtime + if: inputs.readiness_wait != 'normal' + uses: ./.github/actions/run-gate + with: + gate: build + - name: Boot iOS test simulator id: ios-simulator uses: ./.github/actions/boot-ios-test-simulator @@ -130,6 +142,7 @@ jobs: node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json - name: Run iOS simulator replay suite + if: inputs.readiness_wait == 'normal' uses: ./.github/actions/run-gate with: gate: replay-ios @@ -143,22 +156,50 @@ jobs: --report-junit test/artifacts/replays-ios-simulator.junit.xml + - name: Begin iOS readiness comparison + if: inputs.readiness_wait != 'normal' + run: node --experimental-strip-types scripts/replays/readiness-experiment.ts begin ios + + - name: Run iOS readiness comparison + if: inputs.readiness_wait != 'normal' + uses: ./.github/actions/run-gate + with: + gate: replay-ios + args: | + --udid + ${{ steps.ios-simulator.outputs.simulator-udid }} + --debug + --retries + 0 + --artifacts-dir + test/artifacts/replays-ios-simulator + --report-junit + test/artifacts/replays-ios-simulator.junit.xml + + - name: Collect iOS readiness evidence + if: always() && inputs.readiness_wait != 'normal' + run: node --experimental-strip-types scripts/replays/readiness-experiment.ts summarize ios "$AGENT_DEVICE_STATE_DIR" test/artifacts/replays-ios-simulator.junit.xml + - name: Fetch current fixture app + if: inputs.readiness_wait == 'normal' id: fixture-app uses: ./.github/actions/setup-fixture-app with: install: 'false' - name: Report fixture cache source + if: inputs.readiness_wait == 'normal' run: | echo "Fixture app source: ${{ steps.fixture-app.outputs.source }}" >> "$GITHUB_STEP_SUMMARY" - name: Prove selector drag reaches its destination on iOS + if: inputs.readiness_wait == 'normal' run: | xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag.ad --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --session nightly-ios-target-drag --json - name: Run full fixture-backed iOS simulator E2E + if: inputs.readiness_wait == 'normal' env: AGENT_DEVICE_FIXTURE_APP_ID: ${{ steps.fixture-app.outputs.app-id }} AGENT_DEVICE_FIXTURE_APP_PATH: ${{ steps.fixture-app.outputs.app-path }} @@ -170,7 +211,7 @@ jobs: node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts - name: Run iOS physical device replay suite - if: env.IOS_UDID != '' + if: inputs.readiness_wait == 'normal' && env.IOS_UDID != '' env: IOS_UDID: ${{ vars.IOS_UDID }} uses: ./.github/actions/run-gate diff --git a/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts b/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts index d46a0cdf09..36bb45fa91 100644 --- a/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts +++ b/packages/replay-port/src/daemon-port/session-replay-runtime-engine-adapter.ts @@ -203,7 +203,7 @@ export function createAdReplayStepRuntime(params: { }); }, - async observeTarget({ action, token }) { + async observeTarget({ action, token, index }) { const observeOnce = async (): Promise => { const observation = await captureTargetObservation(action); lastObservation = observation; @@ -251,7 +251,13 @@ export function createAdReplayStepRuntime(params: { emitDiagnostic({ level: 'debug', phase: 'interaction_target_readiness', - data: { ...gateWait.readiness, command: action.command }, + data: { + ...gateWait.readiness, + command: action.command, + replaySession: ctx.sessionName, + replayPath: ctx.resolved, + step: index + 1, + }, }); } if (observed.kind === 'done') return observed.result; diff --git a/scripts/replays/android-manual.sh b/scripts/replays/android-manual.sh new file mode 100644 index 0000000000..ea15875117 --- /dev/null +++ b/scripts/replays/android-manual.sh @@ -0,0 +1,62 @@ +#!/bin/sh +set -eu + +READINESS_MODE="$1" +FIXTURE_APK="$2" +FIXTURE_APP_ID="$3" +ANDROID_SETUP_SECONDS="$4" +ANDROID_DIAGNOSTIC="${5:-false}" +ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" +test -n "$ANDROID_SERIAL" +BOOT_FINISHED_AT="$(date +%s)" +pnpm gate build +pnpm clean:daemon + +if [ "$READINESS_MODE" != normal ]; then + LOGCAT_PID= + LOGCAT_EXIT=0 + stop_native_logcat() { + if [ -n "$LOGCAT_PID" ]; then + kill "$LOGCAT_PID" 2>/dev/null || : + wait "$LOGCAT_PID" 2>/dev/null || LOGCAT_EXIT=$? + printf '%s %s\n' "$(date -u +%FT%TZ)" "$LOGCAT_EXIT" > "$NATIVE_DIR/native-logcat-end.txt" + LOGCAT_PID= + fi + } + trap stop_native_logcat 0 + trap 'exit 130' INT + trap 'exit 143' TERM + if [ "$ANDROID_DIAGNOSTIC" = true ]; then + NATIVE_DIR=test/artifacts/readiness-experiment/android-diagnostic + mkdir -p "$NATIVE_DIR" + (ulimit -f 8192; exec timeout --kill-after=2 --signal=TERM 300 adb -s "$ANDROID_SERIAL" logcat -v threadtime -T 1 ActivityTaskManager:I ActivityManager:I AndroidRuntime:E WindowManager:I InputMethodManagerService:V InputMethodManager:V UiAutomationConnection:V '*:S') > "$NATIVE_DIR/native-logcat.stdout.txt" 2> "$NATIVE_DIR/native-logcat.stderr.txt" & + LOGCAT_PID=$! + printf '%s %s\n' "$(date -u +%FT%TZ)" "$LOGCAT_PID" > "$NATIVE_DIR/native-logcat-start.txt" + fi + node --experimental-strip-types scripts/replays/readiness-experiment.ts begin android + REPLAY_EXIT=0 + pnpm gate replay-android --serial "$ANDROID_SERIAL" --debug --retries 0 --artifacts-dir test/artifacts/replays-android --report-junit test/artifacts/replays-android.junit.xml || REPLAY_EXIT=$? + SUMMARY_EXIT=0 + DIAGNOSTIC_ONLY= + if [ "$ANDROID_DIAGNOSTIC" = true ]; then DIAGNOSTIC_ONLY=diagnostic-only; fi + node --experimental-strip-types scripts/replays/readiness-experiment.ts summarize android "$AGENT_DEVICE_STATE_DIR" test/artifacts/replays-android.junit.xml $DIAGNOSTIC_ONLY || SUMMARY_EXIT=$? + stop_native_logcat + DIAGNOSTIC_EXIT=0 + if [ "$ANDROID_DIAGNOSTIC" = true ]; then + node --experimental-strip-types scripts/replays/android-search-diagnostic.ts "$ANDROID_SERIAL" "$NATIVE_DIR" || DIAGNOSTIC_EXIT=$? + echo "Android native diagnostic exit: $DIAGNOSTIC_EXIT; measured replay exit: $REPLAY_EXIT" + fi + if [ "$REPLAY_EXIT" -ne 0 ]; then exit "$REPLAY_EXIT"; fi + if [ "$SUMMARY_EXIT" -ne 0 ]; then exit "$SUMMARY_EXIT"; fi + exit "$DIAGNOSTIC_EXIT" +fi + +adb -s "$ANDROID_SERIAL" install -r "$FIXTURE_APK" +node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag-android.ad --platform android --serial "$ANDROID_SERIAL" --session nightly-android-target-drag --json +pnpm clean:daemon +FULL_STARTED_AT="$(date +%s)" +AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=full AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="$FIXTURE_APK" AGENT_DEVICE_FIXTURE_APP_ID="$FIXTURE_APP_ID" node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts +FULL_FINISHED_AT="$(date +%s)" +echo "Android setup before emulator: $((BOOT_FINISHED_AT - ANDROID_SETUP_SECONDS))s" +echo "Android full scenario wall time: $((FULL_FINISHED_AT - FULL_STARTED_AT))s" +pnpm gate replay-android --retries 2 --artifacts-dir test/artifacts/replays-android --report-junit test/artifacts/replays-android.junit.xml diff --git a/scripts/replays/android-manual.test.ts b/scripts/replays/android-manual.test.ts new file mode 100644 index 0000000000..f968c6c9d2 --- /dev/null +++ b/scripts/replays/android-manual.test.ts @@ -0,0 +1,95 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { parse } from 'yaml'; +import { expect, test } from 'vitest'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { mkdtempForTest } from '@agent-device/test-kit/tmp-dir'; + +test.each([ + [7, false, 7], + [0, true, 2], + [7, true, 7], + [0, true, 11], +] as const)( + 'emulator action preserves suite status %i and separates diagnostic failure', + async (replayExit, diagnostic, expectedExit) => { + const cwd = await mkdtempForTest('android-manual-invocation-'); + const bin = path.join(cwd, 'bin'); + fs.mkdirSync(bin); + const writeTool = (name: string, script: string) => + fs.writeFileSync(path.join(bin, name), `#!/bin/sh\n${script}\n`, { mode: 0o755 }); + writeTool( + 'adb', + String.raw`if [ "$1" = devices ]; then printf "List of devices attached\nemulator-7000\tdevice\n"; else printf '%s\n' "$*"; trap 'touch logger-stopped; exit 0' TERM; for tick in 1 2 3 4 5 6 7 8 9 10; do sleep 0.1; done; fi`, + ); + writeTool('timeout', 'shift; shift; shift; exec "$@"'); + writeTool( + 'pnpm', + String.raw`printf "%s\n" "$*" >> "$READINESS_TEST_LOG" +if [ "$1 $2" = "gate replay-android" ]; then + if [ "$READINESS_TEST_DIAGNOSTIC" = true ]; then read stamp logger < test/artifacts/readiness-experiment/android-diagnostic/native-logcat-start.txt; kill -0 "$logger" || exit 42; sleep 0.1; fi + exit "$READINESS_TEST_REPLAY_EXIT" +fi`, + ); + writeTool( + 'node', + String.raw`if [ "$3" = begin ]; then touch started; sleep 0.1; exit "$READINESS_TEST_BEGIN_EXIT"; fi +if [ "$3" = summarize ]; then touch summarized; exit "$READINESS_TEST_SUMMARY_EXIT"; fi +if [ "$2" = scripts/replays/android-search-diagnostic.ts ]; then read stamp logger < test/artifacts/readiness-experiment/android-diagnostic/native-logcat-start.txt; if kill -0 "$logger" 2>/dev/null; then exit 42; fi; touch diagnosed; exit 2; fi`, + ); + fs.mkdirSync(path.join(cwd, 'scripts/replays'), { recursive: true }); + fs.copyFileSync( + 'scripts/replays/android-manual.sh', + path.join(cwd, 'scripts/replays/android-manual.sh'), + ); + const workflow = parse(fs.readFileSync('.github/workflows/replays-manual.yml', 'utf8')); + const runner = workflow.jobs['manual-android'].steps.find((step: { uses?: string }) => + step.uses?.startsWith('reactivecircus/android-emulator-runner@'), + ); + const script = runner.with.script + .replaceAll(/\$\{\{ inputs.readiness_wait \}\}/g, 'enabled') + .replaceAll(/\$\{\{ inputs.android_diagnostic \}\}/g, String(diagnostic)) + .replaceAll(/\$\{\{ steps.fixture-app.outputs.apk-path \}\}/g, '') + .replaceAll(/\$\{\{ steps.fixture-app.outputs.app-id \}\}/g, '') + .replaceAll(/\$\{\{ steps.android-setup.outputs.seconds \}\}/g, '0'); + const commands = script + .split('\n') + .map((line: string) => line.trim()) + .filter((line: string) => line.length > 0 && !line.startsWith('#')); + const log = path.join(cwd, 'calls.log'); + let exitCode = 0; + for (const command of commands) { + exitCode = runCmdSync('sh', ['-c', command], { + cwd, + allowFailure: true, + env: { + PATH: `${bin}:${process.env.PATH}`, + ANDROID_SERIAL: 'emulator-legacy', + AGENT_DEVICE_STATE_DIR: path.join(cwd, 'state'), + READINESS_TEST_LOG: log, + READINESS_TEST_REPLAY_EXIT: String(replayExit), + READINESS_TEST_DIAGNOSTIC: String(diagnostic), + READINESS_TEST_SUMMARY_EXIT: replayExit === 7 ? '9' : '0', + READINESS_TEST_BEGIN_EXIT: expectedExit === 11 ? '11' : '0', + }, + }).exitCode; + if (exitCode !== 0) break; + } + expect(exitCode).toBe(expectedExit); + expect(fs.existsSync(path.join(cwd, 'started'))).toBe(true); + expect(fs.existsSync(path.join(cwd, 'summarized'))).toBe(expectedExit !== 11); + expect(fs.existsSync(path.join(cwd, 'diagnosed'))).toBe(diagnostic && expectedExit !== 11); + const native = path.join(cwd, 'test/artifacts/readiness-experiment/android-diagnostic'); + expect(fs.existsSync(native)).toBe(diagnostic); + if (diagnostic) { + expect(fs.existsSync(path.join(cwd, 'logger-stopped'))).toBe(true); + expect(fs.readFileSync(path.join(native, 'native-logcat.stdout.txt'), 'utf8')).toContain( + '-s emulator-7000 logcat', + ); + } + if (expectedExit !== 11) + expect(fs.readFileSync(log, 'utf8')).toContain( + 'replay-android --serial emulator-7000 --debug --retries 0', + ); + }, +); diff --git a/scripts/replays/android-search-diagnostic.test.ts b/scripts/replays/android-search-diagnostic.test.ts new file mode 100644 index 0000000000..e68bc402e8 --- /dev/null +++ b/scripts/replays/android-search-diagnostic.test.ts @@ -0,0 +1,58 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test } from 'vitest'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { mkdtempForTest } from '@agent-device/test-kit/tmp-dir'; + +test.each([0, 7])( + 'native diagnostic retains producer evidence and refuses failed probe %i', + async (probeExit) => { + const cwd = await mkdtempForTest('android-search-native-'); + fs.mkdirSync(path.join(cwd, 'bin')); + fs.writeFileSync( + path.join(cwd, 'bin/adb'), + String.raw`#!/bin/sh +printf '%s\n' "$*" >> calls.log +case "$*" in *resolve-activity*) echo 'com.google.android.settings.intelligence/.search.SearchActivity';; *) echo "native producer: $*";; esac +case "$*" in *input_method) exit "$PROBE_EXIT";; esac +`, + { mode: 0o755 }, + ); + const output = path.join(cwd, 'artifacts'); + fs.mkdirSync(output); + fs.writeFileSync(path.join(output, 'native-logcat-end.txt'), '2026-10-12T20:00:00Z 143\n'); + const result = runCmdSync( + process.execPath, + [ + '--experimental-strip-types', + path.resolve('scripts/replays/android-search-diagnostic.ts'), + 'emulator-own', + output, + ], + { + cwd, + allowFailure: true, + timeoutMs: 10_000, + env: { PATH: `${cwd}/bin:${process.env.PATH}`, PROBE_EXIT: String(probeExit) }, + }, + ); + const summary = JSON.parse(fs.readFileSync(path.join(output, 'summary.json'), 'utf8')); + expect(result.exitCode, result.stderr).toBe(probeExit === 0 ? 0 : 2); + expect(summary.complete).toBe(probeExit === 0); + expect(summary.diagnosticOnly).toBe(true); + expect(summary.loggerExit).toBe(143); + expect( + summary.commands.every( + (command: { args: string[] }) => + command.args[0] === '-s' && command.args[1] === 'emulator-own', + ), + ).toBe(true); + expect(fs.readFileSync(path.join(output, 'measured-end-ime.stdout.txt'), 'utf8')).toBe( + 'native producer: -s emulator-own shell dumpsys input_method\n', + ); + expect(fs.readFileSync(path.join(cwd, 'calls.log'), 'utf8')).toContain( + 'shell dumpsys activity exit-info com.google.android.settings.intelligence', + ); + expect(fs.existsSync(path.join(output, 'measured-end-ime.stderr.txt'))).toBe(true); + }, +); diff --git a/scripts/replays/android-search-diagnostic.ts b/scripts/replays/android-search-diagnostic.ts new file mode 100644 index 0000000000..0ad7d1f433 --- /dev/null +++ b/scripts/replays/android-search-diagnostic.ts @@ -0,0 +1,84 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { normalizeError } from '@agent-device/kernel/errors'; + +const [serial, outputDir] = process.argv.slice(2); +if (!serial || !outputDir) { + throw new Error('Usage: android-search-diagnostic.ts '); +} +fs.mkdirSync(outputDir, { recursive: true }); +const startedAt = Date.now(); +const deadline = startedAt + 90_000; +const commands: unknown[] = []; +let probesSucceeded = true; + +function capture(name: string, args: string[]) { + const remaining = deadline - Date.now(); + if (remaining <= 0) throw new Error('Native diagnostic wall-clock budget exhausted'); + let result; + let failure; + try { + result = runCmdSync('adb', ['-s', serial, ...args], { + allowFailure: true, + timeoutMs: Math.min(15_000, remaining), + maxBuffer: 8 * 1024 * 1024, + }); + } catch (error) { + failure = normalizeError(error); + result = { exitCode: -1, stdout: '', stderr: String(error) }; + } + fs.writeFileSync(path.join(outputDir, `${name}.stdout.txt`), result.stdout); + fs.writeFileSync(path.join(outputDir, `${name}.stderr.txt`), result.stderr); + probesSucceeded &&= result.exitCode === 0; + commands.push({ name, args: ['-s', serial, ...args], exitCode: result.exitCode, failure }); + return result.stdout; +} + +let diagnosticError: string | undefined; +let loggerExit: number | undefined; +try { + capture('measured-end-window', ['shell', 'dumpsys', 'window', 'windows']); + capture('measured-end-activity', ['shell', 'dumpsys', 'activity', 'activities']); + capture('measured-end-ime', ['shell', 'dumpsys', 'input_method']); + capture('image', ['shell', 'getprop']); + const resolution = capture('search-resolution', [ + 'shell', + 'cmd', + 'package', + 'resolve-activity', + '--brief', + '-a', + 'android.settings.APP_SEARCH_SETTINGS', + '-c', + 'android.intent.category.DEFAULT', + ]); + const resolvedPackage = /^([A-Za-z0-9_.]+)\/[A-Za-z0-9_.$]+$/m.exec(resolution)?.[1]; + if (!resolvedPackage) throw new Error('Native diagnostic did not resolve the Search provider'); + capture('search-exit-info', ['shell', 'dumpsys', 'activity', 'exit-info', resolvedPackage]); + loggerExit = Number( + fs.readFileSync(path.join(outputDir, 'native-logcat-end.txt'), 'utf8').trim().split(' ')[1], + ); +} catch (error) { + diagnosticError = String(error); +} +const complete = probesSucceeded && !diagnosticError && [0, 124, 143].includes(loggerExit ?? -1); +fs.writeFileSync( + path.join(outputDir, 'summary.json'), + JSON.stringify( + { + head: process.env.GITHUB_SHA, + serial, + diagnosticOnly: true, + startedAt, + finishedAt: Date.now(), + complete, + loggerExit, + diagnosticError, + commands, + }, + null, + 2, + ) + '\n', +); +process.exitCode = complete ? 0 : 2; diff --git a/scripts/replays/readiness-experiment.test.ts b/scripts/replays/readiness-experiment.test.ts new file mode 100644 index 0000000000..36cd39c76c --- /dev/null +++ b/scripts/replays/readiness-experiment.test.ts @@ -0,0 +1,189 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { expect, test, vi } from 'vitest'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { emitDiagnostic, withDiagnosticsScope } from '@agent-device/host-kit/diagnostics'; +import { safeSessionName } from '@agent-device/host-kit/session-paths'; +import { createJunitReplayTestReporter } from '../../src/cli/replay-test/reporters/junit.ts'; +import { mkdtempForTest } from '@agent-device/test-kit/tmp-dir'; +import { baselineSource, countReadiness } from './readiness-experiment.ts'; + +test('baseline disables only the declared default and refuses a changed or duplicated owner', () => { + const declaration = 'const REPLAY_DEFAULT_READINESS_TIMEOUT_MS = 2_000;'; + const refusal = 'Expected exactly one replay readiness default declaration'; + expect(baselineSource(`before\n${declaration}\nafter`)).toBe( + 'before\nconst REPLAY_DEFAULT_READINESS_TIMEOUT_MS = 0;\nafter', + ); + expect(() => baselineSource('const REPLAY_DEFAULT_READINESS_TIMEOUT_MS = 3_000;')).toThrow( + refusal, + ); + expect(() => baselineSource(`${declaration}\n${declaration}`)).toThrow(refusal); +}); + +test('engagement counts actual repeated polls and separates exhausted waits from rescues', () => { + expect( + countReadiness([ + { phase: 'interaction_target_readiness', data: { polls: 1, end: 'done' } }, + { phase: 'interaction_target_readiness', data: { polls: 2, end: 'done' } }, + { phase: 'interaction_target_readiness', data: { polls: 3, end: 'expired' } }, + { phase: 'other', data: { polls: 4, end: 'done' } }, + ]), + ).toEqual({ + observations: 3, + firstPoll: 1, + engaged: 2, + resolved: 1, + incomplete: 1, + targetGate: 0, + dispatch: 2, + }); +}); + +test('CLI evidence uses reporter file identity and deduplicates gate/dispatch rescues; failed dispatch cannot vote', async () => { + const cwd = await mkdtempForTest('readiness-evidence-'); + const script = path.resolve('scripts/replays/readiness-experiment.ts'); + const source = 'packages/replay-port/src/daemon-port/session-replay-action-runtime.ts'; + fs.mkdirSync(path.dirname(path.join(cwd, source)), { recursive: true }); + fs.writeFileSync(path.join(cwd, source), 'const REPLAY_DEFAULT_READINESS_TIMEOUT_MS = 2_000;'); + const corpus = path.join(cwd, 'test/integration/replays/android/emulator'); + fs.mkdirSync(corpus, { recursive: true }); + const file = path.join(corpus, 'navigation.ad'); + fs.writeFileSync(file, 'context platform=android\nclick label=Ready\n'); + const invoke = (...args: string[]) => + runCmdSync(process.execPath, ['--experimental-strip-types', script, ...args], { cwd }); + invoke('prepare', 'enabled', 'a'.repeat(40)); + invoke('begin', 'android'); + const state = path.join(cwd, 'state'); + const session = 'readiness:fixture'; + const requestDir = path.join(state, 'sessions', safeSessionName(session), 'requests'); + fs.mkdirSync(requestDir, { recursive: true }); + const baseMs = Date.now(); + const ts = (offset: number) => new Date(baseMs + offset).toISOString(); + vi.useFakeTimers({ toFake: ['Date'] }); + try { + const outerDir = path.join(state, 'sessions', safeSessionName('readiness:suite'), 'requests'); + fs.mkdirSync(outerDir, { recursive: true }); + await withDiagnosticsScope( + { + session: 'readiness:suite', + command: 'test', + debug: true, + logPath: path.join(outerDir, 'suite.ndjson'), + }, + async () => { + vi.setSystemTime(baseMs + 1); + emitDiagnostic({ + phase: 'interaction_target_readiness', + data: { + polls: 2, + end: 'done', + command: 'click', + replaySession: session, + replayPath: file, + step: 1, + }, + }); + emitDiagnostic({ + phase: 'interaction_target_readiness', + data: { + polls: 2, + end: 'done', + command: 'click', + replaySession: 'unrelated', + replayPath: file, + step: 1, + }, + }); + }, + ); + await withDiagnosticsScope( + { session, command: 'click', debug: true, logPath: path.join(requestDir, 'request.ndjson') }, + async () => { + vi.setSystemTime(baseMs + 3); + emitDiagnostic({ phase: 'interaction_target_readiness', data: { polls: 2, end: 'done' } }); + vi.setSystemTime(baseMs + 6); + emitDiagnostic({ phase: 'interaction_target_readiness', data: { polls: 2, end: 'done' } }); + }, + ); + } finally { + vi.useRealTimers(); + } + const artifacts = path.join(cwd, 'test/artifacts'); + fs.mkdirSync(artifacts, { recursive: true }); + const tracePath = path.join(artifacts, 'replay-timing.ndjson'); + const trace = [ + { type: 'replay_test_attempt_start', session, replayPath: file }, + { type: 'replay_action_start', replayPath: file, ts: ts(2), step: 1, command: 'click' }, + { + type: 'replay_action_stop', + replayPath: file, + ts: ts(4), + step: 1, + command: 'click', + ok: true, + }, + { type: 'replay_action_start', replayPath: file, ts: ts(5), step: 2, command: 'click' }, + { + type: 'replay_action_stop', + replayPath: file, + ts: ts(7), + step: 2, + command: 'click', + ok: false, + }, + ]; + const junit = path.join(artifacts, 'suite.xml'); + const reporter = createJunitReplayTestReporter(junit); + reporter.onSuiteEnd!({ + total: 1, + executed: 1, + passed: 1, + failed: 0, + skipped: 0, + notRun: 0, + durationMs: 50, + failures: [], + tests: [ + { + file, + title: 'A renamed & readable title', + session, + status: 'passed', + durationMs: 50, + attempts: 1, + replayed: 2, + healed: 0, + }, + ], + }); + fs.writeFileSync(tracePath, trace.map((event) => JSON.stringify(event)).join('\n')); + invoke('summarize', 'android', state, junit); + const summaryPath = path.join(artifacts, 'readiness-experiment/android-summary.json'); + let summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8')); + expect(summary.suitePassed).toBe(true); + expect(summary.corpus[0].name).toBe('navigation.ad'); + expect(summary.readiness).toMatchObject({ + engaged: 3, + rescuedActions: 1, + unattributedResolvedWaits: 1, + }); + expect(summary.validForDecision).toBe(false); + trace[4] = { ...trace[4], ok: true }; + fs.writeFileSync(tracePath, trace.map((event) => JSON.stringify(event)).join('\n')); + invoke('summarize', 'android', state, junit); + summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8')); + expect(summary.readiness.rescuedActions).toBe(2); + expect(summary.validForDecision).toBe(true); + invoke('summarize', 'android', state, junit, 'diagnostic-only'); + summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8')); + expect(summary.diagnosticOnly).toBe(true); + expect(summary.suitePassed).toBe(true); + expect(summary.pathProven).toBe(true); + expect(summary.validForDecision).toBe(false); + trace.push(trace[1]!, trace[2]!); + fs.writeFileSync(tracePath, trace.map((event) => JSON.stringify(event)).join('\n')); + invoke('summarize', 'android', state, junit); + summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8')); + expect(summary.validForDecision).toBe(false); + expect(summary.readiness.unattributedResolvedWaits).toBeGreaterThan(0); +}); diff --git a/scripts/replays/readiness-experiment.ts b/scripts/replays/readiness-experiment.ts new file mode 100644 index 0000000000..f3ac92e2a5 --- /dev/null +++ b/scripts/replays/readiness-experiment.ts @@ -0,0 +1,364 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import { pathToFileURL } from 'node:url'; +import { parseXmlDocumentSync } from '@agent-device/xml'; +import { safeSessionName } from '@agent-device/host-kit/session-paths'; + +const DEFAULT_SOURCE = 'packages/replay-port/src/daemon-port/session-replay-action-runtime.ts'; +const DEFAULT_DECLARATION = 'const REPLAY_DEFAULT_READINESS_TIMEOUT_MS = 2_000;'; +const OUTPUT = 'test/artifacts/readiness-experiment'; + +export function baselineSource(source: string): string { + if (source.split(DEFAULT_DECLARATION).length !== 2) { + throw new Error('Expected exactly one replay readiness default declaration'); + } + return source.replace(DEFAULT_DECLARATION, 'const REPLAY_DEFAULT_READINESS_TIMEOUT_MS = 0;'); +} + +export type ReadinessEvent = { + phase?: string; + ts?: string; + session?: string; + data?: { + polls?: number; + end?: string; + command?: string; + replaySession?: string; + replayPath?: string; + step?: number; + }; +}; + +export function countReadiness(events: readonly ReadinessEvent[]) { + const waits = events.filter((event) => event.phase === 'interaction_target_readiness'); + const engaged = waits.filter((event) => (event.data?.polls ?? 0) > 1); + const resolved = engaged.filter((event) => event.data?.end === 'done').length; + return { + observations: waits.length, + firstPoll: waits.filter((event) => event.data?.polls === 1).length, + engaged: engaged.length, + resolved, + incomplete: engaged.length - resolved, + targetGate: engaged.filter((event) => event.data?.command !== undefined).length, + dispatch: engaged.filter((event) => event.data?.command === undefined).length, + }; +} + +function readJson(file: string) { + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +function writeJson(file: string, value: unknown) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`); +} + +function findFiles(dir: string, suffix: string): string[] { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const file = path.join(dir, entry.name); + if (entry.isDirectory()) return findFiles(file, suffix); + return entry.name.endsWith(suffix) ? [file] : []; + }); +} + +function readNdjson(file: string): T[] { + return fs + .readFileSync(file, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + +function prepare(mode: string, head: string) { + if (!['enabled', 'disabled'].includes(mode) || !/^[a-f0-9]{40}$/.test(head)) { + throw new Error('prepare requires enabled/disabled and an exact commit SHA'); + } + const source = fs.readFileSync(DEFAULT_SOURCE, 'utf8'); + const baseline = baselineSource(source); + const applied = mode === 'disabled' ? baseline : source; + if (mode === 'disabled') fs.writeFileSync(DEFAULT_SOURCE, applied); + writeJson(path.join(OUTPUT, 'variant.json'), { + head, + mode, + source: DEFAULT_SOURCE, + sourceSha256: createHash('sha256').update(applied).digest('hex'), + defaultBudgetMs: mode === 'enabled' ? 2_000 : 0, + }); + fs.writeFileSync( + path.join(OUTPUT, 'source.patch'), + mode === 'disabled' + ? `--- a/${DEFAULT_SOURCE}\n+++ b/${DEFAULT_SOURCE}\n@@\n-${DEFAULT_DECLARATION}\n+const REPLAY_DEFAULT_READINESS_TIMEOUT_MS = 0;\n` + : '', + ); +} + +function begin(platform: string) { + if (!['ios', 'android'].includes(platform)) throw new Error('Expected ios/android'); + writeJson(path.join(OUTPUT, `${platform}-start.json`), { startedMs: Date.now() }); +} + +type SuiteCase = { file: string; session: string }; +type ReplayAction = { + session: string; + replayPath: string; + step: number; + command: string; + startedMs: number; + finishedMs: number; + ok: boolean; +}; +type TraceEvent = { + type: string; + ts: string; + replayPath: string; + session: string; + step: number; + command: string; + ok: boolean; +}; +const ACTING_COMMANDS = ['press', 'click', 'longpress']; + +function canonicalFile(file: string): string { + return fs.existsSync(file) ? fs.realpathSync(file) : path.resolve(file); +} + +function readSuite(platform: string, junitPath: string) { + const junit = fs.existsSync(junitPath) ? fs.readFileSync(junitPath, 'utf8') : ''; + const suite = parseXmlDocumentSync(junit) + .find((node) => node.name === 'testsuites') + ?.children.find((node) => node.name === 'testsuite'); + const testcases = suite?.children.filter((node) => node.name === 'testcase') ?? []; + const cases: SuiteCase[] = testcases.map((node) => ({ + file: canonicalFile(node.attributes.file ?? ''), + session: + /^session: (.+)$/m.exec( + node.children.find((child) => child.name === 'system-out')?.text ?? '', + )?.[1] ?? '', + })); + const corpusDir = + platform === 'ios' + ? 'test/integration/replays/ios/simulator' + : 'test/integration/replays/android/emulator'; + const expectedFiles = fs + .readdirSync(corpusDir) + .filter((name) => name.endsWith('.ad')) + .map((name) => canonicalFile(path.join(corpusDir, name))) + .sort(); + const corpusMatches = + JSON.stringify(cases.map((item) => item.file).sort()) === JSON.stringify(expectedFiles); + const caseSessionsValid = + cases.every((item) => item.session.length > 0) && + new Set(cases.map((item) => item.session)).size === cases.length; + const countsPassed = + suite?.attributes.failures === '0' && + suite.attributes.skipped === '0' && + Number(suite.attributes.tests) === cases.length; + return { + cases, + passed: corpusMatches && caseSessionsValid && countsPassed, + replayTimeMs: Number(suite?.attributes.time) * 1_000, + corpus: expectedFiles.map((file) => ({ + name: path.basename(file), + sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex'), + })), + }; +} + +function actionsFromTrace(trace: readonly TraceEvent[], session: string): ReplayAction[] { + return trace + .filter( + (event) => event.type === 'replay_action_start' && ACTING_COMMANDS.includes(event.command), + ) + .flatMap((start) => { + const stops = trace.filter( + (event) => + event.type === 'replay_action_stop' && + event.step === start.step && + event.replayPath === start.replayPath, + ); + if (stops.length !== 1) return []; + const stop = stops[0]!; + return [ + { + session, + replayPath: canonicalFile(start.replayPath), + step: start.step, + command: start.command, + startedMs: Date.parse(start.ts), + finishedMs: Date.parse(stop.ts), + ok: stop.ok === true, + }, + ]; + }); +} + +function loadActions(artifactDir: string, cases: readonly SuiteCase[]) { + const actions: ReplayAction[] = []; + const files: string[] = []; + for (const file of findFiles(artifactDir, 'replay-timing.ndjson')) { + const trace = readNdjson(file); + const attempts = trace.filter((event) => event.type === 'replay_test_attempt_start'); + if (attempts.length !== 1) continue; + const attempt = attempts[0]!; + if ( + !cases.some( + (item) => + item.session === attempt.session && item.file === canonicalFile(attempt.replayPath), + ) + ) + continue; + files.push(file); + actions.push(...actionsFromTrace(trace, attempt.session)); + } + return { actions, files }; +} + +function eventBelongsToCases(event: ReadinessEvent, cases: readonly SuiteCase[]): boolean { + if (event.data?.replaySession !== undefined) { + return cases.some( + (item) => + item.session === event.data?.replaySession && + item.file === canonicalFile(event.data?.replayPath ?? ''), + ); + } + return cases.some((item) => item.session === event.session); +} + +function loadDiagnostics(stateDir: string, cases: readonly SuiteCase[]) { + const events: ReadinessEvent[] = []; + const files: string[] = []; + for (const file of findFiles(path.join(stateDir, 'sessions'), '.ndjson')) { + if (path.basename(path.dirname(file)) !== 'requests') continue; + const selected = readNdjson(file).filter((event) => + eventBelongsToCases(event, cases), + ); + if (selected.length === 0) continue; + files.push(path.relative(stateDir, file)); + events.push(...selected); + } + const complete = + cases.length > 0 && + cases.every((item) => { + const requestDir = path.join('sessions', safeSessionName(item.session), 'requests'); + return files.some((file) => path.dirname(file) === requestDir); + }); + return { events, files, complete }; +} + +function candidatesForWait(event: ReadinessEvent, actions: readonly ReplayAction[]) { + if (event.data?.replaySession !== undefined) { + return actions.filter( + (action) => + action.session === event.data?.replaySession && + action.replayPath === canonicalFile(event.data?.replayPath ?? '') && + action.step === event.data?.step && + action.command === event.data?.command, + ); + } + const observedMs = Date.parse(event.ts ?? ''); + return actions.filter( + (action) => + action.session === event.session && + action.startedMs <= observedMs && + action.finishedMs >= observedMs, + ); +} + +function attributeResolvedWaits( + events: readonly ReadinessEvent[], + actions: readonly ReplayAction[], +) { + const rescued = new Set(); + let unattributedResolvedWaits = 0; + const resolved = events.filter( + (event) => + event.phase === 'interaction_target_readiness' && + (event.data?.polls ?? 0) > 1 && + event.data?.end === 'done', + ); + for (const event of resolved) { + const candidates = candidatesForWait(event, actions); + if (candidates.length !== 1 || !candidates[0]!.ok) { + unattributedResolvedWaits++; + continue; + } + const action = candidates[0]!; + rescued.add(JSON.stringify([action.session, action.replayPath, action.step])); + } + return { rescuedActions: rescued.size, unattributedResolvedWaits }; +} + +function evidenceValidity(params: { + suitePassed: boolean; + pathProven: boolean; + diagnosticsComplete: boolean; + startedMs: number; + finishedMs: number; + replayTimeMs: number; + unattributed: number; +}) { + return ( + params.suitePassed && + params.pathProven && + params.diagnosticsComplete && + Number.isFinite(params.startedMs) && + params.finishedMs > params.startedMs && + Number.isFinite(params.replayTimeMs) && + params.replayTimeMs > 0 && + params.unattributed === 0 + ); +} + +function summarize(platform: string, stateDir: string, junitPath: string, diagnosticOnly = false) { + const finishedMs = Date.now(); + const startPath = path.join(OUTPUT, `${platform}-start.json`); + const startedMs = fs.existsSync(startPath) ? readJson(startPath).startedMs : Number.NaN; + const variant = readJson(path.join(OUTPUT, 'variant.json')); + const suite = readSuite(platform, junitPath); + const trace = loadActions(path.dirname(junitPath), suite.cases); + const diagnostics = loadDiagnostics(stateDir, suite.cases); + const readiness = countReadiness(diagnostics.events); + const attribution = attributeResolvedWaits(diagnostics.events, trace.actions); + const completedActions = trace.actions.filter((action) => action.ok).length; + const pathProven = + completedActions > 0 && + (variant.mode === 'enabled' ? readiness.observations > 0 : readiness.observations === 0); + const summary = { + ...variant, + platform, + diagnosticOnly, + suitePassed: suite.passed, + pathProven, + validForDecision: + !diagnosticOnly && + evidenceValidity({ + suitePassed: suite.passed, + pathProven, + diagnosticsComplete: diagnostics.complete, + startedMs, + finishedMs, + replayTimeMs: suite.replayTimeMs, + unattributed: attribution.unattributedResolvedWaits, + }), + wallTimeMs: Number.isFinite(startedMs) ? finishedMs - startedMs : null, + replayTimeMs: suite.replayTimeMs, + corpus: suite.corpus, + readiness: { ...readiness, ...attribution }, + completedActions, + actionTraceFiles: trace.files, + requestFiles: diagnostics.files, + }; + writeJson(path.join(OUTPUT, `${platform}-summary.json`), summary); + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + const [command, ...args] = process.argv.slice(2); + if (command === 'prepare') prepare(args[0]!, args[1]!); + else if (command === 'begin') begin(args[0]!); + else if (command === 'summarize') + summarize(args[0]!, args[1]!, args[2]!, args[3] === 'diagnostic-only'); + else throw new Error('Expected prepare, begin, or summarize'); +} diff --git a/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts b/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts index ddffb39004..22f8f02945 100644 --- a/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts +++ b/src/daemon/__tests__/replay-target/session-replay-target-verification-runtime.test.ts @@ -221,7 +221,15 @@ test('an annotated click whose target renders on the second capture waits for it }); expect(mockDispatchCommand).toHaveBeenCalledTimes(2); expect(readinessDiagnostics()).toEqual([ - { polls: 2, waitedMs: 200, end: 'done', command: 'click' }, + { + polls: 2, + waitedMs: 200, + end: 'done', + command: 'click', + replaySession: scene.sessionName, + replayPath: scene.filePath, + step: 1, + }, ]); }); @@ -266,7 +274,15 @@ test('the budget the gate hands the dispatch counts from the end of its first ca expect(response.ok).toBe(true); expect(readinessDiagnostics()).toEqual([ - { polls: 9, waitedMs: 2_100, end: 'done', command: 'click' }, + { + polls: 9, + waitedMs: 2_100, + end: 'done', + command: 'click', + replaySession: scene.sessionName, + replayPath: scene.filePath, + step: 1, + }, ]); expect(scene.invoked[0]?.flags?.readinessTimeoutMs).toBe(400); }); diff --git a/test/integration/replays/android/emulator/04-text-input-keyboard.ad b/test/integration/replays/android/emulator/04-text-input-keyboard.ad index 4dbdc1c01c..360704ae75 100644 --- a/test/integration/replays/android/emulator/04-text-input-keyboard.ad +++ b/test/integration/replays/android/emulator/04-text-input-keyboard.ad @@ -4,14 +4,18 @@ context platform=android open settings --relaunch snapshot -i -click @e2 -wait 1000 +click 'text="Search Settings"' +wait 'role=text-field focused=true' 5000 +is focused 'role=text-field' +screenshot "./test/screenshots/replays/android-search-focused.png" snapshot -i keyboard status type "Display" -wait 1000 +wait 'role=text-field value="Display"' 5000 +is focused 'role=text-field value="Display"' +screenshot "./test/screenshots/replays/android-search-typed.png" snapshot -i screenshot "./test/screenshots/replays/android-search-results.png" diff --git a/test/integration/replays/ios/simulator/03-scroll-discovery.ad b/test/integration/replays/ios/simulator/03-scroll-discovery.ad index 2646c29e9b..fd6d9f1250 100644 --- a/test/integration/replays/ios/simulator/03-scroll-discovery.ad +++ b/test/integration/replays/ios/simulator/03-scroll-discovery.ad @@ -2,10 +2,10 @@ context platform=ios target=mobile open com.apple.Preferences --relaunch -scroll down 3 +scroll down --until 'label=Accessibility' wait 500 find text "Accessibility" exists -scroll up 3 +scroll up --until 'label=General' wait 500 find text "General" exists diff --git a/test/integration/replays/ios/simulator/04-text-input-keyboard.ad b/test/integration/replays/ios/simulator/04-text-input-keyboard.ad index a9f04c26d8..5740a652af 100644 --- a/test/integration/replays/ios/simulator/04-text-input-keyboard.ad +++ b/test/integration/replays/ios/simulator/04-text-input-keyboard.ad @@ -3,9 +3,12 @@ context platform=ios target=mobile open com.apple.Preferences --relaunch click "role=searchField || label=Search || label=\"Search\"" -wait 1000 +wait 'editable=true focused=true' +is focused 'editable=true' type "General" -wait 1000 +wait 'editable=true value="General"' +is focused 'editable=true value="General"' +screenshot "./test/screenshots/replays/ios-search-results.png" find text "General" exists diff --git a/vitest.config.ts b/vitest.config.ts index ac655289e7..14ea6794aa 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -103,6 +103,7 @@ export default defineConfig({ include: [ 'src/**/*.test.ts', 'packages/*/src/**/*.test.ts', + 'scripts/replays/*.test.ts', // The maestro conformance projection is test-tree code beside its harness; its own // test stays in the fast lane rather than the node:test conformance script. 'packages/maestro/test/conformance/conformance-selector-projection.test.ts',