diff --git a/.github/workflows/replays-manual.yml b/.github/workflows/replays-manual.yml index 59c76e1d2c..63f22709d6 100644 --- a/.github/workflows/replays-manual.yml +++ b/.github/workflows/replays-manual.yml @@ -58,25 +58,7 @@ jobs: profile: pixel_7 target: google_apis_playstore emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics - script: | - # android-emulator-runner executes this with /usr/bin/sh, which is dash on - # ubuntu runners and rejects `-o pipefail`. The serial assignment below is - # validated by the `test -n` guard instead. - set -eu - ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" - test -n "$ANDROID_SERIAL" - BOOT_FINISHED_AT="$(date +%s)" - pnpm gate build - pnpm clean:daemon - adb -s "$ANDROID_SERIAL" install -r "${{ steps.fixture-app.outputs.apk-path }}" - node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag-android.ad --platform android --serial "$ANDROID_SERIAL" --session nightly-android-target-drag --json - pnpm clean:daemon - FULL_STARTED_AT="$(date +%s)" - AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=full AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts - FULL_FINISHED_AT="$(date +%s)" - echo "Android setup before emulator: $((BOOT_FINISHED_AT - ${{ steps.android-setup.outputs.seconds }}))s" - echo "Android full scenario wall time: $((FULL_FINISHED_AT - FULL_STARTED_AT))s" - pnpm gate replay-android --retries 2 --artifacts-dir test/artifacts/replays-android --report-junit test/artifacts/replays-android.junit.xml + script: sh scripts/replays/android-manual.sh "${{ steps.fixture-app.outputs.apk-path }}" "${{ steps.fixture-app.outputs.app-id }}" "${{ steps.android-setup.outputs.seconds }}" - name: Upload Android artifacts if: always() diff --git a/scripts/replays/android-manual.sh b/scripts/replays/android-manual.sh new file mode 100644 index 0000000000..3f8c418fdd --- /dev/null +++ b/scripts/replays/android-manual.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu + +FIXTURE_APK="$1" +FIXTURE_APP_ID="$2" +SETUP_STARTED_AT="$3" +test -n "$ANDROID_SERIAL" +BOOT_FINISHED_AT="$(date +%s)" + +pnpm gate build +pnpm clean:daemon +adb -s "$ANDROID_SERIAL" install -r "$FIXTURE_APK" +node --experimental-strip-types src/bin.ts replay examples/test-app/replays/drag-android.ad --platform android --serial "$ANDROID_SERIAL" --session nightly-android-target-drag --json +pnpm clean:daemon + +FULL_STARTED_AT="$(date +%s)" +FULL_STATUS=0 +AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_E2E_TIER=full AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="$FIXTURE_APK" AGENT_DEVICE_FIXTURE_APP_ID="$FIXTURE_APP_ID" node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts || FULL_STATUS=$? +FULL_FINISHED_AT="$(date +%s)" +echo "Android setup before emulator: $((BOOT_FINISHED_AT - SETUP_STARTED_AT))s" +echo "Android full scenario wall time: $((FULL_FINISHED_AT - FULL_STARTED_AT))s" +test "$FULL_STATUS" -eq 0 || exit "$FULL_STATUS" +pnpm gate replay-android --retries 2 --artifacts-dir test/artifacts/replays-android --report-junit test/artifacts/replays-android.junit.xml diff --git a/scripts/replays/android-manual.test.ts b/scripts/replays/android-manual.test.ts new file mode 100644 index 0000000000..8d831d3d9c --- /dev/null +++ b/scripts/replays/android-manual.test.ts @@ -0,0 +1,108 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { parse } from 'yaml'; +import { expect, test } from 'vitest'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { mkdtempForTest } from '@agent-device/test-kit/tmp-dir'; + +const pipeline = (serial: string) => [ + 'pnpm|gate build', + 'pnpm|clean:daemon', + `adb|-s ${serial} install -r fixture.apk`, + `node|--experimental-strip-types src/bin.ts replay examples/test-app/replays/drag-android.ad --platform android --serial ${serial} --session nightly-android-target-drag --json`, + 'pnpm|clean:daemon', + 'node|--experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-android-emulator.test.ts', + `env|1|full|${serial}|fixture.apk|com.fixture`, + 'pnpm|gate replay-android --retries 2 --artifacts-dir test/artifacts/replays-android --report-junit test/artifacts/replays-android.junit.xml', +]; + +test.each([ + { serial: 'emulator-7000', fullExit: 0 }, + { serial: 'emulator-8100', fullExit: 0 }, + { serial: 'emulator-8100', fullExit: 13 }, +])( + 'manual action preserves serial $serial and full-suite status $fullExit with timing', + async ({ serial, fullExit }) => { + const cwd = await mkdtempForTest('android-manual-shell-'); + const bin = path.join(cwd, 'bin'); + fs.mkdirSync(bin); + const tool = (name: string, body: string) => + fs.writeFileSync(path.join(bin, name), `#!/bin/sh\n${body}\n`, { mode: 0o755 }); + tool( + 'pnpm', + String.raw`printf 'pnpm|%s\n' "$*" >> calls.log +case "$1 $2" in + 'gate build') exit "$BUILD_EXIT";; + 'gate replay-android') exit "$REPLAY_EXIT";; +esac`, + ); + tool( + 'adb', + String.raw`if [ "$1" = devices ]; then + printf 'List of devices attached\n%s\tdevice\n' "$EXPECTED_SERIAL" + exit 0 +fi +printf 'adb|%s\n' "$*" >> calls.log +test "$1" = -s && test "$2" = "$EXPECTED_SERIAL" || exit 42 +exit "$INSTALL_EXIT"`, + ); + tool( + 'node', + String.raw`printf 'node|%s\n' "$*" >> calls.log +if [ "$3" = replay ]; then exit "$DRAG_EXIT"; fi +printf 'env|%s|%s|%s|%s|%s\n' "$AGENT_DEVICE_ANDROID_E2E" "$AGENT_DEVICE_ANDROID_E2E_TIER" "$AGENT_DEVICE_ANDROID_SERIAL" "$AGENT_DEVICE_FIXTURE_APP_PATH" "$AGENT_DEVICE_FIXTURE_APP_ID" >> calls.log +exit "$FULL_EXIT"`, + ); + tool( + 'date', + String.raw`case "$(cat clock-index 2>/dev/null || echo 0)" in + 0) echo 100; echo 1 > clock-index;; + 1) echo 120; echo 2 > clock-index;; + *) echo 145;; +esac`, + ); + fs.mkdirSync(path.join(cwd, 'scripts/replays'), { recursive: true }); + fs.symlinkSync( + path.resolve('scripts/replays/android-manual.sh'), + path.join(cwd, 'scripts/replays/android-manual.sh'), + ); + const workflow = parse(fs.readFileSync('.github/workflows/replays-manual.yml', 'utf8')); + const runner = workflow.jobs['manual-android'].steps.find((step: { uses?: string }) => + step.uses?.startsWith('reactivecircus/android-emulator-runner@'), + ); + const script: string = runner.with.script + .replaceAll('${{ steps.fixture-app.outputs.apk-path }}', 'fixture.apk') + .replaceAll('${{ steps.fixture-app.outputs.app-id }}', 'com.fixture') + .replaceAll('${{ steps.android-setup.outputs.seconds }}', '90'); + let result = { exitCode: 0, stderr: '', stdout: '' }; + let stdout = ''; + + for (const command of script.split('\n').map((line) => line.trim())) { + if (!command || command.startsWith('#')) continue; + result = runCmdSync('sh', ['-c', command], { + cwd, + allowFailure: true, + timeoutMs: 10_000, + env: { + PATH: `${bin}:${process.env.PATH}`, + ANDROID_SERIAL: serial, + EXPECTED_SERIAL: serial, + BUILD_EXIT: '0', + INSTALL_EXIT: '0', + DRAG_EXIT: '0', + FULL_EXIT: String(fullExit), + REPLAY_EXIT: '0', + }, + }); + stdout += result.stdout; + if (result.exitCode !== 0) break; + } + expect(result.exitCode, result.stderr).toBe(fullExit); + const expected = pipeline(serial); + expect(fs.readFileSync(path.join(cwd, 'calls.log'), 'utf8').trim().split('\n')).toEqual( + fullExit === 0 ? expected : expected.slice(0, -1), + ); + expect(stdout).toContain('Android setup before emulator: 10s'); + expect(stdout).toContain('Android full scenario wall time: 25s'); + }, +); diff --git a/vitest.config.ts b/vitest.config.ts index a799f3723b..030e973b81 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -109,6 +109,7 @@ export default defineConfig({ // The subprocess watchdog self-check: spawns a real node subprocess per case, // and one hangs on purpose (#1414). 'scripts/fuzz/harness.test.ts', + 'scripts/replays/android-manual.test.ts', // The validation fuzz generators' expectation gates (#1781 B2): in-process, no // subprocess or worker, so they ride the fast lane unlike their serialized siblings. 'scripts/fuzz/validation-arbitraries.test.ts',