From b58d5358114e2214ba2ac1740655cafaa5febb01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 20 Oct 2026 18:48:10 +0200 Subject: [PATCH 1/4] fix(android): shorten idle waits for spaced snapshot reads --- android/snapshot-helper/README.md | 7 +++++ .../SnapshotInstrumentation.java | 10 ++++--- packages/contracts/src/interactor-types.ts | 2 ++ .../src/core/snapshot-runtime-binding.ts | 27 ++++++++++++++++++- .../src/snapshot-helper-capture.ts | 7 ++++- .../src/snapshot-helper-session-protocol.ts | 8 +++++- .../src/snapshot-helper-session.ts | 1 + .../src/snapshot-helper-types.ts | 2 ++ packages/platform-android/src/snapshot.ts | 4 +++ src/core/interactors/android.ts | 1 + 10 files changed, 63 insertions(+), 6 deletions(-) diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index c414ce3023..c7bff39255 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -98,6 +98,7 @@ closes that connection; the process stays alive for the next connection: - `snapshot ` — capture and return an XML snapshot, same semantics as the default mode - `snapshot-immediate ` — capture fresh XML without an idle wait when the caller owns settling +- `snapshot-short-idle ` — capture with at most a 100 ms idle wait, capped by the session's configured timeout - `input ` — inject a tap, key, or point-targeted text replacement - `focus ` — read the focused window type and root package after clearing the accessibility cache - `screenshot ` — return a PNG in the `payloadBase64` header @@ -108,6 +109,12 @@ closes that connection; the process stays alive for the next connection: The ready record advertises optional operations in `sessionCommands`. A host must check support before sending them. Text replacement retains a focus tap; the host owns post-write verification. Input failures must not be resent because a mutation may already have reached the app. +The daemon uses the shorter wait for Android `snapshot` and `diff` only when the latest stored +tree is at least 500 ms old, no later operation has retired it, and no freshness recovery or +post-gesture stabilization is pending. Other captures keep their configured wait. Each capture +still reads a fresh tree. An asynchronous app transition can appear in progress; use an explicit +landmark wait when the next command requires a destination screen. + `viewport` and `gesture` responses are headers-only (no body): `kind`, `injectedEvents`, `elapsedMs` for `gesture`, and `x`, `y`, `width`, `height` for `viewport`. `snapshot` responses carry the XML body after the header block, as described below. The response protocol literal is always diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 789db4f26e..b8c4c03736 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -25,6 +25,7 @@ public final class SnapshotInstrumentation extends Instrumentation { private static final long GESTURE_UI_AUTOMATION_CONNECT_TIMEOUT_MS = 8_000; private static final int CHUNK_SIZE = 2 * 1024; private static final long DEFAULT_TIMEOUT_MS = 8_000; + private static final long SHORT_IDLE_TIMEOUT_MS = 100; private static final long ROOT_CAPTURE_STABILIZATION_TIMEOUT_MS = 500; private static final int DEFAULT_MAX_DEPTH = 128; private static final int DEFAULT_MAX_NODES = 5_000; @@ -181,7 +182,7 @@ private void runSnapshotSession( putBaseMetadata( ready, waitForIdleTimeoutMs, waitForIdleQuietMs, timeoutMs, maxDepth, maxNodes); ready.putString("sessionReady", "true"); - ready.putString("sessionCommands", "input,screenshot,focus,snapshot-immediate" + ready.putString("sessionCommands", "input,screenshot,focus,snapshot-immediate,snapshot-short-idle" + (android.os.Build.VERSION.SDK_INT >= 31 ? ",app" : "")); ready.putString("sessionPort", Integer.toString(sessionPort)); sendStatus(2, ready); @@ -223,7 +224,8 @@ private void runSnapshotSession( getConnectedUiAutomationUnchecked()); continue; } - if (!"snapshot".equals(action) && !"snapshot-immediate".equals(action)) { + if (!"snapshot".equals(action) && !"snapshot-immediate".equals(action) + && !"snapshot-short-idle".equals(action)) { SessionResponseWriter.writeSessionError( socket.getOutputStream(), requestId, @@ -235,7 +237,9 @@ private void runSnapshotSession( socket.getOutputStream(), requestId, waitForIdleQuietMs, - "snapshot-immediate".equals(action) ? 0 : waitForIdleTimeoutMs, + "snapshot-immediate".equals(action) ? 0 + : "snapshot-short-idle".equals(action) ? Math.min(waitForIdleTimeoutMs, SHORT_IDLE_TIMEOUT_MS) + : waitForIdleTimeoutMs, timeoutMs, maxDepth, maxNodes); diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index c0a30db5b3..a8c6151006 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -191,6 +191,8 @@ export type SnapshotOptions = BaseSnapshotOptions & { surface?: SessionSurface; /** Internal capture purpose; action outcomes always require the full tree. */ acquisitionIntent?: 'full' | 'surface-observation'; + /** Internal Android observation hint; action and settling captures keep their normal idle wait. */ + androidIdleWait?: 'short'; /** * A one-off read, such as an open's launch observation. It may use a capture host the session * already keeps warm, but it never installs one or leaves running one that it started. It starts diff --git a/packages/daemon/src/core/snapshot-runtime-binding.ts b/packages/daemon/src/core/snapshot-runtime-binding.ts index 12313c63be..8fb3e9a7b2 100644 --- a/packages/daemon/src/core/snapshot-runtime-binding.ts +++ b/packages/daemon/src/core/snapshot-runtime-binding.ts @@ -31,6 +31,9 @@ import { type AdmittedRuntimePlan, } from './session-runtime-admission.ts'; import { resolveSnapshotScope } from './snapshot-capture.ts'; +import { isOutdatedObservation } from './ref-frame.ts'; +import { getActiveAndroidSnapshotFreshness } from './session-snapshot-freshness.ts'; +import { isPostGestureStabilizationPending } from './deferred-interaction-outcome.ts'; import { resolveSessionDevice } from './snapshot-session.ts'; import { selectElementTextOperation, @@ -137,7 +140,16 @@ export async function admitAndBindSnapshotCapture( ); return Object.freeze({ ok: true, - capture: async (input: CaptureSnapshotInput) => await bound.captureSnapshot(input), + capture: async (input: CaptureSnapshotInput) => { + const shortIdleWait = + (command === 'snapshot' || command === 'diff') && + canUseShortAndroidSnapshotIdleWait(session); + return await bound.captureSnapshot( + shortIdleWait + ? { ...input, options: { ...input.options, androidIdleWait: 'short' } } + : input, + ); + }, ...(bound.readTextAtPoint ? { readTextAtPoint: bound.readTextAtPoint } : {}), ...(bound.findText ? { findText: bound.findText } : {}), ...(bound.focusPoint ? { focusPoint: bound.focusPoint } : {}), @@ -145,6 +157,19 @@ export async function admitAndBindSnapshotCapture( }); } +const ANDROID_SNAPSHOT_IDLE_GAP_MS = 500; + +function canUseShortAndroidSnapshotIdleWait(session: SessionState | undefined): boolean { + return ( + session?.device.platform === 'android' && + session.snapshot !== undefined && + !isOutdatedObservation(session.snapshot) && + Date.now() - session.snapshot.createdAt >= ANDROID_SNAPSHOT_IDLE_GAP_MS && + !getActiveAndroidSnapshotFreshness(session) && + !isPostGestureStabilizationPending(session) + ); +} + /** Resolves one plan, inspects its owner facts once, then returns one bound capture closure. */ export async function resolveBoundSnapshotCaptureRuntime( params: SnapshotRuntimeRouteParams, diff --git a/packages/platform-android/src/snapshot-helper-capture.ts b/packages/platform-android/src/snapshot-helper-capture.ts index 97ad5558df..66ced5abe3 100644 --- a/packages/platform-android/src/snapshot-helper-capture.ts +++ b/packages/platform-android/src/snapshot-helper-capture.ts @@ -14,6 +14,7 @@ import { } from './instrumentation-helper.ts'; import { ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, + ANDROID_SNAPSHOT_HELPER_SHORT_IDLE_TIMEOUT_MS, ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS, ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT, @@ -68,7 +69,11 @@ export async function captureAndroidSnapshotWithHelper( result = await options.adb( buildAndroidSnapshotHelperArgs({ ...resolved, - ...(options.skipIdleWait ? { waitForIdleTimeoutMs: 0 } : {}), + waitForIdleTimeoutMs: options.skipIdleWait + ? 0 + : options.shortIdleWait + ? Math.min(resolved.waitForIdleTimeoutMs, ANDROID_SNAPSHOT_HELPER_SHORT_IDLE_TIMEOUT_MS) + : resolved.waitForIdleTimeoutMs, }), { allowFailure: true, diff --git a/packages/platform-android/src/snapshot-helper-session-protocol.ts b/packages/platform-android/src/snapshot-helper-session-protocol.ts index c2ee8cca66..b0365a57fe 100644 --- a/packages/platform-android/src/snapshot-helper-session-protocol.ts +++ b/packages/platform-android/src/snapshot-helper-session-protocol.ts @@ -86,14 +86,20 @@ export async function requestAndroidSnapshotHelperSessionSnapshot(params: { commandTimeoutMs: number; signal?: AbortSignal; skipIdleWait?: boolean; + shortIdleWait?: boolean; }): Promise { const requestId = `snapshot-${Date.now()}-${Math.random().toString(16).slice(2)}`; + const command = params.skipIdleWait + ? 'snapshot-immediate' + : params.shortIdleWait + ? 'snapshot-short-idle' + : 'snapshot'; // The socket waits out the whole command budget the session was given. The device-side // `timeoutMs` bounds only the UiAutomation connect wait, not the traversal after it, so a deadline // built from that wait plus a fixed overhead cut off slow-but-healthy captures (tester-army/e2e#841). const response = await sendAndroidSnapshotHelperSessionCommand( params.port, - `${params.skipIdleWait ? 'snapshot-immediate' : 'snapshot'} ${requestId}`, + `${command} ${requestId}`, params.commandTimeoutMs, params.signal, ); diff --git a/packages/platform-android/src/snapshot-helper-session.ts b/packages/platform-android/src/snapshot-helper-session.ts index dd7cc7e6a8..89cdf3b255 100644 --- a/packages/platform-android/src/snapshot-helper-session.ts +++ b/packages/platform-android/src/snapshot-helper-session.ts @@ -56,6 +56,7 @@ async function captureFromAndroidSnapshotHelperSession(params: { commandTimeoutMs: resolved.commandTimeoutMs, signal: options.signal, skipIdleWait: options.skipIdleWait && session.commands.has('snapshot-immediate'), + shortIdleWait: options.shortIdleWait && session.commands.has('snapshot-short-idle'), }); session.capturedCount += 1; return { diff --git a/packages/platform-android/src/snapshot-helper-types.ts b/packages/platform-android/src/snapshot-helper-types.ts index 945fd87869..11a09482fb 100644 --- a/packages/platform-android/src/snapshot-helper-types.ts +++ b/packages/platform-android/src/snapshot-helper-types.ts @@ -18,6 +18,7 @@ export const ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT = 'uiautomator-xml'; // Keep common snapshots biased toward post-microinteraction reliability. The // value is a max wait; callers that need immediate capture can explicitly pass 0. export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS = 500; +export const ANDROID_SNAPSHOT_HELPER_SHORT_IDLE_TIMEOUT_MS = 100; export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS = 100; export const ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS = 5_000; export const ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS = 5_000; @@ -79,6 +80,7 @@ export type AndroidSnapshotHelperCaptureOptions = { maxNodes?: number; emitChunks?: boolean; skipIdleWait?: boolean; + shortIdleWait?: boolean; }; export type AndroidSnapshotHelperMetadata = { diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index 95645e6a7a..ffcb428f15 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -103,6 +103,7 @@ const HELPER_CONTENT_RECAPTURE_DELAY_MS = 250; const HELPER_CONTENT_RECAPTURE_WINDOW_MS = 10_000; export type AndroidSnapshotOptions = SnapshotOptions & { skipIdleWait?: boolean; + androidIdleWait?: InteractorSnapshotOptions['androidIdleWait']; appBundleId?: string; signal?: AbortSignal; helperArtifact?: AndroidSnapshotHelperArtifact; @@ -447,6 +448,7 @@ async function installAndroidSnapshotHelper( async function captureAndroidUiHierarchyFromHelper(params: { skipIdleWait?: boolean; + androidIdleWait?: InteractorSnapshotOptions['androidIdleWait']; signal?: AbortSignal; adb: AndroidAdbExecutor; adbProvider: AndroidAdbProvider; @@ -462,6 +464,7 @@ async function captureAndroidUiHierarchyFromHelper(params: { signal, }); captureOptions.skipIdleWait = params.skipIdleWait; + captureOptions.shortIdleWait = params.androidIdleWait === 'short'; try { const sessionCapture = await withDiagnosticTimer( 'android_snapshot_helper_session_capture', @@ -558,6 +561,7 @@ async function captureAndroidHelperContentAttempt(params: { } const capture = await captureAndroidUiHierarchyFromHelper({ skipIdleWait: options.skipIdleWait, + androidIdleWait: options.androidIdleWait, signal: options.signal, adb, adbProvider, diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 636056db3d..a95941cba1 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -114,6 +114,7 @@ export function createAndroidInteractor( includeHiddenContentHints: snapshotOptions.includeHiddenContentHints, helperSessionScope: androidHelperSessionScope(snapshotOptions.appBundleId), transient: snapshotOptions.transient, + androidIdleWait: snapshotOptions.androidIdleWait, }), { backend: 'android' }, ); From a886d4a40c05b600f7384b09c05db1929dfc281e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 20 Oct 2026 18:52:46 +0200 Subject: [PATCH 2/4] refactor(android): keep idle policy in snapshot command binding --- .../src/core/snapshot-runtime-binding.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/daemon/src/core/snapshot-runtime-binding.ts b/packages/daemon/src/core/snapshot-runtime-binding.ts index 8fb3e9a7b2..0c0370f7d0 100644 --- a/packages/daemon/src/core/snapshot-runtime-binding.ts +++ b/packages/daemon/src/core/snapshot-runtime-binding.ts @@ -140,16 +140,7 @@ export async function admitAndBindSnapshotCapture( ); return Object.freeze({ ok: true, - capture: async (input: CaptureSnapshotInput) => { - const shortIdleWait = - (command === 'snapshot' || command === 'diff') && - canUseShortAndroidSnapshotIdleWait(session); - return await bound.captureSnapshot( - shortIdleWait - ? { ...input, options: { ...input.options, androidIdleWait: 'short' } } - : input, - ); - }, + capture: async (input: CaptureSnapshotInput) => await bound.captureSnapshot(input), ...(bound.readTextAtPoint ? { readTextAtPoint: bound.readTextAtPoint } : {}), ...(bound.findText ? { findText: bound.findText } : {}), ...(bound.focusPoint ? { focusPoint: bound.focusPoint } : {}), @@ -222,7 +213,12 @@ export async function resolveBoundSnapshotCaptureRuntime( session, device, snapshotScope: resolvedScope.scope, - captureSnapshot: async () => await bound.capture(captureInput), + captureSnapshot: async () => + await bound.capture( + canUseShortAndroidSnapshotIdleWait(session) + ? { ...captureInput, options: { ...captureInput.options, androidIdleWait: 'short' } } + : captureInput, + ), }); } From 0640af5f1357e12caa9bd092a869d93b92aff350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 21 Oct 2026 10:24:54 +0200 Subject: [PATCH 3/4] refactor(android): map capture facts to one idle policy --- packages/contracts/src/interactor-types.ts | 4 ++-- .../daemon/src/core/snapshot-runtime-binding.ts | 5 ++++- .../platform-android/src/fill-verification.ts | 6 +++--- .../src/snapshot-helper-capture.ts | 14 +++++++++----- .../src/snapshot-helper-session-protocol.ts | 16 +++++++++------- .../src/snapshot-helper-session.ts | 4 ++-- .../src/snapshot-helper-types.ts | 3 +-- packages/platform-android/src/snapshot.ts | 13 +++++-------- src/core/interactors/android.ts | 3 ++- 9 files changed, 37 insertions(+), 31 deletions(-) diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index a8c6151006..7ca5aa6924 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -191,8 +191,8 @@ export type SnapshotOptions = BaseSnapshotOptions & { surface?: SessionSurface; /** Internal capture purpose; action outcomes always require the full tree. */ acquisitionIntent?: 'full' | 'surface-observation'; - /** Internal Android observation hint; action and settling captures keep their normal idle wait. */ - androidIdleWait?: 'short'; + /** A stored observation remains current; action and settling captures do not supply this fact. */ + captureBasis?: 'unchanged-since-observation'; /** * A one-off read, such as an open's launch observation. It may use a capture host the session * already keeps warm, but it never installs one or leaves running one that it started. It starts diff --git a/packages/daemon/src/core/snapshot-runtime-binding.ts b/packages/daemon/src/core/snapshot-runtime-binding.ts index 0c0370f7d0..7d341ecc24 100644 --- a/packages/daemon/src/core/snapshot-runtime-binding.ts +++ b/packages/daemon/src/core/snapshot-runtime-binding.ts @@ -216,7 +216,10 @@ export async function resolveBoundSnapshotCaptureRuntime( captureSnapshot: async () => await bound.capture( canUseShortAndroidSnapshotIdleWait(session) - ? { ...captureInput, options: { ...captureInput.options, androidIdleWait: 'short' } } + ? { + ...captureInput, + options: { ...captureInput.options, captureBasis: 'unchanged-since-observation' }, + } : captureInput, ), }); diff --git a/packages/platform-android/src/fill-verification.ts b/packages/platform-android/src/fill-verification.ts index bb28beb373..7a7c447043 100644 --- a/packages/platform-android/src/fill-verification.ts +++ b/packages/platform-android/src/fill-verification.ts @@ -83,7 +83,7 @@ export async function verifyAndroidFilledText( beforeCapture?: AndroidFillTargetCapture | null, ): Promise { const deadline = Deadline.fromTimeoutMs(FILL_VERIFICATION_DEADLINE_MS); - const capture = await captureAndroidUiHierarchy(device, { ...helper, skipIdleWait: true }); + const capture = await captureAndroidUiHierarchy(device, { ...helper, idleWait: 'none' }); let initialObservation: AndroidFillObservation | undefined = parseAndroidFillObservation( capture.xml, capture.metadata, @@ -133,7 +133,7 @@ async function readAndroidFillTargetAtPoint( y: number, helper: AndroidHelperSessionOptions, ): Promise { - const capture = await captureAndroidUiHierarchy(device, { ...helper, skipIdleWait: true }); + const capture = await captureAndroidUiHierarchy(device, { ...helper, idleWait: 'none' }); const observation = parseAndroidFillObservation(capture.xml, capture.metadata); const context = await readAndroidFillVerificationContext(device, observation); const targetInput = inspectAndroidTextAtPoint(observation, x, y, context).targetInput; @@ -360,7 +360,7 @@ async function inspectAndroidFilledText( ): Promise { // Each delay samples the live hierarchy again — settling is what the samples observe, so they // share the helper session but never a capture. - const capture = await captureAndroidUiHierarchy(device, { ...helper, skipIdleWait: true }); + const capture = await captureAndroidUiHierarchy(device, { ...helper, idleWait: 'none' }); return verifyAndroidFilledTextInObservation( parseAndroidFillObservation(capture.xml, capture.metadata), x, diff --git a/packages/platform-android/src/snapshot-helper-capture.ts b/packages/platform-android/src/snapshot-helper-capture.ts index 66ced5abe3..96f97cd68a 100644 --- a/packages/platform-android/src/snapshot-helper-capture.ts +++ b/packages/platform-android/src/snapshot-helper-capture.ts @@ -69,11 +69,15 @@ export async function captureAndroidSnapshotWithHelper( result = await options.adb( buildAndroidSnapshotHelperArgs({ ...resolved, - waitForIdleTimeoutMs: options.skipIdleWait - ? 0 - : options.shortIdleWait - ? Math.min(resolved.waitForIdleTimeoutMs, ANDROID_SNAPSHOT_HELPER_SHORT_IDLE_TIMEOUT_MS) - : resolved.waitForIdleTimeoutMs, + waitForIdleTimeoutMs: + options.idleWait === 'none' + ? 0 + : options.idleWait === 'short' + ? Math.min( + resolved.waitForIdleTimeoutMs, + ANDROID_SNAPSHOT_HELPER_SHORT_IDLE_TIMEOUT_MS, + ) + : resolved.waitForIdleTimeoutMs, }), { allowFailure: true, diff --git a/packages/platform-android/src/snapshot-helper-session-protocol.ts b/packages/platform-android/src/snapshot-helper-session-protocol.ts index b0365a57fe..b9d031d476 100644 --- a/packages/platform-android/src/snapshot-helper-session-protocol.ts +++ b/packages/platform-android/src/snapshot-helper-session-protocol.ts @@ -12,6 +12,7 @@ import { import { ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT, ANDROID_SNAPSHOT_HELPER_PROTOCOL, + type AndroidSnapshotHelperCaptureOptions, type AndroidSnapshotHelperMetadata, type AndroidSnapshotHelperOutput, } from './snapshot-helper-types.ts'; @@ -85,15 +86,16 @@ export async function requestAndroidSnapshotHelperSessionSnapshot(params: { port: number; commandTimeoutMs: number; signal?: AbortSignal; - skipIdleWait?: boolean; - shortIdleWait?: boolean; + idleWait?: AndroidSnapshotHelperCaptureOptions['idleWait']; + commands: ReadonlySet; }): Promise { const requestId = `snapshot-${Date.now()}-${Math.random().toString(16).slice(2)}`; - const command = params.skipIdleWait - ? 'snapshot-immediate' - : params.shortIdleWait - ? 'snapshot-short-idle' - : 'snapshot'; + const requestedCommand = { + full: 'snapshot', + short: 'snapshot-short-idle', + none: 'snapshot-immediate', + }[params.idleWait ?? 'full']; + const command = params.commands.has(requestedCommand) ? requestedCommand : 'snapshot'; // The socket waits out the whole command budget the session was given. The device-side // `timeoutMs` bounds only the UiAutomation connect wait, not the traversal after it, so a deadline // built from that wait plus a fixed overhead cut off slow-but-healthy captures (tester-army/e2e#841). diff --git a/packages/platform-android/src/snapshot-helper-session.ts b/packages/platform-android/src/snapshot-helper-session.ts index 89cdf3b255..b57fa9f91b 100644 --- a/packages/platform-android/src/snapshot-helper-session.ts +++ b/packages/platform-android/src/snapshot-helper-session.ts @@ -55,8 +55,8 @@ async function captureFromAndroidSnapshotHelperSession(params: { port: session.port, commandTimeoutMs: resolved.commandTimeoutMs, signal: options.signal, - skipIdleWait: options.skipIdleWait && session.commands.has('snapshot-immediate'), - shortIdleWait: options.shortIdleWait && session.commands.has('snapshot-short-idle'), + idleWait: options.idleWait, + commands: session.commands, }); session.capturedCount += 1; return { diff --git a/packages/platform-android/src/snapshot-helper-types.ts b/packages/platform-android/src/snapshot-helper-types.ts index 11a09482fb..fbbfe04042 100644 --- a/packages/platform-android/src/snapshot-helper-types.ts +++ b/packages/platform-android/src/snapshot-helper-types.ts @@ -79,8 +79,7 @@ export type AndroidSnapshotHelperCaptureOptions = { maxDepth?: number; maxNodes?: number; emitChunks?: boolean; - skipIdleWait?: boolean; - shortIdleWait?: boolean; + idleWait?: 'full' | 'short' | 'none'; }; export type AndroidSnapshotHelperMetadata = { diff --git a/packages/platform-android/src/snapshot.ts b/packages/platform-android/src/snapshot.ts index ffcb428f15..ddbf8f47ec 100644 --- a/packages/platform-android/src/snapshot.ts +++ b/packages/platform-android/src/snapshot.ts @@ -35,6 +35,7 @@ import { ANDROID_SNAPSHOT_HELPER_CAPTURE_TIMEOUT_MS, ANDROID_SNAPSHOT_HELPER_COMMAND_TIMEOUT_MS, type AndroidHelperSessionScope, + type AndroidSnapshotHelperCaptureOptions, } from './snapshot-helper-types.ts'; import { captureAndroidSnapshotWithHelper, @@ -102,8 +103,7 @@ const HELPER_CONTENT_RECAPTURE_DELAY_MS = 250; */ const HELPER_CONTENT_RECAPTURE_WINDOW_MS = 10_000; export type AndroidSnapshotOptions = SnapshotOptions & { - skipIdleWait?: boolean; - androidIdleWait?: InteractorSnapshotOptions['androidIdleWait']; + idleWait?: AndroidSnapshotHelperCaptureOptions['idleWait']; appBundleId?: string; signal?: AbortSignal; helperArtifact?: AndroidSnapshotHelperArtifact; @@ -447,8 +447,7 @@ async function installAndroidSnapshotHelper( } async function captureAndroidUiHierarchyFromHelper(params: { - skipIdleWait?: boolean; - androidIdleWait?: InteractorSnapshotOptions['androidIdleWait']; + idleWait?: AndroidSnapshotHelperCaptureOptions['idleWait']; signal?: AbortSignal; adb: AndroidAdbExecutor; adbProvider: AndroidAdbProvider; @@ -463,8 +462,7 @@ async function captureAndroidUiHierarchyFromHelper(params: { deviceKey: helperDeviceKey, signal, }); - captureOptions.skipIdleWait = params.skipIdleWait; - captureOptions.shortIdleWait = params.androidIdleWait === 'short'; + captureOptions.idleWait = params.idleWait; try { const sessionCapture = await withDiagnosticTimer( 'android_snapshot_helper_session_capture', @@ -560,8 +558,7 @@ async function captureAndroidHelperContentAttempt(params: { await stopAndroidSnapshotHelperSession(helperDeviceKey); } const capture = await captureAndroidUiHierarchyFromHelper({ - skipIdleWait: options.skipIdleWait, - androidIdleWait: options.androidIdleWait, + idleWait: options.idleWait, signal: options.signal, adb, adbProvider, diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index a95941cba1..33dda07766 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -114,7 +114,8 @@ export function createAndroidInteractor( includeHiddenContentHints: snapshotOptions.includeHiddenContentHints, helperSessionScope: androidHelperSessionScope(snapshotOptions.appBundleId), transient: snapshotOptions.transient, - androidIdleWait: snapshotOptions.androidIdleWait, + idleWait: + snapshotOptions.captureBasis === 'unchanged-since-observation' ? 'short' : 'full', }), { backend: 'android' }, ); From 2b5578286168134f6ddb04cec7077604033641ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 21 Oct 2026 11:07:14 +0200 Subject: [PATCH 4/4] test(android): pin snapshot idle policy across runtime boundaries --- android/snapshot-helper/README.md | 6 +- .../snapshot-runtime-binding.test.ts | 118 +++++++++++++++++- .../__tests__/snapshot-helper-session.test.ts | 43 +++++++ src/core/interactors/android.test.ts | 14 +++ 4 files changed, 177 insertions(+), 4 deletions(-) diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index c7bff39255..c5f026d32f 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -109,9 +109,9 @@ closes that connection; the process stays alive for the next connection: The ready record advertises optional operations in `sessionCommands`. A host must check support before sending them. Text replacement retains a focus tap; the host owns post-write verification. Input failures must not be resent because a mutation may already have reached the app. -The daemon uses the shorter wait for Android `snapshot` and `diff` only when the latest stored -tree is at least 500 ms old, no later operation has retired it, and no freshness recovery or -post-gesture stabilization is pending. Other captures keep their configured wait. Each capture +The daemon uses the shorter wait for Android `snapshot` and `diff` through the ADB helper only +when the latest stored tree is at least 500 ms old, no later operation has retired it, and no +freshness recovery or post-gesture stabilization is pending. Other captures keep their configured wait. Each capture still reads a fresh tree. An asynchronous app transition can appear in progress; use an explicit landmark wait when the next command requires a destination screen. diff --git a/packages/daemon/src/core/__tests__/snapshot-runtime-binding.test.ts b/packages/daemon/src/core/__tests__/snapshot-runtime-binding.test.ts index 6973c0c5d0..4bc1d18f08 100644 --- a/packages/daemon/src/core/__tests__/snapshot-runtime-binding.test.ts +++ b/packages/daemon/src/core/__tests__/snapshot-runtime-binding.test.ts @@ -3,7 +3,7 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import { expect, test, vi } from 'vitest'; import { resolveSnapshotRuntimePlan } from '@agent-device/contracts/platform-runtime-operations'; import { IOS_SIMULATOR } from '@agent-device/test-kit'; -import { makeIosSession } from '../../__tests__/support/session-factories.ts'; +import { makeAndroidSession, makeIosSession } from '../../__tests__/support/session-factories.ts'; import { makeSessionStore } from '../../__tests__/support/store-factory.ts'; import type { BindDeviceRuntime } from '../request-runtime-binding.ts'; import { @@ -14,6 +14,17 @@ import type { DaemonRequest } from '../daemon-request.ts'; import { snapshotRuntimeFixture } from '../../__tests__/support/snapshot-runtime-fixture.ts'; import { ensureDeviceReady } from '../device/device-ready.ts'; import { fakeCreateCommandSurface } from '../../__tests__/support/command-surface-fake.ts'; +import type { + CaptureSnapshotInput, + SnapshotResult, +} from '@agent-device/contracts/snapshot-runtime'; +import type { SessionState } from '../session-state.ts'; +import { markSessionSnapshotOutdated } from '../ref-frame.ts'; +import { + clearAndroidSnapshotFreshness, + markAndroidSnapshotFreshness, +} from '../session-snapshot-freshness.ts'; +import { markDeferredInteractionOutcome } from '../deferred-interaction-outcome.ts'; vi.mock('../device/device-ready.ts', () => ({ ensureDeviceReady: vi.fn(async () => {}) })); @@ -86,3 +97,108 @@ test('sessionless capture readiness follows runtime binding', async () => { expect(resolved.ok).toBe(true); expect(events).toEqual(['bind', 'ready']); }); + +test('snapshot and diff shorten idle only when every observation guard still holds at capture', async () => { + const now = Date.now(); + const clock = vi.spyOn(Date, 'now').mockReturnValue(now); + const cases: { + name: string; + apple?: boolean; + beforeCapture?: (session: SessionState) => void; + short: boolean; + }[] = [ + { name: 'current Android observation at the 500 ms boundary', short: true }, + { name: 'non-Android observation', apple: true, short: false }, + { + name: 'no stored tree', + beforeCapture: (session) => { + session.snapshot = undefined; + }, + short: false, + }, + { name: 'retired observation', beforeCapture: markSessionSnapshotOutdated, short: false }, + { + name: '499 ms old observation', + beforeCapture: (session) => { + session.snapshot = { nodes: [], createdAt: now - 499 }; + }, + short: false, + }, + { + name: 'active freshness recovery', + beforeCapture: (session) => { + markAndroidSnapshotFreshness(session, 'press'); + }, + short: false, + }, + { + name: 'pending gesture stabilization without freshness recovery', + beforeCapture: (session) => { + markDeferredInteractionOutcome({ + session, + command: 'scroll', + positionals: ['down'], + flags: undefined, + }); + clearAndroidSnapshotFreshness(session); + }, + short: false, + }, + ]; + try { + for (const command of ['snapshot', 'diff'] as const) { + for (const scenario of cases) { + const fixture = snapshotRuntimeFixture(); + const capture = vi.fn<(input: CaptureSnapshotInput) => Promise>(async () => + scenario.apple + ? { backend: 'xctest', producer: 'simulator-ax-bridge', nodes: [] } + : { backend: 'android', producer: 'android-uiautomator', nodes: [] }, + ); + const bindDevice: BindDeviceRuntime = async (device, use) => { + const runtime = await fixture.bindDevice(device, use); + return { + ...runtime, + operations: { + ...runtime.operations, + captureSnapshot: capture, + captureSnapshotWithoutActiveApp: capture, + }, + }; + }; + const make = scenario.apple ? makeIosSession : makeAndroidSession; + const sessionStore = makeSessionStore(); + sessionStore.publish( + 'idle-policy', + make('idle-policy', { + appBundleId: 'com.example.app', + snapshot: { nodes: [], createdAt: now - 500 }, + }), + ); + const resolved = await resolveBoundSnapshotCaptureRuntime( + { + createCommandSurface: fakeCreateCommandSurface, + platformServices: fixture.platformServices, + req: { command, positionals: [], session: 'idle-policy', token: 'test-token' }, + sessionName: 'idle-policy', + logPath: '/tmp/idle-policy.log', + sessionStore, + inspectFacts: fixture.inspectFacts, + bindDevice, + }, + command, + ); + expect(resolved.ok, scenario.name).toBe(true); + if (!resolved.ok || !resolved.session) throw new Error('Expected a bound app session'); + scenario.beforeCapture?.(resolved.session); + await resolved.captureSnapshot(); + expect(capture, scenario.name).toHaveBeenCalledOnce(); + expect( + capture.mock.calls[0]?.[0].options?.captureBasis, + `${command}: ${scenario.name}`, + ).toBe(scenario.short ? 'unchanged-since-observation' : undefined); + } + } + } finally { + clock.mockRestore(); + } +}); diff --git a/packages/platform-android/src/__tests__/snapshot-helper-session.test.ts b/packages/platform-android/src/__tests__/snapshot-helper-session.test.ts index 46ef83cb74..38cfa9a472 100644 --- a/packages/platform-android/src/__tests__/snapshot-helper-session.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-helper-session.test.ts @@ -5,6 +5,7 @@ import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-l import { isAndroidSnapshotHelperRuntimeOccupiedError } from '../snapshot-helper-retirement.ts'; import { createSessionProvider, + createPersistentSnapshotHelperProvider, type FakeAndroidProcess, } from './snapshot-helper-session.fixtures.ts'; import path from 'node:path'; @@ -20,6 +21,48 @@ afterEach(async () => { await resetAndroidSnapshotHelperSessions(); }); +test('selects supported idle commands and keeps one helper across full, short and immediate captures', async () => { + const cases = [ + { + commands: ['snapshot-short-idle', 'snapshot-immediate'], + expected: ['snapshot', 'snapshot', 'snapshot-short-idle', 'snapshot-immediate'], + }, + { + commands: ['snapshot-immediate'], + expected: ['snapshot', 'snapshot', 'snapshot', 'snapshot-immediate'], + }, + { commands: [], expected: ['snapshot', 'snapshot', 'snapshot', 'snapshot'] }, + ]; + for (const [index, scenario] of cases.entries()) { + const sessionRequests: string[] = []; + const spawnArgs: (readonly string[])[] = []; + const provider = createPersistentSnapshotHelperProvider({ + calls: [], + spawnArgs, + processes: [], + sessionRequests, + sessionCommands: scenario.commands, + }); + for (const [captureIndex, idleWait] of ( + [undefined, 'full', 'short', 'none'] as const + ).entries()) { + const output = await captureAndroidSnapshotWithHelperSession({ + adb: provider.exec, + adbProvider: provider, + deviceKey: `android:idle-policy-${index}`, + idleWait, + }); + assert.match(output?.xml ?? '', /persistent helper snapshot/); + assert.equal(output?.metadata.sessionReused, captureIndex > 0); + } + assert.deepEqual( + sessionRequests.map((request) => request.split(' ')[0]), + scenario.expected, + ); + assert.equal(spawnArgs.length, 1, 'per-request idle policy must not restart the helper'); + } +}); + test('allows a persistent session snapshot to use the helper command budget', async () => { const calls: (readonly string[])[] = []; const provider = createSessionProvider({ calls, responseDelayMs: 25 }); diff --git a/src/core/interactors/android.test.ts b/src/core/interactors/android.test.ts index 21bb580ab6..e3677fae9f 100644 --- a/src/core/interactors/android.test.ts +++ b/src/core/interactors/android.test.ts @@ -41,6 +41,20 @@ afterEach(() => { vi.resetAllMocks(); }); +test('maps the unchanged-observation fact to short idle and defaults other captures to full idle', async () => { + const nodes = [{ index: 0, identifier: 'target', rect: { x: 0, y: 0, width: 20, height: 20 } }]; + snapshotAndroidMock.mockResolvedValue(makeAndroidSnapshotCapture(nodes)); + for (const scenario of [ + { options: {}, expected: 'full' }, + { options: { captureBasis: 'unchanged-since-observation' }, expected: 'short' }, + ] as const) { + const result = await createAndroidInteractor(device).snapshot(scenario.options); + expect(snapshotAndroidMock.mock.lastCall?.[1]?.idleWait).toBe(scenario.expected); + if ('stage' in result) throw new Error('Android snapshot must be presented'); + expect(result.nodes).toEqual(nodes); + } +}); + test('preserves Android clickability evidence through the interactor snapshot adapter', async () => { const nodes = [{ index: 0, identifier: 'target', rect: { x: 0, y: 0, width: 20, height: 20 } }]; const evidence = {