diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index dceb49ecbd5..7c3ce842eaa 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -1894,6 +1894,7 @@ describe('credential protection', () => { activeElementSecrecy: 'safe', readActiveElementState: {}, readPageActionState: {}, + readDomQuiet: { quietMs: 1_000, loading: false }, }) const pageCall = vi.mocked(contents.executeJavaScript).getMockImplementation() let releaseObservation: (value: unknown) => void = () => {} @@ -3212,6 +3213,113 @@ describe('credential protection', () => { }) }) + it.each([ + ['still rendering', [0, 40, 90, 200], 4], + ['already quiet', [400], 1], + ['never settling', Array.from({ length: 200 }, () => 0), undefined], + ] as const)( + 'observes after an action only once a page %s has gone quiet, within the settle cap', + async (_label, quietReadings, expectedProbes) => { + const contents = await openPage() + const calls: string[] = [] + let probe = 0 + vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => { + if (isPageCall(expression, 'readDomQuiet')) { + calls.push('probe') + const quietMs = quietReadings[Math.min(probe++, quietReadings.length - 1)] + return Promise.resolve({ quietMs, loading: false }) + } + if (isPageCall(expression, 'collectSnapshot')) { + calls.push('snapshot') + return Promise.resolve({ + url: 'https://example.com/', + title: 'Example', + outline: '- heading "Results"', + truncated: false, + refIds: [], + refLineIndexes: {}, + nextElementId: 1_000, + }) + } + if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET) + return Promise.resolve(undefined) + }) + vi.useFakeTimers() + try { + const clicking = driver.executeTool('chat-test', 'browser_click', { + elementId: 0, + observe: {}, + }) + await vi.advanceTimersByTimeAsync(5_000) + + expect(await clicking).toMatchObject({ ok: true, result: { observation: { ok: true } } }) + const probesBeforeSnapshot = calls.indexOf('snapshot') + expect(probesBeforeSnapshot).toBeGreaterThan(0) + if (expectedProbes !== undefined) expect(probesBeforeSnapshot).toBe(expectedProbes) + } finally { + vi.useRealTimers() + } + } + ) + + it('keeps settling through failed quiet probes, and never observes once cancelled', async () => { + const contents = await openPage() + const calls: string[] = [] + let failing = 2 + vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => { + if (isPageCall(expression, 'readDomQuiet')) { + calls.push('probe') + if (failing > 0) { + failing-- + vi.mocked(contents.isLoading).mockReturnValue(true) + return Promise.reject(new Error('Execution context was destroyed')) + } + vi.mocked(contents.isLoading).mockReturnValue(false) + return Promise.resolve({ quietMs: failing === -1 ? 0 : 400, loading: false }) + } + if (isPageCall(expression, 'collectSnapshot')) { + calls.push('snapshot') + return Promise.resolve({ + url: 'https://example.com/', + title: 'Example', + outline: '- heading "Results"', + truncated: false, + refIds: [], + refLineIndexes: {}, + nextElementId: 1_000, + }) + } + if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET) + return Promise.resolve(undefined) + }) + vi.useFakeTimers() + try { + const settled = driver.executeTool('chat-test', 'browser_click', { + elementId: 0, + observe: {}, + }) + await vi.advanceTimersByTimeAsync(5_000) + expect(await settled).toMatchObject({ ok: true, result: { observation: { ok: true } } }) + expect(calls.indexOf('snapshot')).toBe(3) + + calls.length = 0 + failing = -1 + const cancelled = driver.executeTool( + 'chat-test', + 'browser_click', + { elementId: 0, observe: { query: 'Results' } }, + 'click-cancelled' + ) + await vi.advanceTimersByTimeAsync(300) + expect(driver.cancelTool('chat-test', 'click-cancelled')).toBe(true) + await vi.advanceTimersByTimeAsync(5_000) + await cancelled + expect(calls).not.toContain('snapshot') + } finally { + vi.useRealTimers() + } + }) + it('acts on a ref an earlier snapshot of the same page issued', async () => { const contents = await openPage() respondWith(contents, { diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 02ae25dc4b3..5a35e1d789e 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -70,6 +70,7 @@ import { readActiveElementState, readCheckableElementState, readChildFrameElementState, + readDomQuiet, readFormFieldState, readPageActionState, readPageText, @@ -2062,6 +2063,9 @@ async function prepareTypingSurface( const TRANSIENT_PROBE_ERRORS = new Set(['stale', 'not-visible', 'not-editable']) const SETTLE_GRACE_MS = 1_000 const SETTLE_PROBE_INTERVAL_MS = 250 +/** Structural quiet that counts as settled, and the most an observation waits for it. */ +const DOM_QUIET_MS = 150 +const DOM_SETTLE_MAX_MS = 2_000 /** * First-touch page probe with a short settle grace. A live view re-rendering @@ -2096,6 +2100,33 @@ async function execInPageWithSettleGrace( } } +/** + * Lets a page finish rendering what an action started (a route change, a fetched list) before it + * is observed, so the observation is not a half-rendered page. A page already quiet costs one probe. + */ +async function waitForDomQuiet( + target: PageExecutionTarget, + signal: AbortSignal, + executionDeadline?: number +) { + const until = Math.min( + Date.now() + DOM_SETTLE_MAX_MS, + executionDeadline ?? Number.POSITIVE_INFINITY + ) + while (!signal.aborted && Date.now() + 25 < until) { + const probe = await execInPage(target, readDomQuiet, [], false, executionDeadline).catch( + () => null + ) + // A failed probe while the tab loads is a document swapping mid-navigation, so keep waiting; + // one that fails on a loaded tab (a page that cannot be scripted) has nothing to settle. + if (probe === null && !('isLoading' in target && target.isLoading())) return + const state = toRecord(probe) + const quietMs = typeof state.quietMs === 'number' ? state.quietMs : 0 + if (quietMs >= DOM_QUIET_MS && state.loading === false) return + await interruptibleSleep(Math.min(100, Math.max(25, DOM_QUIET_MS - quietMs)), signal) + } +} + function pointFromPrepared(prepared: Record): { x: number; y: number } { return { x: prepared.x as number, y: prepared.y as number } } @@ -5460,15 +5491,27 @@ export async function executeTool( } return result }, - (query) => - executeToolInner( + async (query) => { + await waitForDomQuiet( + session.requireAutomationTab().view.webContents, + executionController.signal, + executionDeadline + ) + assertCurrentExecution() + if (executionController.signal.aborted) { + throw new ToolError( + 'This browser action was cancelled before it could observe the page.' + ) + } + return executeToolInner( query === undefined ? 'browser_snapshot' : 'browser_find', query === undefined ? {} : { query }, assertCurrentExecution, executionDeadline, invocationEpoch, executionController.signal - ), + ) + }, assertCurrentExecution ) const cancellableExecution = Promise.race([execution, cancellation]) diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index 2c394eba4b2..efc9711dfa8 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -17,6 +17,7 @@ import { readActiveElementState, readCheckableElementState, readChildFrameElementState, + readDomQuiet, readPageActionState, readPageText, readSelectElementState, @@ -526,6 +527,28 @@ describe('collectSnapshot', () => { expect(outline).not.toContain('button "x" [ref=999]') }) + it('counts changes inside a shadow root the snapshot walked as page activity', async () => { + vi.useFakeTimers({ toFake: ['performance'] }) + try { + document.body.innerHTML = '
' + const host = visible(document.querySelector('#host') as HTMLDivElement) + const shadow = host.attachShadow({ mode: 'open' }) + shadow.innerHTML = '' + visible(shadow.querySelector('button') as HTMLButtonElement) + collectSnapshot() + await Promise.resolve() + vi.advanceTimersByTime(500) + expect((readDomQuiet() as { quietMs: number }).quietMs).toBeGreaterThanOrEqual(500) + + shadow.append(document.createElement('ul')) + await Promise.resolve() + + expect((readDomQuiet() as { quietMs: number }).quietMs).toBeLessThan(500) + } finally { + vi.useRealTimers() + } + }) + it('marks elements that appeared since the previous snapshot as new', () => { document.body.innerHTML = '' visible(document.querySelector('button') as HTMLButtonElement) diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index a938eba740d..4e726c82ee7 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -56,6 +56,11 @@ declare global { observedRoots: WeakSet }> __simAgentNextElementId?: number + /** When the document last changed structurally (nodes or text), for settling before an observation. */ + __simAgentDomChangedAt?: number + __simAgentDomWatcher?: MutationObserver + /** Adds a document or open shadow root to the settle watcher, once. */ + __simAgentWatchDom: (root: Node) => void /** Every ref this document issued, resolvable while its element keeps that identity. */ __simAgentIssuedRefs?: Map< number, @@ -85,12 +90,37 @@ export function serializePageCall(fn: (...args: never[]) => unknown, args: unkno return `((${String(installPageHelpers)})(), (${String(fn)}).apply(null, ${JSON.stringify(args)}))` } +/** How long the document has gone without a structural change, and whether it is still loading. */ +export function readDomQuiet(): unknown { + return { + quietMs: performance.now() - (window.__simAgentDomChangedAt ?? 0), + loading: document.readyState !== 'complete', + } +} + /** * Installs the helpers page functions share. Page functions are serialized and run standalone, * so the driver runs this installer before each one instead of inlining shared code into every * function. */ export function installPageHelpers(): void { + // Structural changes only: attribute churn (animations, hover styles) would keep a page "busy". + // Snapshots add the same-origin frames and open shadow roots they walk, which a document-level + // observer cannot see into. + if (!window.__simAgentDomWatcher) { + window.__simAgentDomChangedAt = performance.now() + window.__simAgentDomWatcher = new MutationObserver(() => { + window.__simAgentDomChangedAt = performance.now() + }) + } + const watcher = window.__simAgentDomWatcher + const watched = new WeakSet() + window.__simAgentWatchDom = (root: Node): void => { + if (watched.has(root)) return + watched.add(root) + watcher.observe(root, { subtree: true, childList: true, characterData: true }) + } + if (document.documentElement) window.__simAgentWatchDom(document.documentElement) /** * The open modal each document aria-hid together with everything else. MUI's ModalManager * aria-hides every child except the modal's mount node, and a `disablePortal` modal @@ -889,6 +919,7 @@ export function collectSnapshot( try { const innerDoc = (el as HTMLIFrameElement).contentDocument if (innerDoc?.body && isVisible(el)) { + window.__simAgentWatchDom(innerDoc.documentElement) if (!push(`${indent}- iframe:`)) return walk(innerDoc.body.childNodes, childDepth + 1, coveredText) } else if (scopedRoot && !innerDoc && visible) { @@ -901,7 +932,10 @@ export function collectSnapshot( } const shadow = (el as HTMLElement).shadowRoot - if (shadow) walk(shadow.childNodes, childDepth, coveredText) + if (shadow) { + window.__simAgentWatchDom(shadow) + walk(shadow.childNodes, childDepth, coveredText) + } walk(el.childNodes, childDepth, coveredText) } }