diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 7c3ce842eaa..c729da5ebbd 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -3223,14 +3223,18 @@ describe('credential protection', () => { const contents = await openPage() const calls: string[] = [] let probe = 0 + let firstProbeAt = Number.NaN + let snapshotAt = Number.NaN vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => { if (isPageCall(expression, 'readDomQuiet')) { calls.push('probe') + if (Number.isNaN(firstProbeAt)) firstProbeAt = Date.now() const quietMs = quietReadings[Math.min(probe++, quietReadings.length - 1)] return Promise.resolve({ quietMs, loading: false }) } if (isPageCall(expression, 'collectSnapshot')) { calls.push('snapshot') + snapshotAt = Date.now() return Promise.resolve({ url: 'https://example.com/', title: 'Example', @@ -3256,6 +3260,10 @@ describe('credential protection', () => { const probesBeforeSnapshot = calls.indexOf('snapshot') expect(probesBeforeSnapshot).toBeGreaterThan(0) if (expectedProbes !== undefined) expect(probesBeforeSnapshot).toBe(expectedProbes) + else { + expect(snapshotAt - firstProbeAt).toBeGreaterThanOrEqual(1_900) + expect(snapshotAt - firstProbeAt).toBeLessThanOrEqual(2_000) + } } finally { vi.useRealTimers() } diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index efc9711dfa8..dc3da220e3a 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -381,6 +381,148 @@ describe('combobox typing surfaces', () => { expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) + /** + * Answers hit testing with the paint stack (topmost first) over the field's sampled points and + * elsewhere, as `elementsFromPoint` does; `elementFromPoint` is its top. + */ + function stackTo( + overField: Element[], + elsewhere: (x: number, y: number) => Element[], + target: Document | ShadowRoot = document + ): void { + const stackAt = (x: number, y: number) => + x >= 10 && x <= 100 && y <= 20 ? overField : elsewhere(x, y) + Object.defineProperty(target, 'elementsFromPoint', { configurable: true, value: stackAt }) + Object.defineProperty(target, 'elementFromPoint', { + configurable: true, + value: (x: number, y: number) => stackAt(x, y)[0] ?? null, + }) + } + const page = () => [document.body, document.documentElement] + const byId = (id: string) => + document.getElementById(id) as unknown as T + + it('refuses a top-page field behind an unmarked backdrop and panel', () => { + document.body.innerHTML = ` + +
+
Subscribe to our newsletter
+ ` + const input = visible(byId('field')) + register(input) + stackTo([byId('panel'), byId('backdrop'), input, ...page()], () => [ + byId('backdrop'), + ...page(), + ]) + + expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) + }) + + it('refuses a field behind an unmarked modal nested in a full-screen app shell', () => { + document.body.innerHTML = ` +
+ +
Confirm your plan
+
+ ` + const input = visible(byId('field')) + register(input) + stackTo([byId('panel'), byId('backdrop'), input, byId('shell'), ...page()], () => [ + byId('backdrop'), + byId('shell'), + ...page(), + ]) + + expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) + }) + + it('refuses a field behind an unmarked modal inside the same shadow root', () => { + document.body.innerHTML = '
' + const shadow = byId('host').attachShadow({ mode: 'open' }) + shadow.innerHTML = + '
Sign up
' + const input = visible(shadow.querySelector('input') as HTMLInputElement) + const backdrop = shadow.querySelector('#backdrop') as HTMLDivElement + const panel = shadow.querySelector('#panel') as HTMLDivElement + register(input) + stackTo( + [panel, backdrop, input, byId('host'), ...page()], + () => [backdrop, byId('host'), ...page()], + shadow + ) + + expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) + }) + + it('types under covers that do not reach every corner of the viewport', () => { + document.body.innerHTML = ` +
Map
+ + +
Saved to your list
+ + ` + const input = visible(byId('field')) + register(input) + const topHalf = (y: number) => y < window.innerHeight / 2 + for (const [cover, label, elsewhere] of [ + // a fixed header over a full-screen fixed map the field also sits above + [ + 'header', + 'Menu', + (_x: number, y: number): Element[] => [ + ...(topHalf(y) ? [byId('header')] : []), + byId('map'), + ...page(), + ], + ], + // a fixed header with a cookie banner pinned to the bottom corners + [ + 'header', + 'Menu', + (_x: number, y: number): Element[] => [ + topHalf(y) ? byId('header') : byId('banner'), + ...page(), + ], + ], + // a toast whose full-screen wrapper lets the pointer through, so the wrapper is in no stack + ['toast', 'Saved to your list', (): Element[] => page()], + ] as const) { + stackTo([byId(cover), input, byId('map'), ...page()], elsewhere) + expect(focusElementForTyping(0)).toMatchObject({ focused: true, coveredBy: label }) + } + }) + + it('types inside a full-screen fixed app shell that also holds the field', () => { + document.body.innerHTML = ` +
+
Shop for your home
+
+ ` + const input = visible(byId('field')) + register(input) + // the shell paints below the field, or above it when the field sits at a negative z-index + for (const overField of [ + [byId('card'), input, byId('shell'), ...page()], + [byId('card'), byId('shell'), input, ...page()], + ]) { + stackTo(overField, () => [byId('shell'), ...page()]) + expect(focusElementForTyping(0)).toMatchObject({ + focused: true, + coveredBy: 'Shop for your home', + }) + } + }) + + it('refuses a top-page field when no sampled point hits anything', () => { + document.body.innerHTML = '' + const input = visible(document.querySelector('input') as HTMLInputElement) + register(input) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => null }) + + expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) + }) + it('still refuses a covered field inside a frame', () => { const frame = document.createElement('iframe') document.body.append(frame) diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index 4e726c82ee7..e2226ded3d4 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -2040,11 +2040,12 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { coveredByRelatedPopup = true } } - // Top-page typing goes through keyboard focus, so a non-modal overlay does not block it; a modal - // or dialog cover still refuses, and framed fields keep the check for their mapped point. + // Top-page typing goes through keyboard focus, so a non-modal overlay does not block it. A modal + // still does: a dialog, or an unmarked backdrop, which paints above the field and under every + // corner of the viewport. Framed fields keep the check for their mapped point. const topPageField = view === window && window.top === window - const insideModal = (node: Element | null): boolean => { - for (let current = node; current; current = composedParent(current)) { + const insideDialog = (node: Element): boolean => { + for (let current: Element | null = node; current; current = composedParent(current)) { const role = current.getAttribute('role') if ( current.getAttribute('aria-modal') === 'true' || @@ -2057,12 +2058,43 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { } return false } + const behindBackdrop = (): boolean => { + const stackRoot = root as ParentNode & { + elementsFromPoint?: (x: number, y: number) => Element[] + } + if (typeof stackRoot.elementsFromPoint !== 'function') return false + const stackAt = stackRoot.elementsFromPoint.bind(stackRoot) + const aboveField = new Set() + for (const point of blockedPoints) { + for (const element of stackAt(point.x, point.y)) { + if (element === editable || editable.contains(element) || element.contains(editable)) break + aboveField.add(element) + } + } + if (aboveField.size === 0) return false + const width = editable.ownerDocument.documentElement.clientWidth || view.innerWidth + const height = editable.ownerDocument.documentElement.clientHeight || view.innerHeight + const inset = 4 + for (const [x, y] of [ + [inset, inset], + [width - inset, inset], + [inset, height - inset], + [width - inset, height - inset], + ]) { + const corner = new Set(stackAt(x, y)) + for (const element of aboveField) if (!corner.has(element)) aboveField.delete(element) + if (aboveField.size === 0) return false + } + return true + } + const hits = Array.from(new Set(blockedPoints.flatMap((point) => (point.hit ? [point.hit] : [])))) let coveredBy: string | undefined if ( !chosenPoint && topPageField && - blockedPoints.length > 0 && - !blockedPoints.some((point) => insideModal(point.hit)) + hits.length > 0 && + !hits.some(insideDialog) && + !behindBackdrop() ) { chosenPoint = { x: blockedPoints[0].x, y: blockedPoints[0].y } coveredBy = blockerLabel(firstBlocker)