From 675a558593343eb626e14b93181a6117c6e12a97 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 13:14:12 -0800 Subject: [PATCH 1/9] fix(desktop-browser): refuse typing behind an unmarked full-screen modal or when no sampled point lands Top-page typing through a covering element now also refuses when the cover sits in a fixed layer spanning the viewport that does not hold the field, and when every sampled point hits nothing. The never-settling settle test now asserts the cap. --- .../src/main/browser-agent/driver.test.ts | 4 ++ .../main/browser-agent/page-functions.test.ts | 71 +++++++++++++++++++ .../src/main/browser-agent/page-functions.ts | 32 ++++++--- 3 files changed, 97 insertions(+), 10 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 7c3ce842eaa..c7c1c9c5366 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -3223,6 +3223,7 @@ describe('credential protection', () => { const contents = await openPage() const calls: string[] = [] let probe = 0 + let snapshotAt = Number.NaN vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => { if (isPageCall(expression, 'readDomQuiet')) { calls.push('probe') @@ -3231,6 +3232,7 @@ describe('credential protection', () => { } if (isPageCall(expression, 'collectSnapshot')) { calls.push('snapshot') + snapshotAt = Date.now() return Promise.resolve({ url: 'https://example.com/', title: 'Example', @@ -3246,6 +3248,7 @@ describe('credential protection', () => { }) vi.useFakeTimers() try { + const startedAt = Date.now() const clicking = driver.executeTool('chat-test', 'browser_click', { elementId: 0, observe: {}, @@ -3256,6 +3259,7 @@ describe('credential protection', () => { const probesBeforeSnapshot = calls.indexOf('snapshot') expect(probesBeforeSnapshot).toBeGreaterThan(0) if (expectedProbes !== undefined) expect(probesBeforeSnapshot).toBe(expectedProbes) + else expect(snapshotAt - startedAt).toBeLessThanOrEqual(2_000) } finally { vi.useRealTimers() } diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index efc9711dfa8..759157cbf8b 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -381,6 +381,77 @@ describe('combobox typing surfaces', () => { expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) + function coverViewport(el: T): T { + el.getBoundingClientRect = () => + ({ + x: 0, + y: 0, + width: window.innerWidth, + height: window.innerHeight, + top: 0, + left: 0, + right: window.innerWidth, + bottom: window.innerHeight, + }) as DOMRect + return el + } + + it('refuses a top-page field behind an unmarked full-screen modal layer', () => { + document.body.innerHTML = ` + +
Subscribe to our newsletter
+ ` + const input = visible(document.querySelector('input') as HTMLInputElement) + coverViewport(document.querySelector('#backdrop') as HTMLDivElement) + const panel = visible(document.querySelector('#panel') as HTMLDivElement) + register(input) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => panel }) + + expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) + expect(input.value).toBe('') + }) + + it('types under a fixed bar that does not cover the screen', () => { + document.body.innerHTML = + '
Free shipping today
' + const input = visible(document.querySelector('input') as HTMLInputElement) + const bar = visible(document.querySelector('div') as HTMLDivElement) + register(input) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => bar }) + + expect(focusElementForTyping(0)).toMatchObject({ + focused: true, + coveredBy: 'Free shipping today', + }) + }) + + it('types inside a full-screen fixed app shell that also holds the field', () => { + document.body.innerHTML = ` +
+
Shop for your home
+
+ ` + coverViewport(document.querySelector('#shell') as HTMLDivElement) + const input = visible(document.querySelector('input') as HTMLInputElement) + const card = visible(document.querySelector('#card') as HTMLDivElement) + register(input) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => card }) + + expect(focusElementForTyping(0)).toMatchObject({ + focused: true, + coveredBy: 'Shop for your home', + }) + }) + + it('refuses a top-page field when no sampled point hits anything', () => { + document.body.innerHTML = '' + const input = visible(document.querySelector('input') as HTMLInputElement) + register(input) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => null }) + + expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) + }) + it('still refuses a covered field inside a frame', () => { const frame = document.createElement('iframe') document.body.append(frame) diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index 4e726c82ee7..651e1984cc5 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -2040,11 +2040,26 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { coveredByRelatedPopup = true } } - // Top-page typing goes through keyboard focus, so a non-modal overlay does not block it; a modal - // or dialog cover still refuses, and framed fields keep the check for their mapped point. + // Top-page typing goes through keyboard focus, so a non-modal overlay does not block it. A modal + // still refuses: one marked as a dialog, or an unmarked fixed layer spanning the viewport that + // does not also hold the field (a full-screen app shell does). Framed fields keep the check for + // their mapped point, and a field no sampled point reaches refuses. const topPageField = view === window && window.top === window - const insideModal = (node: Element | null): boolean => { - for (let current = node; current; current = composedParent(current)) { + const holdsField = (node: Element): boolean => { + for (let current: Element | null = editable; current; current = composedParent(current)) { + if (current === node) return true + } + return false + } + const spansViewport = (node: Element): boolean => { + if (view.getComputedStyle(node).position !== 'fixed') return false + const box = node.getBoundingClientRect() + const width = Math.min(box.right, view.innerWidth) - Math.max(box.left, 0) + const height = Math.min(box.bottom, view.innerHeight) - Math.max(box.top, 0) + return width >= view.innerWidth * 0.9 && height >= view.innerHeight * 0.9 + } + const insideModal = (node: Element): boolean => { + for (let current: Element | null = node; current; current = composedParent(current)) { const role = current.getAttribute('role') if ( current.getAttribute('aria-modal') === 'true' || @@ -2054,16 +2069,13 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { ) { return true } + if (!holdsField(current) && spansViewport(current)) return true } return false } + const hits = blockedPoints.flatMap((point) => (point.hit ? [point.hit] : [])) let coveredBy: string | undefined - if ( - !chosenPoint && - topPageField && - blockedPoints.length > 0 && - !blockedPoints.some((point) => insideModal(point.hit)) - ) { + if (!chosenPoint && topPageField && hits.length > 0 && !hits.some(insideModal)) { chosenPoint = { x: blockedPoints[0].x, y: blockedPoints[0].y } coveredBy = blockerLabel(firstBlocker) } From 54e669739451eb459931c62a09a7fdbd4a713220 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 13:16:10 -0800 Subject: [PATCH 2/9] refactor(desktop-browser): compute the field's ancestors once, dedupe sampled hits, and time the settle cap from its first probe --- .../src/main/browser-agent/driver.test.ts | 5 ++-- .../main/browser-agent/page-functions.test.ts | 29 +++---------------- .../src/main/browser-agent/page-functions.ts | 23 ++++++++------- 3 files changed, 19 insertions(+), 38 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index c7c1c9c5366..7f7067687b2 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -3223,10 +3223,12 @@ describe('credential protection', () => { const contents = await openPage() const calls: string[] = [] let probe = 0 + let firstProbeAt = Number.NaN let snapshotAt = Number.NaN vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => { if (isPageCall(expression, 'readDomQuiet')) { calls.push('probe') + if (Number.isNaN(firstProbeAt)) firstProbeAt = Date.now() const quietMs = quietReadings[Math.min(probe++, quietReadings.length - 1)] return Promise.resolve({ quietMs, loading: false }) } @@ -3248,7 +3250,6 @@ describe('credential protection', () => { }) vi.useFakeTimers() try { - const startedAt = Date.now() const clicking = driver.executeTool('chat-test', 'browser_click', { elementId: 0, observe: {}, @@ -3259,7 +3260,7 @@ describe('credential protection', () => { const probesBeforeSnapshot = calls.indexOf('snapshot') expect(probesBeforeSnapshot).toBeGreaterThan(0) if (expectedProbes !== undefined) expect(probesBeforeSnapshot).toBe(expectedProbes) - else expect(snapshotAt - startedAt).toBeLessThanOrEqual(2_000) + else expect(snapshotAt - firstProbeAt).toBeLessThanOrEqual(2_000) } finally { vi.useRealTimers() } diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index 759157cbf8b..18a854eb9af 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -66,18 +66,9 @@ function runSerialized(fn: (...args: never[]) => unknown, args: unknown[]): unkn return new Function(`return ${serializePageCall(fn, args)}`)() } -function visible(el: T): T { +function visible(el: T, width = 100, height = 20): T { el.getBoundingClientRect = () => - ({ - x: 0, - y: 0, - width: 100, - height: 20, - top: 0, - left: 0, - right: 100, - bottom: 20, - }) as DOMRect + ({ x: 0, y: 0, width, height, top: 0, left: 0, right: width, bottom: height }) as DOMRect return el } @@ -381,20 +372,8 @@ describe('combobox typing surfaces', () => { expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) - function coverViewport(el: T): T { - el.getBoundingClientRect = () => - ({ - x: 0, - y: 0, - width: window.innerWidth, - height: window.innerHeight, - top: 0, - left: 0, - right: window.innerWidth, - bottom: window.innerHeight, - }) as DOMRect - return el - } + const coverViewport = (el: T): T => + visible(el, window.innerWidth, window.innerHeight) it('refuses a top-page field behind an unmarked full-screen modal layer', () => { document.body.innerHTML = ` diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index 651e1984cc5..8ffaf1e1a51 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -2041,22 +2041,23 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { } } // Top-page typing goes through keyboard focus, so a non-modal overlay does not block it. A modal - // still refuses: one marked as a dialog, or an unmarked fixed layer spanning the viewport that - // does not also hold the field (a full-screen app shell does). Framed fields keep the check for - // their mapped point, and a field no sampled point reaches refuses. + // still does: a dialog, or an unmarked fixed layer spanning the viewport that does not hold the + // field (a full-screen app shell does). Framed fields keep the check for their mapped point. const topPageField = view === window && window.top === window - const holdsField = (node: Element): boolean => { - for (let current: Element | null = editable; current; current = composedParent(current)) { - if (current === node) return true - } - return false + const MODAL_VIEWPORT_COVERAGE = 0.9 + const fieldAncestors = new Set() + for (let current: Element | null = editable; current; current = composedParent(current)) { + fieldAncestors.add(current) } const spansViewport = (node: Element): boolean => { if (view.getComputedStyle(node).position !== 'fixed') return false const box = node.getBoundingClientRect() const width = Math.min(box.right, view.innerWidth) - Math.max(box.left, 0) const height = Math.min(box.bottom, view.innerHeight) - Math.max(box.top, 0) - return width >= view.innerWidth * 0.9 && height >= view.innerHeight * 0.9 + return ( + width >= view.innerWidth * MODAL_VIEWPORT_COVERAGE && + height >= view.innerHeight * MODAL_VIEWPORT_COVERAGE + ) } const insideModal = (node: Element): boolean => { for (let current: Element | null = node; current; current = composedParent(current)) { @@ -2069,11 +2070,11 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { ) { return true } - if (!holdsField(current) && spansViewport(current)) return true + if (!fieldAncestors.has(current) && spansViewport(current)) return true } return false } - const hits = blockedPoints.flatMap((point) => (point.hit ? [point.hit] : [])) + const hits = Array.from(new Set(blockedPoints.flatMap((point) => (point.hit ? [point.hit] : [])))) let coveredBy: string | undefined if (!chosenPoint && topPageField && hits.length > 0 && !hits.some(insideModal)) { chosenPoint = { x: blockedPoints[0].x, y: blockedPoints[0].y } From f716d615b8ef12a2bbe4d938164c90f2035b87fd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 13:16:56 -0800 Subject: [PATCH 3/9] test(desktop-browser): pass only the element to visible() from map --- apps/desktop/src/main/browser-agent/page-functions.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index 18a854eb9af..ea1c2d29ef4 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -200,7 +200,7 @@ describe('serialization contract', () => { it.each(cases)('%s is self-contained', (_name, fn, args) => { document.body.innerHTML = '' - register(...Array.from(document.body.children).map(visible)) + register(...Array.from(document.body.children).map((el) => visible(el))) expect(() => runSerialized(fn, args)).not.toThrow() }) From 27f916a6a117a7295e9cf5028cd990d0234210d9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 13:18:16 -0800 Subject: [PATCH 4/9] test(desktop-browser): drop a typing assertion that cannot fail and bound the settle cap from below --- apps/desktop/src/main/browser-agent/driver.test.ts | 5 ++++- apps/desktop/src/main/browser-agent/page-functions.test.ts | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 7f7067687b2..c729da5ebbd 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -3260,7 +3260,10 @@ describe('credential protection', () => { const probesBeforeSnapshot = calls.indexOf('snapshot') expect(probesBeforeSnapshot).toBeGreaterThan(0) if (expectedProbes !== undefined) expect(probesBeforeSnapshot).toBe(expectedProbes) - else expect(snapshotAt - firstProbeAt).toBeLessThanOrEqual(2_000) + else { + expect(snapshotAt - firstProbeAt).toBeGreaterThanOrEqual(1_900) + expect(snapshotAt - firstProbeAt).toBeLessThanOrEqual(2_000) + } } finally { vi.useRealTimers() } diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index ea1c2d29ef4..0eef764b935 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -387,7 +387,6 @@ describe('combobox typing surfaces', () => { Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => panel }) expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) - expect(input.value).toBe('') }) it('types under a fixed bar that does not cover the screen', () => { From 6a28b9810bd1f7fd128657b819c8e3445f72d16b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 13:30:58 -0800 Subject: [PATCH 5/9] fix(desktop-browser): count a full-screen fixed layer as a modal only when it catches the pointer --- .../main/browser-agent/page-functions.test.ts | 19 +++++++++++++++++++ .../src/main/browser-agent/page-functions.ts | 8 +++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index 0eef764b935..1e065894147 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -403,6 +403,25 @@ describe('combobox typing surfaces', () => { }) }) + it('types under a popup whose full-screen fixed wrapper lets the pointer through', () => { + document.body.innerHTML = ` + +
+
Saved to your list
+
+ ` + const input = visible(document.querySelector('input') as HTMLInputElement) + coverViewport(document.querySelector('#layer') as HTMLDivElement) + const toast = visible(document.querySelector('#toast') as HTMLDivElement) + register(input) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => toast }) + + expect(focusElementForTyping(0)).toMatchObject({ + focused: true, + coveredBy: 'Saved to your list', + }) + }) + it('types inside a full-screen fixed app shell that also holds the field', () => { document.body.innerHTML = `
diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index 8ffaf1e1a51..950932b0834 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -2041,8 +2041,9 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { } } // Top-page typing goes through keyboard focus, so a non-modal overlay does not block it. A modal - // still does: a dialog, or an unmarked fixed layer spanning the viewport that does not hold the - // field (a full-screen app shell does). Framed fields keep the check for their mapped point. + // still does: a dialog, or an unmarked fixed layer spanning the viewport that catches the pointer + // and does not hold the field (a full-screen app shell does). Framed fields keep the check for + // their mapped point. const topPageField = view === window && window.top === window const MODAL_VIEWPORT_COVERAGE = 0.9 const fieldAncestors = new Set() @@ -2050,7 +2051,8 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { fieldAncestors.add(current) } const spansViewport = (node: Element): boolean => { - if (view.getComputedStyle(node).position !== 'fixed') return false + const style = view.getComputedStyle(node) + if (style.position !== 'fixed' || style.pointerEvents === 'none') return false const box = node.getBoundingClientRect() const width = Math.min(box.right, view.innerWidth) - Math.max(box.left, 0) const height = Math.min(box.bottom, view.innerHeight) - Math.max(box.top, 0) From 285e4f0279441926620731966263d3e0de9f3360 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 13:43:14 -0800 Subject: [PATCH 6/9] fix(desktop-browser): detect an unmarked modal by probing the viewport for the layer that covers the field --- .../main/browser-agent/page-functions.test.ts | 119 +++++++++++++----- .../src/main/browser-agent/page-functions.ts | 66 ++++++---- 2 files changed, 130 insertions(+), 55 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index 1e065894147..fe29260882a 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -66,9 +66,18 @@ function runSerialized(fn: (...args: never[]) => unknown, args: unknown[]): unkn return new Function(`return ${serializePageCall(fn, args)}`)() } -function visible(el: T, width = 100, height = 20): T { +function visible(el: T): T { el.getBoundingClientRect = () => - ({ x: 0, y: 0, width, height, top: 0, left: 0, right: width, bottom: height }) as DOMRect + ({ + x: 0, + y: 0, + width: 100, + height: 20, + top: 0, + left: 0, + right: 100, + bottom: 20, + }) as DOMRect return el } @@ -200,7 +209,7 @@ describe('serialization contract', () => { it.each(cases)('%s is self-contained', (_name, fn, args) => { document.body.innerHTML = '' - register(...Array.from(document.body.children).map((el) => visible(el))) + register(...Array.from(document.body.children).map(visible)) expect(() => runSerialized(fn, args)).not.toThrow() }) @@ -372,54 +381,99 @@ describe('combobox typing surfaces', () => { expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) - const coverViewport = (el: T): T => - visible(el, window.innerWidth, window.innerHeight) + /** Answers `elementFromPoint` with `overField` on the field's sampled points, `elsewhere` beyond. */ + function pointTo(overField: Element, elsewhere: (x: number, y: number) => Element): void { + Object.defineProperty(document, 'elementFromPoint', { + configurable: true, + value: (x: number, y: number) => + x >= 10 && x <= 100 && y <= 20 ? overField : elsewhere(x, y), + }) + } + const nearCentre = (x: number, y: number) => + Math.abs(x - window.innerWidth / 2) < 50 && Math.abs(y - window.innerHeight / 2) < 50 - it('refuses a top-page field behind an unmarked full-screen modal layer', () => { + it('refuses a top-page field behind an unmarked backdrop and panel', () => { document.body.innerHTML = ` -
Subscribe to our newsletter
+
+
Subscribe to our newsletter
` const input = visible(document.querySelector('input') as HTMLInputElement) - coverViewport(document.querySelector('#backdrop') as HTMLDivElement) - const panel = visible(document.querySelector('#panel') as HTMLDivElement) + const backdrop = document.querySelector('#backdrop') as HTMLDivElement + const panel = document.querySelector('#panel') as HTMLDivElement register(input) - Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => panel }) + pointTo(panel, (x, y) => (nearCentre(x, y) ? panel : backdrop)) expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) - it('types under a fixed bar that does not cover the screen', () => { - document.body.innerHTML = - '
Free shipping today
' + it('refuses a top-page field behind a backdrop inside a wrapper that lets the pointer through', () => { + document.body.innerHTML = ` + +
+
+
Subscribe to our newsletter
+
+ ` const input = visible(document.querySelector('input') as HTMLInputElement) - const bar = visible(document.querySelector('div') as HTMLDivElement) + const backdrop = document.querySelector('#backdrop') as HTMLDivElement + const panel = document.querySelector('#panel') as HTMLDivElement register(input) - Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => bar }) + pointTo(panel, () => backdrop) - expect(focusElementForTyping(0)).toMatchObject({ - focused: true, - coveredBy: 'Free shipping today', - }) + expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) - it('types under a popup whose full-screen fixed wrapper lets the pointer through', () => { + it('types under a fixed bar or a toast whose wrapper lets the pointer through', () => { document.body.innerHTML = ` -
-
Saved to your list
-
+
Free shipping today
+
Saved to your list
+ ` + const input = visible(document.querySelector('input') as HTMLInputElement) + register(input) + for (const [cover, label] of [ + ['#bar', 'Free shipping today'], + ['#toast', 'Saved to your list'], + ] as const) { + pointTo(document.querySelector(cover) as HTMLDivElement, () => document.body) + expect(focusElementForTyping(0)).toMatchObject({ focused: true, coveredBy: label }) + } + }) + + it('types under a fixed header when a cookie banner pins the bottom corners', () => { + document.body.innerHTML = ` + + +
Results
+ ` const input = visible(document.querySelector('input') as HTMLInputElement) - coverViewport(document.querySelector('#layer') as HTMLDivElement) - const toast = visible(document.querySelector('#toast') as HTMLDivElement) + const header = document.querySelector('#header') as HTMLDivElement + const main = document.querySelector('main') as HTMLElement + const banner = document.querySelector('#banner') as HTMLDivElement register(input) - Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => toast }) + pointTo(header, (x, y) => + nearCentre(x, y) ? main : y < window.innerHeight / 2 ? header : banner + ) - expect(focusElementForTyping(0)).toMatchObject({ - focused: true, - coveredBy: 'Saved to your list', - }) + expect(focusElementForTyping(0)).toMatchObject({ focused: true, coveredBy: 'Menu' }) + }) + + it('types into a fixed search box over a full-screen fixed map when a card covers it', () => { + document.body.innerHTML = ` +
Map
+
+
Try the new app
+ ` + const input = visible(document.querySelector('input') as HTMLInputElement) + register(input) + pointTo( + document.querySelector('#card') as HTMLDivElement, + () => document.querySelector('#map') as HTMLDivElement + ) + + expect(focusElementForTyping(0)).toMatchObject({ focused: true, coveredBy: 'Try the new app' }) }) it('types inside a full-screen fixed app shell that also holds the field', () => { @@ -428,11 +482,10 @@ describe('combobox typing surfaces', () => {
Shop for your home
` - coverViewport(document.querySelector('#shell') as HTMLDivElement) + const shell = document.querySelector('#shell') as HTMLDivElement const input = visible(document.querySelector('input') as HTMLInputElement) - const card = visible(document.querySelector('#card') as HTMLDivElement) register(input) - Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => card }) + pointTo(document.querySelector('#card') as HTMLDivElement, () => shell) expect(focusElementForTyping(0)).toMatchObject({ focused: true, diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index 950932b0834..761a595ec77 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -2041,27 +2041,11 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { } } // Top-page typing goes through keyboard focus, so a non-modal overlay does not block it. A modal - // still does: a dialog, or an unmarked fixed layer spanning the viewport that catches the pointer - // and does not hold the field (a full-screen app shell does). Framed fields keep the check for - // their mapped point. + // still does: a dialog, or unmarked fixed layers that catch the pointer at the viewport's corners + // and centre, include the field's cover, and do not hold the field (a full-screen app shell + // does). Framed fields keep the check for their mapped point. const topPageField = view === window && window.top === window - const MODAL_VIEWPORT_COVERAGE = 0.9 - const fieldAncestors = new Set() - for (let current: Element | null = editable; current; current = composedParent(current)) { - fieldAncestors.add(current) - } - const spansViewport = (node: Element): boolean => { - const style = view.getComputedStyle(node) - if (style.position !== 'fixed' || style.pointerEvents === 'none') return false - const box = node.getBoundingClientRect() - const width = Math.min(box.right, view.innerWidth) - Math.max(box.left, 0) - const height = Math.min(box.bottom, view.innerHeight) - Math.max(box.top, 0) - return ( - width >= view.innerWidth * MODAL_VIEWPORT_COVERAGE && - height >= view.innerHeight * MODAL_VIEWPORT_COVERAGE - ) - } - const insideModal = (node: Element): boolean => { + const insideDialog = (node: Element): boolean => { for (let current: Element | null = node; current; current = composedParent(current)) { const role = current.getAttribute('role') if ( @@ -2072,13 +2056,51 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { ) { return true } - if (!fieldAncestors.has(current) && spansViewport(current)) return true } return false } + const fixedLayerOf = (node: Element | null): Element | null => { + let layer: Element | null = null + for (let current = node; current; current = composedParent(current)) { + if (view.getComputedStyle(current).position === 'fixed') layer = current + } + return layer + } + const screenBlockedByLayer = (covers: Element[]): boolean => { + const documentAt = editable.ownerDocument + const width = documentAt.documentElement.clientWidth || view.innerWidth + const height = documentAt.documentElement.clientHeight || view.innerHeight + const inset = 4 + const probes: Array<[number, number]> = [ + [inset, inset], + [width - inset, inset], + [inset, height - inset], + [width - inset, height - inset], + [width / 2, height / 2], + ] + const layers = new Set() + for (const [x, y] of probes) { + const layer = fixedLayerOf(documentAt.elementFromPoint(x, y)) + if (!layer) return false + layers.add(layer) + } + for (let current: Element | null = editable; current; current = composedParent(current)) { + if (layers.has(current)) return false + } + return covers.some((cover) => { + const layer = fixedLayerOf(cover) + return layer !== null && layers.has(layer) + }) + } const hits = Array.from(new Set(blockedPoints.flatMap((point) => (point.hit ? [point.hit] : [])))) let coveredBy: string | undefined - if (!chosenPoint && topPageField && hits.length > 0 && !hits.some(insideModal)) { + if ( + !chosenPoint && + topPageField && + hits.length > 0 && + !hits.some(insideDialog) && + !screenBlockedByLayer(hits) + ) { chosenPoint = { x: blockedPoints[0].x, y: blockedPoints[0].y } coveredBy = blockerLabel(firstBlocker) } From b038c1b64154358dd82d31598c032bb15841c201 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 14:02:17 -0800 Subject: [PATCH 7/9] fix(desktop-browser): treat an element painted above the field and under every viewport corner as a modal backdrop --- .../main/browser-agent/page-functions.test.ts | 142 ++++++++++-------- .../src/main/browser-agent/page-functions.ts | 53 +++---- 2 files changed, 100 insertions(+), 95 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index fe29260882a..c2cae5edfd4 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -381,111 +381,121 @@ describe('combobox typing surfaces', () => { expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) - /** Answers `elementFromPoint` with `overField` on the field's sampled points, `elsewhere` beyond. */ - function pointTo(overField: Element, elsewhere: (x: number, y: number) => Element): void { - Object.defineProperty(document, 'elementFromPoint', { + /** + * Answers hit testing with the paint stack (topmost first) over the field's sampled points and + * elsewhere, as `elementsFromPoint` does; `elementFromPoint` is its top. + */ + function stackTo( + overField: Element[], + elsewhere: (x: number, y: number) => Element[], + target: Document | ShadowRoot = document + ): void { + const stackAt = (x: number, y: number) => + x >= 10 && x <= 100 && y <= 20 ? overField : elsewhere(x, y) + Object.defineProperty(target, 'elementsFromPoint', { configurable: true, value: stackAt }) + Object.defineProperty(target, 'elementFromPoint', { configurable: true, - value: (x: number, y: number) => - x >= 10 && x <= 100 && y <= 20 ? overField : elsewhere(x, y), + value: (x: number, y: number) => stackAt(x, y)[0] ?? null, }) } - const nearCentre = (x: number, y: number) => - Math.abs(x - window.innerWidth / 2) < 50 && Math.abs(y - window.innerHeight / 2) < 50 + const page = () => [document.body, document.documentElement] + const byId = (id: string) => + document.getElementById(id) as unknown as T it('refuses a top-page field behind an unmarked backdrop and panel', () => { document.body.innerHTML = ` - +
Subscribe to our newsletter
` - const input = visible(document.querySelector('input') as HTMLInputElement) - const backdrop = document.querySelector('#backdrop') as HTMLDivElement - const panel = document.querySelector('#panel') as HTMLDivElement + const input = visible(byId('field')) register(input) - pointTo(panel, (x, y) => (nearCentre(x, y) ? panel : backdrop)) + stackTo([byId('panel'), byId('backdrop'), input, ...page()], () => [ + byId('backdrop'), + ...page(), + ]) expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) - it('refuses a top-page field behind a backdrop inside a wrapper that lets the pointer through', () => { + it('refuses a field behind an unmarked modal nested in a full-screen app shell', () => { document.body.innerHTML = ` - -
-
-
Subscribe to our newsletter
+
+ +
Confirm your plan
` - const input = visible(document.querySelector('input') as HTMLInputElement) - const backdrop = document.querySelector('#backdrop') as HTMLDivElement - const panel = document.querySelector('#panel') as HTMLDivElement + const input = visible(byId('field')) register(input) - pointTo(panel, () => backdrop) + stackTo([byId('panel'), byId('backdrop'), input, byId('shell'), ...page()], () => [ + byId('backdrop'), + byId('shell'), + ...page(), + ]) expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) - it('types under a fixed bar or a toast whose wrapper lets the pointer through', () => { - document.body.innerHTML = ` - -
Free shipping today
-
Saved to your list
- ` - const input = visible(document.querySelector('input') as HTMLInputElement) - register(input) - for (const [cover, label] of [ - ['#bar', 'Free shipping today'], - ['#toast', 'Saved to your list'], - ] as const) { - pointTo(document.querySelector(cover) as HTMLDivElement, () => document.body) - expect(focusElementForTyping(0)).toMatchObject({ focused: true, coveredBy: label }) - } - }) - - it('types under a fixed header when a cookie banner pins the bottom corners', () => { - document.body.innerHTML = ` - - -
Results
- - ` - const input = visible(document.querySelector('input') as HTMLInputElement) - const header = document.querySelector('#header') as HTMLDivElement - const main = document.querySelector('main') as HTMLElement - const banner = document.querySelector('#banner') as HTMLDivElement + it('refuses a field behind an unmarked modal inside the same shadow root', () => { + document.body.innerHTML = '
' + const shadow = byId('host').attachShadow({ mode: 'open' }) + shadow.innerHTML = + '
Sign up
' + const input = visible(shadow.querySelector('input') as HTMLInputElement) + const backdrop = shadow.querySelector('#backdrop') as HTMLDivElement + const panel = shadow.querySelector('#panel') as HTMLDivElement register(input) - pointTo(header, (x, y) => - nearCentre(x, y) ? main : y < window.innerHeight / 2 ? header : banner + stackTo( + [panel, backdrop, input, byId('host'), ...page()], + () => [backdrop, byId('host'), ...page()], + shadow ) - expect(focusElementForTyping(0)).toMatchObject({ focused: true, coveredBy: 'Menu' }) + expect(focusElementForTyping(0)).toMatchObject({ error: 'obstructed' }) }) - it('types into a fixed search box over a full-screen fixed map when a card covers it', () => { + it('types under covers that do not reach every corner of the viewport', () => { document.body.innerHTML = `
Map
-
-
Try the new app
+ + +
Saved to your list
+ ` - const input = visible(document.querySelector('input') as HTMLInputElement) + const input = visible(byId('field')) register(input) - pointTo( - document.querySelector('#card') as HTMLDivElement, - () => document.querySelector('#map') as HTMLDivElement - ) - - expect(focusElementForTyping(0)).toMatchObject({ focused: true, coveredBy: 'Try the new app' }) + const topHalf = (y: number) => y < window.innerHeight / 2 + for (const [cover, label, elsewhere] of [ + // a fixed header over a full-screen fixed map the field also sits above + [ + 'header', + 'Menu', + (_x: number, y: number) => + (topHalf(y) ? [byId('header')] : []).concat(byId('map'), ...page()), + ], + // a fixed header with a cookie banner pinned to the bottom corners + [ + 'header', + 'Menu', + (_x: number, y: number) => [topHalf(y) ? byId('header') : byId('banner'), ...page()], + ], + // a toast whose full-screen wrapper lets the pointer through, so the wrapper is in no stack + ['toast', 'Saved to your list', () => page()], + ] as const) { + stackTo([byId(cover), input, byId('map'), ...page()], elsewhere) + expect(focusElementForTyping(0)).toMatchObject({ focused: true, coveredBy: label }) + } }) it('types inside a full-screen fixed app shell that also holds the field', () => { document.body.innerHTML = `
-
Shop for your home
+
Shop for your home
` - const shell = document.querySelector('#shell') as HTMLDivElement - const input = visible(document.querySelector('input') as HTMLInputElement) + const input = visible(byId('field')) register(input) - pointTo(document.querySelector('#card') as HTMLDivElement, () => shell) + stackTo([byId('card'), input, byId('shell'), ...page()], () => [byId('shell'), ...page()]) expect(focusElementForTyping(0)).toMatchObject({ focused: true, diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index 761a595ec77..e2226ded3d4 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -2041,9 +2041,8 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { } } // Top-page typing goes through keyboard focus, so a non-modal overlay does not block it. A modal - // still does: a dialog, or unmarked fixed layers that catch the pointer at the viewport's corners - // and centre, include the field's cover, and do not hold the field (a full-screen app shell - // does). Framed fields keep the check for their mapped point. + // still does: a dialog, or an unmarked backdrop, which paints above the field and under every + // corner of the viewport. Framed fields keep the check for their mapped point. const topPageField = view === window && window.top === window const insideDialog = (node: Element): boolean => { for (let current: Element | null = node; current; current = composedParent(current)) { @@ -2059,38 +2058,34 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { } return false } - const fixedLayerOf = (node: Element | null): Element | null => { - let layer: Element | null = null - for (let current = node; current; current = composedParent(current)) { - if (view.getComputedStyle(current).position === 'fixed') layer = current + const behindBackdrop = (): boolean => { + const stackRoot = root as ParentNode & { + elementsFromPoint?: (x: number, y: number) => Element[] } - return layer - } - const screenBlockedByLayer = (covers: Element[]): boolean => { - const documentAt = editable.ownerDocument - const width = documentAt.documentElement.clientWidth || view.innerWidth - const height = documentAt.documentElement.clientHeight || view.innerHeight + if (typeof stackRoot.elementsFromPoint !== 'function') return false + const stackAt = stackRoot.elementsFromPoint.bind(stackRoot) + const aboveField = new Set() + for (const point of blockedPoints) { + for (const element of stackAt(point.x, point.y)) { + if (element === editable || editable.contains(element) || element.contains(editable)) break + aboveField.add(element) + } + } + if (aboveField.size === 0) return false + const width = editable.ownerDocument.documentElement.clientWidth || view.innerWidth + const height = editable.ownerDocument.documentElement.clientHeight || view.innerHeight const inset = 4 - const probes: Array<[number, number]> = [ + for (const [x, y] of [ [inset, inset], [width - inset, inset], [inset, height - inset], [width - inset, height - inset], - [width / 2, height / 2], - ] - const layers = new Set() - for (const [x, y] of probes) { - const layer = fixedLayerOf(documentAt.elementFromPoint(x, y)) - if (!layer) return false - layers.add(layer) - } - for (let current: Element | null = editable; current; current = composedParent(current)) { - if (layers.has(current)) return false + ]) { + const corner = new Set(stackAt(x, y)) + for (const element of aboveField) if (!corner.has(element)) aboveField.delete(element) + if (aboveField.size === 0) return false } - return covers.some((cover) => { - const layer = fixedLayerOf(cover) - return layer !== null && layers.has(layer) - }) + return true } const hits = Array.from(new Set(blockedPoints.flatMap((point) => (point.hit ? [point.hit] : [])))) let coveredBy: string | undefined @@ -2099,7 +2094,7 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown { topPageField && hits.length > 0 && !hits.some(insideDialog) && - !screenBlockedByLayer(hits) + !behindBackdrop() ) { chosenPoint = { x: blockedPoints[0].x, y: blockedPoints[0].y } coveredBy = blockerLabel(firstBlocker) From e26a85a0935831f00e8f0fbf56cc36a6449fd35b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 14:02:59 -0800 Subject: [PATCH 8/9] test(desktop-browser): cover a field painted below its own full-screen container --- .../main/browser-agent/page-functions.test.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index c2cae5edfd4..d031938b94b 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -495,12 +495,17 @@ describe('combobox typing surfaces', () => { ` const input = visible(byId('field')) register(input) - stackTo([byId('card'), input, byId('shell'), ...page()], () => [byId('shell'), ...page()]) - - expect(focusElementForTyping(0)).toMatchObject({ - focused: true, - coveredBy: 'Shop for your home', - }) + // the shell paints below the field, or above it when the field sits at a negative z-index + for (const overField of [ + [byId('card'), input, byId('shell'), ...page()], + [byId('card'), byId('shell'), input, ...page()], + ]) { + stackTo(overField, () => [byId('shell'), ...page()]) + expect(focusElementForTyping(0)).toMatchObject({ + focused: true, + coveredBy: 'Shop for your home', + }) + } }) it('refuses a top-page field when no sampled point hits anything', () => { From 2c8244969ce06cecf9f4dfe13674e02357fcfac3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 14:22:29 -0800 Subject: [PATCH 9/9] test(desktop-browser): type the corner stacks as Element arrays --- .../src/main/browser-agent/page-functions.test.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index d031938b94b..dc3da220e3a 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -470,17 +470,23 @@ describe('combobox typing surfaces', () => { [ 'header', 'Menu', - (_x: number, y: number) => - (topHalf(y) ? [byId('header')] : []).concat(byId('map'), ...page()), + (_x: number, y: number): Element[] => [ + ...(topHalf(y) ? [byId('header')] : []), + byId('map'), + ...page(), + ], ], // a fixed header with a cookie banner pinned to the bottom corners [ 'header', 'Menu', - (_x: number, y: number) => [topHalf(y) ? byId('header') : byId('banner'), ...page()], + (_x: number, y: number): Element[] => [ + topHalf(y) ? byId('header') : byId('banner'), + ...page(), + ], ], // a toast whose full-screen wrapper lets the pointer through, so the wrapper is in no stack - ['toast', 'Saved to your list', () => page()], + ['toast', 'Saved to your list', (): Element[] => page()], ] as const) { stackTo([byId(cover), input, byId('map'), ...page()], elsewhere) expect(focusElementForTyping(0)).toMatchObject({ focused: true, coveredBy: label })