diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index da2a3b8918e..e89e96b4dc6 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -3631,6 +3631,9 @@ async function executeToolInner( activation, element: prepared.element, refRecovered: prepared.refRecovered === true, + ...(typeof prepared.clickedThrough === 'string' + ? { clickedThrough: prepared.clickedThrough } + : {}), effectObserved, possibleEffectObserved, effect, diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index 7f30104ff9d..c82eb044edc 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -693,6 +693,57 @@ describe('collectSnapshot', () => { expect(focusElementForTyping(ref)).toMatchObject({ error: 'stale' }) }) + it('clicks through a nested control that fills the whole target', () => { + document.body.innerHTML = '3:00 PM' + const link = visible(document.querySelector('a') as HTMLAnchorElement) + const slot = visible(document.querySelector('span') as HTMLSpanElement) + register(link) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => slot }) + + expect(clickElement(0, false)).toMatchObject({ clickedThrough: '3:00 PM' }) + }) + + it('refuses a nested control that leaves the target edges uncovered', () => { + document.body.innerHTML = 'Open' + const link = visible(document.querySelector('a') as HTMLAnchorElement) + const inner = document.querySelector('span') as HTMLSpanElement + inner.getBoundingClientRect = () => + ({ x: 15, y: 3, width: 70, height: 14, left: 15, top: 3, right: 85, bottom: 17 }) as DOMRect + register(link) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => inner }) + + expect(clickElement(0, false)).toMatchObject({ error: 'nested-control' }) + }) + + it('refuses to click through to a form field or through a synthetic fallback', () => { + document.body.innerHTML = '
' + const wrap = visible(document.querySelector('#wrap') as HTMLDivElement) + const input = visible(document.querySelector('input') as HTMLInputElement) + register(wrap) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => input }) + expect(clickElement(0, false)).toMatchObject({ error: 'nested-control' }) + + document.body.innerHTML = '3:00 PM' + const link = visible(document.querySelector('a') as HTMLAnchorElement) + const slot = visible(document.querySelector('span') as HTMLSpanElement) + register(link) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => slot }) + expect(clickElement(0)).toMatchObject({ error: 'nested-control' }) + }) + + it('still refuses a nested control that covers only part of the target', () => { + document.body.innerHTML = 'Card title ' + const link = visible(document.querySelector('a') as HTMLAnchorElement) + const save = visible(document.querySelector('button') as HTMLButtonElement) + register(link) + Object.defineProperty(document, 'elementFromPoint', { + configurable: true, + value: (x: number) => (x <= 50 ? save : document.body), + }) + + expect(clickElement(0)).toMatchObject({ error: 'nested-control' }) + }) + it('refuses to recover a ref when replacement is ambiguous', () => { document.body.innerHTML = '' const original = visible(document.querySelector('button') as HTMLButtonElement) diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index 1fe3dc93da3..794750a52f6 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -1382,6 +1382,8 @@ export function clickElement( let clientY = 0 let blocker: Element | null = null const blockedHits: Array