diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index da2a3b8918e..e89e96b4dc6 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -3631,6 +3631,9 @@ async function executeToolInner( activation, element: prepared.element, refRecovered: prepared.refRecovered === true, + ...(typeof prepared.clickedThrough === 'string' + ? { clickedThrough: prepared.clickedThrough } + : {}), effectObserved, possibleEffectObserved, effect, diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index 7f30104ff9d..c82eb044edc 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -693,6 +693,57 @@ describe('collectSnapshot', () => { expect(focusElementForTyping(ref)).toMatchObject({ error: 'stale' }) }) + it('clicks through a nested control that fills the whole target', () => { + document.body.innerHTML = '3:00 PM' + const link = visible(document.querySelector('a') as HTMLAnchorElement) + const slot = visible(document.querySelector('span') as HTMLSpanElement) + register(link) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => slot }) + + expect(clickElement(0, false)).toMatchObject({ clickedThrough: '3:00 PM' }) + }) + + it('refuses a nested control that leaves the target edges uncovered', () => { + document.body.innerHTML = 'Open' + const link = visible(document.querySelector('a') as HTMLAnchorElement) + const inner = document.querySelector('span') as HTMLSpanElement + inner.getBoundingClientRect = () => + ({ x: 15, y: 3, width: 70, height: 14, left: 15, top: 3, right: 85, bottom: 17 }) as DOMRect + register(link) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => inner }) + + expect(clickElement(0, false)).toMatchObject({ error: 'nested-control' }) + }) + + it('refuses to click through to a form field or through a synthetic fallback', () => { + document.body.innerHTML = '
' + const wrap = visible(document.querySelector('#wrap') as HTMLDivElement) + const input = visible(document.querySelector('input') as HTMLInputElement) + register(wrap) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => input }) + expect(clickElement(0, false)).toMatchObject({ error: 'nested-control' }) + + document.body.innerHTML = '3:00 PM' + const link = visible(document.querySelector('a') as HTMLAnchorElement) + const slot = visible(document.querySelector('span') as HTMLSpanElement) + register(link) + Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => slot }) + expect(clickElement(0)).toMatchObject({ error: 'nested-control' }) + }) + + it('still refuses a nested control that covers only part of the target', () => { + document.body.innerHTML = 'Card title ' + const link = visible(document.querySelector('a') as HTMLAnchorElement) + const save = visible(document.querySelector('button') as HTMLButtonElement) + register(link) + Object.defineProperty(document, 'elementFromPoint', { + configurable: true, + value: (x: number) => (x <= 50 ? save : document.body), + }) + + expect(clickElement(0)).toMatchObject({ error: 'nested-control' }) + }) + it('refuses to recover a ref when replacement is ambiguous', () => { document.body.innerHTML = '' const original = visible(document.querySelector('button') as HTMLButtonElement) diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index 1fe3dc93da3..794750a52f6 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -1382,6 +1382,8 @@ export function clickElement( let clientY = 0 let blocker: Element | null = null const blockedHits: Array = [] + let firstBlockedPoint: { x: number; y: number } | null = null + let filledByControl: Element | null = null let foundPoint = false const blockerLabel = (element: Element | null): string => ( @@ -1409,6 +1411,7 @@ export function clickElement( break } blocker ??= hit + firstBlockedPoint ??= { x, y } blockedHits.push(hit) } if (foundPoint) break @@ -1511,21 +1514,59 @@ export function clickElement( break } } - if (nested) { - let control = blocker + const nestedControlOf = (hit: Element | null): Element | null => { + let control = hit while (control && control !== el && !isIndependentInteractive(control)) { control = composedParent(control) } + return control && control !== el ? control : null + } + const control = nested ? nestedControlOf(blocker) : null + // One nested control filling the whole target (a time-slot label inside + // its link) is what a person clicking the target hits, so a native click + // on the top page goes there. Form fields and disabled controls keep + // their own refusals, and any uncovered edge leaves the target clickable + // in its own right, so both refuse. + const fillsTarget = (node: Element): boolean => { + const bounds = node.getBoundingClientRect() + return rects.every( + (rect) => + bounds.left <= rect.left + 1 && + bounds.top <= rect.top + 1 && + bounds.right >= rect.right - 1 && + bounds.bottom >= rect.bottom - 1 + ) + } + const controlTag = control ? String(control.tagName || '').toUpperCase() : '' + if ( + control && + firstBlockedPoint && + !dispatchSynthetic && + !focusForKeyboard && + el.ownerDocument.defaultView === window && + window.top === window && + !['INPUT', 'TEXTAREA', 'SELECT'].includes(controlTag) && + !(control as HTMLElement).isContentEditable && + !(control as HTMLButtonElement).disabled && + control.getAttribute('aria-disabled') !== 'true' && + fillsTarget(control) && + blockedHits.every((hit) => hit !== null && nestedControlOf(hit) === control) + ) { + clientX = firstBlockedPoint.x + clientY = firstBlockedPoint.y + filledByControl = control + } else if (nested) { return { error: 'nested-control', blocker: blockerLabel(blocker), - controlId: control && control !== el ? window.__simAgentRefOf(control) : null, + controlId: control ? window.__simAgentRefOf(control) : null, + } + } else { + return { + error: 'obstructed', + blocker: blockerLabel(blocker), + blockerControls: window.__simAgentOverlayControls(blocker), } - } - return { - error: 'obstructed', - blocker: blockerLabel(blocker), - blockerControls: window.__simAgentOverlayControls(blocker), } } @@ -1636,6 +1677,7 @@ export function clickElement( frameDepth, focusSucceeded: focusForKeyboard && el.ownerDocument.activeElement === el, refRecovered: resolved?.recovered === true, + ...(filledByControl ? { clickedThrough: blockerLabel(filledByControl) } : {}), } }