diff --git a/apps/desktop/e2e/browser-page-dialogs.spec.ts b/apps/desktop/e2e/browser-page-dialogs.spec.ts new file mode 100644 index 00000000000..d82a32c2a11 --- /dev/null +++ b/apps/desktop/e2e/browser-page-dialogs.spec.ts @@ -0,0 +1,326 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer, type Server } from 'node:http' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { + type ElectronApplication, + _electron as electron, + expect, + type Page, + test, +} from '@playwright/test' +import type { BrowserPageDialog, BrowserToolName } from '@sim/browser-protocol' +import type { SimDesktopApi } from '@sim/desktop-bridge' +import { getErrorMessage } from '@sim/utils/errors' + +const DESKTOP_DIR = fileURLToPath(new URL('..', import.meta.url)) +const SCOPE = 'browser-page-dialogs-e2e' +const SHELL_FIXTURE = 'Sim fixture

Browser dialogs fixture

' +const FORM_FIXTURE = `form + + +` + +type Bridge = typeof globalThis & { + simDesktop: SimDesktopApi + pageDialog?: BrowserPageDialog | null + boundsTimer?: number +} + +/** + * A page's alert, confirm, and leave-site question belong to whoever is using + * the page: the user on the tab they are looking at, the agent during its own + * action. These checks drive the real shell and native tab views. + */ +test('page dialogs wait for the user on their page and stay automatic for the agent', async () => { + const reportPath = + process.env.DESKTOP_BROWSER_DIALOGS_REPORT_PATH ?? + test.info().outputPath('browser-page-dialogs.json') + const checks: { + name: string + status: 'passed' | 'failed' + durationMs: number + error?: string + }[] = [] + const check = async (name: string, run: () => Promise) => { + const started = Date.now() + try { + await test.step(name, run) + checks.push({ name, status: 'passed', durationMs: Date.now() - started }) + } catch (error) { + checks.push({ + name, + status: 'failed', + durationMs: Date.now() - started, + error: getErrorMessage(error), + }) + throw error + } + } + const calls = new Map() + const server: Server = createServer(async (request, response) => { + const path = new URL(request.url ?? '/', 'http://localhost').pathname + if (path === '/api/desktop/tool/authorize') { + let body = '' + for await (const chunk of request) body += chunk.toString() + const authorization = calls.get(JSON.parse(body).toolCallId) + response.writeHead(authorization ? 200 : 403, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify(authorization ?? {})) + return + } + // The app origin serves the shell; the same server on localhost is the web. + const isSite = request.headers.host?.startsWith('localhost') === true + response.writeHead(200, { + 'Content-Type': 'text/html', + ...(isSite ? {} : { 'Set-Cookie': 'better-auth.session_token=fixture; HttpOnly; Path=/' }), + }) + response.end( + !isSite + ? SHELL_FIXTURE + : path === '/form' || path === '/agent' + ? FORM_FIXTURE + : 'next' + ) + }) + const userData = mkdtempSync(join(tmpdir(), 'sim-browser-dialogs-e2e-')) + let app: ElectronApplication | undefined + let passed = false + try { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Missing fixture address') + const origin = `http://127.0.0.1:${address.port}` + const site = origin.replace('127.0.0.1', 'localhost') + const shellApp = await electron.launch({ + args: [process.env.SIM_DESKTOP_E2E_MAIN ?? '.'], + cwd: DESKTOP_DIR, + env: { ...process.env, SIM_DESKTOP_ORIGIN: origin, SIM_DESKTOP_USER_DATA: userData }, + }) + app = shellApp + // A dialog listener stops Playwright auto-dismissing page dialogs, so the desktop's own + // handling decides their outcome exactly as it does in production. + const leaveDialogsToDesktop = (page: Page) => page.on('dialog', () => {}) + shellApp.context().pages().forEach(leaveDialogsToDesktop) + shellApp.context().on('page', leaveDialogsToDesktop) + const shell = await shellApp.firstWindow() + await expect(shell.getByRole('heading')).toHaveText('Browser dialogs fixture') + await shell.evaluate(async (scope) => { + const bridge = globalThis as Bridge + const api = bridge.simDesktop.browserAgent + await api.activateScope(scope) + const updateBounds = () => + api.setPanelBounds( + { x: 0, y: 80, width: innerWidth, height: innerHeight - 80 }, + null, + scope + ) + updateBounds() + bridge.boundsTimer = window.setInterval(updateBounds, 200) + bridge.pageDialog = null + api.onPageState((state) => { + bridge.pageDialog = state.dialog ?? null + }) + }, SCOPE) + + let callCount = 0 + const execute = async (tool: BrowserToolName, args: Record) => { + const callId = `browser-dialogs-${++callCount}` + calls.set(callId, { chatId: SCOPE, toolName: tool, args }) + const result = await shell.evaluate( + ({ callId, tool, args, scope }) => + (globalThis as Bridge).simDesktop.browserAgent.executeTool(callId, tool, args, scope), + { callId, tool, args, scope: SCOPE } + ) + expect(result.ok).toBe(true) + return result.ok ? result.result : undefined + } + /** Browser-chrome actions are user gestures, so each follows a real click in Sim. */ + const panelAction = async (action: Record) => { + await shell.getByRole('heading').click() + await shell.evaluate( + ({ action, scope }) => + (globalThis as Bridge).simDesktop.browserAgent.panelAction( + action as unknown as Parameters[0], + scope + ), + { action, scope: SCOPE } + ) + } + const pageDialog = () => shell.evaluate(() => (globalThis as Bridge).pageDialog ?? null) + const inPage = (script: string) => + shellApp.evaluate( + async ({ webContents }, { script, url }) => + (await webContents + .getAllWebContents() + .find((contents) => contents.getURL().startsWith(url)) + ?.executeJavaScript(script)) as T, + { script, url: site } + ) + /** Read from the shell: a script cannot run in a page while its dialog is open. */ + const pageTitle = () => + shellApp.evaluate( + ({ webContents }, url) => + webContents + .getAllWebContents() + .find((contents) => contents.getURL().startsWith(url)) + ?.getTitle() ?? null, + site + ) + const pageUrl = () => + shellApp.evaluate( + ({ webContents }, url) => + webContents + .getAllWebContents() + .find((contents) => contents.getURL().startsWith(url)) + ?.getURL() ?? null, + site + ) + /** Trusted input into the page, the way the user's own mouse and keys arrive. */ + const userInput = (selector: string, text = '') => + shellApp.evaluate( + async ({ webContents }, { selector, text, url }) => { + const contents = webContents + .getAllWebContents() + .find((candidate) => candidate.getURL().startsWith(url)) + if (!contents) throw new Error('No page') + const rect: { x: number; y: number } = await contents.executeJavaScript( + `(() => { const { x, y } = document.querySelector(${JSON.stringify(selector)}).getBoundingClientRect(); return { x, y } })()` + ) + const point = { x: Math.round(rect.x + 5), y: Math.round(rect.y + 5) } + contents.sendInputEvent({ type: 'mouseDown', ...point, button: 'left', clickCount: 1 }) + contents.sendInputEvent({ type: 'mouseUp', ...point, button: 'left', clickCount: 1 }) + for (const character of text) + contents.sendInputEvent({ type: 'char', keyCode: character }) + }, + { selector, text, url: site } + ) + + await check('without a renderer that shows dialogs, the shell still answers them', async () => { + await execute('browser_open_url', { url: `${site}/form` }) + await panelAction({ action: 'switch-tab', tabId: '1' }) + await userInput('#delete') + await expect.poll(() => pageTitle()).toBe('confirm:false') + expect(await pageDialog()).toBeNull() + }) + + await panelAction({ action: 'enable-page-dialogs' }) + await inPage("document.title = 'form'") + + await check("the user's confirm waits for their answer", async () => { + await userInput('#delete') + await expect + .poll(pageDialog) + .toMatchObject({ kind: 'confirm', message: 'Delete the report?' }) + expect(await pageTitle()).toBe('form') + const dialog = await pageDialog() + await panelAction({ action: 'respond-dialog', requestId: dialog?.requestId, allowed: true }) + await expect.poll(() => pageTitle()).toBe('confirm:true') + await expect.poll(pageDialog).toBeNull() + }) + + await check("the agent's dialogs never wait on the user", async () => { + await inPage("document.title = 'form'") + const snapshot = await execute('browser_snapshot', {}) + const ref = /button "Delete" \[ref=(\d+)\]/.exec( + String((snapshot as { outline?: string }).outline) + )?.[1] + expect(ref, 'snapshot lists the Delete button').toBeTruthy() + await execute('browser_click', { elementId: Number(ref) }) + await expect.poll(() => pageTitle()).toBe('confirm:false') + expect(await pageDialog()).toBeNull() + await inPage("document.title = 'form'") + await execute('browser_click', { elementId: Number(ref), dialog: { accept: true } }) + await expect.poll(() => pageTitle()).toBe('confirm:true') + expect(await pageDialog()).toBeNull() + }) + + await check('leaving a draft from the URL bar asks, and Stay keeps it', async () => { + await userInput('#draft', 'draft') + await expect + .poll(() => inPage("document.getElementById('draft').value")) + .toBe('draft') + await panelAction({ action: 'navigate', url: `${site}/next` }) + await expect.poll(pageDialog).toMatchObject({ kind: 'beforeunload' }) + const dialog = await pageDialog() + await panelAction({ action: 'respond-dialog', requestId: dialog?.requestId, allowed: false }) + await expect.poll(pageDialog).toBeNull() + expect(await pageUrl()).toBe(`${site}/form`) + expect(await inPage("document.getElementById('draft').value")).toBe('draft') + }) + + await check('Reload of a draft asks too, and Stay keeps it', async () => { + await panelAction({ action: 'reload' }) + await expect.poll(pageDialog).toMatchObject({ kind: 'beforeunload' }) + const dialog = await pageDialog() + await panelAction({ action: 'respond-dialog', requestId: dialog?.requestId, allowed: false }) + await expect.poll(pageDialog).toBeNull() + expect(await inPage("document.getElementById('draft').value")).toBe('draft') + }) + + await check('Leave lets the navigation through without asking again', async () => { + await panelAction({ action: 'navigate', url: `${site}/next` }) + await expect.poll(pageDialog).toMatchObject({ kind: 'beforeunload' }) + const dialog = await pageDialog() + await panelAction({ action: 'respond-dialog', requestId: dialog?.requestId, allowed: true }) + await expect.poll(pageUrl).toBe(`${site}/next`) + expect(await pageDialog()).toBeNull() + }) + /** Opens a page-initiated confirm (no user gesture) on the tab at `path`. */ + const confirmFromPage = (path: string) => + shellApp.evaluate(({ webContents }, url) => { + const contents = webContents.getAllWebContents().find((c) => c.getURL() === url) + if (!contents) throw new Error(`No page at ${url}`) + void contents.executeJavaScript( + "setTimeout(() => { document.title = 'confirm:' + confirm('Continue?') })" + ) + }, `${site}${path}`) + const titleAt = (path: string) => + shellApp.evaluate( + ({ webContents }, url) => + webContents + .getAllWebContents() + .find((c) => c.getURL() === url) + ?.getTitle() ?? null, + `${site}${path}` + ) + + await check( + "a dialog on the agent's page stays automatic until the user takes it", + async () => { + await execute('browser_open_tab', { url: `${site}/agent` }) + const tabs = await shell.evaluate( + async (scope) => + (await (globalThis as Bridge).simDesktop.browserAgent.activateScope(scope)).tabs, + SCOPE + ) + const agentTabId = tabs.find((tab) => tab.url === `${site}/agent`)?.tabId + // The resource strip mirrors the agent's tab on screen without the user claiming it. + await panelAction({ action: 'switch-tab', tabId: agentTabId, claim: false }) + await confirmFromPage('/agent') + await expect.poll(() => titleAt('/agent')).toBe('confirm:false') + expect(await pageDialog()).toBeNull() + } + ) + + await check('a dialog while the browser is off screen stays automatic', async () => { + await shell.evaluate((scope) => { + const bridge = globalThis as Bridge + window.clearInterval(bridge.boundsTimer) + bridge.simDesktop.browserAgent.setPanelBounds(null, null, scope) + }, SCOPE) + await confirmFromPage('/agent') + await expect.poll(() => titleAt('/agent')).toBe('confirm:false') + expect(await pageDialog()).toBeNull() + }) + passed = true + } finally { + mkdirSync(dirname(reportPath), { recursive: true }) + writeFileSync(reportPath, JSON.stringify({ passed, checks }, null, 2)) + await app?.close() + await new Promise((resolve) => server.close(() => resolve())) + rmSync(userData, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/src/main/browser-agent/cdp.test.ts b/apps/desktop/src/main/browser-agent/cdp.test.ts index 821f9b956d4..b951c2470e4 100644 --- a/apps/desktop/src/main/browser-agent/cdp.test.ts +++ b/apps/desktop/src/main/browser-agent/cdp.test.ts @@ -61,11 +61,22 @@ function createOopifFrameFixture() { } } +/** Callbacks for a page with no user to ask, so the shell answers every dialog. */ +const shellAnswersDialogs = { + offerToUser: () => false, + onDialogClosed: () => {}, + claimUserLeave: () => false, +} + describe('browser-agent CDP instrumentation', () => { it('leaves file chooser dialogs native so users can upload files', async () => { const contents = new WebContentsView().webContents - await ensureInstrumented(contents, { onDialog: vi.fn(), dialogResponse: () => null }) + await ensureInstrumented(contents, { + onDialog: vi.fn(), + dialogResponse: () => null, + ...shellAnswersDialogs, + }) expect(contents.debugger.sendCommand).toHaveBeenCalledWith('Page.enable', undefined) expect(contents.debugger.sendCommand).not.toHaveBeenCalledWith( @@ -86,10 +97,18 @@ describe('browser-agent CDP instrumentation', () => { }) await expect( - ensureInstrumented(contents, { onDialog: vi.fn(), dialogResponse: () => null }) + ensureInstrumented(contents, { + onDialog: vi.fn(), + dialogResponse: () => null, + ...shellAnswersDialogs, + }) ).rejects.toThrow('setup acknowledgement lost') await expect( - ensureInstrumented(contents, { onDialog: vi.fn(), dialogResponse: () => null }) + ensureInstrumented(contents, { + onDialog: vi.fn(), + dialogResponse: () => null, + ...shellAnswersDialogs, + }) ).resolves.toBeUndefined() expect(autoAttachAttempts).toBe(2) @@ -98,7 +117,11 @@ describe('browser-agent CDP instrumentation', () => { it('dismisses an OOPIF dialog on the flattened child session', async () => { const contents = new WebContentsView().webContents const onDialog = vi.fn() - await ensureInstrumented(contents, { onDialog, dialogResponse: () => null }) + await ensureInstrumented(contents, { + onDialog, + dialogResponse: () => null, + ...shellAnswersDialogs, + }) const listener = vi .mocked(contents.debugger.on) .mock.calls.find(([event]) => event === 'message')?.[1] as @@ -132,7 +155,7 @@ describe('browser-agent CDP instrumentation', () => { const contents = new WebContentsView().webContents const onDialog = vi.fn() const dialogResponse = vi.fn(() => ({ accept: true })) - await ensureInstrumented(contents, { onDialog, dialogResponse }) + await ensureInstrumented(contents, { onDialog, dialogResponse, ...shellAnswersDialogs }) const listener = vi .mocked(contents.debugger.on) .mock.calls.find(([event]) => event === 'message')?.[1] as @@ -275,7 +298,11 @@ describe('browser-agent CDP instrumentation', () => { async (treeKind) => { const contents = new WebContentsView().webContents const { child, frameTree } = createOopifFrameFixture() - await ensureInstrumented(contents, { onDialog: vi.fn(), dialogResponse: () => null }) + await ensureInstrumented(contents, { + onDialog: vi.fn(), + dialogResponse: () => null, + ...shellAnswersDialogs, + }) const listener = vi .mocked(contents.debugger.on) .mock.calls.find(([event]) => event === 'message')?.[1] as @@ -378,7 +405,11 @@ describe('browser-agent CDP instrumentation', () => { it('falls back to the root target when OOPIF isolated-world creation fails', async () => { const contents = new WebContentsView().webContents const { child, frameTree } = createOopifFrameFixture() - await ensureInstrumented(contents, { onDialog: vi.fn(), dialogResponse: () => null }) + await ensureInstrumented(contents, { + onDialog: vi.fn(), + dialogResponse: () => null, + ...shellAnswersDialogs, + }) const listener = vi .mocked(contents.debugger.on) .mock.calls.find(([event]) => event === 'message')?.[1] as @@ -458,7 +489,11 @@ describe('browser-agent file input handles', () => { async function fileInputFixture(childSession = false) { const contents = new WebContentsView().webContents const { child, frameTree } = createOopifFrameFixture() - await ensureInstrumented(contents, { onDialog: vi.fn(), dialogResponse: () => null }) + await ensureInstrumented(contents, { + onDialog: vi.fn(), + dialogResponse: () => null, + ...shellAnswersDialogs, + }) if (childSession) { const onMessage = vi.mocked(contents.debugger.on).mock.calls[0]?.[1] as | ((event: unknown, method: string, params: unknown, sessionId?: string) => void) diff --git a/apps/desktop/src/main/browser-agent/cdp.ts b/apps/desktop/src/main/browser-agent/cdp.ts index 6bf9f9fbc2d..e0a6ad6630b 100644 --- a/apps/desktop/src/main/browser-agent/cdp.ts +++ b/apps/desktop/src/main/browser-agent/cdp.ts @@ -40,6 +40,20 @@ export interface CdpCallbacks { onDialog: (dialog: PageDialog) => void /** The running action's requested answer; dialogs are dismissed when it has none. */ dialogResponse: () => DialogResponse | null + /** + * Offers an alert or confirm to the user, who answers later through + * `respond`. Returns false when the shell must answer it now instead. + */ + offerToUser: ( + kind: 'alert' | 'confirm', + message: string, + frameUrl: string, + respond: (accept: boolean) => void + ) => boolean + /** The page closed its dialog itself, by navigating away or crashing. */ + onDialogClosed: () => void + /** True when the user, not the shell, decides this beforeunload. */ + claimUserLeave: () => boolean } /** Per-tab callbacks, so a background tab's events reach ITS driver, not the @@ -189,37 +203,65 @@ function handleDebuggerEvent( return } const callbacks = callbacksByContents.get(contents) + if (method === 'Page.javascriptDialogClosed') { + callbacks?.onDialogClosed() + return + } if (method === 'Page.javascriptDialogOpening') { const type = String(params.type ?? 'dialog') const message = String(params.message ?? '').slice(0, 500) - // Dialogs never stay open: beforeunload is accepted (navigation proceeds), - // and alert/confirm follow the running action's requested answer, defaulting - // to dismissal so an unexpected dialog can never block the page. - const accept = type === 'beforeunload' || callbacks?.dialogResponse()?.accept === true - const answer = { accept } - void (async () => { - let handled = false - try { - await send(contents, 'Page.handleJavaScriptDialog', answer, parentSessionId) - handled = true - } catch { - // Some Chromium builds surface an OOPIF's tab-modal dialog on its - // flattened session but accept the answer only on the root target. - if (parentSessionId) { - try { - await send(contents, 'Page.handleJavaScriptDialog', answer) - handled = true - } catch {} - } - } + const requested = callbacks?.dialogResponse() ?? null + // The user answers a dialog on the page they are using, unless an agent + // action asked for a specific answer. Electron decides beforeunload itself + // (will-prevent-unload), so that kind is only acknowledged here. + if (!requested && (type === 'alert' || type === 'confirm')) { + const frameUrl = typeof params.url === 'string' ? params.url : '' + const offered = callbacks?.offerToUser(type, message, frameUrl, (accept) => { + void answerDialog(contents, { accept }, parentSessionId).then((handled) => { + if (!handled) logger.warn('Could not answer page dialog for the user', { type }) + }) + }) + if (offered) return + } + if (type === 'beforeunload' && callbacks?.claimUserLeave()) { + // The user's Leave replays the navigation; this unload stays cancelled. + void answerDialog(contents, { accept: false }, parentSessionId) + return + } + // Otherwise dialogs never stay open: beforeunload is accepted, and alert/confirm + // follow the running action's requested answer, defaulting to dismissal so an + // unexpected dialog can never block the page. + const accept = type === 'beforeunload' || requested?.accept === true + void answerDialog(contents, { accept }, parentSessionId).then((handled) => { if (handled) logger.info('Handled page dialog', { type, accept }) else logger.warn('Could not handle page dialog', { type }) callbacks?.onDialog({ type, message, handled, accepted: handled && accept }) - })() + }) return } } +async function answerDialog( + contents: WebContents, + answer: { accept: boolean }, + parentSessionId: string | undefined +): Promise { + try { + await send(contents, 'Page.handleJavaScriptDialog', answer, parentSessionId) + return true + } catch { + // Some Chromium builds surface an OOPIF's tab-modal dialog on its + // flattened session but accept the answer only on the root target. + if (!parentSessionId) return false + try { + await send(contents, 'Page.handleJavaScriptDialog', answer) + return true + } catch { + return false + } + } +} + interface ProtocolFrame { id: string parentId?: string diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index ba968d28051..dbb2676c1ff 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -553,6 +553,7 @@ function recordNotice(notice: string): void { function pageStateFor(contents: WebContents, tabId: string): BrowserPageState { const issue = session.pageIssueForContents(contents) const mediaPermissionRequest = session.mediaPermissionRequestForContents(contents) + const dialog = session.pageDialogForContents(contents) return { scopeId: session.getBrowserScopeId(), tabId, @@ -563,6 +564,7 @@ function pageStateFor(contents: WebContents, tabId: string): BrowserPageState { canGoForward: session.canGoForward(contents), ...(issue ? { issue } : {}), ...(mediaPermissionRequest ? { mediaPermissionRequest } : {}), + ...(dialog ? { dialog } : {}), } } @@ -614,6 +616,12 @@ function instrumentTab(contents: WebContents): void { const requested = driverScopeState().dialogResponse return requested?.contents === contents ? requested.response : null }), + offerToUser: (kind, message, frameUrl, respond) => + session.withBrowserScope(scopeId, () => + session.offerPageDialogToUser(contents, { kind, message, frameUrl }, respond) + ), + onDialogClosed: inScope(() => session.notePageDialogClosed(contents)), + claimUserLeave: () => session.withBrowserScope(scopeId, () => session.claimUserLeave(contents)), } void (async () => { let lastError: unknown @@ -5382,6 +5390,16 @@ export async function handlePanelAction( } return } + if (action.action === 'enable-page-dialogs') { + session.enablePageDialogs() + return + } + if (action.action === 'respond-dialog') { + if (typeof action.requestId === 'string' && typeof action.allowed === 'boolean') { + session.respondToPageDialog(action.requestId, action.allowed) + } + return + } if (action.action === 'respond-site-permission') { /** Older renderers can still send a response to the retired task-navigation prompt. */ return @@ -5397,8 +5415,11 @@ export async function handlePanelAction( action.url, { agentOwned: false } ) - session.prepareExplicitNavigation(contents) - void contents.loadURL(action.url).catch(() => {}) + const url = action.url + session.navigateForUser(contents, () => { + session.prepareExplicitNavigation(contents) + void contents.loadURL(url).catch(() => {}) + }) session.focusPageForUser(contents) } return @@ -5421,13 +5442,13 @@ export async function handlePanelAction( const contents = tab.view.webContents switch (action.action) { case 'reload': - session.reloadPage(contents) + session.navigateForUser(contents, () => session.reloadPage(contents)) return case 'back': - session.goBack(contents) + session.navigateForUser(contents, () => session.goBack(contents)) return case 'forward': - session.goForward(contents) + session.navigateForUser(contents, () => session.goForward(contents)) return case 'print': contents.print({ printBackground: true }) diff --git a/apps/desktop/src/main/browser-agent/session.ts b/apps/desktop/src/main/browser-agent/session.ts index abbefea8537..ef8079ad003 100644 --- a/apps/desktop/src/main/browser-agent/session.ts +++ b/apps/desktop/src/main/browser-agent/session.ts @@ -9,6 +9,7 @@ import type { BrowserMediaDevice, BrowserMediaPermissionRequest, BrowserOmniboxFocusMode, + BrowserPageDialog, BrowserPageIssue, BrowserTabState, BrowserTabsState, @@ -121,6 +122,12 @@ export interface AgentTab { openerTabId?: string /** A user action asked for this page to take focus once it is on screen. */ pendingUserFocus?: boolean + /** A dialog this page opened that waits on the user, and how to answer it. */ + pageDialog?: { request: BrowserPageDialog; respond: (accept: boolean) => void } + /** Replays the user's browser-chrome navigation if the page asks before unloading. */ + pendingLeave?: () => unknown + /** The user chose Leave; the replayed navigation must not ask again. */ + allowNextUnload?: boolean } interface PendingMediaPermission { @@ -287,6 +294,8 @@ interface BrowserScopeState { * is the only evidence the Browser is the shortcut target while they show. */ browserChromeFocused: boolean + /** The renderer shows page dialogs, so the user can be asked instead of the shell answering. */ + pageDialogsEnabled: boolean automationActive: boolean automationNeedsAttention: boolean /** @@ -315,6 +324,7 @@ function createBrowserScopeState(): BrowserScopeState { focusedBrowserTabId: null, focusedBrowserClearTimer: null, browserChromeFocused: false, + pageDialogsEnabled: false, automationActive: false, automationNeedsAttention: false, findingTabId: null, @@ -1864,6 +1874,124 @@ function applyPendingUserFocus(view: WebContentsView): void { view.webContents.focus() } +/** Lets this scope's renderer show page dialogs from now on. */ +export function enablePageDialogs(): void { + currentScope.pageDialogsEnabled = true +} + +/** + * Whether a dialog on this tab is the user's to answer: the renderer can show + * it, the page is on screen, and it is the user's page rather than the agent's. + * Everything else keeps the shell answering, so the user is never asked about + * work they did not start and a hidden page is never left blocked. + */ +function userOwnsPageDialogs(tab: AgentTab): boolean { + const contents = tab.view.webContents + if (!currentScope.pageDialogsEnabled || contents.isDestroyed()) return false + if (tab.id !== currentScope.activeTabId || !isPanelVisible()) return false + if (getBrowserScopeId() !== getActiveBrowserScopeId()) return false + if (isDispatchingAgentInput(contents)) return false + if (automationTab()?.id !== tab.id) return true + return automationTabClaimedByUser() && !currentScope.automationActive +} + +/** + * Holds a dialog for the user. `frameUrl` is the frame that opened it, so an + * embedded site's dialog is labelled with its own origin, not the page's. + */ +function holdPageDialogForUser( + tab: AgentTab, + { + kind, + message, + frameUrl, + }: { kind: BrowserPageDialog['kind']; message: string; frameUrl: string }, + respond: (accept: boolean) => void +): void { + if (tab.pageDialog) answerPageDialog(tab, false) + let origin = '' + try { + origin = new URL(frameUrl || tab.view.webContents.getURL()).origin + } catch {} + tab.pageDialog = { request: { requestId: generateId(), kind, message, origin }, respond } + events?.onPageStateChanged(tab.view.webContents) +} + +function answerPageDialog(tab: AgentTab, accept: boolean): void { + const dialog = tab.pageDialog + if (!dialog) return + tab.pageDialog = undefined + if (!tab.view.webContents.isDestroyed()) events?.onPageStateChanged(tab.view.webContents) + dialog.respond(accept) +} + +/** + * Hands a page's alert or confirm to the user when it is theirs to answer. + * Returns false when the shell must answer it instead, as it does for agent + * work and for pages the user cannot see. + */ +export function offerPageDialogToUser( + contents: WebContents, + dialog: { kind: 'alert' | 'confirm'; message: string; frameUrl: string }, + respond: (accept: boolean) => void +): boolean { + const tab = tabForContents(contents) + if (!tab || !userOwnsPageDialogs(tab)) return false + holdPageDialogForUser(tab, dialog, respond) + return true +} + +/** The page closed its own dialog, by navigating away or crashing. */ +export function notePageDialogClosed(contents: WebContents): void { + const tab = tabForContents(contents) + if (!tab?.pageDialog || tab.pageDialog.request.kind === 'beforeunload') return + tab.pageDialog = undefined + events?.onPageStateChanged(contents) +} + +/** The user's answer to the exact dialog the renderer showed. */ +export function respondToPageDialog(requestId: string, accept: boolean): void { + const tab = tabs.find((entry) => entry.pageDialog?.request.requestId === requestId) + if (tab) answerPageDialog(tab, accept) +} + +/** The dialog on this page awaiting the user's answer, if any. */ +export function pageDialogForContents(contents: WebContents): BrowserPageDialog | undefined { + return tabForContents(contents)?.pageDialog?.request +} + +/** + * Whether the user decides this unload: true once a prompt holds the user's + * browser-chrome navigation, which is then cancelled until they choose Leave. + */ +export function claimUserLeave(contents: WebContents): boolean { + const tab = tabForContents(contents) + if (!tab || tab.allowNextUnload) return false + if (tab.pageDialog?.request.kind === 'beforeunload') return true + const leave = tab.pendingLeave + tab.pendingLeave = undefined + if (!leave || !userOwnsPageDialogs(tab)) return false + holdPageDialogForUser(tab, { kind: 'beforeunload', message: '', frameUrl: '' }, (accept) => { + if (!accept) return + tab.allowNextUnload = true + leave() + }) + return true +} + +/** + * Runs a navigation the user started from browser chrome. If the page asks + * before unloading, the navigation is held and replayed once the user agrees. + * `navigate` returns false when there was nothing to traverse. + */ +export function navigateForUser(contents: WebContents, navigate: () => unknown): void { + const tab = tabForContents(contents) + if (tab && currentScope.pageDialogsEnabled) tab.pendingLeave = navigate + // Nothing to traverse (Back with no history): no unload will ask, so a later + // page-initiated navigation must not replay this one. + if (navigate() === false && tab) tab.pendingLeave = undefined +} + function focusRendererOmnibox(mode: BrowserOmniboxFocusMode): void { if (getBrowserScopeId() !== getActiveBrowserScopeId()) return const win = panelWindow() @@ -2430,11 +2558,17 @@ function initializeTabView( event.preventDefault() }) - // Pages may hold navigation hostage with beforeunload dialogs nobody can - // see; always let the unload proceed. - contents.on('will-prevent-unload', (event) => { - event.preventDefault() - }) + // A beforeunload is decided twice, by whichever answer lands first: here + // (preventDefault lets the unload proceed) and by the CDP dialog. Both ask + // claimUserLeave, so agent work and page-initiated navigations proceed, and a + // navigation the user started from browser chrome is cancelled and replayed + // only once the user chooses to leave. + contents.on( + 'will-prevent-unload', + bindToBrowserScope(scopeId, (event) => { + if (!claimUserLeave(contents)) event.preventDefault() + }) + ) contents.on( 'render-process-gone', bindToBrowserScope(scopeId, (_event, details) => { @@ -2506,11 +2640,11 @@ function initializeTabView( return } if (shortcut === 'reload') { - reloadPage(contents) + navigateForUser(contents, () => reloadPage(contents)) return } if (shortcut === 'hard-reload') { - hardReloadPage(contents) + navigateForUser(contents, () => hardReloadPage(contents)) return } @@ -2562,6 +2696,16 @@ function initializeTabView( persistBrowserSession() }) ) + contents.on( + 'did-start-navigation', + bindToBrowserScope(scopeId, (details) => { + if (!details.isMainFrame) return + const tab = tabForContents(contents) + if (!tab) return + tab.pendingLeave = undefined + tab.allowNextUnload = false + }) + ) contents.on( 'did-navigate-in-page', bindToBrowserScope(scopeId, (_event, _url, isMainFrame) => { @@ -2630,6 +2774,9 @@ export function hasSession(): boolean { export function setAutomationActive(active: boolean): void { if (currentScope.automationActive === active) return currentScope.automationActive = active + // A dialog left open blocks the page, and the agent must never wait on one. + const automation = automationTab() + if (active && automation?.pageDialog) answerPageDialog(automation, false) applyAutomationTabPolicy() events?.onTabsChanged() } @@ -3325,8 +3472,11 @@ export function switchTab(tabId: string, { claim = true }: { claim?: boolean } = revokeTabMediaPermissions(previousActiveTab, false) previousActiveTab.pendingUserFocus = false } - currentScope.activeTabId = tab.id + // The claim describes the page on screen, so a mirrored switch to another + // page leaves that page unclaimed rather than inheriting the last one's. if (claim) currentScope.visibleTabUserSelected = true + else if (currentScope.activeTabId !== tab.id) currentScope.visibleTabUserSelected = false + currentScope.activeTabId = tab.id promotePendingTabRestore(tab) // Visible selection does not move the automation exemption; the user may // inspect another page while a tool continues in its background tab. @@ -3523,18 +3673,26 @@ export function handleFocusedShortcut( case 'focus-omnibox': focusRendererOmnibox('select') return true - case 'reload-or-clear': - reloadPage(shortcutTab.view.webContents) + case 'reload-or-clear': { + const contents = shortcutTab.view.webContents + navigateForUser(contents, () => reloadPage(contents)) return true - case 'hard-reload': - hardReloadPage(shortcutTab.view.webContents) + } + case 'hard-reload': { + const contents = shortcutTab.view.webContents + navigateForUser(contents, () => hardReloadPage(contents)) return true - case 'back': - goBack(shortcutTab.view.webContents) + } + case 'back': { + const contents = shortcutTab.view.webContents + navigateForUser(contents, () => goBack(contents)) return true - case 'forward': - goForward(shortcutTab.view.webContents) + } + case 'forward': { + const contents = shortcutTab.view.webContents + navigateForUser(contents, () => goForward(contents)) return true + } } const zoomAction = zoomActionForShortcut(shortcut) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-dialog.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-dialog.tsx new file mode 100644 index 00000000000..8a709b9cb08 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-dialog.tsx @@ -0,0 +1,82 @@ +'use client' + +import { useId } from 'react' +import type { BrowserPageDialog } from '@sim/browser-protocol' +import { + ChipConfirmModal, + ChipModal, + ChipModalBody, + ChipModalDescription, + ChipModalFooter, + ChipModalHeader, +} from '@sim/emcn' + +interface BrowserPageDialogModalProps { + dialog: BrowserPageDialog | undefined + open: boolean + onAnswer: (requestId: string, accept: boolean) => void +} + +/** Names the page behind a dialog, so page text can never pass as Sim's own. */ +function dialogSource(origin: string): string { + try { + const { host } = new URL(origin) + if (host) return `${host} says` + } catch {} + return 'This page says' +} + +/** + * The page's alert, confirm, or leave-site question, asked the way a browser + * asks it. Every dismiss path answers the page, which is blocked until then. + */ +export function BrowserPageDialogModal({ dialog, open, onAnswer }: BrowserPageDialogModalProps) { + const messageId = useId() + if (!dialog) return null + const answer = (accept: boolean) => onAnswer(dialog.requestId, accept) + const source = dialogSource(dialog.origin) + + if (dialog.kind === 'alert') { + return ( + !nextOpen && answer(true)} + srTitle={source} + aria-describedby={dialog.message ? messageId : undefined} + > + answer(true)}>{source} + + {dialog.message ? ( + {dialog.message} + ) : null} + + answer(true) }} /> + + ) + } + + if (dialog.kind === 'beforeunload') { + return ( + !nextOpen && answer(false)} + title='Leave site?' + text='Changes you made may not be saved.' + defaultAction='dismiss' + dismissLabel='Stay' + confirm={{ label: 'Leave', onClick: () => answer(true) }} + /> + ) + } + + return ( + !nextOpen && answer(false)} + title={source} + text={dialog.message} + defaultAction='dismiss' + confirm={{ label: 'OK', onClick: () => answer(true), variant: 'primary' }} + /> + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx index fa9fc4b5bd2..36f74125bbc 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-session.tsx @@ -69,6 +69,7 @@ import { addMothershipContext } from '@/lib/mothership/events' import { BrowserDownloads } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-downloads' import { BrowserFindBar } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-find-bar' import { BrowserLoadingBar } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-loading-bar' +import { BrowserPageDialogModal } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-dialog' import { BrowserPageIssueView } from '@/app/workspace/[workspaceId]/home/components/mothership-view/components/resource-content/components/browser-session/browser-page-issue' import { type BrowserPanelOverlay, @@ -523,6 +524,13 @@ export function BrowserSession({ ) }, [permissionRequest, respondToPermission, visible]) + // The shell answers page dialogs itself until this renderer declares it shows them. + useEffect(() => { + sendBrowserPanelAction('enable-page-dialogs', {}, scopeId) + }, [scopeId]) + const answerPageDialog = (requestId: string, accept: boolean) => + sendBrowserPanelAction('respond-dialog', { requestId, allowed: accept }, scopeId) + const permissionModalOpen = shouldShowBrowserPermissionRequest( permissionRequest?.requestId, answeredPermissionRequestId, @@ -1373,6 +1381,11 @@ export function BrowserSession({ open={permissionModalOpen} onDecision={respondToPermission} /> + ) } diff --git a/packages/browser-protocol/src/index.ts b/packages/browser-protocol/src/index.ts index aae31c887e3..4c4c5dd6792 100644 --- a/packages/browser-protocol/src/index.ts +++ b/packages/browser-protocol/src/index.ts @@ -255,6 +255,14 @@ export interface BrowserPanelAction { /** Compatibility response for installed shells with the retired navigation gate. */ | 'respond-site-permission' | 'takeover-done' + /** + * Declares that this renderer shows page dialogs. Until it arrives, the + * shell keeps answering them itself, so an older renderer never leaves a + * page blocked on a dialog nobody can see. + */ + | 'enable-page-dialogs' + /** Answers {@link BrowserPageState.dialog}; `allowed` is OK / Leave. */ + | 'respond-dialog' /** Absolute URL for `navigate` (typed into the panel's URL bar). */ url?: string /** Stable tab id for `switch-tab` and `close-tab`. */ @@ -267,12 +275,26 @@ export interface BrowserPanelAction { claim?: boolean /** Optional free-text instruction submitted with `takeover-done`. */ takeoverResponse?: string - /** Exact pending permission request being answered. */ + /** Exact pending permission request or dialog being answered. */ requestId?: string - /** User decision for a permission response. */ + /** User decision for a permission or dialog response. */ allowed?: boolean } +/** + * A page dialog waiting on the user: `alert` and `confirm` from the page, or + * `beforeunload` when leaving would discard what the page says is unsaved. + * Electron removes `prompt()`, so there is no text-entry kind. + */ +export interface BrowserPageDialog { + requestId: string + kind: 'alert' | 'confirm' | 'beforeunload' + /** The page's own text; empty for `beforeunload`, whose text browsers never show. */ + message: string + /** Origin of the page that opened the dialog, shown so a page cannot pose as Sim. */ + origin: string +} + export type BrowserMediaDevice = 'microphone' | 'camera' /** One document-scoped media request awaiting an explicit user decision. */ @@ -307,6 +329,8 @@ export interface BrowserPageState { mediaPermissionRequest?: BrowserMediaPermissionRequest /** Legacy request from installed shells that still require a site-origin prompt. */ sitePermissionRequest?: BrowserSitePermissionRequest + /** Page dialog awaiting the user's answer. Optional for older shells. */ + dialog?: BrowserPageDialog } /** A recoverable top-level page problem rendered by Sim instead of a blank native view. */