diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index a471258d051..9f3a4daf86a 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -1,4 +1,5 @@ import { BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS } from '@sim/browser-protocol' +import { toRecord } from '@sim/utils/object' import type { MenuItemConstructorOptions, WebContents } from 'electron' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -2097,13 +2098,20 @@ describe('credential protection', () => { for (const [fnName, value] of Object.entries(replies)) { if (isPageCall(expression, fnName)) return Promise.resolve(value) } - if (isPageCall(expression, 'clickElement')) { - return Promise.resolve({ dispatched: false, x: 24, y: 48, element: 'Test' }) - } + if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET) return Promise.resolve(undefined) }) } + function mousePresses(contents: Awaited>): number { + return cdpCalls(contents, 'Input.dispatchMouseEvent').filter( + ([, params]) => toRecord(params).type === 'mousePressed' + ).length + } + + /** What the page reports for an ordinary click target before native dispatch. */ + const CLICK_TARGET = { dispatched: false, x: 24, y: 48, element: 'Test' } + function cdpCalls(contents: Awaited>, method: string): unknown[][] { return vi .mocked(contents.debugger.sendCommand) @@ -3425,6 +3433,217 @@ describe('credential protection', () => { expect(second).not.toMatchObject({ result: { notices: expect.anything() } }) }) + it('runs batched actions in order and returns each result', async () => { + const contents = await openPage() + respondWith(contents, {}) + + const result = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(result).toMatchObject({ + ok: true, + result: { + completed: true, + completedCount: 2, + results: [ + { index: 0, tool: 'browser_click', result: { dispatched: true } }, + { index: 1, tool: 'browser_click', result: { dispatched: true } }, + ], + }, + }) + }) + + it('stops a batch at the first failed action and keeps earlier results', async () => { + const contents = await openPage() + vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => { + if (!isPageCall(expression, 'clickElement')) return Promise.resolve(undefined) + return Promise.resolve( + mousePresses(contents) > 0 ? { error: 'obstructed', blocker: 'IMG' } : CLICK_TARGET + ) + }) + + const result = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(mousePresses(contents)).toBe(1) + expect(result).toMatchObject({ + ok: true, + result: { + completed: false, + completedCount: 1, + stoppedIndex: 1, + stoppedBy: 'failure', + error: expect.stringContaining('covered by IMG'), + }, + }) + }) + + it('stops a batch after an action navigates the page', async () => { + const contents = await openPage() + respondWith(contents, {}) + const sendCommand = vi.mocked(contents.debugger.sendCommand) + const dispatch = sendCommand.getMockImplementation() + sendCommand.mockImplementation((method, params) => { + if (method === 'Input.dispatchMouseEvent' && toRecord(params).type === 'mouseReleased') { + emitContentsEvent(contents, 'did-navigate') + } + return dispatch?.(method, params) ?? Promise.resolve(undefined) + }) + + const result = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(mousePresses(contents)).toBe(1) + expect(result).toMatchObject({ + ok: true, + result: { completed: false, completedCount: 1, stoppedIndex: 1, stoppedBy: 'page-change' }, + }) + }) + + it('stops a batch after an action changes the URL within the document', async () => { + const contents = await openPage() + respondWith(contents, {}) + const url = contents.getURL() + const sendCommand = vi.mocked(contents.debugger.sendCommand) + const dispatch = sendCommand.getMockImplementation() + sendCommand.mockImplementation((method, params) => { + if (method === 'Input.dispatchMouseEvent' && toRecord(params).type === 'mouseReleased') { + vi.mocked(contents.getURL).mockReturnValue(`${url}#next`) + emitContentsEvent(contents, 'did-navigate-in-page') + } + return dispatch?.(method, params) ?? Promise.resolve(undefined) + }) + + const result = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(mousePresses(contents)).toBe(1) + expect(result).toMatchObject({ + ok: true, + result: { completed: false, completedCount: 1, stoppedIndex: 1, stoppedBy: 'page-change' }, + }) + }) + + it('reports a batch cancelled during its first action as an unknown outcome', async () => { + const contents = await openPage() + respondWith(contents, {}) + const sendCommand = vi.mocked(contents.debugger.sendCommand) + const dispatch = sendCommand.getMockImplementation() + sendCommand.mockImplementation((method, params) => + method === 'Input.dispatchKeyEvent' + ? new Promise(() => {}) + : (dispatch?.(method, params) ?? Promise.resolve(undefined)) + ) + + const pending = driver.executeTool( + 'chat-test', + 'browser_batch', + { + actions: [ + { tool: 'browser_press_key', args: { key: 'Enter' } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }, + 'batch-first-call' + ) + await vi.waitFor(() => expect(cdpCalls(contents, 'Input.dispatchKeyEvent')).toHaveLength(1)) + driver.cancelTool('chat-test', 'batch-first-call') + + await expect(pending).resolves.toMatchObject({ + ok: true, + result: { outcomeUnknown: true, doNotRetry: true }, + }) + }) + + it('reports a batch cancelled after an action ran as an unknown outcome', async () => { + const contents = await openPage() + respondWith(contents, {}) + const sendCommand = vi.mocked(contents.debugger.sendCommand) + const dispatch = sendCommand.getMockImplementation() + sendCommand.mockImplementation((method, params) => + method === 'Input.dispatchKeyEvent' + ? new Promise(() => {}) + : (dispatch?.(method, params) ?? Promise.resolve(undefined)) + ) + + const pending = driver.executeTool( + 'chat-test', + 'browser_batch', + { + actions: [ + { tool: 'browser_click', args: { elementId: 0 } }, + { tool: 'browser_press_key', args: { key: 'Enter' } }, + ], + }, + 'batch-call' + ) + await vi.waitFor(() => expect(cdpCalls(contents, 'Input.dispatchKeyEvent')).toHaveLength(1)) + driver.cancelTool('chat-test', 'batch-call') + + await expect(pending).resolves.toMatchObject({ + ok: true, + result: { outcomeUnknown: true, doNotRetry: true }, + }) + }) + + it('rejects batches that name non-action tools or observe per action', async () => { + await openPage() + + const navigation = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_navigate', args: { url: 'https://example.com' } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + const observed = await driver.executeTool('chat-test', 'browser_batch', { + actions: [ + { tool: 'browser_click', args: { elementId: 0, observe: {} } }, + { tool: 'browser_click', args: { elementId: 0 } }, + ], + }) + + expect(navigation).toMatchObject({ + ok: false, + error: expect.stringContaining('Batch action 0'), + }) + expect(observed).toMatchObject({ ok: false, error: expect.stringContaining('cannot observe') }) + }) + + it('keeps element ids valid when an observed action is refused before dispatch', async () => { + const contents = await openPage() + respondWith(contents, { clickElement: { error: 'obstructed', blocker: 'IMG' } }) + + const refused = await driver.executeTool('chat-test', 'browser_click', { + elementId: 0, + observe: {}, + }) + respondWith(contents, {}) + const retried = await driver.executeTool('chat-test', 'browser_click', { elementId: 0 }) + + expect(refused).toEqual({ + ok: false, + error: expect.stringContaining('That element is covered by IMG'), + }) + expect(retried).toMatchObject({ ok: true, result: { dispatched: true } }) + }) + it('invalidates element ids when the active tab changes', async () => { await openPage() await driver.executeTool('chat-test', 'browser_open_tab', {}) diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 9a049a8fcb0..2ba47c613f0 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -144,6 +144,53 @@ type FormField = | { elementId: number; kind: 'select'; value: string } | { elementId: number; kind: 'checked'; checked: boolean } +const MAX_BATCH_ACTIONS = 8 +/** Single-page interactions a batch may run; navigation, observation, and file tools stay separate. */ +const BATCH_ACTION_TOOLS: ReadonlySet = new Set([ + 'browser_click', + 'browser_click_at', + 'browser_type', + 'browser_insert_text', + 'browser_press_key', + 'browser_scroll', + 'browser_select_option', + 'browser_set_checked', + 'browser_hover', +]) + +interface BatchAction { + tool: BrowserToolName + args: Record +} + +function isBatchActionTool(value: unknown): value is BrowserToolName { + return BATCH_ACTION_TOOLS.has(value as BrowserToolName) +} + +function parseBatchActions(params: Record): BatchAction[] { + if (Object.keys(params).some((key) => key !== 'actions')) { + throw new ToolError('A batch accepts only actions; pass observe on the batch itself.') + } + if ( + !Array.isArray(params.actions) || + params.actions.length < 2 || + params.actions.length > MAX_BATCH_ACTIONS + ) { + throw new ToolError(`A batch requires between 2 and ${MAX_BATCH_ACTIONS} actions.`) + } + return params.actions.map((action, index): BatchAction => { + if (!isRecordLike(action) || !isBatchActionTool(action.tool) || !isRecordLike(action.args)) { + throw new ToolError( + `Batch action ${index} must be {tool, args} with tool one of ${[...BATCH_ACTION_TOOLS].join(', ')}.` + ) + } + if ('observe' in action.args) { + throw new ToolError(`Batch action ${index} cannot observe; pass observe on the batch itself.`) + } + return { tool: action.tool, args: action.args } + }) +} + function parseFormFields(params: Record): FormField[] { if (Object.keys(params).some((key) => key !== 'fields')) { throw new ToolError('Form filling accepts only fields; submitting is not supported.') @@ -313,6 +360,14 @@ function createDriverScopeState(): DriverScopeState { function invalidateSnapshot(state = driverScopeState()): void { state.snapshotTabId = null state.snapshotTargets.clear() + cancelPendingSnapshotCapture(state) +} + +/** + * Keeps the current refs but stops any in-flight capture from committing its own. A capture + * clears the refs when it starts, so this is all a failed action with a pending observation needs. + */ +function cancelPendingSnapshotCapture(state: DriverScopeState): void { state.snapshotCaptureEpoch++ } @@ -1038,7 +1093,8 @@ export function browserToolWatchdogMs( tool === 'browser_open_tab' || tool === 'browser_switch_tab' || tool === 'browser_upload_file' || - tool === 'browser_save_download' + tool === 'browser_save_download' || + tool === 'browser_batch' ) { return BROWSER_NAVIGATION_NATIVE_WATCHDOG_MS } @@ -3381,6 +3437,68 @@ async function executeToolInner( } } + case 'browser_batch': { + const actions = parseBatchActions(params) + const results: { index: number; tool: BrowserToolName; result: unknown }[] = [] + const stopped = ( + stoppedIndex: number, + stoppedBy: 'failure' | 'page-change', + error: string + ) => ({ + completed: false, + completedCount: results.length, + stoppedIndex, + stoppedBy, + error, + results, + }) + for (const [index, action] of actions.entries()) { + let tab: ReturnType + let epoch: number + let url: string + let snapshotValid: boolean + let result: unknown + // An action may dispatch input before it returns, so a cancelled or timed-out batch must + // never read as not started once any action has begun. + onActionOutcome?.({ status: 'pending' }) + try { + tab = session.requireAutomationTab() + epoch = navigationEpoch(tab.view.webContents) + url = tab.view.webContents.getURL() + snapshotValid = driverScopeState().snapshotTabId === tab.id + result = await executeToolInner( + action.tool, + action.args, + assertCurrentExecution, + executionDeadline, + invocationEpoch, + signal + ) + } catch (error) { + return stopped( + index, + 'failure', + `Action ${index} (${action.tool}) failed: ${getErrorMessage(error)} Earlier actions already took effect.` + ) + } + results.push({ index, tool: action.tool, result }) + if (index === actions.length - 1) break + if ( + session.automationTab()?.id !== tab.id || + navigationEpoch(tab.view.webContents) !== epoch || + tab.view.webContents.getURL() !== url || + (snapshotValid && driverScopeState().snapshotTabId !== tab.id) + ) { + return stopped( + index + 1, + 'page-change', + `Action ${index} (${action.tool}) changed the page, so the remaining actions did not run. Inspect the page before continuing.` + ) + } + } + return { completed: true, completedCount: results.length, results } + } + case 'browser_fill_form': { const fields = parseFormFields(params) const contents = session.requireAutomationTab().view.webContents @@ -5002,7 +5120,10 @@ export async function executeTool( try { observedResult = await guardedExecution } catch (error) { - if (!actionOutcome) throw error + if (!actionOutcome) { + if (params.observe !== undefined) cancelPendingSnapshotCapture(state) + throw error + } invalidateSnapshot(state) observedResult = actionOutcome.status === 'pending' @@ -5049,12 +5170,7 @@ export async function executeTool( // The watchdog cannot cancel an in-flight renderer promise. Invalidate its // capture token before releasing the queue so a late snapshot cannot // overwrite refs belonging to a newer tab or snapshot. - if ( - tool === 'browser_snapshot' || - tool === 'browser_open_url' || - tool === 'browser_find' || - params.observe !== undefined - ) { + if (tool === 'browser_snapshot' || tool === 'browser_open_url' || tool === 'browser_find') { invalidateSnapshot(state) } const message = String(sanitizeBrowserResult(getErrorMessage(error), undefined, 0, 'error')) diff --git a/apps/desktop/src/main/browser-agent/post-action-observation.ts b/apps/desktop/src/main/browser-agent/post-action-observation.ts index 91e8aa4b616..b00b94f7b2b 100644 --- a/apps/desktop/src/main/browser-agent/post-action-observation.ts +++ b/apps/desktop/src/main/browser-agent/post-action-observation.ts @@ -10,6 +10,7 @@ const OBSERVABLE_ACTIONS: ReadonlySet = new Set([ 'browser_fill_form', 'browser_scroll', 'browser_hover', + 'browser_batch', ]) /** Preserves a dispatched action when its acknowledgement or observation is interrupted. */ diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts index e5da95feb55..5357f638a08 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts @@ -39,6 +39,7 @@ export const TOOL_ICONS: Readonly> = { apply_file_edit: File, auth: Integration, browser: Globe, + browser_batch: Cursor, browser_click: Cursor, browser_click_at: Cursor, browser_close_tab: Cursor, diff --git a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts index 01ddc25be53..6b3546050e0 100644 --- a/apps/sim/lib/mothership/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-catalog-v1.ts @@ -15,6 +15,7 @@ export interface ToolCatalogEntry { | 'apply_file_edit' | 'auth' | 'browser' + | 'browser_batch' | 'browser_click' | 'browser_click_at' | 'browser_close_tab' @@ -159,6 +160,7 @@ export interface ToolCatalogEntry { | 'apply_file_edit' | 'auth' | 'browser' + | 'browser_batch' | 'browser_click' | 'browser_click_at' | 'browser_close_tab' @@ -393,6 +395,107 @@ export const Browser: ToolCatalogEntry = { internal: true, } +export const BrowserBatch: ToolCatalogEntry = { + id: 'browser_batch', + name: 'browser_batch', + route: 'client', + mode: 'async', + parameters: { + additionalProperties: false, + properties: { + actions: { + description: + "Ordered list of 2–8 actions. Each names one action tool and gives exactly that tool's parameters in args, without observe.", + items: { + additionalProperties: false, + properties: { + args: { + description: + 'That tool\'s own parameters, for example {"elementId": 12} for browser_click or {"key": "Enter"} for browser_press_key.', + type: 'object', + }, + tool: { + description: 'The action tool to run.', + enum: [ + 'browser_click', + 'browser_click_at', + 'browser_type', + 'browser_insert_text', + 'browser_press_key', + 'browser_scroll', + 'browser_select_option', + 'browser_set_checked', + 'browser_hover', + ], + type: 'string', + }, + }, + required: ['tool', 'args'], + type: 'object', + }, + maxItems: 8, + minItems: 2, + type: 'array', + }, + observe: { + type: 'object', + description: + 'Observe immediately after this action in the same call. Use {} for a fresh snapshot or {query: string} for matching element refs only. Returned refs replace prior refs. A failed observation does not mean the action failed; inspect its result before retrying.', + properties: { + query: { + type: 'string', + description: + 'Case-insensitive text to find in the resulting page. Omit for the full snapshot. Maximum 4096 characters.', + }, + }, + }, + }, + required: ['actions'], + type: 'object', + }, + resultSchema: { + type: 'object', + properties: { + completed: { type: 'boolean', description: 'True when every action ran and succeeded.' }, + completedCount: { + type: 'number', + description: 'Number of actions that ran and succeeded, from the start of the list.', + }, + error: { + type: 'string', + description: + 'Why the batch stopped early. Earlier actions already took effect; do not repeat them.', + }, + results: { + type: 'array', + description: 'Results of the actions that ran, in order.', + items: { + type: 'object', + properties: { + index: { type: 'number' }, + result: { type: 'object', description: "The action tool's own result." }, + tool: { type: 'string' }, + }, + required: ['index', 'tool', 'result'], + }, + }, + stoppedBy: { + type: 'string', + description: + 'failure when an action failed; page-change when an action changed the page and the rest were skipped.', + enum: ['failure', 'page-change'], + }, + stoppedIndex: { + type: 'number', + description: + 'Zero-based index of the first action that did not run or failed, when the batch stopped early.', + }, + }, + required: ['completed', 'completedCount', 'results'], + }, + clientExecutable: true, +} + export const BrowserClick: ToolCatalogEntry = { id: 'browser_click', name: 'browser_click', @@ -8132,6 +8235,7 @@ export const TOOL_CATALOG: Record = { [ApplyFileEdit.id]: ApplyFileEdit, [Auth.id]: Auth, [Browser.id]: Browser, + [BrowserBatch.id]: BrowserBatch, [BrowserClick.id]: BrowserClick, [BrowserClickAt.id]: BrowserClickAt, [BrowserCloseTab.id]: BrowserCloseTab, diff --git a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts index 54ac6909ade..c17380c3a42 100644 --- a/apps/sim/lib/mothership/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/mothership/generated/tool-schemas-v1.ts @@ -81,6 +81,111 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, + browser_batch: { + parameters: { + additionalProperties: false, + properties: { + actions: { + description: + "Ordered list of 2–8 actions. Each names one action tool and gives exactly that tool's parameters in args, without observe.", + items: { + additionalProperties: false, + properties: { + args: { + description: + 'That tool\'s own parameters, for example {"elementId": 12} for browser_click or {"key": "Enter"} for browser_press_key.', + type: 'object', + }, + tool: { + description: 'The action tool to run.', + enum: [ + 'browser_click', + 'browser_click_at', + 'browser_type', + 'browser_insert_text', + 'browser_press_key', + 'browser_scroll', + 'browser_select_option', + 'browser_set_checked', + 'browser_hover', + ], + type: 'string', + }, + }, + required: ['tool', 'args'], + type: 'object', + }, + maxItems: 8, + minItems: 2, + type: 'array', + }, + observe: { + type: 'object', + description: + 'Observe immediately after this action in the same call. Use {} for a fresh snapshot or {query: string} for matching element refs only. Returned refs replace prior refs. A failed observation does not mean the action failed; inspect its result before retrying.', + properties: { + query: { + type: 'string', + description: + 'Case-insensitive text to find in the resulting page. Omit for the full snapshot. Maximum 4096 characters.', + }, + }, + }, + }, + required: ['actions'], + type: 'object', + }, + resultSchema: { + type: 'object', + properties: { + completed: { + type: 'boolean', + description: 'True when every action ran and succeeded.', + }, + completedCount: { + type: 'number', + description: 'Number of actions that ran and succeeded, from the start of the list.', + }, + error: { + type: 'string', + description: + 'Why the batch stopped early. Earlier actions already took effect; do not repeat them.', + }, + results: { + type: 'array', + description: 'Results of the actions that ran, in order.', + items: { + type: 'object', + properties: { + index: { + type: 'number', + }, + result: { + type: 'object', + description: "The action tool's own result.", + }, + tool: { + type: 'string', + }, + }, + required: ['index', 'tool', 'result'], + }, + }, + stoppedBy: { + type: 'string', + description: + 'failure when an action failed; page-change when an action changed the page and the rest were skipped.', + enum: ['failure', 'page-change'], + }, + stoppedIndex: { + type: 'number', + description: + 'Zero-based index of the first action that did not run or failed, when the batch stopped early.', + }, + }, + required: ['completed', 'completedCount', 'results'], + }, + }, browser_click: { parameters: { type: 'object', diff --git a/apps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts b/apps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts index 1e8d4eab57c..f9dfa10fa2e 100644 --- a/apps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts +++ b/apps/sim/lib/mothership/tools/client/browser-tool-execution.test.ts @@ -110,6 +110,41 @@ describe('executeBrowserToolOnClient', () => { expect(mockExecuteBrowserTool).toHaveBeenCalledTimes(1) }) + it('reports a batch as failed only when one of its actions failed', async () => { + const actions = [ + { tool: 'browser_click', args: { elementId: 1 } }, + { tool: 'browser_click', args: { elementId: 2 } }, + ] + const partial = { + completed: false, + completedCount: 1, + stoppedIndex: 1, + results: [{ index: 0, tool: 'browser_click', result: { dispatched: true } }], + } + const failed = { ...partial, stoppedBy: 'failure', error: 'Action 1 failed' } + const pageChanged = { ...partial, stoppedBy: 'page-change', error: 'Action 0 changed the page' } + const failedCallId = nextToolCallId() + const changedCallId = nextToolCallId() + + mockExecuteBrowserTool.mockResolvedValueOnce(failed).mockResolvedValueOnce(pageChanged) + executeBrowserToolOnClient(failedCallId, 'browser_batch', { actions }, CHAT_SCOPE) + executeBrowserToolOnClient(changedCallId, 'browser_batch', { actions }, CHAT_SCOPE) + await flush() + + expect(mockReportCompletion).toHaveBeenCalledWith( + failedCallId, + 'error', + 'A batched browser action failed; inspect the partial result', + failed + ) + expect(mockReportCompletion).toHaveBeenCalledWith( + changedCallId, + 'success', + 'Browser action completed', + pageChanged + ) + }) + it('preserves every executed completion when a guard result arrives at retention capacity', async () => { const replayClaim = vi .spyOn(BrowserToolReplayLedger.prototype, 'claim') diff --git a/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts b/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts index 23b9587ca58..eb65edd1e11 100644 --- a/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts +++ b/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts @@ -78,6 +78,7 @@ const OBSERVATION_ONLY_BROWSER_TOOLS = { browser_click_at: false, browser_type: false, browser_fill_form: false, + browser_batch: false, browser_insert_text: false, browser_press_key: false, browser_scroll: false, @@ -937,21 +938,25 @@ async function doExecuteBrowserTool( if (cancelled) return const outcomeUnknown = isRecordLike(result) && result.outcomeUnknown === true const effectUnconfirmed = isRecordLike(result) && result.effectObserved === false - const formStopped = + const stoppedMessage = toolName === 'browser_fill_form' && isRecordLike(result) && result.completed === false + ? 'Form filling stopped; inspect the partial result' + : toolName === 'browser_batch' && isRecordLike(result) && result.stoppedBy === 'failure' + ? 'A batched browser action failed; inspect the partial result' + : undefined reportTerminalCompletion( { status: - outcomeUnknown || formStopped + stoppedMessage || outcomeUnknown ? ASYNC_TOOL_CONFIRMATION_STATUS.error : ASYNC_TOOL_CONFIRMATION_STATUS.success, - message: formStopped - ? 'Form filling stopped; inspect the partial result' - : outcomeUnknown + message: + stoppedMessage ?? + (outcomeUnknown ? 'Browser action outcome is unconfirmed; inspect the page before repeating it.' : effectUnconfirmed ? 'Browser input completed; its effect is unconfirmed. Inspect the current state before retrying.' - : 'Browser action completed', + : 'Browser action completed'), data: sanitizeBrowserToolResultForModel(toolName, result), }, 'Failed to report browser tool completion' diff --git a/apps/sim/lib/mothership/tools/tool-display.ts b/apps/sim/lib/mothership/tools/tool-display.ts index b5120c3a177..a390b8873d7 100644 --- a/apps/sim/lib/mothership/tools/tool-display.ts +++ b/apps/sim/lib/mothership/tools/tool-display.ts @@ -676,6 +676,7 @@ const TOOL_TITLES: Record = { browser_drag: 'Dragging element', browser_select_option: 'Selecting option', browser_fill_form: 'Filling form', + browser_batch: 'Running page actions', browser_set_checked: 'Updating control', browser_hover: 'Hovering element', browser_zoom: 'Changing page zoom', diff --git a/packages/browser-protocol/src/index.ts b/packages/browser-protocol/src/index.ts index 926b8b7bc66..aae31c887e3 100644 --- a/packages/browser-protocol/src/index.ts +++ b/packages/browser-protocol/src/index.ts @@ -43,6 +43,7 @@ export const CURRENT_BROWSER_TOOL_NAMES = [ 'browser_click_at', 'browser_type', 'browser_fill_form', + 'browser_batch', 'browser_insert_text', 'browser_press_key', 'browser_scroll', @@ -118,6 +119,7 @@ export function browserToolRendererTimeoutMs( case 'browser_switch_tab': case 'browser_upload_file': case 'browser_save_download': + case 'browser_batch': return BROWSER_NAVIGATION_RENDERER_TIMEOUT_MS case 'browser_wait_for': return (