Skip to content

Commit 4548dfa

Browse files
waleedlatif1waleedlatif1
committed
feat(browser): run planned page actions in order with browser_batch
The desktop runs 2-8 single-page interactions from one authorized call, in order, stopping at the first failed action or after an action navigates, switches tabs, or invalidates the snapshot's element ids. A batch stopped by a failure reports an error status; one stopped by a page change does not. Once an action has run, a cancelled or timed-out batch reports an unknown outcome so the agent does not repeat it.
1 parent b6df969 commit 4548dfa

10 files changed

Lines changed: 492 additions & 10 deletions

File tree

‎apps/desktop/src/main/browser-agent/driver.test.ts‎

Lines changed: 145 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS } from '@sim/browser-protocol'
2+
import { toRecord } from '@sim/utils/object'
23
import type { MenuItemConstructorOptions, WebContents } from 'electron'
34
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
45

@@ -2097,13 +2098,20 @@ describe('credential protection', () => {
20972098
for (const [fnName, value] of Object.entries(replies)) {
20982099
if (isPageCall(expression, fnName)) return Promise.resolve(value)
20992100
}
2100-
if (isPageCall(expression, 'clickElement')) {
2101-
return Promise.resolve({ dispatched: false, x: 24, y: 48, element: 'Test' })
2102-
}
2101+
if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET)
21032102
return Promise.resolve(undefined)
21042103
})
21052104
}
21062105

2106+
function mousePresses(contents: Awaited<ReturnType<typeof openPage>>): number {
2107+
return cdpCalls(contents, 'Input.dispatchMouseEvent').filter(
2108+
([, params]) => toRecord(params).type === 'mousePressed'
2109+
).length
2110+
}
2111+
2112+
/** What the page reports for an ordinary click target before native dispatch. */
2113+
const CLICK_TARGET = { dispatched: false, x: 24, y: 48, element: 'Test' }
2114+
21072115
function cdpCalls(contents: Awaited<ReturnType<typeof openPage>>, method: string): unknown[][] {
21082116
return vi
21092117
.mocked(contents.debugger.sendCommand)
@@ -3425,6 +3433,140 @@ describe('credential protection', () => {
34253433
expect(second).not.toMatchObject({ result: { notices: expect.anything() } })
34263434
})
34273435

3436+
it('runs batched actions in order and returns each result', async () => {
3437+
const contents = await openPage()
3438+
respondWith(contents, {})
3439+
3440+
const result = await driver.executeTool('chat-test', 'browser_batch', {
3441+
actions: [
3442+
{ tool: 'browser_click', args: { elementId: 0 } },
3443+
{ tool: 'browser_click', args: { elementId: 0 } },
3444+
],
3445+
})
3446+
3447+
expect(result).toMatchObject({
3448+
ok: true,
3449+
result: {
3450+
completed: true,
3451+
completedCount: 2,
3452+
results: [
3453+
{ index: 0, tool: 'browser_click', result: { dispatched: true } },
3454+
{ index: 1, tool: 'browser_click', result: { dispatched: true } },
3455+
],
3456+
},
3457+
})
3458+
})
3459+
3460+
it('stops a batch at the first failed action and keeps earlier results', async () => {
3461+
const contents = await openPage()
3462+
vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
3463+
if (!isPageCall(expression, 'clickElement')) return Promise.resolve(undefined)
3464+
return Promise.resolve(
3465+
mousePresses(contents) > 0 ? { error: 'obstructed', blocker: 'IMG' } : CLICK_TARGET
3466+
)
3467+
})
3468+
3469+
const result = await driver.executeTool('chat-test', 'browser_batch', {
3470+
actions: [
3471+
{ tool: 'browser_click', args: { elementId: 0 } },
3472+
{ tool: 'browser_click', args: { elementId: 0 } },
3473+
{ tool: 'browser_click', args: { elementId: 0 } },
3474+
],
3475+
})
3476+
3477+
expect(mousePresses(contents)).toBe(1)
3478+
expect(result).toMatchObject({
3479+
ok: true,
3480+
result: {
3481+
completed: false,
3482+
completedCount: 1,
3483+
stoppedIndex: 1,
3484+
stoppedBy: 'failure',
3485+
error: expect.stringContaining('covered by IMG'),
3486+
},
3487+
})
3488+
})
3489+
3490+
it('stops a batch after an action navigates the page', async () => {
3491+
const contents = await openPage()
3492+
respondWith(contents, {})
3493+
const sendCommand = vi.mocked(contents.debugger.sendCommand)
3494+
const dispatch = sendCommand.getMockImplementation()
3495+
sendCommand.mockImplementation((method, params) => {
3496+
if (method === 'Input.dispatchMouseEvent' && toRecord(params).type === 'mouseReleased') {
3497+
emitContentsEvent(contents, 'did-navigate')
3498+
}
3499+
return dispatch?.(method, params) ?? Promise.resolve(undefined)
3500+
})
3501+
3502+
const result = await driver.executeTool('chat-test', 'browser_batch', {
3503+
actions: [
3504+
{ tool: 'browser_click', args: { elementId: 0 } },
3505+
{ tool: 'browser_click', args: { elementId: 0 } },
3506+
],
3507+
})
3508+
3509+
expect(mousePresses(contents)).toBe(1)
3510+
expect(result).toMatchObject({
3511+
ok: true,
3512+
result: { completed: false, completedCount: 1, stoppedIndex: 1, stoppedBy: 'page-change' },
3513+
})
3514+
})
3515+
3516+
it('reports a batch cancelled after an action ran as an unknown outcome', async () => {
3517+
const contents = await openPage()
3518+
respondWith(contents, {})
3519+
const sendCommand = vi.mocked(contents.debugger.sendCommand)
3520+
const dispatch = sendCommand.getMockImplementation()
3521+
sendCommand.mockImplementation((method, params) =>
3522+
method === 'Input.dispatchKeyEvent'
3523+
? new Promise(() => {})
3524+
: (dispatch?.(method, params) ?? Promise.resolve(undefined))
3525+
)
3526+
3527+
const pending = driver.executeTool(
3528+
'chat-test',
3529+
'browser_batch',
3530+
{
3531+
actions: [
3532+
{ tool: 'browser_click', args: { elementId: 0 } },
3533+
{ tool: 'browser_press_key', args: { key: 'Enter' } },
3534+
],
3535+
},
3536+
'batch-call'
3537+
)
3538+
await vi.waitFor(() => expect(cdpCalls(contents, 'Input.dispatchKeyEvent')).toHaveLength(1))
3539+
driver.cancelTool('chat-test', 'batch-call')
3540+
3541+
await expect(pending).resolves.toMatchObject({
3542+
ok: true,
3543+
result: { outcomeUnknown: true, doNotRetry: true },
3544+
})
3545+
})
3546+
3547+
it('rejects batches that name non-action tools or observe per action', async () => {
3548+
await openPage()
3549+
3550+
const navigation = await driver.executeTool('chat-test', 'browser_batch', {
3551+
actions: [
3552+
{ tool: 'browser_navigate', args: { url: 'https://example.com' } },
3553+
{ tool: 'browser_click', args: { elementId: 0 } },
3554+
],
3555+
})
3556+
const observed = await driver.executeTool('chat-test', 'browser_batch', {
3557+
actions: [
3558+
{ tool: 'browser_click', args: { elementId: 0, observe: {} } },
3559+
{ tool: 'browser_click', args: { elementId: 0 } },
3560+
],
3561+
})
3562+
3563+
expect(navigation).toMatchObject({
3564+
ok: false,
3565+
error: expect.stringContaining('Batch action 0'),
3566+
})
3567+
expect(observed).toMatchObject({ ok: false, error: expect.stringContaining('cannot observe') })
3568+
})
3569+
34283570
it('keeps element ids valid when an observed action is refused before dispatch', async () => {
34293571
const contents = await openPage()
34303572
respondWith(contents, { clickElement: { error: 'obstructed', blocker: 'IMG' } })

‎apps/desktop/src/main/browser-agent/driver.ts‎

Lines changed: 107 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,53 @@ type FormField =
144144
| { elementId: number; kind: 'select'; value: string }
145145
| { elementId: number; kind: 'checked'; checked: boolean }
146146

147+
const MAX_BATCH_ACTIONS = 8
148+
/** Single-page interactions a batch may run; navigation, observation, and file tools stay separate. */
149+
const BATCH_ACTION_TOOLS: ReadonlySet<BrowserToolName> = new Set([
150+
'browser_click',
151+
'browser_click_at',
152+
'browser_type',
153+
'browser_insert_text',
154+
'browser_press_key',
155+
'browser_scroll',
156+
'browser_select_option',
157+
'browser_set_checked',
158+
'browser_hover',
159+
])
160+
161+
interface BatchAction {
162+
tool: BrowserToolName
163+
args: Record<string, unknown>
164+
}
165+
166+
function isBatchActionTool(value: unknown): value is BrowserToolName {
167+
return BATCH_ACTION_TOOLS.has(value as BrowserToolName)
168+
}
169+
170+
function parseBatchActions(params: Record<string, unknown>): BatchAction[] {
171+
if (Object.keys(params).some((key) => key !== 'actions')) {
172+
throw new ToolError('A batch accepts only actions; pass observe on the batch itself.')
173+
}
174+
if (
175+
!Array.isArray(params.actions) ||
176+
params.actions.length < 2 ||
177+
params.actions.length > MAX_BATCH_ACTIONS
178+
) {
179+
throw new ToolError(`A batch requires between 2 and ${MAX_BATCH_ACTIONS} actions.`)
180+
}
181+
return params.actions.map((action, index): BatchAction => {
182+
if (!isRecordLike(action) || !isBatchActionTool(action.tool) || !isRecordLike(action.args)) {
183+
throw new ToolError(
184+
`Batch action ${index} must be {tool, args} with tool one of ${[...BATCH_ACTION_TOOLS].join(', ')}.`
185+
)
186+
}
187+
if ('observe' in action.args) {
188+
throw new ToolError(`Batch action ${index} cannot observe; pass observe on the batch itself.`)
189+
}
190+
return { tool: action.tool, args: action.args }
191+
})
192+
}
193+
147194
function parseFormFields(params: Record<string, unknown>): FormField[] {
148195
if (Object.keys(params).some((key) => key !== 'fields')) {
149196
throw new ToolError('Form filling accepts only fields; submitting is not supported.')
@@ -1046,7 +1093,8 @@ export function browserToolWatchdogMs(
10461093
tool === 'browser_open_tab' ||
10471094
tool === 'browser_switch_tab' ||
10481095
tool === 'browser_upload_file' ||
1049-
tool === 'browser_save_download'
1096+
tool === 'browser_save_download' ||
1097+
tool === 'browser_batch'
10501098
) {
10511099
return BROWSER_NAVIGATION_NATIVE_WATCHDOG_MS
10521100
}
@@ -3389,6 +3437,64 @@ async function executeToolInner(
33893437
}
33903438
}
33913439

3440+
case 'browser_batch': {
3441+
const actions = parseBatchActions(params)
3442+
const results: { index: number; tool: BrowserToolName; result: unknown }[] = []
3443+
const stopped = (
3444+
stoppedIndex: number,
3445+
stoppedBy: 'failure' | 'page-change',
3446+
error: string
3447+
) => ({
3448+
completed: false,
3449+
completedCount: results.length,
3450+
stoppedIndex,
3451+
stoppedBy,
3452+
error,
3453+
results,
3454+
})
3455+
for (const [index, action] of actions.entries()) {
3456+
let tab: ReturnType<typeof session.requireAutomationTab>
3457+
let epoch: number
3458+
let snapshotValid: boolean
3459+
let result: unknown
3460+
// Once an action has run, a cancelled or timed-out batch must not read as never started.
3461+
if (index > 0) onActionOutcome?.({ status: 'pending' })
3462+
try {
3463+
tab = session.requireAutomationTab()
3464+
epoch = navigationEpoch(tab.view.webContents)
3465+
snapshotValid = driverScopeState().snapshotTabId === tab.id
3466+
result = await executeToolInner(
3467+
action.tool,
3468+
action.args,
3469+
assertCurrentExecution,
3470+
executionDeadline,
3471+
invocationEpoch,
3472+
signal
3473+
)
3474+
} catch (error) {
3475+
return stopped(
3476+
index,
3477+
'failure',
3478+
`Action ${index} (${action.tool}) failed: ${getErrorMessage(error)} Earlier actions already took effect.`
3479+
)
3480+
}
3481+
results.push({ index, tool: action.tool, result })
3482+
if (index === actions.length - 1) break
3483+
if (
3484+
session.automationTab()?.id !== tab.id ||
3485+
navigationEpoch(tab.view.webContents) !== epoch ||
3486+
(snapshotValid && driverScopeState().snapshotTabId !== tab.id)
3487+
) {
3488+
return stopped(
3489+
index + 1,
3490+
'page-change',
3491+
`Action ${index} (${action.tool}) changed the page, so the remaining actions did not run. Inspect the page before continuing.`
3492+
)
3493+
}
3494+
}
3495+
return { completed: true, completedCount: results.length, results }
3496+
}
3497+
33923498
case 'browser_fill_form': {
33933499
const fields = parseFormFields(params)
33943500
const contents = session.requireAutomationTab().view.webContents

‎apps/desktop/src/main/browser-agent/post-action-observation.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const OBSERVABLE_ACTIONS: ReadonlySet<BrowserToolName> = new Set([
1010
'browser_fill_form',
1111
'browser_scroll',
1212
'browser_hover',
13+
'browser_batch',
1314
])
1415

1516
/** Preserves a dispatched action when its acknowledgement or observation is interrupted. */

‎apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ export const TOOL_ICONS: Readonly<Record<string, IconComponent>> = {
3939
apply_file_edit: File,
4040
auth: Integration,
4141
browser: Globe,
42+
browser_batch: Cursor,
4243
browser_click: Cursor,
4344
browser_click_at: Cursor,
4445
browser_close_tab: Cursor,

0 commit comments

Comments
 (0)