fix(desktop): yield the page only for background calls that act on it

- Only a call from the background executor yields the page to the user.
  A call from the chat view the user is watching acts exactly as before:
  there the user steers the agent directly.
- A background call that only reads the page (snapshot, read text, list
  tabs) never waits for the user, since it cannot collide with their input.
- Which browser tools only observe the page now lives in
  @sim/browser-protocol, shared by the chat view's replay policy and the
  driver.
- The prevent-sleep switch shows only when Sim runs chats on this device
  in the background.
- Tests: a chat-view call acts at once under user input; a background call
  waits, and a background read does not; a user who starts working
  mid-action stops it before its next input; sign-out releases the sleep
  blocker while a call is still running.

Author: waleedlatif1

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -79,6 +79,102 @@ describe('executeTool', () => {
     driver = freshDriver()
   })
 
+  /** The user presses a key in the page the agent drives, as the browser reports it. */
+  function typeInAgentPage(contents: WebContents) {
+    const calls: ReadonlyArray<readonly unknown[]> = vi.mocked(contents.on).mock.calls
+    const listener = calls.find(([event]) => event === 'before-input-event')?.[1]
+    if (typeof listener !== 'function') throw new Error('No input listener on the agent page')
+    listener({}, { type: 'keyDown', isAutoRepeat: false })
+  }
+
+  it('lets a call from the chat view act at once while the user works in the page', async () => {
+    await driver.executeTool('chat-test', 'browser_open_tab', {})
+    typeInAgentPage(session.requireTab().view.webContents)
+    expect(session.msSinceUserIntervention()).not.toBeNull()
+    vi.useFakeTimers()
+    try {
+      let settled = false
+      const opening = driver.executeTool('chat-test', 'browser_open_tab', {}).then((result) => {
+        settled = true
+        return result
+      })
+      await vi.advanceTimersByTimeAsync(100)
+
+      expect(settled).toBe(true)
+      expect((await opening).ok).toBe(true)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('makes a background call wait until the user leaves the page alone', async () => {
+    await driver.executeTool('chat-test', 'browser_open_tab', {})
+    typeInAgentPage(session.requireTab().view.webContents)
+    vi.useFakeTimers()
+    try {
+      let settled = false
+      const opening = driver
+        .executeTool('chat-test', 'browser_open_tab', {}, 'background-1', undefined, {
+          background: true,
+        })
+        .then((result) => {
+          settled = true
+          return result
+        })
+      await vi.advanceTimersByTimeAsync(1_000)
+      expect(settled).toBe(false)
+
+      await vi.advanceTimersByTimeAsync(4_000)
+      expect((await opening).ok).toBe(true)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('stops a background action before its next input once the user starts working mid-action', async () => {
+    await driver.executeTool('chat-test', 'browser_open_tab', {})
+    const contents = session.requireTab().view.webContents
+    // The user presses a key while the action reads the page before acting on it.
+    const probe = vi.mocked(contents.executeJavaScript).getMockImplementation()
+    vi.mocked(contents.executeJavaScript).mockImplementation(async (...args) => {
+      typeInAgentPage(contents)
+      return probe ? probe(...args) : undefined
+    })
+
+    const result = await driver.executeTool(
+      'chat-test',
+      'browser_press_key',
+      { key: 'Enter' },
+      'background-key',
+      undefined,
+      { background: true }
+    )
+
+    expect(result.ok).toBe(false)
+    expect(result.error).toContain('the user started working in this page')
+  })
+
+  it('never makes a background read wait for the user', async () => {
+    await driver.executeTool('chat-test', 'browser_open_tab', {})
+    typeInAgentPage(session.requireTab().view.webContents)
+    vi.useFakeTimers()
+    try {
+      let settled = false
+      void driver
+        .executeTool('chat-test', 'browser_list_tabs', {}, 'background-read', undefined, {
+          background: true,
+        })
+        .then(() => {
+          settled = true
+        })
+      await vi.advanceTimersByTimeAsync(100)
+
+      expect(settled).toBe(true)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('returns ok:false instead of throwing for tool-level failures', async () => {
     // No session exists, so any page-dependent tool fails with guidance.
     const result = await driver.executeTool('chat-test', 'browser_click', { elementId: 1 })

apps/desktop/src/main/browser-agent/driver.ts

@@ -19,6 +19,7 @@
 import {
   BROWSER_DATA_KINDS,
   BROWSER_NAVIGATION_NATIVE_WATCHDOG_MS,
+  BROWSER_TOOL_OBSERVES_ONLY,
   BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS,
   BROWSER_UPLOAD_MAX_FILES,
   type BrowserDataKind,
@@ -5135,12 +5136,23 @@ async function yieldToUser(
   logger.info('Browser automation resumed after the user stopped', { toolCallId })
 }
 
+/** How a browser call reached the driver. */
+interface BrowserToolExecutionOptions {
+  /**
+   * Run by the background executor while the user may be working in the same page, so it yields
+   * the page to them. A call from the chat view the user is watching never does: there the user
+   * steers the agent directly, as they always have.
+   */
+  background?: boolean
+}
+
 export async function executeTool(
   scopeId: string,
   tool: BrowserToolName,
   params: Record<string, unknown>,
   toolCallId?: string,
-  authorizationBoundary?: BrowserToolQueueBoundary
+  authorizationBoundary?: BrowserToolQueueBoundary,
+  options: BrowserToolExecutionOptions = {}
 ): Promise<{ ok: boolean; result?: unknown; error?: string }> {
   const resolvedScopeId = resolveDriverScopeId(scopeId)
   if (authorizationBoundary) {
@@ -5231,7 +5243,11 @@ export async function executeTool(
           session.setAutomationActive(true)
         }
         try {
-          const yieldsToUser = tool !== 'browser_request_takeover'
+          // Only a background call acting on the page yields; reading it cannot collide with the user.
+          const yieldsToUser =
+            options.background === true &&
+            tool !== 'browser_request_takeover' &&
+            !BROWSER_TOOL_OBSERVES_ONLY[tool]
           const watchdogMs = browserToolWatchdogMs(tool, params)
           if (yieldsToUser && isUserWorkingInPage()) {
             await yieldToUser(

apps/desktop/src/main/desktop-executor/service.test.ts

@@ -12,6 +12,8 @@ import { createDesktopExecutorService, deviceName } from '@/main/desktop-executo
 /** Sim's device routes, with registration answers held until the test releases them. */
 function fakeSim(protocolVersion = 1) {
   const requests: string[] = []
+  /** Calls the inbox offers; a claim answers for the first one. */
+  const offered: string[] = []
   /**
    * Answers to pending registrations: enabled or not, an HTTP status Sim fails with, or
    * `'offline'` for a request that never reached Sim.
@@ -37,25 +39,60 @@ function fakeSim(protocolVersion = 1) {
         reconcileMs: 10_000,
       })
     }
-    if (path === '/api/desktop/inbox') return Response.json({ items: [] })
+    if (path === '/api/desktop/inbox') {
+      return Response.json({
+        items: offered.map((toolCallId) => ({
+          kind: 'call',
+          toolCallId,
+          toolName: 'terminal',
+          chatId: 'chat-a',
+          workspaceId: 'ws-1',
+          createdAt: new Date().toISOString(),
+        })),
+      })
+    }
+    if (path === '/api/desktop/tool/claim') {
+      const toolCallId = offered.shift()
+      if (!toolCallId) return Response.json({ error: 'gone' }, { status: 404 })
+      return Response.json({
+        toolName: 'terminal',
+        args: { operation: 'run', args: { command: 'sleep 600' } },
+        chatId: 'chat-a',
+        workspaceId: 'ws-1',
+        executionToken: `token-${toolCallId}`,
+      })
+    }
+    if (path === '/api/desktop/tool/lease') return Response.json({ renewed: true })
     return new Promise<Response>((_resolve, reject) =>
       init.signal?.addEventListener('abort', () => reject(new Error('aborted')))
     )
   })
-  return { fetch, requests, registrations }
+  return { fetch, requests, registrations, offered }
 }
 
 async function service(protocolVersion = 1, userDataPath?: string) {
   const sim = fakeSim(protocolVersion)
+  /** What the sleep blocker was told, in order. */
+  const busy: boolean[] = []
   const desktopExecutor = createDesktopExecutorService({
     userDataPath: userDataPath ?? (await mkdtemp(join(tmpdir(), 'sim-executor-service-'))),
     origin: () => 'https://sim.test',
     appSession: () => ({ fetch: sim.fetch }),
     preferences: () => ({ browserEnabled: true, terminalEnabled: true }),
     accountDataAvailable: () => true,
-    runner: { run: vi.fn(), cancel: vi.fn() },
+    // A command that runs until it is stopped.
+    runner: {
+      run: (_call, signal) =>
+        new Promise((resolve) =>
+          signal.addEventListener('abort', () =>
+            resolve({ status: 'cancelled', message: 'Stopped.' })
+          )
+        ),
+      cancel: async () => {},
+    },
+    onBusyChange: (value) => busy.push(value),
   })
-  return { sim, desktopExecutor }
+  return { sim, desktopExecutor, busy }
 }
 
 describe('desktop executor registration', () => {
@@ -177,6 +214,19 @@ describe('desktop executor registration', () => {
 
     expect(sim.requests.filter((request) => request.includes('/api/desktop/devices'))).toEqual([])
   })
+
+  it('releases the sleep blocker at sign-out while a call is still running', async () => {
+    const { sim, desktopExecutor, busy } = await service()
+    sim.offered.push('call-1')
+    desktopExecutor.start()
+    await vi.waitFor(() => expect(sim.registrations).toHaveLength(1))
+    sim.registrations[0]?.(true)
+    await vi.waitFor(() => expect(busy).toEqual([true]))
+
+    await desktopExecutor.signOut()
+
+    expect(busy).toEqual([true, false])
+  })
 })
 
 describe('device name', () => {

apps/desktop/src/main/index.ts

@@ -605,7 +605,10 @@ function main(): void {
       preferences: () => desktopSettings.getPreferences(),
       accountDataAvailable,
       browser: {
-        executeTool: executeAgentBrowserTool,
+        executeTool: (scopeId, tool, params, toolCallId) =>
+          executeAgentBrowserTool(scopeId, tool, params, toolCallId, undefined, {
+            background: true,
+          }),
         cancelTool: cancelAgentBrowserTool,
         hasSession: hasBrowserScopeSession,
         restoreScope: restoreAgentBrowserScope,

apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx

@@ -33,6 +33,8 @@ export function Desktop() {
   const workspaceId = params.workspaceId as string
   const [preferences, setPreferences] = useState<DesktopPreferences | null>(null)
   const [pendingPreference, setPendingPreference] = useState<DesktopPreferenceKey | null>(null)
+  /** Whether Sim runs chats on this device in the background; only then can sleep matter. */
+  const [runsInBackground, setRunsInBackground] = useState(false)
   const updateState = useDesktopUpdateState()
   const shellVersion = getDesktopShellVersion()
 
@@ -46,6 +48,10 @@ export function Desktop() {
       .getPreferences()
       .then(setPreferences)
       .catch(() => toast.error('Could not load desktop settings'))
+    void bridge.desktopExecutor
+      ?.getDevice()
+      .then((device) => setRunsInBackground(device !== null))
+      .catch(() => setRunsInBackground(false))
   }, [router, workspaceId])
 
   const updatePreference = async (key: DesktopPreferenceKey, value: boolean) => {
@@ -73,7 +79,8 @@ export function Desktop() {
 
   const notificationsDisabled =
     !preferences.notificationsEnabled || pendingPreference === 'notificationsEnabled'
-  const supportsPreventSleep = Boolean(getDesktopBridge()?.settings.setPreventSleepWhileRunning)
+  const supportsPreventSleep =
+    runsInBackground && Boolean(getDesktopBridge()?.settings.setPreventSleepWhileRunning)
 
   return (
     <SettingsPanel>

apps/sim/lib/mothership/tools/client/browser-tool-execution.ts

@@ -7,7 +7,11 @@
  * browser and reports the outcome via the confirm endpoint, which wakes the
  * server-side waiter.
  */
-import { type BrowserToolName, browserToolRendererTimeoutMs } from '@sim/browser-protocol'
+import {
+  BROWSER_TOOL_OBSERVES_ONLY,
+  type BrowserToolName,
+  browserToolRendererTimeoutMs,
+} from '@sim/browser-protocol'
 import {
   browserSessionClosedCompletion,
   browserToolCompletion,
@@ -44,41 +48,7 @@ const logger = createLogger('CopilotBrowserToolExecution')
  * reload cannot cause a page or external side effect. Every stateful current
  * tool and the retired takeover flow remain fail-closed.
  */
-const OBSERVATION_ONLY_BROWSER_TOOLS = {
-  browser_navigate: false,
-  browser_open_url: false,
-  browser_go_back: false,
-  browser_go_forward: false,
-  browser_reload: false,
-  browser_open_tab: false,
-  browser_switch_tab: false,
-  browser_close_tab: false,
-  browser_list_tabs: true,
-  browser_list_sessions: true,
-  browser_list_downloads: true,
-  browser_save_download: false,
-  browser_wait_for: true,
-  browser_snapshot: true,
-  browser_find: true,
-  browser_read_text: true,
-  browser_screenshot: true,
-  browser_extract: true,
-  browser_click: false,
-  browser_click_at: false,
-  browser_type: false,
-  browser_fill_form: false,
-  browser_batch: false,
-  browser_insert_text: false,
-  browser_press_key: false,
-  browser_scroll: false,
-  browser_select_option: false,
-  browser_set_checked: false,
-  browser_upload_file: false,
-  browser_hover: false,
-  browser_drag: false,
-  browser_zoom: false,
-  browser_request_takeover: false,
-} as const satisfies Readonly<Record<BrowserToolName, boolean>>
+const OBSERVATION_ONLY_BROWSER_TOOLS = BROWSER_TOOL_OBSERVES_ONLY
 
 /** Tool events older than this are replays, not live instructions — never act on them. */
 const MAX_EVENT_AGE_MS = 120_000

packages/browser-protocol/src/index.ts

@@ -150,6 +150,46 @@ export function isCurrentBrowserToolName(name: string): name is CurrentBrowserTo
   return CURRENT_BROWSER_TOOL_NAME_SET.has(name)
 }
 
+/**
+ * Which browser tools only observe the page (read it, list it, wait on it) and never act on it.
+ * Repeating one cannot cause a side effect, and one can never collide with what the user is doing.
+ */
+export const BROWSER_TOOL_OBSERVES_ONLY = {
+  browser_navigate: false,
+  browser_open_url: false,
+  browser_go_back: false,
+  browser_go_forward: false,
+  browser_reload: false,
+  browser_open_tab: false,
+  browser_switch_tab: false,
+  browser_close_tab: false,
+  browser_list_tabs: true,
+  browser_list_sessions: true,
+  browser_list_downloads: true,
+  browser_save_download: false,
+  browser_wait_for: true,
+  browser_snapshot: true,
+  browser_find: true,
+  browser_read_text: true,
+  browser_screenshot: true,
+  browser_extract: true,
+  browser_click: false,
+  browser_click_at: false,
+  browser_type: false,
+  browser_fill_form: false,
+  browser_batch: false,
+  browser_insert_text: false,
+  browser_press_key: false,
+  browser_scroll: false,
+  browser_select_option: false,
+  browser_set_checked: false,
+  browser_upload_file: false,
+  browser_hover: false,
+  browser_drag: false,
+  browser_zoom: false,
+  browser_request_takeover: false,
+} as const satisfies Readonly<Record<BrowserToolName, boolean>>
+
 export function isBrowserTheme(value: unknown): value is BrowserTheme {
   return typeof value === 'string' && BROWSER_THEME_SET.has(value)
 }