From b05971da575e4012467a19a7d9c9dd0348f17613 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 5 Oct 2026 23:21:40 -0700 Subject: [PATCH 1/6] feat(desktop): keep the machine awake for background chats and yield the page to the user - Prevent sleep: while any chat has desktop work claimed on this machine, the app holds a prevent-app-suspension power blocker, released as soon as the last result is delivered. On by default, with a switch in Desktop settings that notes a closed lid still sleeps the computer. The switch is a new optional bridge method, so older shells simply do not show it. - Takeover: when the user clicks, scrolls or types in the page an agent drives, that chat's next browser action waits, marked as needing attention, until the user has left the page alone for a few seconds, then resumes in the same page. A Stop still cancels it while it waits. --- apps/desktop/e2e/background-executor.spec.ts | 77 +++++++++++++++++++ apps/desktop/src/main/browser-agent/driver.ts | 29 ++++++- .../desktop/src/main/browser-agent/session.ts | 23 ++++++ apps/desktop/src/main/config.ts | 3 + .../src/main/desktop-executor/service.ts | 3 + .../desktop-executor/sleep-blocker.test.ts | 63 +++++++++++++++ .../main/desktop-executor/sleep-blocker.ts | 45 +++++++++++ apps/desktop/src/main/desktop-settings.ts | 10 +++ apps/desktop/src/main/index.ts | 9 +++ apps/desktop/src/main/ipc.test.ts | 1 + apps/desktop/src/main/ipc.ts | 9 +++ apps/desktop/src/preload/index.ts | 2 + .../settings/components/desktop/desktop.tsx | 24 ++++++ packages/desktop-bridge/src/index.ts | 10 +++ 14 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 apps/desktop/src/main/desktop-executor/sleep-blocker.test.ts create mode 100644 apps/desktop/src/main/desktop-executor/sleep-blocker.ts diff --git a/apps/desktop/e2e/background-executor.spec.ts b/apps/desktop/e2e/background-executor.spec.ts index 4dd67c50458..f40d6c499f2 100644 --- a/apps/desktop/e2e/background-executor.spec.ts +++ b/apps/desktop/e2e/background-executor.spec.ts @@ -359,6 +359,11 @@ async function launch(userData: string): Promise<{ app: ElectronApplication; win SIM_DESKTOP_USER_DATA: userData, }, }) + // A dialog listener stops Playwright auto-dismissing page dialogs, so the desktop's own handling + // decides their outcome exactly as it does in production. + const leaveDialogsToDesktop = (page: Page) => page.on('dialog', () => {}) + app.context().pages().forEach(leaveDialogsToDesktop) + app.context().on('page', leaveDialogsToDesktop) const window = await app.firstWindow() return { app, window } } @@ -730,6 +735,78 @@ test.describe('background executor', () => { expect(device).toEqual({ deviceId, protocolVersion: 1 }) }) }) + + test('I: the agent yields its page while the user works in it, then takes it back', async () => { + app = (await launch(mkdtempSync(join(tmpdir(), 'sim-executor-i-')))).app + const deviceId = await registeredDevice() + const opened = sim.issue(deviceId, CHAT_A, 'browser_open_url', { + url: `${sim.origin}/counter?chat=I`, + }) + const outline = ((await settled(opened)).data?.snapshot as { outline: string }).outline + const button = refFor(outline, 'Count visit') + + const typeInAgentPage = () => + app?.evaluate(({ webContents }) => { + const page = webContents + .getAllWebContents() + .find((contents) => contents.getURL().includes('/counter?chat=I')) + page?.sendInputEvent({ type: 'keyDown', keyCode: 'Tab' }) + page?.sendInputEvent({ type: 'keyUp', keyCode: 'Tab' }) + }) + await typeInAgentPage() + const click = sim.issue(deviceId, CHAT_A, 'browser_click', { elementId: button }) + for (let i = 0; i < 4; i++) { + await sleep(500) + await typeInAgentPage() + } + const lastUserInputAt = Date.now() + + await check('I: the click waits until the user stops, then runs once', async () => { + const completion = await settled(click, 30_000) + expect(completion.status, completion.message).toBe('success') + expect(completion.at - lastUserInputAt).toBeGreaterThanOrEqual(3_000) + await expect.poll(() => sim.hits.get('I')).toBe(1) + }) + }) + + test('J: the machine stays awake only while a chat has work running', async () => { + app = (await launch(mkdtempSync(join(tmpdir(), 'sim-executor-j-')))).app + await app.evaluate(({ powerSaveBlocker }) => { + const log: string[] = [] + const active = new Set() + let next = 1 + const target = globalThis as typeof globalThis & { __sleepBlocks?: string[] } + target.__sleepBlocks = log + powerSaveBlocker.start = (type) => { + log.push(`start:${type}`) + active.add(next) + return next++ + } + powerSaveBlocker.stop = (id) => { + log.push('stop') + active.delete(id) + return true + } + powerSaveBlocker.isStarted = (id) => active.has(id) + }) + const deviceId = await registeredDevice() + const sleepLog = () => + app?.evaluate(() => (globalThis as { __sleepBlocks?: string[] }).__sleepBlocks ?? []) + + const run = sim.issue(deviceId, CHAT_B, 'terminal', { + operation: 'run', + args: { command: 'sleep 3; echo awake', waitSeconds: 30 }, + }) + await check('J: a blocker is held while the command runs', async () => { + await expect.poll(sleepLog, { timeout: 15_000 }).toEqual(['start:prevent-app-suspension']) + }) + await check('J: it is released once the result is delivered', async () => { + await settled(run) + await expect + .poll(sleepLog, { timeout: 10_000 }) + .toEqual(['start:prevent-app-suspension', 'stop']) + }) + }) }) function readFileSafe(path: string): string { diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index 711fa28c7a8..cad875d655d 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -32,7 +32,7 @@ import { import type { BrowserDownloadsState, BrowserToolbarCommand } from '@sim/desktop-bridge' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { sleep } from '@sim/utils/helpers' +import { interruptibleSleep, sleep } from '@sim/utils/helpers' import { isRecordLike, omit, toArray, toRecord } from '@sim/utils/object' import type { BrowserWindow, MenuItemConstructorOptions, WebContents, WebFrameMain } from 'electron' import { Menu } from 'electron' @@ -5094,6 +5094,30 @@ function withNotices(result: unknown): unknown { return { value: result, notices } } +/** Quiet time after the user's last click or keystroke before the agent takes the page back. */ +const USER_TAKEOVER_IDLE_MS = 4_000 + +function isUserWorkingInPage(): boolean { + const since = session.msSinceUserIntervention() + return since !== null && since < USER_TAKEOVER_IDLE_MS +} + +/** + * The user and the agent never act in the same page at once. While the user is clicking, typing + * or scrolling in the tab the agent drives, the agent's next action waits, marked as needing + * attention, and resumes once the user has left the page alone for a few seconds. + */ +async function yieldToUser(toolCallId: string | undefined, signal: AbortSignal): Promise { + session.setAutomationNeedsAttention(true) + logger.info('Browser automation yielding to the user', { toolCallId }) + while (isUserWorkingInPage()) { + await interruptibleSleep(250, signal) + if (signal.aborted) throw new ToolError('This browser action was cancelled.') + } + session.setAutomationNeedsAttention(false) + logger.info('Browser automation resumed after the user stopped', { toolCallId }) +} + export async function executeTool( scopeId: string, tool: BrowserToolName, @@ -5190,6 +5214,9 @@ export async function executeTool( session.setAutomationActive(true) } try { + if (tool !== 'browser_request_takeover' && isUserWorkingInPage()) { + await yieldToUser(toolCallId, executionController.signal) + } const response = dialogResponse(tool, params) state.dialogResponse = response ? { contents: session.requireAutomationTab().view.webContents, response } diff --git a/apps/desktop/src/main/browser-agent/session.ts b/apps/desktop/src/main/browser-agent/session.ts index abbefea8537..ea7d77dea38 100644 --- a/apps/desktop/src/main/browser-agent/session.ts +++ b/apps/desktop/src/main/browser-agent/session.ts @@ -298,6 +298,8 @@ interface BrowserScopeState { */ findingTabId: string | null findingRequestId: number | null + /** When the user last clicked, scrolled or typed in the tab the agent drives. */ + userInterventionAt: number | null } function createBrowserScopeState(): BrowserScopeState { @@ -319,6 +321,7 @@ function createBrowserScopeState(): BrowserScopeState { automationNeedsAttention: false, findingTabId: null, findingRequestId: null, + userInterventionAt: null, } } @@ -2368,6 +2371,7 @@ function initializeTabView( return } const tab = tabs.find((entry) => entry.view.webContents === contents) + if (tab?.id === currentScope.automationTabId) noteUserIntervention() if (tab?.id === currentScope.activeTabId) { currentScope.visibleTabUserSelected = true if (mouse.type === 'mouseDown') tab.lastRealUserGestureAt = Date.now() @@ -2484,6 +2488,13 @@ function initializeTabView( 'before-input-event', bindToBrowserScope(scopeId, (event, input) => { const tab = tabs.find((entry) => entry.view === view) + if ( + !isDispatchingAgentInput(contents) && + input.type === 'keyDown' && + tab?.id === currentScope.automationTabId + ) { + noteUserIntervention() + } if (!isDispatchingAgentInput(contents) && tab?.id === currentScope.activeTabId) { currentScope.visibleTabUserSelected = true if (input.type === 'keyDown' && !input.isAutoRepeat) tab.lastRealUserGestureAt = Date.now() @@ -3117,8 +3128,20 @@ export function claimActiveTabForUser(): AgentTab | null { return tab } +/** The user just acted in the page the agent drives; the agent yields until they stop. */ +function noteUserIntervention(): void { + currentScope.userInterventionAt = Date.now() +} + +/** How long ago the user last acted in the current scope's agent tab, or null if never. */ +export function msSinceUserIntervention(): number | null { + const at = currentScope.userInterventionAt + return at === null ? null : Math.max(0, Date.now() - at) +} + /** Explicit hand-back after takeover lets automation resume in the same page. */ export function returnAutomationTabToAgent(): void { + currentScope.userInterventionAt = null if (currentScope.activeTabId === currentScope.automationTabId) { currentScope.visibleTabUserSelected = false } diff --git a/apps/desktop/src/main/config.ts b/apps/desktop/src/main/config.ts index 30c00528b10..d4b5497bb5c 100644 --- a/apps/desktop/src/main/config.ts +++ b/apps/desktop/src/main/config.ts @@ -104,6 +104,8 @@ export interface DesktopSettings { /** Whether omnibox typing may request live Google search completions. */ browserSearchSuggestionsEnabled?: boolean terminalEnabled?: boolean + /** Keep the machine awake while a chat is running desktop work in the background. */ + preventSleepWhileRunning?: boolean /** Device-wide browser page appearance; `app` follows Sim. */ browserTheme?: 'app' | 'light' | 'dark' /** Device-wide default zoom for built-in browser pages. */ @@ -236,6 +238,7 @@ const DEFAULT_SETTINGS: DesktopSettings = { browserEnabled: true, browserSearchSuggestionsEnabled: true, terminalEnabled: true, + preventSleepWhileRunning: true, } export interface ConfigStore { diff --git a/apps/desktop/src/main/desktop-executor/service.ts b/apps/desktop/src/main/desktop-executor/service.ts index c0dd079d36c..1dd6689274a 100644 --- a/apps/desktop/src/main/desktop-executor/service.ts +++ b/apps/desktop/src/main/desktop-executor/service.ts @@ -57,6 +57,8 @@ export interface DesktopExecutorServiceDeps { accountDataAvailable: () => boolean runner: DesktopToolRunner onApprovals?: (items: DesktopApprovalItem[]) => void + /** Whether any chat has desktop work claimed on this machine changed. */ + onBusyChange?: (busy: boolean) => void } export interface DesktopExecutorService { @@ -229,6 +231,7 @@ export function createDesktopExecutorService( leaseRenewMs: nextTiming.leaseRenewMs, onUnregistered: handleUnrecognized, ...(deps.onApprovals ? { onApprovals: deps.onApprovals } : {}), + ...(deps.onBusyChange ? { onBusyChange: deps.onBusyChange } : {}), }) await executor.recover() // Signed out while recovering: sign-out already disposed this executor. diff --git a/apps/desktop/src/main/desktop-executor/sleep-blocker.test.ts b/apps/desktop/src/main/desktop-executor/sleep-blocker.test.ts new file mode 100644 index 00000000000..7954b224db3 --- /dev/null +++ b/apps/desktop/src/main/desktop-executor/sleep-blocker.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from 'vitest' +import { createSleepBlocker } from '@/main/desktop-executor/sleep-blocker' + +function harness(enabled = true) { + let nextId = 1 + const active = new Set() + const powerSaveBlocker = { + start: vi.fn(() => { + const id = nextId++ + active.add(id) + return id + }), + stop: vi.fn((id: number) => { + active.delete(id) + }), + isStarted: (id: number) => active.has(id), + } + let preference = enabled + const blocker = createSleepBlocker({ enabled: () => preference, powerSaveBlocker }) + return { + blocker, + powerSaveBlocker, + active, + setPreference: (value: boolean) => { + preference = value + blocker.refresh() + }, + } +} + +describe('keeping the machine awake for background work', () => { + it('holds one blocker only while a chat has work running', () => { + const { blocker, powerSaveBlocker, active } = harness() + + blocker.setBusy(true) + blocker.setBusy(true) + expect(active.size).toBe(1) + expect(powerSaveBlocker.start).toHaveBeenCalledWith('prevent-app-suspension') + + blocker.setBusy(false) + expect(active.size).toBe(0) + }) + + it('never holds one while the user has switched it off, and follows the switch live', () => { + const { blocker, active, setPreference } = harness(false) + + blocker.setBusy(true) + expect(active.size).toBe(0) + + setPreference(true) + expect(active.size).toBe(1) + setPreference(false) + expect(active.size).toBe(0) + }) + + it('does not start one for an idle machine when the switch turns on', () => { + const { active, setPreference } = harness(false) + + setPreference(true) + + expect(active.size).toBe(0) + }) +}) diff --git a/apps/desktop/src/main/desktop-executor/sleep-blocker.ts b/apps/desktop/src/main/desktop-executor/sleep-blocker.ts new file mode 100644 index 00000000000..218bf813390 --- /dev/null +++ b/apps/desktop/src/main/desktop-executor/sleep-blocker.ts @@ -0,0 +1,45 @@ +/** + * Keeps the machine from idle-sleeping while a chat has desktop work running in the background, + * and only then. It cannot keep a closed laptop awake: the OS sleeps it anyway, and the work + * resumes reporting when it wakes. + */ + +interface PowerSaveBlocker { + start(type: 'prevent-app-suspension'): number + stop(id: number): void + isStarted(id: number): boolean +} + +interface SleepBlockerDeps { + /** The user's "prevent sleep while a chat is running" switch. */ + enabled: () => boolean + powerSaveBlocker: PowerSaveBlocker +} + +export function createSleepBlocker(deps: SleepBlockerDeps) { + let busy = false + let blockerId: number | null = null + + const apply = () => { + const wanted = busy && deps.enabled() + const held = blockerId !== null && deps.powerSaveBlocker.isStarted(blockerId) + if (wanted && !held) { + blockerId = deps.powerSaveBlocker.start('prevent-app-suspension') + } else if (!wanted && blockerId !== null) { + if (held) deps.powerSaveBlocker.stop(blockerId) + blockerId = null + } + } + + return { + /** Whether any chat has desktop work claimed on this machine. */ + setBusy(next: boolean): void { + busy = next + apply() + }, + /** Re-applies the user's switch to the work running now. */ + refresh(): void { + apply() + }, + } +} diff --git a/apps/desktop/src/main/desktop-settings.ts b/apps/desktop/src/main/desktop-settings.ts index f14e0157fc9..37c88b27056 100644 --- a/apps/desktop/src/main/desktop-settings.ts +++ b/apps/desktop/src/main/desktop-settings.ts @@ -37,6 +37,7 @@ export interface DesktopSettingsService { getPreferences(): DesktopPreferences setPreference(key: DesktopPreferenceKey, value: boolean): DesktopPreferences setBrowserSearchSuggestionsEnabled(enabled: boolean): DesktopPreferences + setPreventSleepWhileRunning(enabled: boolean): DesktopPreferences setAppearancePreference( key: DesktopAppearanceSettingKey, value: DesktopAppearanceTheme @@ -60,6 +61,8 @@ interface DesktopSettingsServiceDeps { setBrowserEnabled: (enabled: boolean) => void /** Ends every open agent shell when the surface is turned off. */ setTerminalEnabled: (enabled: boolean) => void + /** Starts or stops keeping the machine awake for background work already running. */ + setPreventSleepWhileRunning: (enabled: boolean) => void /** Repaints current browser tabs when their persisted appearance changes. */ setBrowserTheme: (theme: DesktopAppearanceTheme) => void /** Applies a new default zoom to current and future browser tabs. */ @@ -93,6 +96,7 @@ function readPreferences( browserEnabled: config.get('browserEnabled') ?? true, browserSearchSuggestionsEnabled: config.get('browserSearchSuggestionsEnabled') ?? true, terminalEnabled: config.get('terminalEnabled') ?? true, + preventSleepWhileRunning: config.get('preventSleepWhileRunning') ?? true, browserTheme: isDesktopAppearanceTheme(browserTheme) ? browserTheme : 'app', browserDefaultZoom: isDesktopZoomPercent(browserDefaultZoom) ? browserDefaultZoom : 100, browserDownloadDirectory: @@ -175,6 +179,12 @@ export function createDesktopSettingsService( deps.config.flush() return read() }, + setPreventSleepWhileRunning(enabled) { + deps.config.set('preventSleepWhileRunning', enabled) + deps.config.flush() + deps.setPreventSleepWhileRunning(enabled) + return read() + }, setAppearancePreference(key, value) { const previousBrowserTheme = key === 'browserTheme' ? read().browserTheme : undefined deps.config.set(key, value) diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 990fa6724aa..a6e33a99329 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -9,6 +9,7 @@ import { dialog, Notification, net, + powerSaveBlocker, session, shell, } from 'electron' @@ -63,6 +64,7 @@ import { DesktopChatSessionStore } from '@/main/desktop-chat-session-store' import { createApprovalNotifier } from '@/main/desktop-executor/approval-notifier' import { createDesktopToolRunner } from '@/main/desktop-executor/runner' import { createDesktopExecutorService } from '@/main/desktop-executor/service' +import { createSleepBlocker } from '@/main/desktop-executor/sleep-blocker' import { createDesktopSettingsService } from '@/main/desktop-settings' import { attachDownloadHandling } from '@/main/downloads' import { createAuthFlow, createConnectFlow, createHandoffManager } from '@/main/handoff' @@ -546,6 +548,7 @@ function main(): void { } desktopExecutor.refreshRegistration() }, + setPreventSleepWhileRunning: () => sleepBlocker.refresh(), setBrowserTheme: setAgentBrowserTheme, setBrowserDefaultZoom: setAgentBrowserDefaultZoom, setTerminalDefaultZoom: (zoom) => { @@ -572,6 +575,11 @@ function main(): void { }, }) + const sleepBlocker = createSleepBlocker({ + enabled: () => desktopSettings.getPreferences().preventSleepWhileRunning ?? true, + powerSaveBlocker, + }) + const approvalNotifier = createApprovalNotifier({ preferences: () => desktopSettings.getPreferences(), focusedChatId: () => { @@ -592,6 +600,7 @@ function main(): void { preferences: () => desktopSettings.getPreferences(), accountDataAvailable, onApprovals: (items) => approvalNotifier.update(items), + onBusyChange: (busy) => sleepBlocker.setBusy(busy), runner: createDesktopToolRunner({ preferences: () => desktopSettings.getPreferences(), accountDataAvailable, diff --git a/apps/desktop/src/main/ipc.test.ts b/apps/desktop/src/main/ipc.test.ts index 4a596f77a65..0c037b5a5b0 100644 --- a/apps/desktop/src/main/ipc.test.ts +++ b/apps/desktop/src/main/ipc.test.ts @@ -311,6 +311,7 @@ describe('registerIpcHandlers', () => { getPreferences: vi.fn(() => DEFAULT_DESKTOP_PREFERENCES), setPreference: vi.fn(), setBrowserSearchSuggestionsEnabled: vi.fn(), + setPreventSleepWhileRunning: vi.fn(), setAppearancePreference: vi.fn(), setBrowserDefaultZoom: vi.fn(), setTerminalDefaultZoom: vi.fn(), diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index cd5b81d8400..41d9b409650 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -811,6 +811,15 @@ export function registerIpcHandlers(deps: IpcDeps): void { ? deps.settings.setBrowserSearchSuggestionsEnabled(enabled) : deps.settings.getPreferences(), }, + 'desktop:settings:set-prevent-sleep': { + kind: 'invoke', + gate: 'app-origin', + denied: null, + handler: (enabled) => + typeof enabled === 'boolean' + ? deps.settings.setPreventSleepWhileRunning(enabled) + : deps.settings.getPreferences(), + }, 'desktop:settings:set-appearance': { kind: 'invoke', gate: 'app-origin', diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 6518ff00592..6071925e7dd 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -179,6 +179,8 @@ const api: SimDesktopApi = { getPreferences: (): Promise => ipcRenderer.invoke('desktop:settings:get'), setPreference: (key: DesktopPreferenceKey, value: boolean): Promise => ipcRenderer.invoke('desktop:settings:set', key, value), + setPreventSleepWhileRunning: (enabled: boolean): Promise => + ipcRenderer.invoke('desktop:settings:set-prevent-sleep', enabled), setBrowserSearchSuggestionsEnabled: (enabled: boolean): Promise => ipcRenderer.invoke('desktop:settings:set-browser-search-suggestions', enabled), notify: (payload: DesktopNotificationPayload): Promise => diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx index 20d72828869..02f6d793a1c 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx @@ -7,6 +7,7 @@ import { useParams, useRouter } from 'next/navigation' import { getDesktopBridge, getDesktopShellVersion } from '@/lib/desktop' import { SettingsPanel } from '@/app/workspace/[workspaceId]/settings/components/settings-panel' import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' +import { useDesktopPreferenceMutation } from '@/hooks/use-desktop-preference-mutation' import { useDesktopUpdateState } from '@/hooks/use-desktop-update-state' interface PreferenceRowProps { @@ -60,12 +61,19 @@ export function Desktop() { } } + const { pending: preventSleepPending, mutate: setPreventSleep } = useDesktopPreferenceMutation( + async (bridge, enabled: boolean) => bridge.settings.setPreventSleepWhileRunning?.(enabled), + 'Could not update desktop settings', + setPreferences + ) + if (!preferences) { return null } const notificationsDisabled = !preferences.notificationsEnabled || pendingPreference === 'notificationsEnabled' + const supportsPreventSleep = Boolean(getDesktopBridge()?.settings.setPreventSleepWhileRunning) return ( @@ -90,6 +98,22 @@ export function Desktop() { disabled={pendingPreference !== null} onCheckedChange={(checked) => void updatePreference('launchAtLogin', checked)} /> + {supportsPreventSleep && ( +
+
+ +

+ Closing the lid still puts your computer to sleep +

+
+ void setPreventSleep(checked)} + /> +
+ )} + /** + * Keeps the machine awake while a chat runs desktop work in the background. Optional for + * compatibility with installed shells that predate the background executor. + */ + setPreventSleepWhileRunning?(enabled: boolean): Promise notify(payload: DesktopNotificationPayload): Promise /** Overrides the appearance requested by browser pages. */ setBrowserTheme(theme: DesktopAppearanceTheme): Promise From 168421cc7568fc6866ce5715827529c96e88fcdb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 08:32:35 -0700 Subject: [PATCH 2/6] fix(desktop): give the page back to the user mid-action - A browser action now rechecks, right before each input it sends, whether the user is working in the page. Between a batch's steps or during a click's target probes, the user wins and the action stops before its next input. - The wait for the user is bounded inside the action's own deadline. A user who keeps working gets the model a clear "not run" instead of a timeout with an unknown outcome. - The user's activity belongs to the tab they acted in. Once the agent drives another tab (the old one closed or replaced, the session reopened), that activity no longer holds anything up. - `createSleepBlocker` is documented. --- apps/desktop/e2e/background-executor.spec.ts | 38 ++++++++++++++++ apps/desktop/src/main/browser-agent/driver.ts | 45 +++++++++++++++---- .../desktop/src/main/browser-agent/session.ts | 29 +++++++----- .../main/desktop-executor/sleep-blocker.ts | 5 +++ 4 files changed, 97 insertions(+), 20 deletions(-) diff --git a/apps/desktop/e2e/background-executor.spec.ts b/apps/desktop/e2e/background-executor.spec.ts index f40d6c499f2..7f3a6be737a 100644 --- a/apps/desktop/e2e/background-executor.spec.ts +++ b/apps/desktop/e2e/background-executor.spec.ts @@ -769,6 +769,44 @@ test.describe('background executor', () => { }) }) + test('I: an action the user never stops working long enough for does not run', async () => { + app = (await launch(mkdtempSync(join(tmpdir(), 'sim-executor-i2-')))).app + const deviceId = await registeredDevice() + const opened = sim.issue(deviceId, CHAT_A, 'browser_open_url', { + url: `${sim.origin}/counter?chat=I2`, + }) + const outline = ((await settled(opened)).data?.snapshot as { outline: string }).outline + const button = refFor(outline, 'Count visit') + const typeInAgentPage = () => + app?.evaluate(({ webContents }) => { + const page = webContents + .getAllWebContents() + .find((contents) => contents.getURL().includes('/counter?chat=I2')) + page?.sendInputEvent({ type: 'keyDown', keyCode: 'Tab' }) + page?.sendInputEvent({ type: 'keyUp', keyCode: 'Tab' }) + }) + + await typeInAgentPage() + const click = sim.issue(deviceId, CHAT_A, 'browser_click', { elementId: button }) + let typing = true + const keepTyping = (async () => { + while (typing) { + await typeInAgentPage() + await sleep(1_000) + } + })() + + await check('I: the click reports it never ran, instead of timing out', async () => { + const completion = await settled(click, 60_000) + typing = false + await keepTyping + expect(completion.status).toBe('error') + expect(completion.message).toContain('Not run: the user kept working in this page') + expect(completion.data).not.toMatchObject({ outcomeUnknown: true }) + expect(sim.hits.get('I2') ?? 0).toBe(0) + }) + }) + test('J: the machine stays awake only while a chat has work running', async () => { app = (await launch(mkdtempSync(join(tmpdir(), 'sim-executor-j-')))).app await app.evaluate(({ powerSaveBlocker }) => { diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index cad875d655d..d348b40f3c5 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -5096,6 +5096,12 @@ function withNotices(result: unknown): unknown { /** Quiet time after the user's last click or keystroke before the agent takes the page back. */ const USER_TAKEOVER_IDLE_MS = 4_000 +/** The longest an action waits for the user to finish; past it, the action does not run. */ +const USER_TAKEOVER_MAX_WAIT_MS = 30_000 +const USER_KEPT_WORKING = + 'Not run: the user kept working in this page, so this browser action never started and nothing was sent to the page. Ask the user whether they are done before acting in this page again.' +const USER_TOOK_OVER_MID_ACTION = + 'Stopped: the user started working in this page, so this browser action stopped before sending its next input. Earlier steps may have taken effect; inspect the page before continuing.' function isUserWorkingInPage(): boolean { const since = session.msSinceUserIntervention() @@ -5107,14 +5113,25 @@ function isUserWorkingInPage(): boolean { * or scrolling in the tab the agent drives, the agent's next action waits, marked as needing * attention, and resumes once the user has left the page alone for a few seconds. */ -async function yieldToUser(toolCallId: string | undefined, signal: AbortSignal): Promise { +async function yieldToUser( + toolCallId: string | undefined, + signal: AbortSignal, + maxWaitMs: number +): Promise { session.setAutomationNeedsAttention(true) logger.info('Browser automation yielding to the user', { toolCallId }) - while (isUserWorkingInPage()) { - await interruptibleSleep(250, signal) - if (signal.aborted) throw new ToolError('This browser action was cancelled.') + // Bounded well inside the action's own deadline, so a user who keeps working gets the model a + // clear "not run" rather than a timeout with an unknown outcome. + const deadline = Date.now() + maxWaitMs + try { + while (isUserWorkingInPage()) { + if (Date.now() >= deadline) throw new ToolError(USER_KEPT_WORKING) + await interruptibleSleep(250, signal) + if (signal.aborted) throw new ToolError('This browser action was cancelled.') + } + } finally { + session.setAutomationNeedsAttention(false) } - session.setAutomationNeedsAttention(false) logger.info('Browser automation resumed after the user stopped', { toolCallId }) } @@ -5214,20 +5231,32 @@ export async function executeTool( session.setAutomationActive(true) } try { - if (tool !== 'browser_request_takeover' && isUserWorkingInPage()) { - await yieldToUser(toolCallId, executionController.signal) + const yieldsToUser = tool !== 'browser_request_takeover' + const watchdogMs = browserToolWatchdogMs(tool, params) + if (yieldsToUser && isUserWorkingInPage()) { + await yieldToUser( + toolCallId, + executionController.signal, + watchdogMs === null + ? USER_TAKEOVER_MAX_WAIT_MS + : Math.min(USER_TAKEOVER_MAX_WAIT_MS, Math.floor(watchdogMs / 2)) + ) } const response = dialogResponse(tool, params) state.dialogResponse = response ? { contents: session.requireAutomationTab().view.webContents, response } : null const executionEpoch = ++state.toolExecutionEpoch - const watchdogMs = browserToolWatchdogMs(tool, params) const executionDeadline = watchdogMs === null ? undefined : Date.now() + watchdogMs + // Checked right before each input the action sends: a user who starts working in the + // page mid-action (between a batch's steps, during a click's target probes) wins it. const assertCurrentExecution = () => { if (state.toolExecutionEpoch !== executionEpoch) { throw new ToolError('This browser action expired before it could dispatch input.') } + if (yieldsToUser && isUserWorkingInPage()) { + throw new ToolError(USER_TOOK_OVER_MID_ACTION) + } } let actionOutcome: BrowserActionOutcome | undefined const execution = withPostActionObservation( diff --git a/apps/desktop/src/main/browser-agent/session.ts b/apps/desktop/src/main/browser-agent/session.ts index ea7d77dea38..8cc86bd9b0b 100644 --- a/apps/desktop/src/main/browser-agent/session.ts +++ b/apps/desktop/src/main/browser-agent/session.ts @@ -298,8 +298,11 @@ interface BrowserScopeState { */ findingTabId: string | null findingRequestId: number | null - /** When the user last clicked, scrolled or typed in the tab the agent drives. */ - userInterventionAt: number | null + /** + * When the user last clicked, scrolled or typed in the tab the agent drives, and in which tab: + * activity in a tab the agent no longer drives (closed, replaced) holds nothing up. + */ + userIntervention: { tabId: string; at: number } | null } function createBrowserScopeState(): BrowserScopeState { @@ -321,7 +324,7 @@ function createBrowserScopeState(): BrowserScopeState { automationNeedsAttention: false, findingTabId: null, findingRequestId: null, - userInterventionAt: null, + userIntervention: null, } } @@ -2371,7 +2374,7 @@ function initializeTabView( return } const tab = tabs.find((entry) => entry.view.webContents === contents) - if (tab?.id === currentScope.automationTabId) noteUserIntervention() + if (tab && tab.id === currentScope.automationTabId) noteUserIntervention(tab.id) if (tab?.id === currentScope.activeTabId) { currentScope.visibleTabUserSelected = true if (mouse.type === 'mouseDown') tab.lastRealUserGestureAt = Date.now() @@ -2491,9 +2494,10 @@ function initializeTabView( if ( !isDispatchingAgentInput(contents) && input.type === 'keyDown' && - tab?.id === currentScope.automationTabId + tab && + tab.id === currentScope.automationTabId ) { - noteUserIntervention() + noteUserIntervention(tab.id) } if (!isDispatchingAgentInput(contents) && tab?.id === currentScope.activeTabId) { currentScope.visibleTabUserSelected = true @@ -3129,19 +3133,20 @@ export function claimActiveTabForUser(): AgentTab | null { } /** The user just acted in the page the agent drives; the agent yields until they stop. */ -function noteUserIntervention(): void { - currentScope.userInterventionAt = Date.now() +function noteUserIntervention(tabId: string): void { + currentScope.userIntervention = { tabId, at: Date.now() } } -/** How long ago the user last acted in the current scope's agent tab, or null if never. */ +/** How long ago the user last acted in the tab the agent drives now, or null if never. */ export function msSinceUserIntervention(): number | null { - const at = currentScope.userInterventionAt - return at === null ? null : Math.max(0, Date.now() - at) + const intervention = currentScope.userIntervention + if (!intervention || intervention.tabId !== currentScope.automationTabId) return null + return Math.max(0, Date.now() - intervention.at) } /** Explicit hand-back after takeover lets automation resume in the same page. */ export function returnAutomationTabToAgent(): void { - currentScope.userInterventionAt = null + currentScope.userIntervention = null if (currentScope.activeTabId === currentScope.automationTabId) { currentScope.visibleTabUserSelected = false } diff --git a/apps/desktop/src/main/desktop-executor/sleep-blocker.ts b/apps/desktop/src/main/desktop-executor/sleep-blocker.ts index 218bf813390..0b6a2ca82aa 100644 --- a/apps/desktop/src/main/desktop-executor/sleep-blocker.ts +++ b/apps/desktop/src/main/desktop-executor/sleep-blocker.ts @@ -16,6 +16,11 @@ interface SleepBlockerDeps { powerSaveBlocker: PowerSaveBlocker } +/** + * Holds one `prevent-app-suspension` power blocker while the executor is busy and the switch is + * on, and releases it as soon as either stops being true. `setBusy` follows the executor; + * `refresh` follows the switch. + */ export function createSleepBlocker(deps: SleepBlockerDeps) { let busy = false let blockerId: number | null = null From dc952f1afc9dcab927f3ac2ece3629e7b12b8342 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 12:31:28 -0700 Subject: [PATCH 3/6] test(desktop): read the blockers the OS holds instead of asserting the mock was called --- .../desktop-executor/sleep-blocker.test.ts | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/apps/desktop/src/main/desktop-executor/sleep-blocker.test.ts b/apps/desktop/src/main/desktop-executor/sleep-blocker.test.ts index 7954b224db3..870cb9e08bb 100644 --- a/apps/desktop/src/main/desktop-executor/sleep-blocker.test.ts +++ b/apps/desktop/src/main/desktop-executor/sleep-blocker.test.ts @@ -1,25 +1,25 @@ -import { describe, expect, it, vi } from 'vitest' +import { describe, expect, it } from 'vitest' import { createSleepBlocker } from '@/main/desktop-executor/sleep-blocker' function harness(enabled = true) { let nextId = 1 - const active = new Set() + /** The blockers the OS holds now, by id, with the kind each one prevents. */ + const active = new Map() const powerSaveBlocker = { - start: vi.fn(() => { + start: (type: 'prevent-app-suspension') => { const id = nextId++ - active.add(id) + active.set(id, type) return id - }), - stop: vi.fn((id: number) => { + }, + stop: (id: number) => { active.delete(id) - }), + }, isStarted: (id: number) => active.has(id), } let preference = enabled const blocker = createSleepBlocker({ enabled: () => preference, powerSaveBlocker }) return { blocker, - powerSaveBlocker, active, setPreference: (value: boolean) => { preference = value @@ -30,12 +30,11 @@ function harness(enabled = true) { describe('keeping the machine awake for background work', () => { it('holds one blocker only while a chat has work running', () => { - const { blocker, powerSaveBlocker, active } = harness() + const { blocker, active } = harness() blocker.setBusy(true) blocker.setBusy(true) - expect(active.size).toBe(1) - expect(powerSaveBlocker.start).toHaveBeenCalledWith('prevent-app-suspension') + expect([...active.values()]).toEqual(['prevent-app-suspension']) blocker.setBusy(false) expect(active.size).toBe(0) From 7dd5b05bce96205d1a8d56962914c1f91dc53f76 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 13:58:25 -0700 Subject: [PATCH 4/6] fix(desktop): yield the page only for background calls that act on it - Only a call from the background executor yields the page to the user. A call from the chat view the user is watching acts exactly as before: there the user steers the agent directly. - A background call that only reads the page (snapshot, read text, list tabs) never waits for the user, since it cannot collide with their input. - Which browser tools only observe the page now lives in @sim/browser-protocol, shared by the chat view's replay policy and the driver. - The prevent-sleep switch shows only when Sim runs chats on this device in the background. - Tests: a chat-view call acts at once under user input; a background call waits, and a background read does not; a user who starts working mid-action stops it before its next input; sign-out releases the sleep blocker while a call is still running. --- .../src/main/browser-agent/driver.test.ts | 96 +++++++++++++++++++ apps/desktop/src/main/browser-agent/driver.ts | 20 +++- .../src/main/desktop-executor/service.test.ts | 58 ++++++++++- apps/desktop/src/main/index.ts | 5 +- .../settings/components/desktop/desktop.tsx | 9 +- .../tools/client/browser-tool-execution.ts | 42 ++------ packages/browser-protocol/src/index.ts | 40 ++++++++ 7 files changed, 226 insertions(+), 44 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 364a94aeb4a..a01264688a5 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -79,6 +79,102 @@ describe('executeTool', () => { driver = freshDriver() }) + /** The user presses a key in the page the agent drives, as the browser reports it. */ + function typeInAgentPage(contents: WebContents) { + const calls: ReadonlyArray = vi.mocked(contents.on).mock.calls + const listener = calls.find(([event]) => event === 'before-input-event')?.[1] + if (typeof listener !== 'function') throw new Error('No input listener on the agent page') + listener({}, { type: 'keyDown', isAutoRepeat: false }) + } + + it('lets a call from the chat view act at once while the user works in the page', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + typeInAgentPage(session.requireTab().view.webContents) + expect(session.msSinceUserIntervention()).not.toBeNull() + vi.useFakeTimers() + try { + let settled = false + const opening = driver.executeTool('chat-test', 'browser_open_tab', {}).then((result) => { + settled = true + return result + }) + await vi.advanceTimersByTimeAsync(100) + + expect(settled).toBe(true) + expect((await opening).ok).toBe(true) + } finally { + vi.useRealTimers() + } + }) + + it('makes a background call wait until the user leaves the page alone', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + typeInAgentPage(session.requireTab().view.webContents) + vi.useFakeTimers() + try { + let settled = false + const opening = driver + .executeTool('chat-test', 'browser_open_tab', {}, 'background-1', undefined, { + background: true, + }) + .then((result) => { + settled = true + return result + }) + await vi.advanceTimersByTimeAsync(1_000) + expect(settled).toBe(false) + + await vi.advanceTimersByTimeAsync(4_000) + expect((await opening).ok).toBe(true) + } finally { + vi.useRealTimers() + } + }) + + it('stops a background action before its next input once the user starts working mid-action', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + const contents = session.requireTab().view.webContents + // The user presses a key while the action reads the page before acting on it. + const probe = vi.mocked(contents.executeJavaScript).getMockImplementation() + vi.mocked(contents.executeJavaScript).mockImplementation(async (...args) => { + typeInAgentPage(contents) + return probe ? probe(...args) : undefined + }) + + const result = await driver.executeTool( + 'chat-test', + 'browser_press_key', + { key: 'Enter' }, + 'background-key', + undefined, + { background: true } + ) + + expect(result.ok).toBe(false) + expect(result.error).toContain('the user started working in this page') + }) + + it('never makes a background read wait for the user', async () => { + await driver.executeTool('chat-test', 'browser_open_tab', {}) + typeInAgentPage(session.requireTab().view.webContents) + vi.useFakeTimers() + try { + let settled = false + void driver + .executeTool('chat-test', 'browser_list_tabs', {}, 'background-read', undefined, { + background: true, + }) + .then(() => { + settled = true + }) + await vi.advanceTimersByTimeAsync(100) + + expect(settled).toBe(true) + } finally { + vi.useRealTimers() + } + }) + it('returns ok:false instead of throwing for tool-level failures', async () => { // No session exists, so any page-dependent tool fails with guidance. const result = await driver.executeTool('chat-test', 'browser_click', { elementId: 1 }) diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index d348b40f3c5..8e6922a6a7b 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -19,6 +19,7 @@ import { BROWSER_DATA_KINDS, BROWSER_NAVIGATION_NATIVE_WATCHDOG_MS, + BROWSER_TOOL_OBSERVES_ONLY, BROWSER_TOOL_QUEUE_WAIT_TIMEOUT_MS, BROWSER_UPLOAD_MAX_FILES, type BrowserDataKind, @@ -5135,12 +5136,23 @@ async function yieldToUser( logger.info('Browser automation resumed after the user stopped', { toolCallId }) } +/** How a browser call reached the driver. */ +interface BrowserToolExecutionOptions { + /** + * Run by the background executor while the user may be working in the same page, so it yields + * the page to them. A call from the chat view the user is watching never does: there the user + * steers the agent directly, as they always have. + */ + background?: boolean +} + export async function executeTool( scopeId: string, tool: BrowserToolName, params: Record, toolCallId?: string, - authorizationBoundary?: BrowserToolQueueBoundary + authorizationBoundary?: BrowserToolQueueBoundary, + options: BrowserToolExecutionOptions = {} ): Promise<{ ok: boolean; result?: unknown; error?: string }> { const resolvedScopeId = resolveDriverScopeId(scopeId) if (authorizationBoundary) { @@ -5231,7 +5243,11 @@ export async function executeTool( session.setAutomationActive(true) } try { - const yieldsToUser = tool !== 'browser_request_takeover' + // Only a background call acting on the page yields; reading it cannot collide with the user. + const yieldsToUser = + options.background === true && + tool !== 'browser_request_takeover' && + !BROWSER_TOOL_OBSERVES_ONLY[tool] const watchdogMs = browserToolWatchdogMs(tool, params) if (yieldsToUser && isUserWorkingInPage()) { await yieldToUser( diff --git a/apps/desktop/src/main/desktop-executor/service.test.ts b/apps/desktop/src/main/desktop-executor/service.test.ts index ce81c6cc1fb..20ff6168e68 100644 --- a/apps/desktop/src/main/desktop-executor/service.test.ts +++ b/apps/desktop/src/main/desktop-executor/service.test.ts @@ -12,6 +12,8 @@ import { createDesktopExecutorService, deviceName } from '@/main/desktop-executo /** Sim's device routes, with registration answers held until the test releases them. */ function fakeSim(protocolVersion = 1) { const requests: string[] = [] + /** Calls the inbox offers; a claim answers for the first one. */ + const offered: string[] = [] /** * Answers to pending registrations: enabled or not, an HTTP status Sim fails with, or * `'offline'` for a request that never reached Sim. @@ -37,25 +39,60 @@ function fakeSim(protocolVersion = 1) { reconcileMs: 10_000, }) } - if (path === '/api/desktop/inbox') return Response.json({ items: [] }) + if (path === '/api/desktop/inbox') { + return Response.json({ + items: offered.map((toolCallId) => ({ + kind: 'call', + toolCallId, + toolName: 'terminal', + chatId: 'chat-a', + workspaceId: 'ws-1', + createdAt: new Date().toISOString(), + })), + }) + } + if (path === '/api/desktop/tool/claim') { + const toolCallId = offered.shift() + if (!toolCallId) return Response.json({ error: 'gone' }, { status: 404 }) + return Response.json({ + toolName: 'terminal', + args: { operation: 'run', args: { command: 'sleep 600' } }, + chatId: 'chat-a', + workspaceId: 'ws-1', + executionToken: `token-${toolCallId}`, + }) + } + if (path === '/api/desktop/tool/lease') return Response.json({ renewed: true }) return new Promise((_resolve, reject) => init.signal?.addEventListener('abort', () => reject(new Error('aborted'))) ) }) - return { fetch, requests, registrations } + return { fetch, requests, registrations, offered } } async function service(protocolVersion = 1, userDataPath?: string) { const sim = fakeSim(protocolVersion) + /** What the sleep blocker was told, in order. */ + const busy: boolean[] = [] const desktopExecutor = createDesktopExecutorService({ userDataPath: userDataPath ?? (await mkdtemp(join(tmpdir(), 'sim-executor-service-'))), origin: () => 'https://sim.test', appSession: () => ({ fetch: sim.fetch }), preferences: () => ({ browserEnabled: true, terminalEnabled: true }), accountDataAvailable: () => true, - runner: { run: vi.fn(), cancel: vi.fn() }, + // A command that runs until it is stopped. + runner: { + run: (_call, signal) => + new Promise((resolve) => + signal.addEventListener('abort', () => + resolve({ status: 'cancelled', message: 'Stopped.' }) + ) + ), + cancel: async () => {}, + }, + onBusyChange: (value) => busy.push(value), }) - return { sim, desktopExecutor } + return { sim, desktopExecutor, busy } } describe('desktop executor registration', () => { @@ -177,6 +214,19 @@ describe('desktop executor registration', () => { expect(sim.requests.filter((request) => request.includes('/api/desktop/devices'))).toEqual([]) }) + + it('releases the sleep blocker at sign-out while a call is still running', async () => { + const { sim, desktopExecutor, busy } = await service() + sim.offered.push('call-1') + desktopExecutor.start() + await vi.waitFor(() => expect(sim.registrations).toHaveLength(1)) + sim.registrations[0]?.(true) + await vi.waitFor(() => expect(busy).toEqual([true])) + + await desktopExecutor.signOut() + + expect(busy).toEqual([true, false]) + }) }) describe('device name', () => { diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index a6e33a99329..a2e0b614f37 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -605,7 +605,10 @@ function main(): void { preferences: () => desktopSettings.getPreferences(), accountDataAvailable, browser: { - executeTool: executeAgentBrowserTool, + executeTool: (scopeId, tool, params, toolCallId) => + executeAgentBrowserTool(scopeId, tool, params, toolCallId, undefined, { + background: true, + }), cancelTool: cancelAgentBrowserTool, hasSession: hasBrowserScopeSession, restoreScope: restoreAgentBrowserScope, diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx index 02f6d793a1c..b6a19bf86ca 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx @@ -33,6 +33,8 @@ export function Desktop() { const workspaceId = params.workspaceId as string const [preferences, setPreferences] = useState(null) const [pendingPreference, setPendingPreference] = useState(null) + /** Whether Sim runs chats on this device in the background; only then can sleep matter. */ + const [runsInBackground, setRunsInBackground] = useState(false) const updateState = useDesktopUpdateState() const shellVersion = getDesktopShellVersion() @@ -46,6 +48,10 @@ export function Desktop() { .getPreferences() .then(setPreferences) .catch(() => toast.error('Could not load desktop settings')) + void bridge.desktopExecutor + ?.getDevice() + .then((device) => setRunsInBackground(device !== null)) + .catch(() => setRunsInBackground(false)) }, [router, workspaceId]) const updatePreference = async (key: DesktopPreferenceKey, value: boolean) => { @@ -73,7 +79,8 @@ export function Desktop() { const notificationsDisabled = !preferences.notificationsEnabled || pendingPreference === 'notificationsEnabled' - const supportsPreventSleep = Boolean(getDesktopBridge()?.settings.setPreventSleepWhileRunning) + const supportsPreventSleep = + runsInBackground && Boolean(getDesktopBridge()?.settings.setPreventSleepWhileRunning) return ( diff --git a/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts b/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts index cb5586a8e0d..129f8e56b56 100644 --- a/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts +++ b/apps/sim/lib/mothership/tools/client/browser-tool-execution.ts @@ -7,7 +7,11 @@ * browser and reports the outcome via the confirm endpoint, which wakes the * server-side waiter. */ -import { type BrowserToolName, browserToolRendererTimeoutMs } from '@sim/browser-protocol' +import { + BROWSER_TOOL_OBSERVES_ONLY, + type BrowserToolName, + browserToolRendererTimeoutMs, +} from '@sim/browser-protocol' import { browserSessionClosedCompletion, browserToolCompletion, @@ -44,41 +48,7 @@ const logger = createLogger('CopilotBrowserToolExecution') * reload cannot cause a page or external side effect. Every stateful current * tool and the retired takeover flow remain fail-closed. */ -const OBSERVATION_ONLY_BROWSER_TOOLS = { - browser_navigate: false, - browser_open_url: false, - browser_go_back: false, - browser_go_forward: false, - browser_reload: false, - browser_open_tab: false, - browser_switch_tab: false, - browser_close_tab: false, - browser_list_tabs: true, - browser_list_sessions: true, - browser_list_downloads: true, - browser_save_download: false, - browser_wait_for: true, - browser_snapshot: true, - browser_find: true, - browser_read_text: true, - browser_screenshot: true, - browser_extract: true, - browser_click: false, - browser_click_at: false, - browser_type: false, - browser_fill_form: false, - browser_batch: false, - browser_insert_text: false, - browser_press_key: false, - browser_scroll: false, - browser_select_option: false, - browser_set_checked: false, - browser_upload_file: false, - browser_hover: false, - browser_drag: false, - browser_zoom: false, - browser_request_takeover: false, -} as const satisfies Readonly> +const OBSERVATION_ONLY_BROWSER_TOOLS = BROWSER_TOOL_OBSERVES_ONLY /** Tool events older than this are replays, not live instructions — never act on them. */ const MAX_EVENT_AGE_MS = 120_000 diff --git a/packages/browser-protocol/src/index.ts b/packages/browser-protocol/src/index.ts index 4aab432ff1c..569b824ba96 100644 --- a/packages/browser-protocol/src/index.ts +++ b/packages/browser-protocol/src/index.ts @@ -150,6 +150,46 @@ export function isCurrentBrowserToolName(name: string): name is CurrentBrowserTo return CURRENT_BROWSER_TOOL_NAME_SET.has(name) } +/** + * Which browser tools only observe the page (read it, list it, wait on it) and never act on it. + * Repeating one cannot cause a side effect, and one can never collide with what the user is doing. + */ +export const BROWSER_TOOL_OBSERVES_ONLY = { + browser_navigate: false, + browser_open_url: false, + browser_go_back: false, + browser_go_forward: false, + browser_reload: false, + browser_open_tab: false, + browser_switch_tab: false, + browser_close_tab: false, + browser_list_tabs: true, + browser_list_sessions: true, + browser_list_downloads: true, + browser_save_download: false, + browser_wait_for: true, + browser_snapshot: true, + browser_find: true, + browser_read_text: true, + browser_screenshot: true, + browser_extract: true, + browser_click: false, + browser_click_at: false, + browser_type: false, + browser_fill_form: false, + browser_batch: false, + browser_insert_text: false, + browser_press_key: false, + browser_scroll: false, + browser_select_option: false, + browser_set_checked: false, + browser_upload_file: false, + browser_hover: false, + browser_drag: false, + browser_zoom: false, + browser_request_takeover: false, +} as const satisfies Readonly> + export function isBrowserTheme(value: unknown): value is BrowserTheme { return typeof value === 'string' && BROWSER_THEME_SET.has(value) } From d6e0938aa4d382feafd3e022185ea4093779d833 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 16:57:20 -0700 Subject: [PATCH 5/6] test(desktop): give the notification tests the prevent-sleep setter the settings service now requires --- apps/desktop/src/main/desktop-settings.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/desktop/src/main/desktop-settings.test.ts b/apps/desktop/src/main/desktop-settings.test.ts index fdc2e135d8f..436125a513c 100644 --- a/apps/desktop/src/main/desktop-settings.test.ts +++ b/apps/desktop/src/main/desktop-settings.test.ts @@ -29,6 +29,7 @@ function service(window: { url: string; focused: boolean } | null) { setBrowserTheme: vi.fn(), setBrowserDefaultZoom: vi.fn(), setTerminalDefaultZoom: vi.fn(), + setPreventSleepWhileRunning: vi.fn(), getDefaultBrowserDownloadDirectory: () => '/tmp', chooseBrowserDownloadDirectory: async () => null, }) From 8a624c911f711661323c9778f27c3a846478e2f2 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 6 Oct 2026 16:57:21 -0700 Subject: [PATCH 6/6] test(desktop): show the prevent-sleep switch only on a device that runs chats in the background --- .../components/desktop/desktop.test.tsx | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.test.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.test.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.test.tsx new file mode 100644 index 00000000000..64a7b951fd8 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.test.tsx @@ -0,0 +1,84 @@ +/** + * @vitest-environment jsdom + */ +import { act, type ReactNode } from 'react' +import { emcnMock } from '@sim/testing/mocks/emcn.mock' +import { libDesktopMock, libDesktopMockFns } from '@sim/testing/mocks/lib-desktop.mock' +import { nextNavigationMock } from '@sim/testing/mocks/next-navigation.mock' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@sim/emcn', () => ({ + ...emcnMock, + Label: ({ children, htmlFor }: { children?: ReactNode; htmlFor?: string }) => ( + + ), + Switch: ({ id, checked }: { id: string; checked: boolean }) => ( +