Merge remote-tracking branch 'origin/staging' into feat/desktop-windows-v2

Author: waleedlatif1

apps/desktop/README.md

@@ -267,7 +267,8 @@ group with the **Computer Use** restriction. Older desktop builds omit this opti
 continue to work.
 
 The user grants Accessibility and Screen Recording through macOS; every app is then available except
-Sim, System Settings, password managers and terminals (shell access has its own Terminal control),
+Sim, System Settings, password managers, terminals and scripting apps (shell access has its own
+Terminal control),
 which the helper refuses. With **Ask before using a new
 app** on, each app's first use is approved for the task or persistently, and persistent approvals are
 listed in Settings and can be revoked. Sign-out and deployment changes clear opt-in, the ask choice

apps/desktop/native/computer-use/ComputerUse.swift

@@ -142,15 +142,18 @@ func elementRect(_ element: AXUIElement) -> CGRect? {
           AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil }
     return CGRect(origin: position, size: size)
 }
-/// Apps computer use never operates, lowercased: macOS security settings, password managers, and
-/// terminals, whose shell access has its own Terminal control.
+/// Apps computer use never operates, lowercased: macOS security settings, password managers, and apps
+/// whose primary purpose is running shell commands or scripts (terminals, Script Editor, Automator,
+/// Shortcuts), since shell access goes through the Terminal tool's own opt-in. Apps that merely embed a
+/// terminal, such as editors, stay usable.
 let PROTECTED_APPS: Set<String> = [
     "com.apple.systempreferences", "com.apple.keychainaccess", "com.apple.passwords",
     "com.1password.1password", "com.agilebits.onepassword7", "com.agilebits.onepassword-osx", "com.bitwarden.desktop",
     "org.keepassxc.keepassxc", "com.lastpass.lastpass", "in.sinew.enpass-desktop", "me.proton.pass.electron",
     "com.markmcguill.strongbox", "com.markmcguill.strongbox.mac",
     "com.apple.terminal", "com.googlecode.iterm2", "dev.warp.warp-stable", "com.mitchellh.ghostty",
     "net.kovidgoyal.kitty", "org.alacritty", "com.github.wez.wezterm", "co.zeit.hyper", "org.tabby",
+    "com.apple.scripteditor2", "com.apple.automator", "com.apple.shortcuts",
 ]
 /// Whether computer use refuses an app: Sim's own apps and `PROTECTED_APPS`. Bundle identifiers match
 /// case-insensitively in Launch Services.
@@ -161,7 +164,7 @@ func isProtectedApp(_ bundle: String) -> Bool {
 func appFor(_ bundleId: String?, launch: Bool = false) async throws -> NSRunningApplication {
     let bundle = try required(bundleId, "bundleId")
     guard !isProtectedApp(bundle) else {
-        throw ComputerError("protected_app", "Computer use does not operate Sim, System Settings, password managers, or terminals; use the terminal tool for shell work.")
+        throw ComputerError("protected_app", "Computer use does not operate Sim, System Settings, password managers, terminals, or scripting apps; use the terminal tool for shell work.")
     }
     var apps = NSRunningApplication.runningApplications(withBundleIdentifier: bundle).filter { !$0.isTerminated }
     if apps.isEmpty && launch {

apps/desktop/native/computer-use/README.md

@@ -10,7 +10,7 @@ Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` return
 
 The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized. The menu bar and menus are read in place, so an open menu's items and their submenus come first; items of closed menus are read last, capped at 80 menu nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. An Accessibility failure counts as sent unless Accessibility rejected the request before the app saw it (unsupported attribute or action, illegal argument). Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise.
 
-Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected, and an element target uses its own window; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps, System Settings, password managers and terminals are protected targets (`PROTECTED_APPS`, matched case-insensitively); shell work belongs to the separate Terminal control. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`.
+Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected, and an element target uses its own window; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps, System Settings, password managers, terminals and scripting apps (Script Editor, Automator, Shortcuts) are protected targets (`PROTECTED_APPS`, matched case-insensitively); shell work belongs to the separate Terminal control. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`.
 
 AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A hit on a window's own background counts as that window. Background tabs and minimized windows share bounds with the window in front, so they take no coordinate input. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves the item's own menu open with the app highlighting that item (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly.
 

apps/desktop/native/computer-use/tests/NativeTests.swift

@@ -26,7 +26,7 @@ import AppKit
         expectError("unsupported_key") { _ = try parseKey("F99") }
         expectError("invalid_arguments") { _ = try parseKey("Hyper+A") }
         expectError("invalid_arguments") { _ = try required(Optional<String>.none, "example") }
-        for bundle in ["com.apple.systempreferences", "com.apple.Terminal", "COM.GOOGLECODE.ITERM2", "net.kovidgoyal.kitty", "com.1password.1password", "com.apple.Passwords", "com.bitwarden.desktop", "ai.sim.desktop", "com.simstudio.computer-use"] {
+        for bundle in ["com.apple.systempreferences", "com.apple.Terminal", "COM.GOOGLECODE.ITERM2", "net.kovidgoyal.kitty", "com.1password.1password", "com.apple.Passwords", "com.bitwarden.desktop", "ai.sim.desktop", "com.simstudio.computer-use", "com.apple.ScriptEditor2", "com.apple.Automator", "com.apple.shortcuts"] {
             precondition(isProtectedApp(bundle), bundle)
         }
         for bundle in ["com.apple.TextEdit", "com.apple.finder", "com.apple.Safari"] { precondition(!isProtectedApp(bundle), bundle) }

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -1894,6 +1894,7 @@ describe('credential protection', () => {
         activeElementSecrecy: 'safe',
         readActiveElementState: {},
         readPageActionState: {},
+        readDomQuiet: { quietMs: 1_000, loading: false },
       })
       const pageCall = vi.mocked(contents.executeJavaScript).getMockImplementation()
       let releaseObservation: (value: unknown) => void = () => {}
@@ -3212,6 +3213,113 @@ describe('credential protection', () => {
     })
   })
 
+  it.each([
+    ['still rendering', [0, 40, 90, 200], 4],
+    ['already quiet', [400], 1],
+    ['never settling', Array.from({ length: 200 }, () => 0), undefined],
+  ] as const)(
+    'observes after an action only once a page %s has gone quiet, within the settle cap',
+    async (_label, quietReadings, expectedProbes) => {
+      const contents = await openPage()
+      const calls: string[] = []
+      let probe = 0
+      vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
+        if (isPageCall(expression, 'readDomQuiet')) {
+          calls.push('probe')
+          const quietMs = quietReadings[Math.min(probe++, quietReadings.length - 1)]
+          return Promise.resolve({ quietMs, loading: false })
+        }
+        if (isPageCall(expression, 'collectSnapshot')) {
+          calls.push('snapshot')
+          return Promise.resolve({
+            url: 'https://example.com/',
+            title: 'Example',
+            outline: '- heading "Results"',
+            truncated: false,
+            refIds: [],
+            refLineIndexes: {},
+            nextElementId: 1_000,
+          })
+        }
+        if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET)
+        return Promise.resolve(undefined)
+      })
+      vi.useFakeTimers()
+      try {
+        const clicking = driver.executeTool('chat-test', 'browser_click', {
+          elementId: 0,
+          observe: {},
+        })
+        await vi.advanceTimersByTimeAsync(5_000)
+
+        expect(await clicking).toMatchObject({ ok: true, result: { observation: { ok: true } } })
+        const probesBeforeSnapshot = calls.indexOf('snapshot')
+        expect(probesBeforeSnapshot).toBeGreaterThan(0)
+        if (expectedProbes !== undefined) expect(probesBeforeSnapshot).toBe(expectedProbes)
+      } finally {
+        vi.useRealTimers()
+      }
+    }
+  )
+
+  it('keeps settling through failed quiet probes, and never observes once cancelled', async () => {
+    const contents = await openPage()
+    const calls: string[] = []
+    let failing = 2
+    vi.mocked(contents.executeJavaScript).mockImplementation((expression: string) => {
+      if (isPageCall(expression, 'readDomQuiet')) {
+        calls.push('probe')
+        if (failing > 0) {
+          failing--
+          vi.mocked(contents.isLoading).mockReturnValue(true)
+          return Promise.reject(new Error('Execution context was destroyed'))
+        }
+        vi.mocked(contents.isLoading).mockReturnValue(false)
+        return Promise.resolve({ quietMs: failing === -1 ? 0 : 400, loading: false })
+      }
+      if (isPageCall(expression, 'collectSnapshot')) {
+        calls.push('snapshot')
+        return Promise.resolve({
+          url: 'https://example.com/',
+          title: 'Example',
+          outline: '- heading "Results"',
+          truncated: false,
+          refIds: [],
+          refLineIndexes: {},
+          nextElementId: 1_000,
+        })
+      }
+      if (isPageCall(expression, 'clickElement')) return Promise.resolve(CLICK_TARGET)
+      return Promise.resolve(undefined)
+    })
+    vi.useFakeTimers()
+    try {
+      const settled = driver.executeTool('chat-test', 'browser_click', {
+        elementId: 0,
+        observe: {},
+      })
+      await vi.advanceTimersByTimeAsync(5_000)
+      expect(await settled).toMatchObject({ ok: true, result: { observation: { ok: true } } })
+      expect(calls.indexOf('snapshot')).toBe(3)
+
+      calls.length = 0
+      failing = -1
+      const cancelled = driver.executeTool(
+        'chat-test',
+        'browser_click',
+        { elementId: 0, observe: { query: 'Results' } },
+        'click-cancelled'
+      )
+      await vi.advanceTimersByTimeAsync(300)
+      expect(driver.cancelTool('chat-test', 'click-cancelled')).toBe(true)
+      await vi.advanceTimersByTimeAsync(5_000)
+      await cancelled
+      expect(calls).not.toContain('snapshot')
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('acts on a ref an earlier snapshot of the same page issued', async () => {
     const contents = await openPage()
     respondWith(contents, {

apps/desktop/src/main/browser-agent/driver.ts

@@ -70,6 +70,7 @@ import {
   readActiveElementState,
   readCheckableElementState,
   readChildFrameElementState,
+  readDomQuiet,
   readFormFieldState,
   readPageActionState,
   readPageText,
@@ -2065,6 +2066,9 @@ async function prepareTypingSurface(
 const TRANSIENT_PROBE_ERRORS = new Set(['stale', 'not-visible', 'not-editable'])
 const SETTLE_GRACE_MS = 1_000
 const SETTLE_PROBE_INTERVAL_MS = 250
+/** Structural quiet that counts as settled, and the most an observation waits for it. */
+const DOM_QUIET_MS = 150
+const DOM_SETTLE_MAX_MS = 2_000
 
 /**
  * First-touch page probe with a short settle grace. A live view re-rendering
@@ -2099,6 +2103,33 @@ async function execInPageWithSettleGrace<Args extends unknown[], Result>(
   }
 }
 
+/**
+ * Lets a page finish rendering what an action started (a route change, a fetched list) before it
+ * is observed, so the observation is not a half-rendered page. A page already quiet costs one probe.
+ */
+async function waitForDomQuiet(
+  target: PageExecutionTarget,
+  signal: AbortSignal,
+  executionDeadline?: number
+) {
+  const until = Math.min(
+    Date.now() + DOM_SETTLE_MAX_MS,
+    executionDeadline ?? Number.POSITIVE_INFINITY
+  )
+  while (!signal.aborted && Date.now() + 25 < until) {
+    const probe = await execInPage(target, readDomQuiet, [], false, executionDeadline).catch(
+      () => null
+    )
+    // A failed probe while the tab loads is a document swapping mid-navigation, so keep waiting;
+    // one that fails on a loaded tab (a page that cannot be scripted) has nothing to settle.
+    if (probe === null && !('isLoading' in target && target.isLoading())) return
+    const state = toRecord(probe)
+    const quietMs = typeof state.quietMs === 'number' ? state.quietMs : 0
+    if (quietMs >= DOM_QUIET_MS && state.loading === false) return
+    await interruptibleSleep(Math.min(100, Math.max(25, DOM_QUIET_MS - quietMs)), signal)
+  }
+}
+
 function pointFromPrepared(prepared: Record<string, unknown>): { x: number; y: number } {
   return { x: prepared.x as number, y: prepared.y as number }
 }
@@ -5463,15 +5494,27 @@ export async function executeTool(
               }
               return result
             },
-            (query) =>
-              executeToolInner(
+            async (query) => {
+              await waitForDomQuiet(
+                session.requireAutomationTab().view.webContents,
+                executionController.signal,
+                executionDeadline
+              )
+              assertCurrentExecution()
+              if (executionController.signal.aborted) {
+                throw new ToolError(
+                  'This browser action was cancelled before it could observe the page.'
+                )
+              }
+              return executeToolInner(
                 query === undefined ? 'browser_snapshot' : 'browser_find',
                 query === undefined ? {} : { query },
                 assertCurrentExecution,
                 executionDeadline,
                 invocationEpoch,
                 executionController.signal
-              ),
+              )
+            },
             assertCurrentExecution
           )
           const cancellableExecution = Promise.race([execution, cancellation])

apps/desktop/src/main/browser-agent/page-functions.test.ts

@@ -17,6 +17,7 @@ import {
   readActiveElementState,
   readCheckableElementState,
   readChildFrameElementState,
+  readDomQuiet,
   readPageActionState,
   readPageText,
   readSelectElementState,
@@ -526,6 +527,28 @@ describe('collectSnapshot', () => {
     expect(outline).not.toContain('button "x" [ref=999]')
   })
 
+  it('counts changes inside a shadow root the snapshot walked as page activity', async () => {
+    vi.useFakeTimers({ toFake: ['performance'] })
+    try {
+      document.body.innerHTML = '<div id="host"></div>'
+      const host = visible(document.querySelector('#host') as HTMLDivElement)
+      const shadow = host.attachShadow({ mode: 'open' })
+      shadow.innerHTML = '<button>Load</button>'
+      visible(shadow.querySelector('button') as HTMLButtonElement)
+      collectSnapshot()
+      await Promise.resolve()
+      vi.advanceTimersByTime(500)
+      expect((readDomQuiet() as { quietMs: number }).quietMs).toBeGreaterThanOrEqual(500)
+
+      shadow.append(document.createElement('ul'))
+      await Promise.resolve()
+
+      expect((readDomQuiet() as { quietMs: number }).quietMs).toBeLessThan(500)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('marks elements that appeared since the previous snapshot as new', () => {
     document.body.innerHTML = '<button>Compose</button>'
     visible(document.querySelector('button') as HTMLButtonElement)