fix(desktop): inbox persistence order, desktop-only overdue sweep, and ungated pickup windows (#8685)

* fix(desktop): scope the overdue sweep to desktop calls and keep ungated pickup windows

- The overdue listing and the per-call deadline read only consider desktop
  tool calls, and the settlement checks the call is one the desktop runs, so
  a bound run's workflow call or Sim-files VFS read is never failed with the
  desktop's not-started result.
- The cron scans only runs inside the inbox's horizon, oldest calls first,
  within its batch limit.
- Recording a decision clears the pickup deadline only for a call that was
  gated; a call that was never gated keeps the window it was offered with.

* fix(desktop): limit the overdue sweep over desktop calls only, and bound it by deadline

The sweep's tool-name filter admitted read, grep and glob calls on Sim's own
files, which the settlement then skipped, so enough of them could fill every
batch and starve real desktop calls. Queries now use the SQL form of
isDesktopToolCall (a desktop tool by name, or a VFS read of a granted local
folder) before their limit.

The sweep's horizon is now on the deadline that lapsed, not on the run's
start: Sim does not enforce a run's wall clock, so a long-lived run's
recently overdue call is still settled.

* fix(desktop): hand a device its calls in the order they were persisted

The inbox ordered calls by created_at, then tool_call_id. Calls of one turn
can be persisted in the same millisecond, and then the tie broke on random
ids: a device could run a click before the type the model emitted first.

Calls now carry persist_seq, a strictly increasing number assigned on insert
(pre-persist writes them in emission order), and the inbox and the overdue
sweep order by it. The migration adds the column without a default and then
sets the default, so existing rows are not rewritten; rows persisted before it
have no position and sort first, as the oldest.

* fix(desktop): reach every overdue desktop call, and declare the desktop tool names as const

A 24 h horizon on the lapsed deadline meant a call the backstop missed for a
day was never settled. The sweep starts from the few unsettled (pending or
running) calls in persistence order, so it needs no horizon to stay small.

The desktop tool names are a literal array declared as const, and the lookup
set is derived from it.

* test(mothership): give the hand-built tool call table the persistence sequence column

Author: waleedlatif1

apps/sim/lib/desktop/application/executor.integration.ts

@@ -865,6 +865,28 @@ describe.runIf(Boolean(redisUrl))('desktop background executor protocol', () =>
       ])
     })
 
+    it('lists calls persisted in the same millisecond in the order they were persisted', async () => {
+      const desktop = await signedInDesktop()
+      const run = await boundRun(desktop)
+      const sameMillisecond = new Date()
+      /** Ids that sort in reverse, so neither the clock nor the id can produce this order. */
+      const persisted = ['type-zz', 'click-mm', 'submit-aa'].map(
+        (name) => `${name}-${generateId()}`
+      )
+      for (const toolCallId of persisted) {
+        await db.insert(copilotAsyncToolCalls).values({
+          runId: run.runId,
+          toolCallId,
+          toolName: 'browser_click',
+          args: { ref: 'e1' },
+          createdAt: sameMillisecond,
+          pickupDeadlineAt: sql`now() + interval '1 minute'`,
+        })
+      }
+
+      expect((await inbox(desktop)).items.map((item) => item.toolCallId)).toEqual(persisted)
+    })
+
     it('lists pending calls in persistence order even when their doorbell was never heard', async () => {
       const desktop = await signedInDesktop()
       const run = await boundRun(desktop)

apps/sim/lib/desktop/executor/bound-turn.integration.ts

@@ -731,6 +731,138 @@ describe.runIf(Boolean(redisUrl))("a turn bound to a desktop's background execut
     TURN_WAIT_MS
   )
 
+  it(
+    'leaves a bound run’s calls the desktop never runs to their own path',
+    async () => {
+      const desktop = await signedInDesktop()
+      const run = await boundRun(desktop)
+      const longAgo = new Date(Date.now() - 600_000)
+      const workflowCall = generateId()
+      const simFileRead = generateId()
+      await db.insert(copilotAsyncToolCalls).values([
+        {
+          runId: run.runId,
+          toolCallId: workflowCall,
+          toolName: 'run_workflow',
+          args: {},
+          createdAt: longAgo,
+        },
+        {
+          runId: run.runId,
+          toolCallId: simFileRead,
+          toolName: 'read',
+          args: { path: 'workspace/notes.md' },
+          createdAt: longAgo,
+        },
+      ])
+
+      await runCleanupStaleExecutions()
+
+      expect((await storedCall(workflowCall)).status).toBe('pending')
+      expect((await storedCall(simFileRead)).status).toBe('pending')
+    },
+    TURN_WAIT_MS
+  )
+
+  it(
+    'reaches an abandoned desktop call however many older Sim-file reads the run left pending',
+    async () => {
+      const desktop = await signedInDesktop()
+      const run = await boundRun(desktop)
+      await db.insert(copilotAsyncToolCalls).values(
+        Array.from({ length: 201 }, () => ({
+          runId: run.runId,
+          toolCallId: generateId(),
+          toolName: 'read',
+          args: { path: 'workspace/notes.md' },
+          createdAt: new Date(Date.now() - 900_000),
+        }))
+      )
+      const abandoned = generateId()
+      await db.insert(copilotAsyncToolCalls).values({
+        runId: run.runId,
+        toolCallId: abandoned,
+        toolName: 'browser_click',
+        args: { ref: 'e1' },
+        createdAt: new Date(Date.now() - 600_000),
+      })
+
+      await runCleanupStaleExecutions()
+
+      expect((await storedCall(abandoned)).status).toBe('failed')
+    },
+    TURN_WAIT_MS
+  )
+
+  it(
+    'still settles a call that stayed overdue for days, however long the backstop missed it',
+    async () => {
+      const desktop = await signedInDesktop()
+      const run = await boundRun(desktop)
+      const toolCallId = generateId()
+      await db.insert(copilotAsyncToolCalls).values({
+        runId: run.runId,
+        toolCallId,
+        toolName: 'browser_click',
+        args: { ref: 'e1' },
+        createdAt: new Date(Date.now() - 2 * 24 * 3_600_000),
+      })
+
+      await runCleanupStaleExecutions()
+
+      expect((await storedCall(toolCallId)).status).toBe('failed')
+    },
+    TURN_WAIT_MS
+  )
+
+  it(
+    'settles a call whose window lapsed recently on a run that started long ago',
+    async () => {
+      const desktop = await signedInDesktop()
+      const run = await boundRun(desktop)
+      await db
+        .update(copilotRuns)
+        .set({ startedAt: new Date(Date.now() - 2 * 24 * 3_600_000) })
+        .where(eq(copilotRuns.id, run.runId))
+      const toolCallId = generateId()
+      await db.insert(copilotAsyncToolCalls).values({
+        runId: run.runId,
+        toolCallId,
+        toolName: 'browser_click',
+        args: { ref: 'e1' },
+        createdAt: new Date(Date.now() - 600_000),
+      })
+
+      await runCleanupStaleExecutions()
+
+      expect((await storedCall(toolCallId)).status).toBe('failed')
+    },
+    TURN_WAIT_MS
+  )
+
+  it(
+    'keeps the pickup window of a call that was never gated when a decision is posted for it',
+    async () => {
+      const desktop = await signedInDesktop()
+      const run = await boundRun(desktop)
+      await desktop.pull()
+
+      const { toolCallId, answer, context } = await agentCalls(run, 'browser_click', { ref: 'e1' })
+      const offeredRow = await offered(toolCallId)
+      const decided = await post(toolPermissionPOST, '/api/copilot/tool-permission', {
+        decisions: [{ toolCallId, decision: 'allow' }],
+      })
+      expect(decided.status).toBe(200)
+
+      expect((await storedCall(toolCallId)).pickupDeadlineAt).toEqual(offeredRow.pickupDeadlineAt)
+      const { executionToken } = await desktop.claim(toolCallId)
+      await desktop.complete(toolCallId, executionToken, { clicked: true })
+      await answer
+      expect(resultOf(context, toolCallId)).toMatchObject({ success: true })
+    },
+    TURN_WAIT_MS
+  )
+
   it(
     'settles a call Sim never offered once its pickup window would have closed',
     async () => {

apps/sim/lib/desktop/executor/repository.ts

@@ -31,7 +31,8 @@ import {
   isTerminalAsyncStatus,
 } from '@/lib/mothership/async-runs/lifecycle'
 import { DESKTOP_TOOL_PICKUP_GRACE_MS } from '@/lib/mothership/constants'
-import { DESKTOP_TOOL_CALL_NAMES } from '@/lib/mothership/tools/desktop-tools'
+import { NAMED_DESKTOP_TOOL_NAMES } from '@/lib/mothership/tools/desktop-tools'
+import { USER_LOCAL_VFS_ROOT } from '@/lib/mothership/tools/local-filesystem'
 
 const LIVE_RUN_STATUSES: CopilotRunStatus[] = ['active', 'paused_waiting_for_tool', 'resuming']
 
@@ -61,8 +62,51 @@ function pickupOverdueAt(at: SQL) {
     sql`${pickupDeadline} <= ${at}`
   )
 }
+
+function isUserLocalVfsPath(path: SQL) {
+  return sql`(${path} = ${USER_LOCAL_VFS_ROOT} OR ${path} LIKE ${`${USER_LOCAL_VFS_ROOT}/%`})`
+}
+
+/**
+ * The SQL form of `isDesktopToolCall`, so a query limits only over calls the desktop runs: a
+ * desktop tool by name, or a VFS read of a granted local folder (not a read of Sim's own files).
+ */
+const isDesktopToolCallRow = or(
+  inArray(copilotAsyncToolCalls.toolName, [...NAMED_DESKTOP_TOOL_NAMES]),
+  and(
+    inArray(copilotAsyncToolCalls.toolName, ['read', 'grep']),
+    isUserLocalVfsPath(sql`${copilotAsyncToolCalls.args}->>'path'`)
+  ),
+  and(
+    eq(copilotAsyncToolCalls.toolName, 'glob'),
+    isUserLocalVfsPath(sql`${copilotAsyncToolCalls.args}->>'pattern'`)
+  )
+)
 const INBOX_ROW_LIMIT = 500
 
+/**
+ * Persistence order, which follows the order the model emitted the calls in: calls of one turn
+ * can share a millisecond, so the timestamp alone cannot order them. Rows persisted before the
+ * sequence existed have none and come first, as they are the oldest.
+ */
+const persistOrder = [
+  sql`${copilotAsyncToolCalls.persistSeq} ASC NULLS FIRST`,
+  asc(copilotAsyncToolCalls.createdAt),
+  asc(copilotAsyncToolCalls.toolCallId),
+]
+
+function comparePersistOrder(
+  a: { persistSeq: number | null; createdAt: Date; toolCallId: string },
+  b: { persistSeq: number | null; createdAt: Date; toolCallId: string }
+): number {
+  if (a.persistSeq !== b.persistSeq) {
+    if (a.persistSeq === null) return -1
+    if (b.persistSeq === null) return 1
+    return a.persistSeq - b.persistSeq
+  }
+  return a.createdAt.getTime() - b.createdAt.getTime() || a.toolCallId.localeCompare(b.toolCallId)
+}
+
 export interface DesktopDeviceRegistration {
   id: string
   userId: string
@@ -157,7 +201,7 @@ export async function touchDesktopDevice(deviceId: string): Promise<void> {
  * out the other: unclaimed calls on its recent open runs that are offered or waiting for the
  * user's decision, and calls it claimed that Sim settled without its result and it has not yet
  * acknowledged (cancel items). A call the device is still running is never listed: it already
- * holds it. Ordered by persistence time, the order the device claims in.
+ * holds it. Ordered by persistence, the order the device claims in.
  */
 export async function listDesktopInboxRows(identity: Omit<DesktopDeviceIdentity, 'sessionId'>) {
   const rowsWhere = (state: SQL | undefined) =>
@@ -171,6 +215,7 @@ export async function listDesktopInboxRows(identity: Omit<DesktopDeviceIdentity,
         permissionDecision: copilotAsyncToolCalls.permissionDecision,
         claimed: sql<boolean>`${copilotAsyncToolCalls.executionOwnerToken} IS NOT NULL`,
         createdAt: copilotAsyncToolCalls.createdAt,
+        persistSeq: copilotAsyncToolCalls.persistSeq,
         chatId: copilotRuns.chatId,
         chatTitle: copilotChats.title,
         workspaceId: copilotRuns.workspaceId,
@@ -183,11 +228,11 @@ export async function listDesktopInboxRows(identity: Omit<DesktopDeviceIdentity,
           eq(copilotRuns.desktopDeviceId, identity.deviceId),
           eq(copilotRuns.userId, identity.userId),
           sql`${copilotRuns.startedAt} > now() - make_interval(hours => ${DESKTOP_INBOX_HORIZON_HOURS})`,
-          inArray(copilotAsyncToolCalls.toolName, [...DESKTOP_TOOL_CALL_NAMES]),
+          isDesktopToolCallRow,
           state
         )
       )
-      .orderBy(asc(copilotAsyncToolCalls.createdAt), asc(copilotAsyncToolCalls.toolCallId))
+      .orderBy(...persistOrder)
       .limit(INBOX_ROW_LIMIT)
   const [waiting, cancelled] = await Promise.all([
     rowsWhere(
@@ -207,10 +252,7 @@ export async function listDesktopInboxRows(identity: Omit<DesktopDeviceIdentity,
       )
     ),
   ])
-  return [...waiting, ...cancelled].sort(
-    (a, b) =>
-      a.createdAt.getTime() - b.createdAt.getTime() || a.toolCallId.localeCompare(b.toolCallId)
-  )
+  return [...waiting, ...cancelled].sort(comparePersistOrder)
 }
 
 export type DesktopInboxRow = Awaited<ReturnType<typeof listDesktopInboxRows>>[number]
@@ -336,12 +378,14 @@ export async function offerDesktopToolCall(input: {
 /**
  * A bound desktop call with the deadlines only Sim enforces, read on the database clock every CAS
  * uses: whether its pickup window closed while it is unclaimed, and whether its lease lapsed while
- * it runs. Null for a call on a run no device is bound to.
+ * it runs. Null for a call on a run no device is bound to, and for a tool the desktop never runs.
  */
 export async function getDesktopToolCallDeadlines(toolCallId: string) {
   const [row] = await db
     .select({
       toolCallId: copilotAsyncToolCalls.toolCallId,
+      toolName: copilotAsyncToolCalls.toolName,
+      args: copilotAsyncToolCalls.args,
       runId: copilotRuns.id,
       userId: copilotRuns.userId,
       deviceId: copilotRuns.desktopDeviceId,
@@ -361,7 +405,11 @@ export async function getDesktopToolCallDeadlines(toolCallId: string) {
     .innerJoin(copilotRuns, eq(copilotRuns.id, copilotAsyncToolCalls.runId))
     .leftJoin(desktopDevices, eq(desktopDevices.id, copilotRuns.desktopDeviceId))
     .where(
-      and(eq(copilotAsyncToolCalls.toolCallId, toolCallId), isNotNull(copilotRuns.desktopDeviceId))
+      and(
+        eq(copilotAsyncToolCalls.toolCallId, toolCallId),
+        isNotNull(copilotRuns.desktopDeviceId),
+        isDesktopToolCallRow
+      )
     )
     .limit(1)
   return row?.deviceId ? { ...row, deviceId: row.deviceId } : null
@@ -374,7 +422,9 @@ export type DesktopToolCallDeadlines = NonNullable<
 /**
  * Bound desktop calls a deadline passed for at least `slackMs` ago: unclaimed past their pickup
  * deadline (offered or not), or claimed by the executor with a lapsed lease. A live waiter settles
- * these within its 5 s poll, so anything this finds lost its waiter.
+ * these within its 5 s poll, so anything this finds lost its waiter. The scan starts from the
+ * few unsettled calls (pending or running), in persistence order, so however long a call stayed
+ * overdue it is still reached.
  */
 export async function listOverdueDesktopToolCalls(input: { slackMs: number; limit: number }) {
   const overdue = sql`clock_timestamp() - ${input.slackMs} * interval '1 millisecond'`
@@ -385,6 +435,7 @@ export async function listOverdueDesktopToolCalls(input: { slackMs: number; limi
     .where(
       and(
         isNotNull(copilotRuns.desktopDeviceId),
+        isDesktopToolCallRow,
         or(
           pickupOverdueAt(overdue),
           and(
@@ -397,6 +448,7 @@ export async function listOverdueDesktopToolCalls(input: { slackMs: number; limi
         )
       )
     )
+    .orderBy(...persistOrder)
     .limit(input.limit)
   return rows.map((row) => row.toolCallId)
 }

apps/sim/lib/mothership/async-runs/browser-download-claim.integration.ts

@@ -48,6 +48,7 @@ describe('browser download admission with PostgreSQL', () => {
       result jsonb, error text, permission_decision text, permission_decided_at timestamp,
       permission_requested_at timestamp,
       pickup_deadline_at timestamp with time zone,
+      persist_seq bigint,
       claimed_at timestamp, claimed_by text, execution_started_at timestamp,
       execution_settled_at timestamp, execution_owner_token text,
       execution_lease_expires_at timestamptz, execution_revoked_at timestamptz,

apps/sim/lib/mothership/async-runs/repository.ts

@@ -1738,8 +1738,9 @@ export async function recordToolPermissionDecision(
         .set({
           permissionDecision: decision,
           permissionDecidedAt: now,
-          // No pickup window runs while the user decides: an allowed call is offered afresh.
-          pickupDeadlineAt: null,
+          // No pickup window runs while the user decides: a gated call is offered afresh once
+          // allowed. A call that was never gated keeps the window it was offered with.
+          pickupDeadlineAt: sql`CASE WHEN ${copilotAsyncToolCalls.permissionRequestedAt} IS NULL THEN ${copilotAsyncToolCalls.pickupDeadlineAt} END`,
           updatedAt: now,
         })
         .where(

apps/sim/lib/mothership/request/tools/desktop-wait.test.ts

@@ -98,6 +98,8 @@ describe('settleAbandonedDesktopToolCalls', () => {
       .mockRejectedValueOnce(new Error('connection reset'))
       .mockResolvedValueOnce({
         toolCallId: 'overdue',
+        toolName: 'browser_click',
+        args: { ref: 'e1' },
         runId: 'run-1',
         userId: 'user-1',
         deviceId: 'device-1',

apps/sim/lib/mothership/request/tools/desktop-wait.ts

@@ -1,6 +1,7 @@
 import { createLogger } from '@sim/logger'
 import { toError } from '@sim/utils/errors'
 import { interruptibleSleep } from '@sim/utils/helpers'
+import { toRecordOrNull } from '@sim/utils/object'
 import { ringDesktopInbox } from '@/lib/desktop/executor/doorbell'
 import { isDesktopPresent } from '@/lib/desktop/executor/presence'
 import {
@@ -23,6 +24,7 @@ import {
   type ClientToolSettlementGuard,
   settleClientToolCall,
 } from '@/lib/mothership/request/tools/client-settlement.server'
+import { isDesktopToolCall } from '@/lib/mothership/tools/desktop-tools'
 import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
 const logger = createLogger('CopilotDesktopToolWait')
@@ -230,7 +232,9 @@ async function readDesktopPresence(deviceId: string): Promise<boolean | null> {
  */
 async function settleOverdueDesktopToolCall(toolCallId: string): Promise<boolean> {
   const call = await getDesktopToolCallDeadlines(toolCallId)
-  if (!call) return false
+  // A VFS read of Sim's own files shares its tool name with a local read, but no desktop runs it.
+  if (!call || !isDesktopToolCall(call.toolName, toRecordOrNull(call.args) ?? undefined))
+    return false
   if (call.status === ASYNC_TOOL_STATUS.pending) {
     const present = await readDesktopPresence(call.deviceId)
     // Before its pickup window closes, a call fails early only when its device is known to be away: