Skip to content

Commit e152d13

Browse files
committed
feat(desktop): bind turns to a desktop and supervise their calls in the background
Wires the run loop to the background executor protocol, still inert until a desktop that speaks it registers with the flag on: - A desktop composer whose shell has a background executor sends its device id and protocol version. Admission binds the run to that device only when the flag is on for the user and the device is registered, as an executor, to the caller's own session; otherwise the turn stays with the chat view. - On a bound run every desktop call is persisted pending. Once it may run, a supervisor offers it to the device and rings its doorbell, fails it at once as not started when the device is offline, fails it as not started when no one claims it within 15 s, and fails it as outcome unknown when a claimed call's lease lapses. Every failure is sealed like a device result and woken through the confirmation channel. A gated call rings an approval doorbell. - The resume gate leaves bound desktop calls to that supervisor, so a long terminal command lives as long as the device renews its lease, and the Sim tool lease sweep no longer claims executor leases. - The chat view's authorize and confirm routes refuse a bound run's calls, and Stop rings the device so it cancels what it is running.
1 parent 574d70f commit e152d13

18 files changed

Lines changed: 988 additions & 4 deletions

File tree

‎apps/sim/app/api/copilot/confirm/route.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,26 @@ describe('Copilot Confirm API Route', () => {
131131
expect(JSON.stringify(publishToolConfirmation.mock.calls)).not.toContain('resolved-secret')
132132
})
133133

134+
it("refuses a chat view's failure for a call a desktop's background executor owns", async () => {
135+
getAsyncToolCall.mockResolvedValue({
136+
...existingRow,
137+
toolName: 'browser_click',
138+
status: 'pending',
139+
claimedBy: null,
140+
})
141+
getRunSegment.mockResolvedValue({ id: 'run-1', userId: 'user-1', desktopDeviceId: 'device-1' })
142+
143+
const response = await POST(
144+
createMockPostRequest({
145+
toolCallId: 'tool-call-123',
146+
status: 'error',
147+
message: 'The desktop refused this claim',
148+
})
149+
)
150+
151+
expect(response.status).toBe(409)
152+
})
153+
134154
it('atomically detaches a live background confirmation', async () => {
135155
const response = await POST(
136156
createMockPostRequest({

‎apps/sim/app/api/copilot/confirm/route.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@ import { isBrowserToolName, isCurrentBrowserToolName } from '@sim/browser-protoc
22
import { createLogger } from '@sim/logger'
33
import { isTerminalToolName } from '@sim/terminal-protocol'
44
import { getErrorMessage, toError } from '@sim/utils/errors'
5-
import { isPlainRecord } from '@sim/utils/object'
5+
import { isPlainRecord, toRecord } from '@sim/utils/object'
66
import { type NextRequest, NextResponse } from 'next/server'
77
import { copilotConfirmContract } from '@/lib/api/contracts/copilot'
88
import { parseRequest, validationErrorResponse } from '@/lib/api/server'
99
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
10+
import { isDesktopExecutorTool } from '@/lib/desktop/executor/tools'
1011
import {
1112
ASYNC_TOOL_CONFIRMATION_STATUS,
1213
ASYNC_TOOL_STATUS,
@@ -228,6 +229,20 @@ export const POST = withRouteHandler((req: NextRequest) => {
228229
return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
229230
}
230231

232+
if (
233+
run.desktopDeviceId &&
234+
isDesktopExecutorTool(existing.toolName, toRecord(existing.args))
235+
) {
236+
span.setAttribute(TraceAttr.CopilotConfirmOutcome, CopilotConfirmOutcome.Forbidden)
237+
return NextResponse.json(
238+
{
239+
error:
240+
"This chat's desktop actions report through the desktop app's background executor",
241+
},
242+
{ status: 409 }
243+
)
244+
}
245+
231246
const isWorkflowTool = isWorkflowToolName(existing.toolName || '')
232247
const workflowId = isWorkflowTool
233248
? resolveWorkflowToolTargetId(existing.args, run.workflowId)

‎apps/sim/app/api/desktop/tool/authorize/route.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,26 @@ describe('desktop tool authorization', () => {
7272
})
7373
})
7474

75+
it("refuses a chat view's claim on a run bound to a desktop's background executor", async () => {
76+
getAsyncToolCall.mockResolvedValueOnce({
77+
toolCallId: 'bound-click',
78+
runId: 'run-1',
79+
status: 'pending',
80+
toolName: 'browser_click',
81+
args: { ref: 'e1' },
82+
})
83+
getRunSegment.mockResolvedValueOnce({
84+
id: 'run-1',
85+
chatId: 'chat-1',
86+
userId: 'user-1',
87+
status: 'active',
88+
desktopDeviceId: 'device-1',
89+
})
90+
91+
const response = await POST(request('bound-click'))
92+
expect(response.status).toBe(409)
93+
})
94+
7595
it('rejects retired browser tools retained only for history', async () => {
7696
getAsyncToolCall.mockResolvedValueOnce({
7797
toolCallId: 'retired-browser-tool',

‎apps/sim/app/api/desktop/tool/authorize/route.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
4747
if (run.status === 'complete' || run.status === 'error' || run.status === 'cancelled') {
4848
return createNotFoundResponse('Pending client tool call not found')
4949
}
50+
// The device's background executor claims a bound run's calls through its own fenced route.
51+
if (run.desktopDeviceId)
52+
return NextResponse.json(
53+
{ error: "This chat's desktop actions run in the desktop app's background executor" },
54+
{ status: 409 }
55+
)
5056

5157
const args = isRecordLike(toolCall.args) ? (toolCall.args as Record<string, unknown>) : {}
5258
const isBrowserTool = isCurrentBrowserToolName(toolCall.toolName)

‎apps/sim/lib/desktop/application/executor.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import {
3232
import {
3333
acknowledgeDesktopCallResult,
3434
claimOfferedDesktopCall,
35+
getBindableDesktopDevice,
3536
getBoundDesktopCall,
3637
getBoundDesktopDevice,
3738
listDesktopInboxRows,
@@ -70,6 +71,31 @@ async function isDesktopBackgroundExecutorEnabled(userId: string): Promise<boole
7071
return isFeatureEnabled('mothership-desktop-background-executor', { userId })
7172
}
7273

74+
/**
75+
* The device a new turn binds to: the composer's own, but only while the executor is on for this
76+
* user and the device is registered to this very session as an executor. Anything else leaves
77+
* the turn to the chat view, as before the executor existed.
78+
*/
79+
export async function resolveTurnDesktopDevice(
80+
principal: SessionPrincipal,
81+
deviceId: string
82+
): Promise<string | null> {
83+
if (!(await isDesktopBackgroundExecutorEnabled(principal.userId))) return null
84+
const device = await getBindableDesktopDevice({
85+
deviceId,
86+
userId: principal.userId,
87+
sessionId: principal.sessionId,
88+
})
89+
if (!device) {
90+
logger.warn('Turn not bound: its desktop is not registered to this session', {
91+
userId: principal.userId,
92+
deviceId,
93+
})
94+
return null
95+
}
96+
return device.id
97+
}
98+
7399
export interface RegisterDesktopDeviceInput extends DeviceInput {
74100
name: string
75101
appVersion: string

‎apps/sim/lib/desktop/executor/repository.ts‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,3 +410,147 @@ export async function acknowledgeDesktopCallResult(input: {
410410
return { outcome: 'superseded', status: row.status }
411411
})
412412
}
413+
414+
/** A device the caller may bind a new turn to: theirs, on this session, and able to execute. */
415+
export async function getBindableDesktopDevice(identity: DesktopDeviceIdentity) {
416+
const [row] = await db
417+
.select({ id: desktopDevices.id })
418+
.from(desktopDevices)
419+
.where(
420+
and(
421+
eq(desktopDevices.id, identity.deviceId),
422+
eq(desktopDevices.userId, identity.userId),
423+
eq(desktopDevices.sessionId, identity.sessionId),
424+
isNull(desktopDevices.revokedAt),
425+
sql`coalesce((${desktopDevices.capabilities} ->> 'executor')::int, 0) >= 1`
426+
)
427+
)
428+
.limit(1)
429+
return row ?? null
430+
}
431+
432+
/** The device a run's desktop calls are bound to, or null for a run the chat view serves. */
433+
export async function getRunDesktopDeviceId(runId: string): Promise<string | null> {
434+
const [row] = await db
435+
.select({ desktopDeviceId: copilotRuns.desktopDeviceId })
436+
.from(copilotRuns)
437+
.where(eq(copilotRuns.id, runId))
438+
.limit(1)
439+
return row?.desktopDeviceId ?? null
440+
}
441+
442+
/**
443+
* Offers a pending call to its device by opening its pickup window. Only an unowned pending call
444+
* on a bound run can be offered, and only once.
445+
*/
446+
export async function offerDesktopCall(input: {
447+
toolCallId: string
448+
runId: string
449+
pickupGraceMs: number
450+
}): Promise<boolean> {
451+
const [row] = await db
452+
.update(copilotAsyncToolCalls)
453+
.set({
454+
executionLeaseExpiresAt: sql`clock_timestamp() + ${input.pickupGraceMs} * interval '1 millisecond'`,
455+
})
456+
.where(
457+
and(
458+
eq(copilotAsyncToolCalls.toolCallId, input.toolCallId),
459+
eq(copilotAsyncToolCalls.runId, input.runId),
460+
eq(copilotAsyncToolCalls.status, ASYNC_TOOL_STATUS.pending),
461+
isNull(copilotAsyncToolCalls.executionOwnerToken),
462+
isNull(copilotAsyncToolCalls.executionLeaseExpiresAt),
463+
sql`EXISTS (SELECT 1 FROM ${copilotRuns} r WHERE r.id = ${copilotAsyncToolCalls.runId}
464+
AND r.desktop_device_id IS NOT NULL)`
465+
)
466+
)
467+
.returning({ toolCallId: copilotAsyncToolCalls.toolCallId })
468+
return Boolean(row)
469+
}
470+
471+
/** Where a supervised call stands, measured on the database clock every CAS uses. */
472+
export async function getDesktopCallState(toolCallId: string) {
473+
const [row] = await db
474+
.select({
475+
status: copilotAsyncToolCalls.status,
476+
ownerToken: copilotAsyncToolCalls.executionOwnerToken,
477+
result: copilotAsyncToolCalls.result,
478+
msUntilLeaseEnd: sql<
479+
number | null
480+
>`(extract(epoch from (${copilotAsyncToolCalls.executionLeaseExpiresAt} - clock_timestamp())) * 1000)::float8`,
481+
})
482+
.from(copilotAsyncToolCalls)
483+
.where(eq(copilotAsyncToolCalls.toolCallId, toolCallId))
484+
.limit(1)
485+
return row ?? null
486+
}
487+
488+
interface DesktopCallFailure {
489+
toolCallId: string
490+
runId: string
491+
result: AsyncCompletionData
492+
error: string
493+
}
494+
495+
/**
496+
* Fails a call nobody claimed: the inverse CAS of the claim, so exactly one of the two wins.
497+
* `deadlinePassed` limits it to a call whose pickup window has closed.
498+
*/
499+
export async function failUnclaimedDesktopCall(
500+
input: DesktopCallFailure & { deadlinePassed: boolean }
501+
): Promise<boolean> {
502+
const [row] = await db
503+
.update(copilotAsyncToolCalls)
504+
.set({
505+
status: ASYNC_TOOL_STATUS.failed,
506+
result: sanitizeValueForJsonb(input.result),
507+
error: input.error,
508+
completedAt: sql`now()`,
509+
updatedAt: sql`now()`,
510+
})
511+
.where(
512+
and(
513+
eq(copilotAsyncToolCalls.toolCallId, input.toolCallId),
514+
eq(copilotAsyncToolCalls.runId, input.runId),
515+
eq(copilotAsyncToolCalls.status, ASYNC_TOOL_STATUS.pending),
516+
isNull(copilotAsyncToolCalls.executionOwnerToken),
517+
input.deadlinePassed
518+
? sql`${copilotAsyncToolCalls.executionLeaseExpiresAt} <= clock_timestamp()`
519+
: undefined
520+
)
521+
)
522+
.returning({ toolCallId: copilotAsyncToolCalls.toolCallId })
523+
return Boolean(row)
524+
}
525+
526+
/**
527+
* Fails a claimed call whose lease lapsed with this token still on it. The token stays on the row,
528+
* so the device's late result is answered as superseded and acknowledges the cancellation.
529+
*/
530+
export async function failLapsedDesktopCall(
531+
input: DesktopCallFailure & { ownerToken: string }
532+
): Promise<boolean> {
533+
const [row] = await db
534+
.update(copilotAsyncToolCalls)
535+
.set({
536+
status: ASYNC_TOOL_STATUS.failed,
537+
result: sanitizeValueForJsonb(input.result),
538+
error: input.error,
539+
claimedBy: null,
540+
claimedAt: null,
541+
completedAt: sql`now()`,
542+
updatedAt: sql`now()`,
543+
})
544+
.where(
545+
and(
546+
eq(copilotAsyncToolCalls.toolCallId, input.toolCallId),
547+
eq(copilotAsyncToolCalls.runId, input.runId),
548+
eq(copilotAsyncToolCalls.status, ASYNC_TOOL_STATUS.running),
549+
eq(copilotAsyncToolCalls.executionOwnerToken, input.ownerToken),
550+
isNull(copilotAsyncToolCalls.executionRevokedAt),
551+
sql`${copilotAsyncToolCalls.executionLeaseExpiresAt} <= clock_timestamp()`
552+
)
553+
)
554+
.returning({ toolCallId: copilotAsyncToolCalls.toolCallId })
555+
return Boolean(row)
556+
}

0 commit comments

Comments
 (0)