feat(desktop): import local files from the background executor

apps/desktop/e2e/background-executor.spec.ts

@@ -1,5 +1,6 @@
 import { execFileSync } from 'node:child_process'
-import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
+import { createHash } from 'node:crypto'
+import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
 import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
@@ -17,7 +18,8 @@ import { sleep } from '@sim/utils/helpers'
 
 /**
  * The Sim desktop app's background executor against a fixture Sim that speaks the executor's
- * device protocol (register, inbox, doorbell, claim, lease, complete) the way Sim's routes do.
+ * device protocol (register, inbox, doorbell, claim, lease, complete, import) the way Sim's
+ * routes do.
  * The window navigates, reloads and leaves the chats while their calls run: nothing in this
  * suite depends on a chat view, which is the point. Each scenario's checks land in a JSON
  * report at BACKGROUND_EXECUTOR_REPORT_PATH.
@@ -66,12 +68,26 @@ interface ReportCheck {
 
 const report: ReportCheck[] = []
 
+interface ImportedEntry {
+  toolCallId: string
+  kind: string
+  sourceName: string
+  relativePath: string
+  sha256?: string
+  bytes?: number
+}
+
+function sha256(bytes: Buffer): string {
+  return createHash('sha256').update(bytes).digest('hex')
+}
+
 class FixtureSim {
   readonly calls = new Map<string, FixtureCall>()
   readonly devices = new Map<string, { name: string; platform: string }>()
   readonly requests: string[] = []
   readonly hits = new Map<string, number>()
   readonly streams = new Map<string, Set<ServerResponse>>()
+  readonly imported: ImportedEntry[] = []
   enabled = true
   offline = false
   droppedWhileOffline = 0
@@ -100,6 +116,7 @@ class FixtureSim {
     this.devices.clear()
     this.requests.length = 0
     this.hits.clear()
+    this.imported.length = 0
     this.enabled = true
     this.offline = false
     this.droppedWhileOffline = 0
@@ -156,12 +173,60 @@ class FixtureSim {
     }
   }
 
+  private async raw(request: IncomingMessage): Promise<Buffer> {
+    const chunks: Buffer[] = []
+    for await (const chunk of request) chunks.push(Buffer.from(chunk))
+    return Buffer.concat(chunks)
+  }
+
   private async body(request: IncomingMessage): Promise<Record<string, unknown>> {
-    let text = ''
-    for await (const chunk of request) text += chunk.toString()
+    const text = (await this.raw(request)).toString()
     return text ? (JSON.parse(text) as Record<string, unknown>) : {}
   }
 
+  /** Stores one entry of a claimed, running import, as Sim's import route does. */
+  private async importEntry(
+    url: URL,
+    request: IncomingMessage,
+    response: ServerResponse
+  ): Promise<void> {
+    const query = Object.fromEntries(url.searchParams)
+    const call = this.calls.get(query.toolCallId ?? '')
+    if (
+      request.method !== 'PUT' ||
+      !call ||
+      call.deviceId !== query.deviceId ||
+      call.toolName !== 'import_local_files' ||
+      call.token !== request.headers['x-sim-execution-token'] ||
+      call.status !== 'running'
+    ) {
+      this.json(response, 404, { error: 'Desktop import not found' })
+      return
+    }
+    // As Sim's route does: a file must declare its length, and arrive whole.
+    if (query.kind === 'file' && request.headers['content-length'] === undefined) {
+      this.json(response, 411, { error: 'A file import must declare its length' })
+      return
+    }
+    const content = await this.raw(request)
+    if (query.kind === 'file' && content.length !== Number(request.headers['content-length'])) {
+      this.json(response, 400, { error: 'The file did not arrive whole' })
+      return
+    }
+    const entry: ImportedEntry = {
+      toolCallId: call.toolCallId,
+      kind: query.kind ?? '',
+      sourceName: query.sourceName ?? '',
+      relativePath: query.relativePath ?? '',
+      ...(query.kind === 'file' ? { sha256: sha256(content), bytes: content.length } : {}),
+    }
+    this.imported.push(entry)
+    this.json(response, 200, {
+      id: `entry-${this.imported.length}`,
+      name: entry.relativePath.split('/').at(-1) || entry.sourceName,
+    })
+  }
+
   private json(response: ServerResponse, status: number, body: unknown): void {
     response.writeHead(status, { 'Content-Type': 'application/json' })
     response.end(JSON.stringify(body))
@@ -261,6 +326,10 @@ class FixtureSim {
       this.json(response, 200, { items })
       return
     }
+    if (path === '/api/desktop/tool/import') {
+      await this.importEntry(url, request, response)
+      return
+    }
     if (path.startsWith('/api/desktop/tool/')) {
       const body = await this.body(request)
       const call = this.calls.get(String(body.toolCallId))
@@ -585,6 +654,67 @@ test.describe('background executor', () => {
     })
   })
 
+  test('D: a folder import lands in Sim while the user is in another chat', async () => {
+    const userData = mkdtempSync(join(tmpdir(), 'sim-executor-d-'))
+    const launched = await launch(userData)
+    app = launched.app
+    const deviceId = await registeredDevice()
+    const source = join(userData, 'Reports')
+    mkdirSync(join(source, 'q3'), { recursive: true })
+    writeFileSync(join(source, 'notes.txt'), 'remember the numbers')
+    // Larger than one 8 MB read, so the file crosses Electron in several chunks.
+    const large = Buffer.alloc(9 * 1024 * 1024, 7)
+    writeFileSync(join(source, 'q3', 'export.bin'), large)
+    await launched.window.goto(`${sim.origin}/workspace/${WORKSPACE}/chat/${CHAT_C}`)
+
+    const call = sim.issue(deviceId, CHAT_B, 'import_local_files', {
+      path: source,
+      targetWorkspaceId: WORKSPACE,
+      folderId: 'folder-e2e',
+    })
+
+    await check('D: the import completes with every entry it stored', async () => {
+      const completion = await settled(call, 60_000)
+      expect(completion.status).toBe('success')
+      expect(completion.data).toMatchObject({
+        success: true,
+        workspaceId: WORKSPACE,
+        folders: [
+          { id: 'entry-1', relativePath: '' },
+          { id: 'entry-3', relativePath: 'q3' },
+        ],
+        files: [
+          { id: 'entry-2', relativePath: 'notes.txt' },
+          { id: 'entry-4', relativePath: 'q3/export.bin' },
+        ],
+      })
+    })
+
+    await check('D: Sim received the tree with the bytes on disk, once', async () => {
+      expect(sim.imported).toEqual([
+        { toolCallId: call, kind: 'directory', sourceName: 'Reports', relativePath: '' },
+        {
+          toolCallId: call,
+          kind: 'file',
+          sourceName: 'Reports',
+          relativePath: 'notes.txt',
+          sha256: sha256(Buffer.from('remember the numbers')),
+          bytes: 20,
+        },
+        { toolCallId: call, kind: 'directory', sourceName: 'Reports', relativePath: 'q3' },
+        {
+          toolCallId: call,
+          kind: 'file',
+          sourceName: 'Reports',
+          relativePath: 'q3/export.bin',
+          sha256: sha256(large),
+          bytes: large.length,
+        },
+      ])
+      expect(sim.requireCall(call).claims).toBe(1)
+    })
+  })
+
   test('E: Stop from another chat stops a running browser wait and terminal command', async () => {
     const userData = mkdtempSync(join(tmpdir(), 'sim-executor-e-'))
     app = (await launch(userData)).app

apps/desktop/src/main/desktop-executor/client.test.ts

@@ -1,3 +1,4 @@
+import { DESKTOP_IMPORT_TOKEN_HEADER } from '@sim/desktop-bridge'
 import { describe, expect, it } from 'vitest'
 import { createDesktopExecutorClient, UnsendableRequestError } from '@/main/desktop-executor/client'
 
@@ -48,4 +49,38 @@ describe('desktop executor client', () => {
     const loneSurrogate = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/
     expect(loneSurrogate.test(sentMessage)).toBe(false)
   })
+
+  it('sends an import entry with its execution token in a header, never in the URL', async () => {
+    const received: Array<{ url: string; token: string | null }> = []
+    const client = createDesktopExecutorClient({
+      origin: () => 'https://sim.test',
+      fetch: async (url, init) => {
+        received.push({ url, token: new Headers(init.headers).get(DESKTOP_IMPORT_TOKEN_HEADER) })
+        return Response.json({ id: 'file-1', name: 'notes.txt' })
+      },
+      deviceId: '00000000-0000-4000-8000-000000000000',
+    })
+
+    await client.importEntry(
+      {
+        call: {
+          toolCallId: 'call-1',
+          toolName: 'import_local_files',
+          args: {},
+          chatId: 'chat-1',
+          workspaceId: 'ws-1',
+          executionToken: 'secret-token-1',
+        },
+        kind: 'file',
+        sourceName: 'notes.txt',
+        relativePath: '',
+        content: new Blob(['hello']),
+      },
+      new AbortController().signal
+    )
+
+    expect(received).toHaveLength(1)
+    expect(received[0]?.token).toBe('secret-token-1')
+    expect(decodeURIComponent(received[0]?.url ?? '')).not.toContain('secret-token-1')
+  })
 })

apps/desktop/src/main/desktop-executor/client.ts

@@ -3,6 +3,7 @@
  * own session cookie, which is the session the device registered under; Sim refuses any other.
  */
 
+import { DESKTOP_IMPORT_TOKEN_HEADER } from '@sim/desktop-bridge'
 import { getErrorMessage } from '@sim/utils/errors'
 import { parseRetryAfter } from '@sim/utils/retry'
 import { truncateAtCodePoint } from '@sim/utils/string'
@@ -13,14 +14,19 @@ import {
   type DesktopCompletionRequest,
   type DesktopDeviceRegistration,
   type DesktopExecutorTiming,
+  type DesktopImportEntryRequest,
+  type DesktopImportedEntry,
   type DesktopInboxItem,
   parseClaim,
   parseCompletionOutcome,
+  parseImportedEntry,
   parseInbox,
   parseRegistration,
 } from '@/main/desktop-executor/protocol'
 
 const REQUEST_TIMEOUT_MS = 15_000
+/** An import entry carries up to a 64 MB file, so it gets longer than a control request. */
+const IMPORT_TIMEOUT_MS = 300_000
 
 /** A request Sim answered with a failure, or that never got an answer (`status` 0). */
 export class DeviceRequestError extends Error {
@@ -73,6 +79,10 @@ export interface DesktopExecutorClient {
   claim(toolCallId: string): Promise<ClaimedDesktopCall>
   renewLease(toolCallId: string, executionToken: string): Promise<void>
   complete(request: DesktopCompletionRequest): Promise<DesktopCompletionOutcome>
+  importEntry(
+    request: DesktopImportEntryRequest,
+    signal: AbortSignal
+  ): Promise<DesktopImportedEntry>
 }
 
 interface DesktopExecutorClientOptions {
@@ -97,21 +107,27 @@ export function createDesktopExecutorClient(
   const { deviceId } = options
 
   async function send(
-    method: 'GET' | 'POST',
+    method: 'GET' | 'POST' | 'PUT',
     path: string,
-    body?: Record<string, unknown>,
-    signal?: AbortSignal
+    body?: Record<string, unknown> | Blob,
+    signal?: AbortSignal,
+    timeoutMs = REQUEST_TIMEOUT_MS,
+    extraHeaders: Record<string, string> = {}
   ): Promise<Response> {
-    const encoded = body ? encode(body) : undefined
-    const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS)
+    const timeout = AbortSignal.timeout(timeoutMs)
+    const raw = body instanceof Blob
+    const encoded = body === undefined ? undefined : raw ? body : encode(body)
     let response: Response
     try {
       response = await options.fetch(`${options.origin()}${path}`, {
         method,
         credentials: 'include',
         headers: {
           Accept: 'application/json',
-          ...(body ? { 'Content-Type': 'application/json' } : {}),
+          ...(body
+            ? { 'Content-Type': raw ? 'application/octet-stream' : 'application/json' }
+            : {}),
+          ...extraHeaders,
         },
         ...(encoded !== undefined ? { body: encoded } : {}),
         signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
@@ -204,5 +220,25 @@ export function createDesktopExecutorClient(
       if (!outcome) throw malformed('completion')
       return outcome
     },
+    async importEntry({ call, kind, sourceName, relativePath, content }, signal) {
+      const query = new URLSearchParams({
+        deviceId,
+        toolCallId: call.toolCallId,
+        kind,
+        sourceName,
+        relativePath,
+      })
+      const response = await send(
+        'PUT',
+        `/api/desktop/tool/import?${query}`,
+        content,
+        signal,
+        IMPORT_TIMEOUT_MS,
+        { [DESKTOP_IMPORT_TOKEN_HEADER]: call.executionToken }
+      )
+      const entry = parseImportedEntry(await response.json().catch(() => null))
+      if (!entry) throw malformed('import')
+      return entry
+    },
   }
 }

apps/desktop/src/main/desktop-executor/executor.test.ts

@@ -67,6 +67,9 @@ class FakeSim {
     openInboxStream: async () => {
       throw new Error('not used')
     },
+    importEntry: async () => {
+      throw new Error('not used')
+    },
     claim: async (toolCallId) => {
       this.claims.push(toolCallId)
       const error = this.claimError?.(toolCallId)

apps/desktop/src/main/desktop-executor/protocol.ts

@@ -1,6 +1,6 @@
 /**
  * The device side of Sim's background executor protocol (`/api/desktop/devices`, `/inbox`,
- * `/inbox/stream`, `/tool/claim`, `/tool/lease`, `/tool/complete`). Sim's contracts are the
+ * `/inbox/stream`, `/tool/claim`, `/tool/lease`, `/tool/complete`, `/tool/import`). Sim's contracts are the
  * source of truth; responses are parsed defensively here because a malformed one must never
  * reach a tool.
  */
@@ -62,6 +62,25 @@ export interface ClaimedDesktopCall {
 
 export type DesktopCompletionOutcome = 'recorded' | 'duplicate' | 'superseded'
 
+/**
+ * One entry of a claimed import, stored under the call's target folder: `sourceName` is the
+ * import source's own name and `relativePath` the entry's place inside it (`''` for the source).
+ */
+export interface DesktopImportEntryRequest {
+  call: ClaimedDesktopCall
+  kind: 'file' | 'directory'
+  sourceName: string
+  relativePath: string
+  /** A file's bytes; a directory has none. */
+  content?: Blob
+}
+
+/** What Sim stored an import entry as: the file, or the folder it reused or created. */
+export interface DesktopImportedEntry {
+  id: string
+  name: string
+}
+
 export interface DesktopCompletionRequest {
   toolCallId: string
   executionToken: string
@@ -175,3 +194,13 @@ export function parseCompletionOutcome(body: unknown): DesktopCompletionOutcome
     ? body.outcome
     : null
 }
+
+/**
+ * Reads Sim's answer to an import entry: the id and name it stored the entry as. Null when either
+ * is missing or empty, which the client reports as a malformed response.
+ */
+export function parseImportedEntry(body: unknown): DesktopImportedEntry | null {
+  if (!isRecordLike(body) || typeof body.id !== 'string' || typeof body.name !== 'string')
+    return null
+  return body.id && body.name ? { id: body.id, name: body.name } : null
+}