fix: preserve desktop observations through recovery and report app focus

Author: Sg312

apps/desktop/native/computer-use/ComputerUse.swift

@@ -363,11 +363,25 @@ func requireForegroundTarget(pid: pid_t, windowID: String, point: CGPoint, frame
     }
     guard let top, (top[kCGWindowNumber as String] as? NSNumber)?.stringValue == windowID else { throw ComputerError("window_occluded", "The target point is covered by another window; bring the intended window forward and observe again.") }
 }
-func postMouse(pid: pid_t, type: CGEventType, point: CGPoint, button: CGMouseButton, count: Int = 1, windowID: String? = nil, frame: CGRect? = nil, release: Bool = false) throws {
+final class CoordinateDispatch {
+    private(set) var hasDispatched = false
+    func willDispatch() { hasDispatched = true }
+    func classify(_ error: ComputerError) -> ComputerError {
+        ComputerError(error.code, error.message, dispatchState: hasDispatched ? nil : "not_started")
+    }
+}
+@MainActor
+func coordinateAction(_ operation: (CoordinateDispatch) async throws -> Void) async throws {
+    let dispatch = CoordinateDispatch()
+    do { try await operation(dispatch) }
+    catch let error as ComputerError { throw dispatch.classify(error) }
+}
+func postMouse(pid: pid_t, type: CGEventType, point: CGPoint, button: CGMouseButton, count: Int = 1, windowID: String? = nil, frame: CGRect? = nil, release: Bool = false, dispatch: CoordinateDispatch) throws {
     guard let windowID, let frame else { throw ComputerError("invalid_window", "Mouse input requires an observed app window.") }
     if !release { try requireForegroundTarget(pid: pid, windowID: windowID, point: point, frame: frame) }
     guard let source = CGEventSource(stateID: .privateState), let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else { throw ComputerError("input_failed", "Could not create mouse event.") }
     event.setIntegerValueField(.mouseEventClickState, value: Int64(count))
+    dispatch.willDispatch()
     event.post(tap: .cghidEventTap)
 }
 
@@ -482,6 +496,7 @@ final class Driver {
                 } catch let error as ComputerError { result["screenshotError"] = String((error.code + ": " + error.message).prefix(2000)) }
                 catch { result["screenshotError"] = String(("capture_failed: " + String(describing: error)).prefix(2000)) }
             }
+            result["isActive"] = app.isActive
             return result
         case "click", "type_text", "input_sequence", "press_key", "scroll", "drag", "set_value", "perform_action": break
         default: throw ComputerError("unknown_method", "Unknown computer-use method.")
@@ -508,13 +523,15 @@ final class Driver {
             guard ["left", "right"].contains(button), (1...3).contains(count) else { throw ComputerError("invalid_arguments", "Invalid button or clickCount.") }
             if p.elementId != nil && button == "left" && count == 1 { try axCheck(AXUIElementPerformAction(try snapshot.element(p.elementId), kAXPressAction as CFString)) }
             else {
-                let position = try point(p, app: app, snapshot: snapshot); let mouse: CGMouseButton = button == "right" ? .right : .left
-                let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId")
-                let frame = try required(snapshot.windowFrames[windowID], "window frame")
-                for click in 1...count {
-                    try checkCancellation()
-                    try postMouse(pid: pid, type: button == "right" ? .rightMouseDown : .leftMouseDown, point: position, button: mouse, count: click, windowID: windowID, frame: frame)
-                    try postMouse(pid: pid, type: button == "right" ? .rightMouseUp : .leftMouseUp, point: position, button: mouse, count: click, windowID: windowID, frame: frame, release: true)
+                try await coordinateAction { dispatch in
+                    let position = try point(p, app: app, snapshot: snapshot); let mouse: CGMouseButton = button == "right" ? .right : .left
+                    let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId")
+                    let frame = try required(snapshot.windowFrames[windowID], "window frame")
+                    for click in 1...count {
+                        try checkCancellation()
+                        try postMouse(pid: pid, type: button == "right" ? .rightMouseDown : .leftMouseDown, point: position, button: mouse, count: click, windowID: windowID, frame: frame, dispatch: dispatch)
+                        try postMouse(pid: pid, type: button == "right" ? .rightMouseUp : .leftMouseUp, point: position, button: mouse, count: click, windowID: windowID, frame: frame, release: true, dispatch: dispatch)
+                    }
                 }
             }
         case "type_text", "input_sequence":
@@ -594,27 +611,31 @@ final class Driver {
             }
             if !selectedAll { try postKey(pid: pid, code: code, flags: flags) }
         case "scroll":
-            let position = try point(p, app: app, snapshot: snapshot); let dx = p.deltaX ?? 0; let dy = p.deltaY ?? 0
-            guard dx.isFinite, dy.isFinite, abs(dx) <= 10000, abs(dy) <= 10000 else { throw ComputerError("invalid_arguments", "Scroll deltas exceed bounds.") }
-            guard let event = CGEvent(scrollWheelEvent2Source: CGEventSource(stateID: .privateState), units: .pixel, wheelCount: 2, wheel1: Int32(-dy), wheel2: Int32(-dx), wheel3: 0) else { throw ComputerError("input_failed", "Could not create scroll event.") }
-            let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId")
-            let frame = try required(snapshot.windowFrames[windowID], "window frame")
-            try requireForegroundTarget(pid: pid, windowID: windowID, point: position, frame: frame)
-            event.location = position; event.post(tap: .cghidEventTap)
+            try await coordinateAction { dispatch in
+                let position = try point(p, app: app, snapshot: snapshot); let dx = p.deltaX ?? 0; let dy = p.deltaY ?? 0
+                guard dx.isFinite, dy.isFinite, abs(dx) <= 10000, abs(dy) <= 10000 else { throw ComputerError("invalid_arguments", "Scroll deltas exceed bounds.") }
+                guard let event = CGEvent(scrollWheelEvent2Source: CGEventSource(stateID: .privateState), units: .pixel, wheelCount: 2, wheel1: Int32(-dy), wheel2: Int32(-dx), wheel3: 0) else { throw ComputerError("input_failed", "Could not create scroll event.") }
+                let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId")
+                let frame = try required(snapshot.windowFrames[windowID], "window frame")
+                try requireForegroundTarget(pid: pid, windowID: windowID, point: position, frame: frame)
+                event.location = position; dispatch.willDispatch(); event.post(tap: .cghidEventTap)
+            }
         case "drag":
-            let start = try point(p, app: app, snapshot: snapshot); let end = try point(p, app: app, snapshot: snapshot, end: true)
-            let windowID = try required(p.windowId, "windowId"); let frame = try required(snapshot.windowFrames[windowID], "window frame")
-            try postMouse(pid: pid, type: .leftMouseDown, point: start, button: .left, windowID: windowID, frame: frame)
-            var lastPoint = start
-            defer { try? postMouse(pid: pid, type: .leftMouseUp, point: lastPoint, button: .left, windowID: windowID, frame: frame, release: true) }
-            try await Task.sleep(for: .milliseconds(10))
-            for step in 1...10 {
-                try checkCancellation()
-                let amount = CGFloat(step) / 10
-                let position = CGPoint(x: start.x + (end.x - start.x) * amount, y: start.y + (end.y - start.y) * amount)
-                try postMouse(pid: pid, type: .leftMouseDragged, point: position, button: .left, windowID: windowID, frame: frame)
-                lastPoint = position
+            try await coordinateAction { dispatch in
+                let start = try point(p, app: app, snapshot: snapshot); let end = try point(p, app: app, snapshot: snapshot, end: true)
+                let windowID = try required(p.windowId, "windowId"); let frame = try required(snapshot.windowFrames[windowID], "window frame")
+                try postMouse(pid: pid, type: .leftMouseDown, point: start, button: .left, windowID: windowID, frame: frame, dispatch: dispatch)
+                var lastPoint = start
+                defer { try? postMouse(pid: pid, type: .leftMouseUp, point: lastPoint, button: .left, windowID: windowID, frame: frame, release: true, dispatch: dispatch) }
                 try await Task.sleep(for: .milliseconds(10))
+                for step in 1...10 {
+                    try checkCancellation()
+                    let amount = CGFloat(step) / 10
+                    let position = CGPoint(x: start.x + (end.x - start.x) * amount, y: start.y + (end.y - start.y) * amount)
+                    try postMouse(pid: pid, type: .leftMouseDragged, point: position, button: .left, windowID: windowID, frame: frame, dispatch: dispatch)
+                    lastPoint = position
+                    try await Task.sleep(for: .milliseconds(10))
+                }
             }
         default: throw ComputerError("unknown_method", "Unknown method.")
         }

apps/desktop/native/computer-use/tests/NativeTests.swift

@@ -6,7 +6,7 @@ import AppKit
         catch let error as ComputerError { precondition(error.code == code) }
         catch { fatalError("Unexpected \(error)") }
     }
-    static func main() throws {
+    @MainActor static func main() async throws {
         let frame = CGRect(x: -1920, y: 120, width: 800, height: 600)
         try requireObservedFrame(frame, current: frame)
         expectError("stale_window") { try requireObservedFrame(frame, current: frame.offsetBy(dx: 1, dy: 0)) }
@@ -52,6 +52,23 @@ import AppKit
         let submitFocus = try keyMayChangeFocus("Enter"); precondition(!submitFocus)
         let searchFocus = try keyMayChangeFocus("Cmd+K"); precondition(searchFocus)
         let selectFocus = try keyMayChangeFocus("Cmd+A"); precondition(!selectFocus)
+        do {
+            try await coordinateAction { _ in throw ComputerError("foreground_required", "preflight") }
+            fatalError("Expected pre-dispatch rejection")
+        } catch let error as ComputerError {
+            precondition(error.code == "foreground_required" && error.dispatchState == "not_started")
+        }
+        for code in ["foreground_required", "window_occluded", "cancelled"] {
+            do {
+                try await coordinateAction { dispatch in
+                    dispatch.willDispatch()
+                    throw ComputerError(code, "after mouse down", dispatchState: "not_started")
+                }
+                fatalError("Expected partial dispatch rejection")
+            } catch let error as ComputerError {
+                precondition(error.code == code && error.dispatchState == nil)
+            }
+        }
         cancellationRequested = 1
         expectError("cancelled") { try checkCancellation() }
         cancellationRequested = 0

apps/sim/lib/mothership/async-runs/repository.ts

@@ -1415,6 +1415,7 @@ export async function replaceTerminalAsyncToolCallResult(input: {
   status: AsyncTerminalStatus
   result: AsyncCompletionData | null
   error: string | null
+  expectedResult?: AsyncCompletionData
 }) {
   return await withDbSpan(
     TraceSpan.CopilotAsyncRunsMarkAsyncToolStatus,
@@ -1437,7 +1438,10 @@ export async function replaceTerminalAsyncToolCallResult(input: {
         .where(
           and(
             eq(copilotAsyncToolCalls.toolCallId, input.toolCallId),
-            eq(copilotAsyncToolCalls.status, input.status)
+            eq(copilotAsyncToolCalls.status, input.status),
+            input.expectedResult !== undefined
+              ? sql`${copilotAsyncToolCalls.result} = ${JSON.stringify(sanitizeValueForJsonb(input.expectedResult))}::jsonb`
+              : undefined
           )
         )
         .returning()

apps/sim/lib/mothership/async-runs/tool-identity.integration.ts

@@ -208,6 +208,40 @@ describe.skipIf(!databaseUrl)('Copilot tool identity with PostgreSQL', () => {
     ).resolves.toBeNull()
   })
 
+  it('allows only one projected receipt to replace the exact JSONB source across competing waiters', async () => {
+    await createCurrentCalls()
+    const source = { completion: 'sealed-source', context: { run: 'bound', user: 'bound' } }
+    await completeAsyncToolCall({ toolCallId: firstId, status: 'completed', result: source })
+    const receipts = [
+      { __sealedClientToolProjectionV1: 'first-authenticated-projection' },
+      { __sealedClientToolProjectionV1: 'second-authenticated-projection' },
+    ]
+    const attempts = await Promise.all(
+      receipts.map((result) =>
+        replaceTerminalAsyncToolCallResult({
+          toolCallId: firstId,
+          status: 'completed',
+          error: null,
+          result,
+          expectedResult: { context: { user: 'bound', run: 'bound' }, completion: 'sealed-source' },
+        })
+      )
+    )
+    expect(attempts.filter(Boolean)).toHaveLength(1)
+    const winner = attempts.find((row) => row !== null)
+    expect((await getAsyncToolCall(firstId))?.result).toEqual(winner?.result)
+    expect(
+      await replaceTerminalAsyncToolCallResult({
+        toolCallId: firstId,
+        status: 'completed',
+        result: { success: true },
+        error: null,
+        expectedResult: source,
+      })
+    ).toBeNull()
+    expect((await getAsyncToolCall(firstId))?.result).toEqual(winner?.result)
+  })
+
   describe.skipIf(!redisUrl)('with actual Redis confirmation and permission channels', () => {
     it('wakes only the matching waiter and reads its durable terminal result', async () => {
       await createCurrentCalls()

apps/sim/lib/mothership/generated/computer-use.ts

@@ -101,7 +101,7 @@ export const ComputerUseSchema = z.union([
       .boolean()
       .optional()
       .describe(
-        "Set true to explicitly activate this app before the keyboard batch. Defaults to false; editor, window, and geometry checks still apply.",
+        "Explicitly bring this app to the foreground before the batch. Defaults to false; leave omitted for background input. An inactive app alone does not require activation. Do not set true when the user requires background operation. Editor, window, and geometry checks still apply.",
       ),
     steps: z
       .array(InputSequenceStep)
@@ -159,7 +159,12 @@ const ComputerUseNodeSchema = z.strictObject({
   label: z.string().max(8192).optional(),
   value: z.string().max(32_000).optional(),
   enabled: z.boolean().optional(),
-  focused: z.boolean().optional(),
+  focused: z
+    .boolean()
+    .optional()
+    .describe(
+      "Accessibility focus within this app; does not imply the app is foreground or an item is selected.",
+    ),
   editable: z.boolean().optional(),
   placeholder: z.string().max(1024).optional(),
   actions: z.array(z.string().max(128)).max(128),
@@ -180,6 +185,12 @@ export const ComputerUseScreenshotSchema = z.strictObject({
 export const ComputerUseSnapshotSchema = z.strictObject({
   kind: z.literal("state"),
   bundleId: BundleId,
+  isActive: z
+    .boolean()
+    .optional()
+    .describe(
+      "Whether this app was foreground when observed. Omitted means unknown; inactive apps may still support background controls.",
+    ),
   snapshotId: SnapshotId,
   windowId: WindowId,
   windows: z.array(ComputerUseWindowSchema).max(100),

apps/sim/lib/mothership/request/handlers/types.ts

@@ -20,6 +20,7 @@ import { CopilotDegradedReason } from '@/lib/mothership/generated/trace-attribut
 import { recordDegraded } from '@/lib/mothership/request/metrics'
 import { markToolResultSeen } from '@/lib/mothership/request/sse-utils'
 import { setTerminalToolCallState } from '@/lib/mothership/request/tool-call-state'
+import { toolStatusOutput } from '@/lib/mothership/request/tools/tool-status-output'
 import type {
   ContentBlock,
   ExecutionContext,
@@ -299,7 +300,7 @@ export async function emitSyntheticToolResult(
         mode: MothershipStreamV1ToolMode.async,
         phase: MothershipStreamV1ToolPhase.result,
         success,
-        output: resultPayload,
+        output: toolStatusOutput(resultPayload),
         ...(syntheticStatus ? { status: syntheticStatus } : {}),
         ...(!success && completion?.message ? { error: completion.message } : {}),
       },

apps/sim/lib/mothership/request/tools/client-completion-seal.server.ts

@@ -1,7 +1,10 @@
 import { generateId } from '@sim/utils/id'
 import { isPlainRecord } from '@sim/utils/object'
 import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption'
-import type { AsyncCompletionData } from '@/lib/mothership/async-runs/lifecycle'
+import type {
+  AsyncCompletionData,
+  AsyncTerminalCompletionSnapshot,
+} from '@/lib/mothership/async-runs/lifecycle'
 import {
   isResolvedSecretTraceProvenanceV1,
   type ResolvedSecretTraceProvenanceV1,
@@ -10,6 +13,7 @@ import {
 
 export const SEALED_CLIENT_TOOL_COMPLETION_FIELD = '__sealedClientToolCompletionV1'
 export const SEALED_CLIENT_TOOL_CONTEXT_FIELD = '__sealedClientToolContextV1'
+export const SEALED_CLIENT_TOOL_PROJECTION_FIELD = '__sealedClientToolProjectionV1'
 
 interface ClientToolBinding {
   toolCallId: string
@@ -47,7 +51,10 @@ type ReportUnsealFailure = (reason: ClientToolUnsealFailureReason) => void
 /** Reads a sealed record while exposing only the guard that refused it, never its contents. */
 async function readSealedRecord(
   value: unknown,
-  field: typeof SEALED_CLIENT_TOOL_COMPLETION_FIELD | typeof SEALED_CLIENT_TOOL_CONTEXT_FIELD,
+  field:
+    | typeof SEALED_CLIENT_TOOL_COMPLETION_FIELD
+    | typeof SEALED_CLIENT_TOOL_CONTEXT_FIELD
+    | typeof SEALED_CLIENT_TOOL_PROJECTION_FIELD,
   reportFailure?: ReportUnsealFailure
 ): Promise<Record<string, unknown> | null> {
   if (!isPlainRecord(value)) {
@@ -188,3 +195,48 @@ export async function unsealClientToolContext(
     provenance: context.provenance,
   }
 }
+
+interface ProjectedClientToolCompletion extends ClientToolBinding {
+  status: 'success' | 'error' | 'cancelled'
+  message: string
+  data: AsyncCompletionData
+}
+
+/** Mint only after raw completion and provenance have passed model-safe projection. */
+export async function sealProjectedClientToolCompletion(
+  content: ProjectedClientToolCompletion
+): Promise<Record<typeof SEALED_CLIENT_TOOL_PROJECTION_FIELD, string>> {
+  const { encrypted } = await encryptSecret(
+    JSON.stringify({
+      purpose: 'projected-client-tool-completion-v1',
+      ...content,
+    })
+  )
+  return { [SEALED_CLIENT_TOOL_PROJECTION_FIELD]: encrypted }
+}
+
+/** An authenticated projected snapshot survives process handoff without replaying raw provenance. */
+export async function unsealProjectedClientToolCompletion(
+  value: unknown,
+  expected: ClientToolBinding & { status: AsyncTerminalCompletionSnapshot['status'] },
+  reportFailure?: ReportUnsealFailure
+): Promise<AsyncTerminalCompletionSnapshot | null> {
+  const content = await readSealedRecord(value, SEALED_CLIENT_TOOL_PROJECTION_FIELD, reportFailure)
+  if (!content) return null
+  if (!bindingMatches(content, expected) || content.status !== expected.status) {
+    reportFailure?.('binding-mismatch')
+    return null
+  }
+  if (
+    content.purpose !== 'projected-client-tool-completion-v1' ||
+    (content.status !== 'success' &&
+      content.status !== 'error' &&
+      content.status !== 'cancelled') ||
+    typeof content.message !== 'string' ||
+    !Object.hasOwn(content, 'data')
+  ) {
+    reportFailure?.('invalid-content')
+    return null
+  }
+  return { status: content.status, message: content.message, data: content.data }
+}