improvement(provenance): aggregate and attribute unrecorded durable reads

apps/sim/app/api/knowledge/secret-provenance.ts

@@ -248,6 +248,8 @@ export async function finalizeKnowledgePersistedResponse(options: {
     registry,
     documents: options.documents,
     chunks: options.chunks,
+    ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}),
+    actorUserId: options.userId,
   })
   return finalizeKnowledgeRegistryResponse({
     request: options.request,

apps/sim/app/api/memory/secret-provenance.test.ts

@@ -18,6 +18,7 @@ vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({
   reportUnrecordedDurableProvenance: mockReport,
 }))
 
+import { memoryListQuerySchema } from '@/lib/api/contracts/memory'
 import { AuthType } from '@/lib/auth/hybrid'
 import {
   PRIVATE_SECRET_PROVENANCE_BUNDLE_V1,
@@ -341,3 +342,11 @@ describe('memory write secret provenance', () => {
     expect(mockReport).not.toHaveBeenCalled()
   })
 })
+
+describe('memory list query contract', () => {
+  it('rejects a limit past the page ceiling and keeps the default below it', () => {
+    expect(memoryListQuerySchema.safeParse({ limit: '2000' }).success).toBe(false)
+    expect(memoryListQuerySchema.parse({})).toMatchObject({ limit: 50 })
+    expect(memoryListQuerySchema.parse({ limit: '1000' })).toMatchObject({ limit: 1000 })
+  })
+})

apps/sim/executor/handlers/agent/agent-handler.ts

@@ -1680,6 +1680,7 @@ export class AgentBlockHandler implements BlockHandler {
               identity,
               registry: ctx.resolvedSecretTraceRegistry,
               view: 'opaque',
+              ...(ctx.userId ? { actorUserId: ctx.userId } : {}),
             })
             if (!safe) {
               unsafeGeneratedDocumentFiles.add(`${file.key}:${file.id}`)

apps/sim/executor/handlers/mothership/mothership-handler.ts

@@ -689,7 +689,7 @@ async function buildMothershipFileAttachments(
   )
   const modelSafe = await areModelSafeWorkspaceFileKeys(
     userFiles.map((file) => file.key).filter((key): key is string => Boolean(key)),
-    { workspaceId: ctx.workspaceId }
+    { workspaceId: ctx.workspaceId, ...(ctx.userId ? { actorUserId: ctx.userId } : {}) }
   )
   if (!modelSafe) throw new Error(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE)
 

apps/sim/lib/api/contracts/memory.ts

@@ -34,7 +34,13 @@ export const agentMemoryDataSchemaContract = agentMemoryDataSchema
 export const memoryListQuerySchema = z.object({
   workspaceId: z.string().optional(),
   query: z.string().nullable().optional(),
-  limit: z.coerce.number().int().min(1).optional().default(50),
+  limit: z.coerce
+    .number()
+    .int()
+    .min(1)
+    .max(1000, 'Cannot list more than 1000 memories per request')
+    .optional()
+    .default(50),
 })
 
 export const memoryMessageSchema = z

apps/sim/lib/copilot/tools/handlers/vfs.ts

@@ -126,6 +126,7 @@ async function canReturnWorkspaceFileValue(
         registry: context.resolvedSecretTraceRegistry,
         view: provenanceView,
         value,
+        actorUserId: context.userId,
       }))
     ) {
       return false

apps/sim/lib/execution/mounted-file-secret-provenance.ts

@@ -89,6 +89,14 @@ export async function createMountedFileSecretProvenanceScanner(
 
   return {
     hasSecrets,
+    /**
+     * A scan that cannot finish yields `unknown` — a taint — where the registry's per-value scan
+     * over-approximates instead. The asymmetry is deliberate: that scan only narrows a candidate
+     * set that is already a sound answer, while this one decides whether egress redaction of these
+     * entries would suffice for these bytes — a claim that cannot be made for content the same
+     * matcher just failed on. Reaching the event bound takes an eight-plus-character literal
+     * occurring ~a million times, so only degenerate content pays the refusal.
+     */
     scan(buffer) {
       const matched = new Map<string, WorkspaceFileSecretProvenanceEntry>()
       try {

apps/sim/lib/knowledge/application/search.ts

@@ -15,6 +15,10 @@ import { OrchestrationError } from '@/lib/core/orchestration/types'
 import { PlatformEvents } from '@/lib/core/telemetry'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance'
+import {
+  isDurableSecretProvenanceEnforced,
+  reportUnrecordedDurableProvenance,
+} from '@/lib/execution/durable-secret-provenance-enforcement'
 import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case'
 import {
   KnowledgeUsageLimitExceededError,
@@ -488,6 +492,8 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({
       }
     })
     if (registry && provenanceSnapshot) {
+      const knowledgeEnforced = isDurableSecretProvenanceEnforced('knowledge')
+      let unrecordedCount = provenanceSnapshot.unrecordedCount
       for (const [documentId, document] of Object.entries(provenanceSnapshot.documentMetadata)) {
         const renderedMetadata = results
           .filter((result) => result.documentId === documentId)
@@ -496,18 +502,34 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({
             sourceUrl: result.sourceUrl,
             metadata: result.metadata,
           }))
+        if (renderedMetadata.length === 0) continue
+        if (document.provenance.status === 'unknown' && !knowledgeEnforced) unrecordedCount += 1
         if (
-          renderedMetadata.length > 0 &&
           !(await importDurableSecretProvenance(
             registry,
             document.provenance,
             renderedMetadata,
-            'knowledge'
+            'knowledge',
+            { reportUnrecorded: false }
           ))
         ) {
           registry.markIncomplete('knowledge-result-provenance-unavailable')
         }
       }
+      /**
+       * One entry for the whole search — chunks and rendered metadata are one read. Skipped when
+       * the registry latched: a latched read never reaches a model, and this entry exists to say a
+       * fail-open read went ahead unvouched.
+       */
+      if (unrecordedCount > 0 && !registry.isPermanentlyIncomplete()) {
+        reportUnrecordedDurableProvenance({
+          surface: 'knowledge',
+          cause: 'durable-provenance-unknown',
+          affectedCount: unrecordedCount,
+          workspaceId: context.workspaceId,
+          actorUserId: userId,
+        })
+      }
     }
     const cost = baseCost
       ? {

apps/sim/lib/knowledge/secret-provenance.test.ts

@@ -1,24 +1,34 @@
 /**
  * @vitest-environment node
  */
-import { document } from '@sim/db/schema'
+import { document, embedding } from '@sim/db/schema'
 import { queueTableRows, resetDbChainMock } from '@sim/testing'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import { hashDurableSecretProvenanceValue } from '@/lib/execution/durable-secret-provenance'
 import {
   createKnowledgeDocumentSourceValue,
+  importKnowledgePersistedResponseSecretProvenance,
+  importKnowledgeSearchResultSecretProvenance,
   loadKnowledgeDocumentSecretRegistry,
   readBoundKnowledgeDocumentSecretProvenance,
 } from '@/lib/knowledge/secret-provenance'
+import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
-const { mockDecryptSecret } = vi.hoisted(() => ({
+const { mockDecryptSecret, mockIsEnforced, mockReport } = vi.hoisted(() => ({
   mockDecryptSecret: vi.fn(),
+  mockIsEnforced: vi.fn(() => false),
+  mockReport: vi.fn(),
 }))
 
 vi.mock('@/lib/core/security/encryption', () => ({
   decryptSecret: mockDecryptSecret,
 }))
 
+vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({
+  isDurableSecretProvenanceEnforced: mockIsEnforced,
+  reportUnrecordedDurableProvenance: mockReport,
+}))
+
 const DOCUMENT_SOURCE = createKnowledgeDocumentSourceValue({
   filename: 'source.pdf',
   fileUrl: '/api/files/serve/workspace%2Fworkspace-1%2Fsource.pdf?context=workspace',
@@ -39,6 +49,7 @@ describe('knowledge durable secret provenance', () => {
     resetDbChainMock()
     queueTableRows(document, [DOCUMENT_ROW])
     mockDecryptSecret.mockResolvedValue({ decrypted: 'tracked-secret' })
+    mockIsEnforced.mockReturnValue(false)
   })
 
   it('uses the same explicit source shape for joined rows and persisted writes', () => {
@@ -130,3 +141,108 @@ describe('knowledge durable secret provenance', () => {
     })
   })
 })
+
+describe('knowledge unrecorded-read reporting', () => {
+  const SCOPE = { userId: 'user-1', workspaceId: 'workspace-1' }
+  const UNRECORDED_DOCUMENT_ROW = {
+    id: 'doc-1',
+    ...DOCUMENT_SOURCE,
+    secretProvenanceVersion: 1,
+    provenanceSourceHash: null,
+    status: 'unknown',
+    entries: null,
+  }
+  const UNRECORDED_CHUNK_ROW = {
+    id: 'chunk-1',
+    documentId: 'doc-1',
+    content: 'chunk text',
+    chunkHash: 'stale',
+    secretProvenanceVersion: 1,
+    provenanceContentHash: null,
+    status: 'unknown',
+    entries: null,
+  }
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    mockIsEnforced.mockReturnValue(false)
+  })
+
+  it('reports one aggregated entry per read, naming workspace, actor, and count', async () => {
+    queueTableRows(document, [UNRECORDED_DOCUMENT_ROW])
+    queueTableRows(embedding, [UNRECORDED_CHUNK_ROW])
+    const registry = new ResolvedSecretTraceRegistry([], SCOPE)
+
+    await expect(
+      importKnowledgePersistedResponseSecretProvenance({
+        registry,
+        documents: [{ id: 'doc-1', source: DOCUMENT_SOURCE, value: {} }],
+        chunks: [{ id: 'chunk-1', documentId: 'doc-1', content: 'chunk text', value: {} }],
+        workspaceId: 'workspace-1',
+        actorUserId: 'user-1',
+      })
+    ).resolves.toBe(true)
+
+    expect(registry.isPermanentlyIncomplete()).toBe(false)
+    expect(mockReport).toHaveBeenCalledTimes(1)
+    expect(mockReport).toHaveBeenCalledWith({
+      surface: 'knowledge',
+      cause: 'durable-provenance-unknown',
+      affectedCount: 2,
+      workspaceId: 'workspace-1',
+      actorUserId: 'user-1',
+    })
+  })
+
+  /** A fault return fails the read closed, so no unvouched record reached anything to report. */
+  it('reports nothing when the read fails closed on a missing row', async () => {
+    queueTableRows(document, [])
+    const registry = new ResolvedSecretTraceRegistry([], SCOPE)
+
+    await expect(
+      importKnowledgePersistedResponseSecretProvenance({
+        registry,
+        documents: [{ id: 'doc-1', source: DOCUMENT_SOURCE, value: {} }],
+        workspaceId: 'workspace-1',
+        actorUserId: 'user-1',
+      })
+    ).resolves.toBe(false)
+
+    expect(mockReport).not.toHaveBeenCalled()
+  })
+
+  it('latches without reporting once the surface is enforced', async () => {
+    mockIsEnforced.mockReturnValue(true)
+    queueTableRows(document, [UNRECORDED_DOCUMENT_ROW])
+    const registry = new ResolvedSecretTraceRegistry([], SCOPE)
+
+    await expect(
+      importKnowledgePersistedResponseSecretProvenance({
+        registry,
+        documents: [{ id: 'doc-1', source: DOCUMENT_SOURCE, value: {} }],
+        workspaceId: 'workspace-1',
+        actorUserId: 'user-1',
+      })
+    ).resolves.toBe(false)
+
+    expect(registry.isPermanentlyIncomplete()).toBe(true)
+    expect(mockReport).not.toHaveBeenCalled()
+  })
+
+  /** The search read spans chunks and rendered metadata, so its caller owns the one report. */
+  it('returns the unrecorded count from a search import instead of reporting it', async () => {
+    queueTableRows(embedding, [{ ...UNRECORDED_CHUNK_ROW, documentId: DOCUMENT_ROW.id }])
+    queueTableRows(document, [DOCUMENT_ROW])
+    const registry = new ResolvedSecretTraceRegistry([], SCOPE)
+
+    const snapshot = await importKnowledgeSearchResultSecretProvenance({
+      registry,
+      results: [{ id: 'chunk-1', documentId: DOCUMENT_ROW.id, content: 'chunk text' }],
+    })
+
+    expect(snapshot.imported).toBe(true)
+    expect(snapshot.unrecordedCount).toBe(1)
+    expect(mockReport).not.toHaveBeenCalled()
+  })
+})