improvement(search): log the shape of each search and the source, kind, and size of each read

apps/sim/lib/knowledge/application/search-diagnostics.ts

@@ -3,10 +3,18 @@ import type { knowledgeOperations } from '@/lib/knowledge/application/operations
 import type { SearchKnowledgeInput } from '@/lib/knowledge/application/search'
 import {
   measureSearchStage,
+  type SearchDiagnosticMetadata,
   type SearchStage,
   withSearchDiagnostics,
 } from '@/lib/knowledge/search/diagnostics'
 
+function scopeKind(input: {
+  workspaceId?: string | null
+  organizationId?: string | null
+}): SearchDiagnosticMetadata['scopeKind'] {
+  return input.organizationId ? 'organization' : input.workspaceId ? 'workspace' : undefined
+}
+
 /** Preserve the operation and authorization contract while timing the whole use case. */
 export function instrumentSearchUseCase<
   I extends Omit<SearchKnowledgeInput, 'knowledgeBaseIds' | 'workspaceId' | 'organizationId'> & {
@@ -25,11 +33,7 @@ export function instrumentSearchUseCase<
         {
           surface: args.input.surface ?? 'other',
           principalKind: args.principal.kind,
-          scopeKind: args.input.organizationId
-            ? 'organization'
-            : args.input.workspaceId
-              ? 'workspace'
-              : undefined,
+          scopeKind: scopeKind(args.input),
           topK: args.input.topK,
           documentFilterCount: args.input.filters?.documentIds?.length ?? 0,
           hasSourceFilter: Boolean(args.input.filters?.source),
@@ -48,3 +52,35 @@ export function instrumentSearchUseCase<
       ),
   }
 }
+
+/**
+ * Times a document read as one trace, so each read logs its own summary: the source, the
+ * document's kind and size, and the window returned. A read inside a traced tool call adds to
+ * that call's trace instead.
+ */
+export function instrumentReadUseCase<
+  I extends {
+    surface?: SearchDiagnosticMetadata['surface']
+    workspaceId?: string | null
+    organizationId?: string | null
+    startOffset?: number
+  },
+  R,
+>(
+  useCase: AuthorizingUseCase<typeof knowledgeOperations.readDocument, I, R>
+): AuthorizingUseCase<typeof knowledgeOperations.readDocument, I, R> {
+  return {
+    ...useCase,
+    execute: (args) =>
+      withSearchDiagnostics(
+        {
+          operation: 'read_document',
+          ...(args.input.surface ? { surface: args.input.surface } : {}),
+          principalKind: args.principal.kind,
+          scopeKind: scopeKind(args.input),
+          continued: Boolean(args.input.startOffset),
+        },
+        () => measureSearchStage('document_read', () => useCase.execute(args))
+      ),
+  }
+}

apps/sim/lib/knowledge/mcp/server.ts

@@ -297,6 +297,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe
               principal,
               input: {
                 ...options,
+                surface: 'mcp',
                 documentId: id,
                 organizationId,
                 scaledWindow: true,

apps/sim/lib/knowledge/search/diagnostics.test.ts

@@ -1,13 +1,17 @@
 import { createDeferred } from '@sim/testing/helpers'
 import { getMockLogger } from '@sim/testing/mocks/logger.mock'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { OrchestrationError } from '@/lib/core/orchestration/types'
 import {
+  describeSearchRequest,
+  documentKindLabel,
   measureSearchStage,
   recordSearchProviderResult,
   withSearchDiagnostics,
   withSearchProviderDiagnostics,
   withSearchRequestDiagnostics,
 } from '@/lib/knowledge/search/diagnostics'
+import { LiveReadError } from '@/lib/sim-search/live/read-error'
 
 const logs = getMockLogger('KnowledgeSearchDiagnostics')
 
@@ -171,4 +175,112 @@ describe('search pipeline diagnostics', () => {
     )
     expect(JSON.stringify(logs.info.mock.calls)).not.toContain('private preview content')
   })
+
+  it('describes a search request without its text, modifiers, cursor, or account', () => {
+    const shape = describeSearchRequest({
+      query: '  private launch   plan ',
+      filters: {
+        startDate: '2026-10-30T00:00:00Z',
+        endDate: '2026-11-13T12:00:00Z',
+        sortBy: 'newest',
+      },
+      nativeQueries: [
+        {
+          provider: 'slack',
+          query: 'private decision',
+          modifiers: 'in:#private-channel',
+          cursor: 'private-cursor',
+          accountId: 'private-account',
+          termClauses: ['private term'],
+        },
+        { provider: 'jira', query: '', kind: 'issues' },
+      ],
+    })
+    expect(shape).toEqual({
+      queryTermCount: 3,
+      sortBy: 'newest',
+      dateWindowDays: 14.5,
+      nativeQueries: [
+        {
+          provider: 'slack',
+          hasText: true,
+          hasModifiers: true,
+          hasCursor: true,
+          hasProject: false,
+          targetsAccount: true,
+          keywordOnly: false,
+          termClauseCount: 1,
+        },
+        {
+          provider: 'jira',
+          kind: 'issues',
+          hasText: false,
+          hasModifiers: false,
+          hasCursor: false,
+          hasProject: false,
+          targetsAccount: false,
+          keywordOnly: false,
+          termClauseCount: 0,
+        },
+      ],
+    })
+    expect(JSON.stringify(shape)).not.toContain('private')
+  })
+
+  it('keeps a document kind only when it is a bounded token', () => {
+    expect(documentKindLabel('file')).toBe('file')
+    expect(documentKindLabel('application/vnd.google-apps.document')).toBe(
+      'application/vnd.google-apps.document'
+    )
+    expect(documentKindLabel('Quarterly plan for a private customer')).toBe('other')
+    expect(documentKindLabel('ProjectAtlas_Q3')).toBe('other')
+    expect(documentKindLabel('atlas-launch-notes')).toBe('other')
+    expect(documentKindLabel('notes/atlas')).toBe('other')
+    expect(documentKindLabel(undefined)).toBeUndefined()
+  })
+
+  it('names the class of a failure that ends a trace, never its message', async () => {
+    const failures = [
+      new OrchestrationError('not_found', 'private document title is gone'),
+      new LiveReadError('private provider text', true),
+      new Error('private unexpected text'),
+    ]
+    for (const error of failures)
+      await expect(
+        withSearchDiagnostics({ operation: 'read_document' }, async () => {
+          throw error
+        })
+      ).rejects.toBe(error)
+    const summaries = logs.info.mock.calls
+      .filter(([message]) => message === 'Knowledge search completed')
+      .slice(-3)
+      .map(([, fields]) => fields?.failure)
+    expect(summaries).toEqual([
+      { reason: 'not_found' },
+      { reason: 'provider_retryable' },
+      { reason: 'unexpected_error' },
+    ])
+    const logged = JSON.stringify(logs.info.mock.calls)
+    for (const error of failures) expect(logged).not.toContain(error.message)
+  })
+
+  it('keeps the failure class when an outer tool call turns the error into a result', async () => {
+    await withSearchDiagnostics({ surface: 'copilot' }, async () => {
+      try {
+        await withSearchDiagnostics({ operation: 'read_document' }, async () => {
+          throw new OrchestrationError('not_found', 'private document title is gone')
+        })
+      } catch {
+        return { success: false }
+      }
+    })
+    expect(logs.info).toHaveBeenLastCalledWith(
+      'Knowledge search completed',
+      expect.objectContaining({
+        surface: 'copilot',
+        outcome: 'error',
+        failure: { reason: 'not_found' },
+      })
+    )
+  })
 })