From 34d7ac1a13f5048fdb0b5c374bcc13c3da082024 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:43:45 -0800 Subject: [PATCH 1/2] improvement(search): log the shape of each search and the source, kind, and size of each read --- .../application/search-diagnostics.ts | 46 ++++- apps/sim/lib/knowledge/mcp/server.ts | 1 + .../lib/knowledge/search/diagnostics.test.ts | 109 ++++++++++++ apps/sim/lib/knowledge/search/diagnostics.ts | 164 ++++++++++++++++-- .../lib/sim-search/live/application.test.ts | 54 ++++++ apps/sim/lib/sim-search/live/application.ts | 116 ++++++++----- apps/sim/lib/sim-search/live/dates.ts | 5 +- .../sim/lib/sim-search/live/providers.test.ts | 46 +++++ apps/sim/lib/sim-search/live/read-error.ts | 5 + apps/sim/lib/sim-search/live/slack.ts | 14 +- 10 files changed, 488 insertions(+), 72 deletions(-) diff --git a/apps/sim/lib/knowledge/application/search-diagnostics.ts b/apps/sim/lib/knowledge/application/search-diagnostics.ts index a1e444d726c..1b2a289d393 100644 --- a/apps/sim/lib/knowledge/application/search-diagnostics.ts +++ b/apps/sim/lib/knowledge/application/search-diagnostics.ts @@ -3,10 +3,18 @@ import type { knowledgeOperations } from '@/lib/knowledge/application/operations import type { SearchKnowledgeInput } from '@/lib/knowledge/application/search' import { measureSearchStage, + type SearchDiagnosticMetadata, type SearchStage, withSearchDiagnostics, } from '@/lib/knowledge/search/diagnostics' +function scopeKind(input: { + workspaceId?: string | null + organizationId?: string | null +}): SearchDiagnosticMetadata['scopeKind'] { + return input.organizationId ? 'organization' : input.workspaceId ? 'workspace' : undefined +} + /** Preserve the operation and authorization contract while timing the whole use case. */ export function instrumentSearchUseCase< I extends Omit & { @@ -25,11 +33,7 @@ export function instrumentSearchUseCase< { surface: args.input.surface ?? 'other', principalKind: args.principal.kind, - scopeKind: args.input.organizationId - ? 'organization' - : args.input.workspaceId - ? 'workspace' - : undefined, + scopeKind: scopeKind(args.input), topK: args.input.topK, documentFilterCount: args.input.filters?.documentIds?.length ?? 0, hasSourceFilter: Boolean(args.input.filters?.source), @@ -48,3 +52,35 @@ export function instrumentSearchUseCase< ), } } + +/** + * Times a document read as one trace, so each read logs its own summary: the source, the + * document's kind and size, and the window returned. A read inside a traced tool call adds to + * that call's trace instead. + */ +export function instrumentReadUseCase< + I extends { + surface?: SearchDiagnosticMetadata['surface'] + workspaceId?: string | null + organizationId?: string | null + startOffset?: number + }, + R, +>( + useCase: AuthorizingUseCase +): AuthorizingUseCase { + return { + ...useCase, + execute: (args) => + withSearchDiagnostics( + { + operation: 'read_document', + ...(args.input.surface ? { surface: args.input.surface } : {}), + principalKind: args.principal.kind, + scopeKind: scopeKind(args.input), + continued: Boolean(args.input.startOffset), + }, + () => measureSearchStage('document_read', () => useCase.execute(args)) + ), + } +} diff --git a/apps/sim/lib/knowledge/mcp/server.ts b/apps/sim/lib/knowledge/mcp/server.ts index b4dc2b4aa2c..80641cd8ed3 100644 --- a/apps/sim/lib/knowledge/mcp/server.ts +++ b/apps/sim/lib/knowledge/mcp/server.ts @@ -297,6 +297,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe principal, input: { ...options, + surface: 'mcp', documentId: id, organizationId, scaledWindow: true, diff --git a/apps/sim/lib/knowledge/search/diagnostics.test.ts b/apps/sim/lib/knowledge/search/diagnostics.test.ts index 59fe303e715..44aabc5b6e4 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.test.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.test.ts @@ -1,13 +1,17 @@ import { createDeferred } from '@sim/testing/helpers' import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { + describeSearchRequest, + documentKindLabel, measureSearchStage, recordSearchProviderResult, withSearchDiagnostics, withSearchProviderDiagnostics, withSearchRequestDiagnostics, } from '@/lib/knowledge/search/diagnostics' +import { LiveReadError } from '@/lib/sim-search/live/read-error' const logs = getMockLogger('KnowledgeSearchDiagnostics') @@ -171,4 +175,109 @@ describe('search pipeline diagnostics', () => { ) expect(JSON.stringify(logs.info.mock.calls)).not.toContain('private preview content') }) + + it('describes a search request without its text, modifiers, cursor, or account', () => { + const shape = describeSearchRequest({ + query: ' private launch plan ', + filters: { + startDate: '2026-10-30T00:00:00Z', + endDate: '2026-11-13T12:00:00Z', + sortBy: 'newest', + }, + nativeQueries: [ + { + provider: 'slack', + query: 'private decision', + modifiers: 'in:#private-channel', + cursor: 'private-cursor', + accountId: 'private-account', + termClauses: ['private term'], + }, + { provider: 'jira', query: '', kind: 'issues' }, + ], + }) + expect(shape).toEqual({ + queryTermCount: 3, + sortBy: 'newest', + dateWindowDays: 14.5, + nativeQueries: [ + { + provider: 'slack', + hasText: true, + hasModifiers: true, + hasCursor: true, + hasProject: false, + targetsAccount: true, + keywordOnly: false, + termClauseCount: 1, + }, + { + provider: 'jira', + kind: 'issues', + hasText: false, + hasModifiers: false, + hasCursor: false, + hasProject: false, + targetsAccount: false, + keywordOnly: false, + termClauseCount: 0, + }, + ], + }) + expect(JSON.stringify(shape)).not.toContain('private') + }) + + it('keeps a document kind only when it is a bounded token', () => { + expect(documentKindLabel('file')).toBe('file') + expect(documentKindLabel('application/vnd.google-apps.document')).toBe( + 'application/vnd.google-apps.document' + ) + expect(documentKindLabel('Quarterly plan for a private customer')).toBe('other') + expect(documentKindLabel(undefined)).toBeUndefined() + }) + + it('names the class of a failure that ends a trace, never its message', async () => { + const failures = [ + new OrchestrationError('not_found', 'private document title is gone'), + new LiveReadError('private provider text', true), + new Error('private unexpected text'), + ] + for (const error of failures) + await expect( + withSearchDiagnostics({ operation: 'read_document' }, async () => { + throw error + }) + ).rejects.toBe(error) + const summaries = logs.info.mock.calls + .filter(([message]) => message === 'Knowledge search completed') + .slice(-3) + .map(([, fields]) => fields?.failure) + expect(summaries).toEqual([ + { reason: 'not_found' }, + { reason: 'provider_retryable' }, + { reason: 'unexpected_error' }, + ]) + const logged = JSON.stringify(logs.info.mock.calls) + for (const error of failures) expect(logged).not.toContain(error.message) + }) + + it('keeps the failure class when an outer tool call turns the error into a result', async () => { + await withSearchDiagnostics({ surface: 'copilot' }, async () => { + try { + await withSearchDiagnostics({ operation: 'read_document' }, async () => { + throw new OrchestrationError('not_found', 'private document title is gone') + }) + } catch { + return { success: false } + } + }) + expect(logs.info).toHaveBeenLastCalledWith( + 'Knowledge search completed', + expect.objectContaining({ + surface: 'copilot', + outcome: 'error', + failure: { reason: 'not_found' }, + }) + ) + }) }) diff --git a/apps/sim/lib/knowledge/search/diagnostics.ts b/apps/sim/lib/knowledge/search/diagnostics.ts index 64ca7f4808a..815f8ab4760 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.ts @@ -2,7 +2,13 @@ import { AsyncLocalStorage } from 'node:async_hooks' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { toRecord } from '@sim/utils/object' -import type { LiveSearchProvider } from '@/lib/api/contracts/mothership-assistant-tools' +import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge' +import type { + LiveSearchProvider, + NativeSearchQuery, +} from '@/lib/api/contracts/mothership-assistant-tools' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { nativeDateBounds } from '@/lib/sim-search/live/dates' const logger = createLogger('KnowledgeSearchDiagnostics', { logLevel: 'INFO' }) const PROGRESS_INTERVAL_MS = 5000 @@ -109,6 +115,80 @@ export interface SearchDiagnosticMetadata { accountCount?: number searchedAccountCount?: number nativeQueryCount?: number + /** Words in the plain query; the words themselves are never logged. */ + queryTermCount?: number + sortBy?: WorkspaceSearchFilters['sortBy'] + /** Days between a date filter's bounds, when it has both. */ + dateWindowDays?: number + /** One entry per native query, in request order, so `queryIndex` resolves to a provider. */ + nativeQueries?: NativeQueryShape[] + /** The source a read came from; a search reports its sources on each provider summary. */ + provider?: LiveSearchProvider + /** The provider's type for the document read, such as `message` or `file`. */ + documentKind?: string + /** Characters in the whole document text a read found, not just the window it returned. */ + contentCharacters?: number + returnedCharacters?: number + /** Whether a read continued from an earlier window rather than starting at the beginning. */ + continued?: boolean + hasMore?: boolean +} + +/** What a native query asked for, without its text, modifiers, cursor, or account. */ +interface NativeQueryShape { + provider: LiveSearchProvider + kind?: NativeSearchQuery['kind'] + browse?: NativeSearchQuery['browse'] + hasText: boolean + hasModifiers: boolean + hasCursor: boolean + hasProject: boolean + targetsAccount: boolean + keywordOnly: boolean + termClauseCount: number +} + +const DAY_MS = 86_400_000 + +/** The content-free shape of a search request: counts, flags, and the enums it selected. */ +export function describeSearchRequest(input: { + query: string + filters?: WorkspaceSearchFilters + nativeQueries?: readonly NativeSearchQuery[] +}): SearchDiagnosticMetadata { + const { start, end } = nativeDateBounds({ filters: input.filters }) + return { + queryTermCount: input.query.split(/\s+/).filter(Boolean).length, + sortBy: input.filters?.sortBy, + ...(start && end + ? { dateWindowDays: Math.round(((Date.parse(end) - Date.parse(start)) / DAY_MS) * 10) / 10 } + : {}), + ...(input.nativeQueries?.length + ? { + nativeQueries: input.nativeQueries.map((query) => ({ + provider: query.provider, + kind: query.kind, + browse: query.browse, + hasText: Boolean(query.query), + hasModifiers: Boolean(query.modifiers?.trim()), + hasCursor: Boolean(query.cursor), + hasProject: Boolean(query.project), + targetsAccount: Boolean(query.accountId), + keywordOnly: Boolean(query.keywordOnly), + termClauseCount: query.termClauses?.length ?? 0, + })), + } + : {}), + } +} + +/** + * A provider's document type reduced to a bounded token. Kinds are code-defined except a few + * passed through from a provider, such as a Drive MIME type, so anything else becomes `other`. + */ +export function documentKindLabel(kind: string | undefined): string | undefined { + if (!kind) return undefined + return /^[\w.+/-]{1,80}$/.test(kind) ? kind : 'other' } interface StageTiming { @@ -123,6 +203,8 @@ interface SearchTrace { startedAt: number metadata: SearchDiagnosticMetadata stages: Partial> + /** The first failure thrown through the trace, kept even when an outer caller swallows it. */ + failure?: ReturnType active: Map< symbol, { stage: SearchStage; startedAt: number; provider?: LiveSearchProvider; accountIndex?: number } @@ -136,6 +218,23 @@ const currentQueries = new AsyncLocalStorage() const roundMs = (value: number) => Math.round(value * 100) / 100 const MAX_PROVIDER_REQUEST_DIAGNOSTICS = 128 + +/** + * Events a provider adapter reports while it searches, counted on its provider summary. Each is + * a fixed code; an adapter never passes provider data through one. + */ +type SearchProviderSignal = + /** Channels a Slack search with text listed alongside its messages. */ + | 'slack.channels_listed' + /** An empty search filtered with `in:#name` named a channel that exists. */ + | 'slack.channel_filter_exists' + /** An empty search filtered with `in:#name` named no searchable channel. */ + | 'slack.channel_filter_missing' + /** The channel lookup had another page and no exact match, so it ruled nothing out. */ + | 'slack.channel_filter_unresolved' + /** The channel lookup failed; the search kept its results without a note. */ + | 'slack.channel_lookup_failed' + const FAILURE_REASONS = [ 'provider_validation', 'provider_access_denied', @@ -148,6 +247,10 @@ const FAILURE_REASONS = [ 'mcp_payload_too_large', 'mcp_payload_unreadable', 'mcp_account_required', + /** A read failed in a way the same read may get past if retried, such as a timeout. */ + 'provider_retryable', + /** A read failed in a way retrying the same read will not change. */ + 'provider_error', /** The account never got to search: no slot freed with enough of the search deadline left. */ 'not_started', ] as const @@ -181,7 +284,8 @@ const ERROR_CODES = new Set([ interface ProviderMetadata { provider: LiveSearchProvider - accountIndex: number + /** The account's position among those a search fanned out to; a read has one account. */ + accountIndex?: number } interface ProviderResult { @@ -234,6 +338,22 @@ interface ProviderTrace { requests: RequestDiagnostic[] requestCount: number failedRequestCount: number + signals: Partial> +} + +/** Failure classes recognized by error name when no diagnostic code or domain class applies. */ +const NAMED_FAILURES: Record = { + AbortError: 'aborted', + PayloadSizeLimitError: 'response_size_limit', + SyntaxError: 'invalid_json', + TimeoutError: 'timeout', +} + +function failureReason(error: unknown, record: Record): string { + const coded = FAILURE_REASONS.find((candidate) => candidate === record.diagnosticCode) + if (coded) return coded + if (error instanceof OrchestrationError) return error.code + return (typeof record.name === 'string' && NAMED_FAILURES[record.name]) || 'unexpected_error' } /** Error messages, stacks, response bodies, and arbitrary provider codes never enter logs. */ @@ -246,17 +366,7 @@ function describeFailure(error: unknown) { [-32700, -32600, -32601, -32602, -32603, -32000, -32001].includes(value) ? value : undefined - const reason = - FAILURE_REASONS.find((candidate) => candidate === record.diagnosticCode) ?? - (record.name === 'AbortError' - ? 'aborted' - : record.name === 'PayloadSizeLimitError' - ? 'response_size_limit' - : record.name === 'SyntaxError' - ? 'invalid_json' - : record.name === 'TimeoutError' - ? 'timeout' - : 'unexpected_error') + const reason = failureReason(error, record) const httpStatus = typeof record.httpStatus === 'number' && Number.isInteger(record.httpStatus) && @@ -292,6 +402,7 @@ export async function withSearchProviderDiagnostics( requests: [], requestCount: 0, failedRequestCount: 0, + signals: {}, } return providers.run(provider, async () => { let failure: ReturnType | undefined @@ -304,6 +415,7 @@ export async function withSearchProviderDiagnostics( provider.closed = true logger.info('Live search provider completed', { searchId: root.searchId, + operation: root.metadata.operation, ...metadata, startedAfterMs: roundMs(provider.startedAt - root.startedAt), elapsedMs: roundMs(performance.now() - provider.startedAt), @@ -320,6 +432,7 @@ export async function withSearchProviderDiagnostics( failedRequestCount: provider.failedRequestCount, pendingRequestCount: provider.pendingRequestCount, omittedRequestCount: provider.requestCount - provider.requests.length, + ...(Object.keys(provider.signals).length ? { signals: provider.signals } : {}), }) } }) @@ -333,6 +446,12 @@ export function recordSearchProviderResult(result: ProviderResult, error?: unkno }) } +/** Counts a provider adapter's event on the summary of the provider it runs under. */ +export function recordSearchProviderSignal(signal: SearchProviderSignal, count = 1): void { + const provider = providers.getStore() + if (provider) provider.signals[signal] = (provider.signals[signal] ?? 0) + count +} + /** Native query ordinals distinguish parallel requests without retaining their query text. */ export function withSearchQueryDiagnostics( queryIndex: number | undefined, @@ -436,9 +555,15 @@ export async function withSearchDiagnostics( metadata: SearchDiagnosticMetadata, run: () => Promise ): Promise { - if (traces.getStore()) { + const outer = traces.getStore() + if (outer) { annotateSearchDiagnostics(metadata) - return run() + try { + return await run() + } catch (error) { + outer.failure ??= describeFailure(error) + throw error + } } const trace: SearchTrace = { searchId: generateId(), @@ -472,9 +597,16 @@ export async function withSearchDiagnostics( ? 'partial' : 'success' return result + } catch (error) { + trace.failure ??= describeFailure(error) + throw error } finally { clearInterval(timer) - logger.info('Knowledge search completed', { ...snapshot(), outcome }) + logger.info('Knowledge search completed', { + ...snapshot(), + outcome, + ...(trace.failure ? { failure: trace.failure } : {}), + }) } }) } diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 4c3403096a4..006547ca81a 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -1189,6 +1189,60 @@ describe('authorized live retrieval', () => { }) ).rejects.toBe('user_stop:test') }) + describe('read diagnostics', () => { + async function searchedDocumentId() { + const [result] = (await searchLiveKnowledge.execute({ principal, input })).results + if (!result) throw new Error('Expected the search fixture to return a result') + diagnosticsLogger.info.mockClear() + return result.documentId + } + const logged = (message: string) => + diagnosticsLogger.info.mock.calls.filter(([logged]) => logged === message).map(([, f]) => f) + + it('logs the source, kind, and size of each read without its title or text', async () => { + const documentId = await searchedDocumentId() + mocks.read.mockResolvedValue({ + ...document, + kind: 'file', + title: 'Private roadmap', + content: 'private text '.repeat(2_500), + }) + await readLiveDocument.execute({ + principal, + input: { + workspaceId: 'workspace', + surface: 'mcp', + documentId, + limit: 3, + scaledWindow: true, + resultSecretRegistry: new ResolvedSecretTraceRegistry([]), + }, + }) + expect(logged('Knowledge search completed')).toEqual([ + expect.objectContaining({ + operation: 'read_document', + surface: 'mcp', + continued: false, + provider: 'google_drive', + documentKind: 'file', + contentCharacters: 32_500, + returnedCharacters: 20_000, + hasMore: true, + outcome: 'success', + stages: expect.objectContaining({ + document_read: expect.objectContaining({ count: 1 }), + }), + }), + ]) + expect(logged('Live search provider completed')).toEqual([ + expect.objectContaining({ operation: 'read_document', provider: 'google_drive' }), + ]) + const text = JSON.stringify(diagnosticsLogger.info.mock.calls) + expect(text).not.toContain('Private roadmap') + expect(text).not.toContain('private text') + }) + }) + describe('read window', () => { const content = 'x'.repeat(30_000) async function readWindow(extra: Record, text = content) { diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index ca6415593db..5e613bd81f0 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -38,9 +38,14 @@ import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/au import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import type { SearchKnowledgeInput } from '@/lib/knowledge/application/search' -import { instrumentSearchUseCase } from '@/lib/knowledge/application/search-diagnostics' +import { + instrumentReadUseCase, + instrumentSearchUseCase, +} from '@/lib/knowledge/application/search-diagnostics' import { annotateSearchDiagnostics, + describeSearchRequest, + documentKindLabel, measureSearchStage, recordSearchProviderResult, withSearchProviderDiagnostics, @@ -483,6 +488,9 @@ const liveKnowledgeSearch = defineAuthorizedKnowledgeUseCase({ queries?.some((query) => !query.query) ) input = { ...input, filters: withImpliedListingBound(input.filters, new Date()) } + annotateSearchDiagnostics( + describeSearchRequest({ query: input.query, filters: input.filters, nativeQueries: queries }) + ) if ( (!input.query.trim() && !hasDateBounds(input.filters) && @@ -996,6 +1004,7 @@ export const searchLiveKnowledge = instrumentSearchUseCase( ) export type LiveReadInput = ResourceOwner & { + surface?: SearchKnowledgeInput['surface'] documentId: string filters?: WorkspaceSearchFilters limit: number @@ -1059,7 +1068,7 @@ function liveReadFailure(error: unknown, provider: string, deadline?: AbortSigna return error } -export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ +const liveDocumentRead = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readDocument, resolveContext: ({ input }: { input: LiveReadInput }) => resolveKnowledgeOwnerContext(input), async execute({ principal, input }) { @@ -1087,56 +1096,59 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ /** Cancellation by the caller passes through unclassified. */ const readFailure = (error: unknown, deadline?: AbortSignal) => input.signal?.aborted ? error : liveReadFailure(error, reference.provider, deadline) + annotateSearchDiagnostics({ provider: reference.provider }) const deadline = AbortSignal.timeout(READ_DEADLINE_MS) const signal = input.signal ? AbortSignal.any([input.signal, deadline]) : deadline - const [resolved, policies] = await Promise.all([ - waitWithAbort(resolveLiveAccount(input, userId, reference.account, signal), signal), - loadLiveSearchPolicies(input), - ]).catch((error: unknown) => { + const { resolved, document } = await withSearchProviderDiagnostics( + { provider: reference.provider }, + async () => { + const [resolved, policies] = await Promise.all([ + measureSearchStage('live.resolve', () => + waitWithAbort(resolveLiveAccount(input, userId, reference.account, signal), signal) + ), + measureSearchStage('live.policies', () => loadLiveSearchPolicies(input)), + ]) + if (resolved.account.provider !== reference.provider) + throw new OrchestrationError('not_found', 'Document account changed') + const pool = createPinnedConnectionPool() + let session: LiveAccountSession | undefined + try { + const opened = await measureSearchStage('live.session', () => + openLiveAccountSession({ owner: input, userId, resolved, policies, signal, pool }) + ) + session = opened + if (!(await measureSearchStage('live.verify', () => opened.verify(reference)))) + throw new OrchestrationError( + 'not_found', + 'Document is outside your organization’s search scope' + ) + const document = await measureSearchStage('live.read', () => + opened.read(reference, input.filters) + ) + /** Readers degrade section failures to warnings, so the signal decides cancellation. */ + signal.throwIfAborted() + const current = await measureSearchStage('live.verify', () => + opened.verifyCurrent(document) + ) + /** A verifier may report a check cut short by cancellation as a normal result. */ + signal.throwIfAborted() + if (!current) + throw new OrchestrationError( + 'not_found', + 'Document is outside your organization’s search scope' + ) + return { resolved, document } + } finally { + try { + await session?.close() + } finally { + pool.destroy() + } + } + } + ).catch((error: unknown) => { throw readFailure(error, deadline) }) - if (resolved.account.provider !== reference.provider) - throw new OrchestrationError('not_found', 'Document account changed') - const pool = createPinnedConnectionPool() - let document: NativeDocument - let session: LiveAccountSession | undefined - try { - session = await openLiveAccountSession({ - owner: input, - userId, - resolved, - policies, - signal, - pool, - }) - if (!(await session.verify(reference))) - throw new OrchestrationError( - 'not_found', - 'Document is outside your organization’s search scope' - ) - const currentSession = session - document = await measureSearchStage('live.read', () => - currentSession.read(reference, input.filters) - ) - /** Readers degrade section failures to warnings, so the signal decides cancellation. */ - signal.throwIfAborted() - const current = await session.verifyCurrent(document) - /** A verifier may report a check cut short by cancellation as a normal result. */ - signal.throwIfAborted() - if (!current) - throw new OrchestrationError( - 'not_found', - 'Document is outside your organization’s search scope' - ) - } catch (error) { - throw readFailure(error, deadline) - } finally { - try { - await session?.close() - } finally { - pool.destroy() - } - } if (!matchesLiveFilters(document, input.documentId, reference.provider, input.filters)) throw new OrchestrationError('not_found', 'Document is outside the selected search filters') const content = safeContent(document.content, input.resultSecretRegistry) @@ -1147,6 +1159,12 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ 'This document changed. Read again from the beginning' ) const end = readWindowEnd(content, start, input.scaledWindow) + annotateSearchDiagnostics({ + documentKind: documentKindLabel(document.kind ?? reference.kind), + contentCharacters: content.length, + returnedCharacters: end - start, + hasMore: end < content.length, + }) return { documentId: input.documentId, knowledgeBaseId: '', @@ -1177,6 +1195,8 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ }, }) +export const readLiveDocument = instrumentReadUseCase(liveDocumentRead) + /** Metadata only; the provider's token is still reauthorized at search/read time. */ export const listLiveSearchAccounts = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listPersonalSearchIntegrations, diff --git a/apps/sim/lib/sim-search/live/dates.ts b/apps/sim/lib/sim-search/live/dates.ts index 7018ed714f9..a240dc54495 100644 --- a/apps/sim/lib/sim-search/live/dates.ts +++ b/apps/sim/lib/sim-search/live/dates.ts @@ -48,7 +48,10 @@ export function withImpliedListingBound( } /** Native bounds may be widened for provider precision; returned metadata is checked exactly. */ -export function nativeDateBounds(input: NativeSearchInput): { start?: string; end?: string } { +export function nativeDateBounds(input: Pick): { + start?: string + end?: string +} { const filters = input.filters const start = [filters?.startDate, filters?.modifiedAfter] .filter((value): value is string => Boolean(value)) diff --git a/apps/sim/lib/sim-search/live/providers.test.ts b/apps/sim/lib/sim-search/live/providers.test.ts index 50c69722f2a..c6bc58d8d76 100644 --- a/apps/sim/lib/sim-search/live/providers.test.ts +++ b/apps/sim/lib/sim-search/live/providers.test.ts @@ -1,4 +1,9 @@ +import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { describe, expect, it, vi } from 'vitest' +import { + withSearchDiagnostics, + withSearchProviderDiagnostics, +} from '@/lib/knowledge/search/diagnostics' import { readGitHub, searchGitHub } from '@/lib/sim-search/live/github' import { searchCalendar, searchDrive } from '@/lib/sim-search/live/google' import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' @@ -517,6 +522,47 @@ describe('Slack channels', () => { }) }) + it('counts what each channel lookup concluded on the Slack provider summary', async () => { + const api = client() + const filtered = (modifiers: string) => + searchSlack(api, { + ...input, + query: '', + scopes, + filters: { startDate: '2026-10-30T00:00:00Z' }, + native: { provider: 'slack', query: '', modifiers }, + }) + api.json + .mockResolvedValueOnce(rts([channel('C1', 'launch-team'), channel('C2', 'launch-ops')])) + .mockResolvedValueOnce(rts([])) + .mockResolvedValueOnce(rts([channel('C1', 'launch-team')])) + .mockResolvedValueOnce(rts([])) + .mockResolvedValueOnce(rts([])) + .mockResolvedValueOnce(rts([])) + .mockResolvedValueOnce({ ...rts([]), response_metadata: { next_cursor: 'more' } }) + .mockResolvedValueOnce(rts([])) + .mockRejectedValueOnce(new Error('lookup failed')) + await withSearchDiagnostics({ surface: 'mcp' }, () => + withSearchProviderDiagnostics({ provider: 'slack', accountIndex: 0 }, async () => { + await searchSlack(api, { ...input, scopes, native: { provider: 'slack', query: 'launch' } }) + await filtered('in:#launch-team') + await filtered('in:#launch-tem') + await filtered('in:#launch') + await filtered('in:#launch-crew') + }) + ) + const summary = getMockLogger('KnowledgeSearchDiagnostics').info.mock.calls.find( + ([message]) => message === 'Live search provider completed' + )?.[1] + expect(summary?.signals).toEqual({ + 'slack.channels_listed': 2, + 'slack.channel_filter_exists': 1, + 'slack.channel_filter_missing': 1, + 'slack.channel_filter_unresolved': 1, + 'slack.channel_lookup_failed': 1, + }) + }) + it('keeps the empty search when the channel lookup fails', async () => { const api = client() api.json diff --git a/apps/sim/lib/sim-search/live/read-error.ts b/apps/sim/lib/sim-search/live/read-error.ts index f2b9bbec5ba..f0fb306d680 100644 --- a/apps/sim/lib/sim-search/live/read-error.ts +++ b/apps/sim/lib/sim-search/live/read-error.ts @@ -1,9 +1,13 @@ +import type { SearchFailureReason } from '@/lib/knowledge/search/diagnostics' + /** * A provider-side read failure whose message tells the caller what to do next, and whether the * same read may succeed if retried. Messages are written by Search code, never copied from a * provider response body. */ export class LiveReadError extends Error { + readonly diagnosticCode: SearchFailureReason + constructor( message: string, readonly retryable: boolean, @@ -11,5 +15,6 @@ export class LiveReadError extends Error { ) { super(message) this.name = 'LiveReadError' + this.diagnosticCode = retryable ? 'provider_retryable' : 'provider_error' } } diff --git a/apps/sim/lib/sim-search/live/slack.ts b/apps/sim/lib/sim-search/live/slack.ts index d01a35f5b8b..8db571be21c 100644 --- a/apps/sim/lib/sim-search/live/slack.ts +++ b/apps/sim/lib/sim-search/live/slack.ts @@ -1,3 +1,4 @@ +import { recordSearchProviderSignal } from '@/lib/knowledge/search/diagnostics' import { dateSortDirection, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates' import { array, NativeSearchError, object, string } from '@/lib/sim-search/live/http' import { joinMessages } from '@/lib/sim-search/live/pages' @@ -102,13 +103,21 @@ async function channelFilterNote( if (input.signal?.aborted) throw error return undefined }) - if (!data) return undefined + if (!data) { + recordSearchProviderSignal('slack.channel_lookup_failed') + return undefined + } const channels = searchableChannels(data, input) const named = channels.find((channel) => channel.name.toLowerCase() === name) - if (named) + if (named) { + recordSearchProviderSignal('slack.channel_filter_exists') return `${channelLabel(named)} exists; none of its messages matched this query and date window.` + } /** A lookup with another page has not ruled the name out, so it only names what it found. */ const complete = !string(object(data.response_metadata).next_cursor) + recordSearchProviderSignal( + complete ? 'slack.channel_filter_missing' : 'slack.channel_filter_unresolved' + ) const similar = channels.slice(0, MAX_LISTED_CHANNELS).map(channelLabel).join(', ') return joinMessages([ complete ? `No channel named #${name} can be searched here.` : '', @@ -301,6 +310,7 @@ async function searchSlackPage( }) } const listed = listChannels ? searchableChannels(data, input).slice(0, MAX_LISTED_CHANNELS) : [] + if (listed.length) recordSearchProviderSignal('slack.channels_listed', listed.length) const nextCursor = string(object(data.response_metadata).next_cursor) || undefined /** Only a search with no result on any page can say what its channel filter matched. */ const filtered = From d03fd9f2df45bcec759162b9427519730784c668 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 13 Nov 2026 22:53:24 -0800 Subject: [PATCH 2/2] improvement(search): log a document kind only as a type name or MIME type --- apps/sim/lib/knowledge/search/diagnostics.test.ts | 3 +++ apps/sim/lib/knowledge/search/diagnostics.ts | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/knowledge/search/diagnostics.test.ts b/apps/sim/lib/knowledge/search/diagnostics.test.ts index 44aabc5b6e4..021c86fa509 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.test.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.test.ts @@ -233,6 +233,9 @@ describe('search pipeline diagnostics', () => { 'application/vnd.google-apps.document' ) expect(documentKindLabel('Quarterly plan for a private customer')).toBe('other') + expect(documentKindLabel('ProjectAtlas_Q3')).toBe('other') + expect(documentKindLabel('atlas-launch-notes')).toBe('other') + expect(documentKindLabel('notes/atlas')).toBe('other') expect(documentKindLabel(undefined)).toBeUndefined() }) diff --git a/apps/sim/lib/knowledge/search/diagnostics.ts b/apps/sim/lib/knowledge/search/diagnostics.ts index 815f8ab4760..8bc855b8778 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.ts @@ -182,13 +182,18 @@ export function describeSearchRequest(input: { } } +/** Lowercase type names such as `message` or `merge_requests`, as provider adapters define them. */ +const DOCUMENT_KIND = /^[a-z][a-z0-9_]{0,31}$/ +/** A registered MIME type, which Drive reports as a file's kind. */ +const MIME_TYPE = /^(application|audio|font|image|model|text|video)\/[a-z0-9][a-z0-9.+-]{0,79}$/ + /** - * A provider's document type reduced to a bounded token. Kinds are code-defined except a few - * passed through from a provider, such as a Drive MIME type, so anything else becomes `other`. + * A provider's document type when it is a type name or MIME type. A kind passed through from a + * provider in any other form, which could carry free text, becomes `other`. */ export function documentKindLabel(kind: string | undefined): string | undefined { if (!kind) return undefined - return /^[\w.+/-]{1,80}$/.test(kind) ? kind : 'other' + return DOCUMENT_KIND.test(kind) || MIME_TYPE.test(kind) ? kind : 'other' } interface StageTiming {