fix(knowledge): share the indexed-text bound, cut by code point, and reject oversized filenames and tags at the document APIs

Author: waleedlatif1

apps/sim/lib/api/contracts/knowledge/documents.test.ts

@@ -5,11 +5,14 @@ import { describe, expect, it } from 'vitest'
 import { z } from 'zod'
 import {
   bulkCreateDocumentsBodySchema,
+  createDocumentBodySchema,
   documentDataSchema,
   listKnowledgeDocumentsQuerySchema,
   parseDocumentTagFiltersParam,
+  updateDocumentBodySchema,
   upsertDocumentBodySchema,
 } from '@/lib/api/contracts/knowledge/documents'
+import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants'
 import { getDocumentIndexingStatus } from '@/lib/knowledge/documents/types'
 
 describe('document processing response compatibility', () => {
@@ -255,3 +258,35 @@ describe('internal document processingOptions', () => {
     })
   })
 })
+
+describe('document filename and tag bounds', () => {
+  const base = { fileUrl: 'https://example.com/a.txt', fileSize: 1, mimeType: 'text/plain' }
+  const atLimit = 'a'.repeat(MAX_DOCUMENT_INDEXED_TEXT_LENGTH)
+  const overLimit = `${atLimit}a`
+
+  it('accepts a filename and tag exactly at the indexed-text limit', () => {
+    expect(
+      createDocumentBodySchema.safeParse({ ...base, filename: atLimit, tag1: atLimit }).success
+    ).toBe(true)
+  })
+
+  it('rejects a filename over the limit on create, upsert, and update with a descriptive message', () => {
+    for (const schema of [createDocumentBodySchema, upsertDocumentBodySchema, updateDocumentBodySchema]) {
+      const result = schema.safeParse({ ...base, filename: overLimit })
+      expect(result.success).toBe(false)
+      expect(result.error?.issues[0]?.message).toBe(
+        `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
+      )
+    }
+  })
+
+  it('rejects a tag value over the limit on create and update', () => {
+    for (const schema of [createDocumentBodySchema, updateDocumentBodySchema]) {
+      const result = schema.safeParse({ ...base, filename: 'a.txt', tag3: overLimit })
+      expect(result.success).toBe(false)
+      expect(result.error?.issues[0]?.message).toBe(
+        `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
+      )
+    }
+  })
+})

apps/sim/lib/api/contracts/knowledge/documents.ts

@@ -15,7 +15,11 @@ import {
 import { privateSecretProvenanceBundleSchema } from '@/lib/api/contracts/primitives'
 import { defineRouteContract } from '@/lib/api/contracts/types'
 import { PRIVATE_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata'
-import { getFieldTypeForSlot, MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE } from '@/lib/knowledge/constants'
+import {
+  getFieldTypeForSlot,
+  MAX_DOCUMENT_INDEXED_TEXT_LENGTH,
+  MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE,
+} from '@/lib/knowledge/constants'
 import { DOCUMENT_PROCESSING_STATUSES } from '@/lib/knowledge/documents/types'
 import { getOperatorsForFieldType, isValidFilterValue } from '@/lib/knowledge/filters/types'
 import { knowledgeDocumentUploadMetadataSchema } from '@/lib/knowledge/upload-metadata'
@@ -115,18 +119,26 @@ export function parseDocumentTagFiltersParam(
   return z.array(documentTagFilterSchema).parse(JSON.parse(value))
 }
 
+/** A text tag value that fits its index row; see {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH}. */
+const documentTagValueSchema = z
+  .string()
+  .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`)
+
 export const createDocumentBodySchema = z.object({
-  filename: z.string().min(1, 'Filename is required'),
+  filename: z
+    .string()
+    .min(1, 'Filename is required')
+    .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`),
   fileUrl: knowledgeDocumentFileUrlSchema,
   fileSize: z.number().min(1, 'File size must be greater than 0'),
   mimeType: z.string().min(1, 'MIME type is required'),
-  tag1: z.string().optional(),
-  tag2: z.string().optional(),
-  tag3: z.string().optional(),
-  tag4: z.string().optional(),
-  tag5: z.string().optional(),
-  tag6: z.string().optional(),
-  tag7: z.string().optional(),
+  tag1: documentTagValueSchema.optional(),
+  tag2: documentTagValueSchema.optional(),
+  tag3: documentTagValueSchema.optional(),
+  tag4: documentTagValueSchema.optional(),
+  tag5: documentTagValueSchema.optional(),
+  tag6: documentTagValueSchema.optional(),
+  tag7: documentTagValueSchema.optional(),
   documentTagsData: z.string().optional(),
 })
 
@@ -165,7 +177,10 @@ export type SingleCreateDocumentBody = z.input<typeof singleCreateDocumentBodySc
 
 export const upsertDocumentBodySchema = z.object({
   documentId: z.string().optional(),
-  filename: z.string().min(1, 'Filename is required'),
+  filename: z
+    .string()
+    .min(1, 'Filename is required')
+    .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`),
   fileUrl: knowledgeDocumentFileUrlSchema,
   fileSize: z.number().min(1, 'File size must be greater than 0'),
   mimeType: z.string().min(1, 'MIME type is required'),
@@ -196,7 +211,10 @@ export const bulkCreateDocumentsResponseSchema = z.object({
 })
 
 export const updateDocumentBodySchema = z.object({
-  filename: z.string().min(1, 'Filename is required').optional(),
+  filename: z
+    .string()
+    .min(1, 'Filename is required')
+    .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`).optional(),
   enabled: z.boolean().optional(),
   chunkCount: z.number().min(0).optional(),
   tokenCount: z.number().min(0).optional(),
@@ -205,13 +223,13 @@ export const updateDocumentBodySchema = z.object({
   processingError: z.string().optional(),
   markFailedDueToTimeout: z.boolean().optional(),
   retryProcessing: z.boolean().optional(),
-  tag1: z.string().optional(),
-  tag2: z.string().optional(),
-  tag3: z.string().optional(),
-  tag4: z.string().optional(),
-  tag5: z.string().optional(),
-  tag6: z.string().optional(),
-  tag7: z.string().optional(),
+  tag1: documentTagValueSchema.optional(),
+  tag2: documentTagValueSchema.optional(),
+  tag3: documentTagValueSchema.optional(),
+  tag4: documentTagValueSchema.optional(),
+  tag5: documentTagValueSchema.optional(),
+  tag6: documentTagValueSchema.optional(),
+  tag7: documentTagValueSchema.optional(),
   number1: z.string().optional(),
   number2: z.string().optional(),
   number3: z.string().optional(),

apps/sim/lib/knowledge/connectors/sync-persistence.test.ts

@@ -29,7 +29,9 @@ vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({
 }))
 vi.mock('@/connectors/registry.server', () => ({
   CONNECTOR_REGISTRY: {
-    fixture: { mapTags: (metadata: Record<string, unknown>) => ({ label: metadata.label }) },
+    fixture: {
+      mapTags: (metadata: Record<string, unknown>) => ({ label: metadata.label, owner: metadata.owner }),
+    },
   },
 }))
 
@@ -434,21 +436,22 @@ describe('organization source cache persistence', () => {
 })
 
 describe('resolveTagMapping', () => {
-  it('bounds a mapped tag value that would exceed its index row limit', () => {
+  it('bounds a mapped tag value that would exceed its index row limit and keeps a short one intact', () => {
     const tags = resolveTagMapping(
       'fixture',
-      { label: 'y'.repeat(5000) },
-      { tagSlotMapping: { label: 'tag1' } }
+      { label: 'y'.repeat(5000), owner: 'Purchasing' },
+      { tagSlotMapping: { label: 'tag1', owner: 'tag2' } }
     )
     expect(tags?.tag1).toBe(`${'y'.repeat(512)}...`)
+    expect(tags?.tag2).toBe('Purchasing')
   })
 
-  it('keeps a short mapped tag value intact', () => {
+  it('cuts by code point so a bounded value never ends in half a surrogate pair', () => {
     const tags = resolveTagMapping(
       'fixture',
-      { label: 'Purchasing' },
+      { label: '\u{1F600}'.repeat(600) },
       { tagSlotMapping: { label: 'tag1' } }
     )
-    expect(tags?.tag1).toBe('Purchasing')
+    expect(tags?.tag1).toBe(`${'\u{1F600}'.repeat(512)}...`)
   })
 })

apps/sim/lib/knowledge/connectors/sync-persistence.ts

@@ -3,7 +3,6 @@ import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/
 import { createLogger } from '@sim/logger'
 import { chunkArray } from '@sim/utils/helpers'
 import { generateId } from '@sim/utils/id'
-import { truncate } from '@sim/utils/string'
 import { and, eq, exists, inArray, isNull, lt, or, sql } from 'drizzle-orm'
 import { getInternalApiBaseUrl } from '@/lib/core/utils/urls'
 import type { DbOrTx } from '@/lib/db/types'
@@ -27,6 +26,7 @@ import {
 } from '@/lib/knowledge/documents/storage-upload'
 import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key'
 import { getFileMetadataByKeys } from '@/lib/uploads/server/metadata'
+import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants'
 import { CONNECTOR_REGISTRY } from '@/connectors/registry.server'
 import type { DocumentTags, ExternalDocument } from '@/connectors/types'
 
@@ -175,16 +175,15 @@ export async function persistDocumentAcls(
 const MAX_SAFE_TITLE_LENGTH = 200
 
 /**
- * A document's filename and text tags sit under btree indexes, and Postgres refuses an index
- * row past about 2.7 KB. A source title or tag value beyond that fails the row and, with it,
- * every sync that lists the document again. 512 characters keeps a four-byte-per-character
- * value inside the ceiling.
+ * Source titles and mapped tag values are untrusted machine input with no caller to refuse them,
+ * so they are cut to {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH} by code point, never inside a
+ * surrogate pair.
  */
-const MAX_INDEXED_TEXT_LENGTH = 512
-
-/** Bounds a source-supplied value that lands in an indexed text column. */
 function boundIndexedText(value: string): string {
-  return truncate(value, MAX_INDEXED_TEXT_LENGTH)
+  if (value.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value
+  const points = Array.from(value)
+  if (points.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value
+  return `${points.slice(0, MAX_DOCUMENT_INDEXED_TEXT_LENGTH).join('')}...`
 }
 
 function sanitizeStorageTitle(title: string): string {
@@ -265,7 +264,7 @@ export function resolveTagMapping(
   for (const [semanticKey, slot] of Object.entries(mapping)) {
     const value = semanticTags[semanticKey]
     ;(result as Record<string, unknown>)[slot] =
-      typeof value === 'string' ? boundIndexedText(value) : value != null ? value : null
+      typeof value === 'string' ? boundIndexedText(value) : (value ?? null)
   }
   return result
 }

apps/sim/lib/knowledge/constants.ts

@@ -3,6 +3,14 @@ import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants'
 /** Max character length for a knowledge base description, enforced at every layer (UI, internal API, v1 API). */
 export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000
 
+/**
+ * Max character length for a document's filename and text tag values. Both sit under btree
+ * indexes, and Postgres refuses an index row past about 2.7 KB (SQLSTATE 54000); 512 characters
+ * keeps a four-byte-per-character value inside that ceiling. Connectors truncate source titles to
+ * it; the document APIs reject longer input.
+ */
+export const MAX_DOCUMENT_INDEXED_TEXT_LENGTH = 512
+
 /** Hard bound for path-indexed knowledge folder trees and recursive cascades. */
 export const MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE = MAX_FOLDERS_PER_WORKSPACE
 

apps/sim/lib/knowledge/upload-metadata.test.ts

@@ -7,8 +7,19 @@ import {
   knowledgeDocumentUploadMetadataSchema,
   persistedKnowledgeDocumentUploadMetadataSchema,
 } from '@/lib/knowledge/upload-metadata'
+import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants'
 
 describe('knowledgeDocumentUploadMetadataSchema', () => {
+  it('rejects a tag value that would not fit its index row', () => {
+    const result = knowledgeDocumentUploadMetadataSchema.safeParse({
+      tag1: 'a'.repeat(MAX_DOCUMENT_INDEXED_TEXT_LENGTH + 1),
+    })
+    expect(result.success).toBe(false)
+    expect(result.error?.issues[0]?.message).toBe(
+      `Knowledge document tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
+    )
+  })
+
   it('rejects a recipe outside the accepted set', () => {
     const result = knowledgeDocumentUploadMetadataSchema.safeParse({
       processingOptions: { recipe: 'totally-bogus-recipe' },

apps/sim/lib/knowledge/upload-metadata.ts

@@ -1,5 +1,6 @@
 import { z } from 'zod'
 import type { RecursiveRecipe } from '@/lib/chunkers/types'
+import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants'
 
 /**
  * Recipes the recursive chunker implements. Mirrors `RecursiveRecipe`; the
@@ -35,7 +36,10 @@ const LANGUAGE_TAG_SHAPE = /^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$/
 
 const knowledgeDocumentUploadTagSchema = z
   .string()
-  .max(1000, 'Knowledge document tag values cannot exceed 1000 characters')
+  .max(
+    MAX_DOCUMENT_INDEXED_TEXT_LENGTH,
+    `Knowledge document tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
+  )
   .optional()
 
 const knowledgeDocumentUploadTagShape = {