Skip to content

Commit 4753b99

Browse files
committed
fix(mothership): preserve content revisions and archive budgets
1 parent f7480fd commit 4753b99

6 files changed

Lines changed: 154 additions & 12 deletions

File tree

‎apps/sim/lib/knowledge/__integration__/content-delivery.integration.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
/** Real PostgreSQL content admission, pagination, captured metadata, and streamed ZIP readback. */
2+
import { mkdtempSync } from 'node:fs'
3+
import { rm } from 'node:fs/promises'
4+
import { tmpdir } from 'node:os'
5+
import path from 'node:path'
26
import * as audit from '@sim/audit'
37
import { db } from '@sim/db'
48
import {
@@ -18,6 +22,14 @@ import { eq, inArray } from 'drizzle-orm'
1822
import { PostgresJsPreparedQuery } from 'drizzle-orm/postgres-js/session'
1923
import JSZip from 'jszip'
2024
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest'
25+
26+
const fixtureStorage = vi.hoisted(() => ({ root: '' }))
27+
vi.mock('@/lib/uploads/core/setup.server', () => ({
28+
get UPLOAD_DIR_SERVER() {
29+
return fixtureStorage.root
30+
},
31+
}))
32+
2133
import { observeContentDelivery } from '@/lib/core/application'
2234
import { encryptSecret } from '@/lib/core/security/encryption'
2335
import type { DurableSecretProvenance } from '@/lib/execution/durable-secret-provenance'
@@ -28,11 +40,13 @@ import {
2840
import { admitKnowledgeChunks } from '@/lib/knowledge/application/admit-knowledge-content'
2941
import { KNOWLEDGE_DELEGATION_AUDIENCE } from '@/lib/knowledge/application/authorization'
3042
import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks'
43+
import * as knowledgeContexts from '@/lib/knowledge/application/contexts'
3144
import { listKnowledgeDocuments } from '@/lib/knowledge/application/documents'
3245
import {
3346
exportKnowledgeBase,
3447
type KnowledgeBaseExportBundle,
3548
} from '@/lib/knowledge/application/exports'
49+
import { readOriginalKnowledgeDocument } from '@/lib/knowledge/application/read-original-document'
3650
import {
3751
readDetailedKnowledgeTagUsage,
3852
readKnowledgeTagUsage,
@@ -48,6 +62,7 @@ import { buildKnowledgeBundleArchive } from '@/lib/knowledge/transfer/export-arc
4862
import { embeddingVectorValues } from '@/lib/knowledge/vector-columns'
4963
import { createCopilotChatPrincipal } from '@/lib/mothership/auth/application-delegation'
5064
import { createRegistryContentGate } from '@/lib/mothership/content-delivery/registry-gate'
65+
import { uploadFile } from '@/lib/uploads/core/storage-service'
5166
import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
5267

5368
const SECRET = 'fixture-knowledge-token-41875'
@@ -173,6 +188,7 @@ async function drainAudits() {
173188
}
174189

175190
beforeAll(async () => {
191+
fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-knowledge-delivery-'))
176192
await seedKnowledgeAclFixture(ids)
177193
const record = audit.recordAudit
178194
vi.spyOn(audit, 'recordAudit').mockImplementation((entry) => {
@@ -208,6 +224,7 @@ afterAll(async () => {
208224
await deleteWorkspaceFixture(db, eq(workspace.id, ids.workspaceId))
209225
await db.delete(organization).where(eq(organization.id, ids.organizationId))
210226
await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId]))
227+
await rm(fixtureStorage.root, { recursive: true, force: true })
211228
await db.$client.end()
212229
})
213230

@@ -444,6 +461,68 @@ describe('knowledge content delivery', () => {
444461
}
445462
)
446463

464+
it('binds an original read to the captured storage key across a concurrent replacement', async () => {
465+
const doc = await seedDocument({ original: false, content: SECRET, provenance: exact })
466+
const oldKey = `kb/${generateId()}.txt`
467+
const newKey = `kb/${generateId()}.txt`
468+
for (const [key, content] of [
469+
[oldKey, SECRET],
470+
[newKey, 'replacement public content'],
471+
]) {
472+
await uploadFile({
473+
file: Buffer.from(content),
474+
fileName: 'fixture.txt',
475+
contentType: 'text/plain',
476+
context: 'knowledge-base',
477+
customKey: key,
478+
preserveKey: true,
479+
persistMetadata: false,
480+
})
481+
}
482+
await db.update(document).set({ storageKey: oldKey }).where(eq(document.id, doc.id))
483+
const input = {
484+
knowledgeBaseId: ids.knowledgeBaseId,
485+
documentId: doc.id,
486+
assertedWorkspaceId: ids.workspaceId,
487+
}
488+
const matched = await inChat(() => readOriginalKnowledgeDocument.execute({ principal, input }))
489+
expect(matched.buffer?.toString()).toBe(SECRET)
490+
expect(matched.hasRecordedSecrets).toBe(true)
491+
const resolve = knowledgeContexts.resolveCanonicalActiveKnowledgeDocumentContext
492+
let replaced = false
493+
const interleave = vi
494+
.spyOn(knowledgeContexts, 'resolveCanonicalActiveKnowledgeDocumentContext')
495+
.mockImplementation(async (...args) => {
496+
const context = await resolve(...args)
497+
await db.transaction(async (tx) => {
498+
const [row] = await tx
499+
.update(document)
500+
.set({ storageKey: newKey })
501+
.where(eq(document.id, doc.id))
502+
.returning()
503+
await replaceKnowledgeDocumentSecretProvenanceInTx(
504+
tx,
505+
doc.id,
506+
createKnowledgeDocumentSourceValue(row),
507+
{ status: 'exact', entries: [] }
508+
)
509+
})
510+
replaced = true
511+
return context
512+
})
513+
try {
514+
await expect(
515+
inChat(() => readOriginalKnowledgeDocument.execute({ principal, input }))
516+
).rejects.toThrow(/provenance|model/i)
517+
expect(replaced).toBe(true)
518+
} finally {
519+
interleave.mockRestore()
520+
}
521+
const current = await inChat(() => readOriginalKnowledgeDocument.execute({ principal, input }))
522+
expect(current.buffer?.toString()).toBe('replacement public content')
523+
expect(current.hasRecordedSecrets).toBe(false)
524+
})
525+
447526
it('exports clean and legacy originals with chunks through the existing ZIP stream', async () => {
448527
const docs = [
449528
await seedDocument(),

‎apps/sim/lib/knowledge/application/read-original-document.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ describe('original knowledge document read', () => {
8989
)
9090
workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('read')
9191
mocks.provenance.mockResolvedValue({
92+
storageKey: document.storageKey,
9293
source: { filename: document.filename, fileUrl: document.fileUrl },
9394
provenance: { status: 'exact', entries: [] },
9495
})
@@ -104,6 +105,7 @@ describe('original knowledge document read', () => {
104105
},
105106
})
106107
mocks.provenance.mockResolvedValue({
108+
storageKey: document.storageKey,
107109
source: { filename: document.filename, fileUrl: 'https://example.test/private' },
108110
provenance: { status: 'exact', entries: [] },
109111
})
@@ -144,6 +146,7 @@ describe('original knowledge document read', () => {
144146
'refuses Chat an original its gate refuses, before reading it: %j',
145147
async (provenance) => {
146148
mocks.provenance.mockResolvedValue({
149+
storageKey: document.storageKey,
147150
source: { filename: document.filename, fileUrl: document.fileUrl },
148151
provenance,
149152
})
@@ -156,6 +159,7 @@ describe('original knowledge document read', () => {
156159
})
157160
it('classifies a concurrent source replacement as unknown, refusing it before reading', async () => {
158161
mocks.provenance.mockResolvedValue({
162+
storageKey: document.storageKey,
159163
source: { filename: 'changed', fileUrl: document.fileUrl },
160164
provenance: { status: 'exact', entries: [] },
161165
})
@@ -164,6 +168,7 @@ describe('original knowledge document read', () => {
164168
})
165169
it('reads for a caller outside Chat without classifying, flagging recorded secrets', async () => {
166170
mocks.provenance.mockResolvedValue({
171+
storageKey: document.storageKey,
167172
source: { filename: document.filename, fileUrl: document.fileUrl },
168173
provenance: { status: 'exact', entries: [{ secret: 'private' }] },
169174
})

‎apps/sim/lib/knowledge/application/read-original-document.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,9 @@ export const readOriginalKnowledgeDocument = defineAuthorizedKnowledgeUseCase({
5353
)
5454
const snapshot = await loadKnowledgeDocumentDurableSecretProvenance(doc.id)
5555
const provenance =
56+
snapshot.storageKey === doc.storageKey &&
5657
hashDurableSecretProvenanceValue(snapshot.source) ===
57-
hashDurableSecretProvenanceValue(createKnowledgeDocumentSourceValue(doc))
58+
hashDurableSecretProvenanceValue(createKnowledgeDocumentSourceValue(doc))
5859
? snapshot.provenance
5960
: ({ status: 'unknown' } as const)
6061
await requireAdmittedContent(principal, {

‎apps/sim/lib/mothership/__integration__/content-delivery.integration.ts‎

Lines changed: 54 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ import {
103103
userTableRowSecretProvenance,
104104
workflowTest,
105105
workspaceFileSearchRevision,
106+
workspaceFiles,
106107
} from '@sim/db/schema'
107108
import { sha256Hex } from '@sim/security/hash'
108109
import { eq, sql } from 'drizzle-orm'
@@ -210,11 +211,18 @@ function noteExercised(method: string, route: string) {
210211
}
211212

212213
describe('content reaching Chat is admitted per item', () => {
214+
let redis: NonNullable<ReturnType<typeof getRedisClient>>
213215
let fixture: ContentFixture
214216
let connector: { connectorId: string; documentId: string }
215217

216218
beforeAll(async () => {
217219
fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-content-delivery-'))
220+
const client = getRedisClient()
221+
if (!client)
222+
throw new Error(
223+
'Set TEST_REDIS_URL to a disposable local Redis or use bun run test:integration'
224+
)
225+
redis = client
218226
// Compile the lazy tool graph before request deadlines and per-case provider fixtures exist.
219227
await matchV2Route('/api/v2/tools/file_get_content/execute')?.load()
220228
fixture = await seedContentFixture()
@@ -273,7 +281,7 @@ describe('content reaching Chat is admitted per item', () => {
273281
workspaceId: fixture.ids.workspaceId,
274282
type: 'mothership',
275283
})
276-
await getRedisClient()?.set(
284+
await redis.set(
277285
`mothership:sandbox-resources:${token}:context`,
278286
JSON.stringify({
279287
toolCallId: generateId(),
@@ -608,6 +616,51 @@ describe('content reaching Chat is admitted per item', () => {
608616
expect(refused.body).not.toContain('gamma')
609617
})
610618

619+
it.each(['clean', 'entries'] as const)(
620+
'applies the archive declared-size budget after %s contributor admission',
621+
async (kind) => {
622+
const input = await getWorkspaceFile(ws(), fixture.files[kind], { throwOnError: true })
623+
if (!input) throw new Error('Missing archive contributor fixture')
624+
const source = `await getFileBase64('${input.id}'); pdf.addPage().drawText('fixture render');`
625+
const documentIds: string[] = []
626+
for (let index = 0; index < (kind === 'clean' ? 6 : 5); index += 1) {
627+
const doc = await uploadWorkspaceFile(
628+
ws(),
629+
fixture.ids.aliceId,
630+
Buffer.from(
631+
index === 5 ? 'throw new Error("archive budget checked too late")' : source
632+
),
633+
`archive-budget-${generateId()}.pdf`,
634+
'text/x-python-pdf',
635+
{
636+
notifyWorkspaceChange: false,
637+
secretProvenance: { status: 'exact', entries: [] },
638+
}
639+
)
640+
// Declared source sizes exercise aggregate accounting without large fixture allocations.
641+
await db
642+
.update(workspaceFiles)
643+
.set({ sizeBytes: 50 * 1024 * 1024 })
644+
.where(eq(workspaceFiles.id, doc.id))
645+
documentIds.push(doc.id)
646+
}
647+
const result = await cli(
648+
'GET',
649+
`/api/v2/files/bulk-download?workspaceId=${ws()}&fileIds=${fixture.files.clean},${documentIds}`
650+
)
651+
if (kind === 'clean') {
652+
expect(result.status, result.body).toBe(400)
653+
expect(result.body).toMatch(/download limit/i)
654+
} else {
655+
expect(result.status, result.body).toBe(200)
656+
expect(result.body).toContain(CANARY.clean)
657+
expect(result.body).toContain('sim-download-report.json')
658+
for (const id of documentIds) expect(result.body).toContain(id)
659+
expect(result.body).not.toContain('%PDF-1.7')
660+
}
661+
}
662+
)
663+
611664
it('shares only clean files publicly on Chat’s behalf (#9, D1)', async () => {
612665
const share = (fileId: string) =>
613666
cli('PATCH', `/api/v2/files/${fileId}/share`, {

‎apps/sim/lib/workspace-files/application/download-workspace-file-items.ts‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import { fileOperations } from '@/lib/workspace-files/application/operations'
3131
import { parseWorkspaceFileFolderDisplayPath } from '@/lib/workspace-files/folder-display-path'
3232
import { MAX_ZIP_DOWNLOAD_FILES } from '@/lib/workspace-files/limits'
3333

34-
export const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024
34+
const MAX_ZIP_DOWNLOAD_BYTES = 250 * 1024 * 1024
3535
const MAX_REQUESTED_FILE_IDS = 1_000
3636
const MAX_REQUESTED_FOLDER_IDS = 1_000
3737

@@ -104,6 +104,14 @@ function validationError(message: string): never {
104104
throw new OrchestrationError('validation', message)
105105
}
106106

107+
function validateDeclaredBytes(bytes: number): void {
108+
if (bytes > MAX_ZIP_DOWNLOAD_BYTES) {
109+
validationError(
110+
`Selected files total ${formatFileSize(bytes)}, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`
111+
)
112+
}
113+
}
114+
107115
async function executeDownloadWorkspaceFileItems({
108116
input,
109117
context,
@@ -183,16 +191,11 @@ async function executeDownloadWorkspaceFileItems({
183191
'opaque'
184192
)
185193

186-
const declaredBytes = filesToZip.reduce((sum, file) => sum + file.size, 0)
187-
if (declaredBytes > MAX_ZIP_DOWNLOAD_BYTES) {
188-
validationError(
189-
`Selected files total ${formatFileSize(declaredBytes)}, which exceeds the ${formatFileSize(MAX_ZIP_DOWNLOAD_BYTES)} download limit.`
190-
)
191-
}
192-
193194
const reservedForStreamed = filesToZip
194195
.filter((file) => !needsRenderedArtifact(file.type, file.name))
195196
.reduce((sum, file) => sum + file.size, 0)
197+
let declaredBytes = reservedForStreamed
198+
validateDeclaredBytes(declaredBytes)
196199
const renderedDocuments = new Map<string, Buffer>()
197200
const pendingNames: string[] = []
198201
let renderedBytes = 0
@@ -211,6 +214,8 @@ async function executeDownloadWorkspaceFileItems({
211214
filesToZip = filesToZip.filter((admitted) => admitted.id !== file.id)
212215
continue
213216
}
217+
declaredBytes += file.size
218+
validateDeclaredBytes(declaredBytes)
214219
renderedBytes += buffer.length
215220
renderedDocuments.set(file.id, buffer)
216221
} catch (error) {
@@ -240,7 +245,7 @@ async function executeDownloadWorkspaceFileItems({
240245
folderPaths,
241246
renderedDocuments,
242247
/** Counted over what is zipped, after any rendered document whose contributors were refused. */
243-
declaredBytes: filesToZip.reduce((sum, file) => sum + file.size, 0),
248+
declaredBytes,
244249
}
245250
}
246251

‎scripts/check-unused-exports.baseline.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5280,7 +5280,6 @@
52805280
"apps/sim/lib/workspace-events/constants.ts#SIM_RULE_EVENT_TYPES",
52815281
"apps/sim/lib/workspace-files/application/csv-preview-workspace-file.ts#CsvPreviewWorkspaceFileInput",
52825282
"apps/sim/lib/workspace-files/application/delete-workspace-file.ts#DeleteWorkspaceFileInput",
5283-
"apps/sim/lib/workspace-files/application/download-workspace-file-items.ts#MAX_ZIP_DOWNLOAD_BYTES",
52845283
"apps/sim/lib/workspace-files/application/download-workspace-file.ts#DownloadWorkspaceFileInput",
52855284
"apps/sim/lib/workspace-files/application/extract-workspace-file.ts#ExtractWorkspaceFileInput",
52865285
"apps/sim/lib/workspace-files/application/extract-workspace-file.ts#ExtractWorkspaceFileResult",

0 commit comments

Comments
 (0)