Skip to content

Commit 9b1a95d

Browse files
committed
fix(mothership): preserve protection across content transformations
1 parent dee7a08 commit 9b1a95d

22 files changed

Lines changed: 663 additions & 78 deletions

‎.claude/rules/sim-content-delivery.md‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,13 @@ the workflow executor see byte-identical results.
3737
does not undo the write: audit and effects run, then the operation answers with the refusal.
3838
Settled tool results use the same deferral, but do not claim that a mutation was saved.
3939
- A source is `{ kind, id, provenance, view }`. `view` is `'complete'` (text whose every entry
40-
may appear), `{ derived: value }` (only entries occurring in `value`), or `'opaque'` (bytes no
41-
redaction reaches: binary, base64, archives, images, public links).
40+
may appear), `{ derived: value }` (only entries occurring in `value`), `'opaque'` (bytes no
41+
redaction reaches: binary, base64, archives, images, public links), or `'excerpt'` (text already
42+
cut before admission, where a secret fragment cannot be redacted). Both gates withhold an
43+
excerpt with known secret entries; raw opaque bytes may still enter the workbench with history.
44+
- `redactBeforeCutting` projects complete text before selecting a line window. The session gate
45+
creates the existing redactor from the operation's bounded evidence only when a cut needs it;
46+
ordinary byte reads do not decrypt a second time or reload history.
4247
- Gates, installed by the surface: the **registry gate** (Chat tools, CLI, mounts) imports entries
4348
into the registry egress projects with; the **session gate** (run_code sandbox proxy) records
4449
them into the workbench history. Neither admits `unknown`, with one exception: the CLI's direct

‎.cursor/rules/sim-content-delivery.mdc‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,13 @@ the workflow executor see byte-identical results.
2828
does not undo the write: audit and effects run, then the operation answers with the refusal.
2929
Settled tool results use the same deferral, but do not claim that a mutation was saved.
3030
- A source is `{ kind, id, provenance, view }`. `view` is `'complete'` (text whose every entry
31-
may appear), `{ derived: value }` (only entries occurring in `value`), or `'opaque'` (bytes no
32-
redaction reaches: binary, base64, archives, images, public links).
31+
may appear), `{ derived: value }` (only entries occurring in `value`), `'opaque'` (bytes no
32+
redaction reaches: binary, base64, archives, images, public links), or `'excerpt'` (text already
33+
cut before admission, where a secret fragment cannot be redacted). Both gates withhold an
34+
excerpt with known secret entries; raw opaque bytes may still enter the workbench with history.
35+
- `redactBeforeCutting` projects complete text before selecting a line window. The session gate
36+
creates the existing redactor from the operation's bounded evidence only when a cut needs it;
37+
ordinary byte reads do not decrypt a second time or reload history.
3338
- Gates, installed by the surface: the **registry gate** (Chat tools, CLI, mounts) imports entries
3439
into the registry egress projects with; the **session gate** (run_code sandbox proxy) records
3540
them into the workbench history. Neither admits `unknown`, with one exception: the CLI's direct

‎apps/sim/lib/core/application/content-delivery.ts‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@ export type ContentSourceKind =
2323
/**
2424
* How an item's content leaves the use case. `complete` is text in which any of its secrets may
2525
* appear; `derived` is text built from part of it, so only secrets occurring in `value` count;
26-
* `opaque` is bytes no text redaction reaches — binary, base64, archives, images, public links.
26+
* `opaque` is bytes no text redaction reaches — binary, base64, archives, images, public links;
27+
* `excerpt` is text already cut before admission, so a secret fragment cannot be redacted.
2728
*/
28-
export type ContentView = 'complete' | 'opaque' | { derived: unknown }
29+
export type ContentView = 'complete' | 'opaque' | 'excerpt' | { derived: unknown }
2930

3031
/** One stored item or tool result whose content a use case is about to return. */
3132
export interface ContentSource {
@@ -43,10 +44,13 @@ export interface ContentSource {
4344
export interface ContentDeliveryGate {
4445
admit(sources: readonly ContentSource[]): Promise<readonly boolean[]>
4546
/**
46-
* The text as this surface's egress will show it, or `undefined` when it cannot be shown. A gate
47-
* whose surface redacts (Chat's registry) implements it; see {@link redactBeforeCutting}.
47+
* Redacts text before cutting, or returns `undefined` when it cannot be shown. The current
48+
* operation's bounded evidence lets a surface create a redactor only when a cut needs one.
4849
*/
49-
project?(text: string): string | undefined
50+
project?(
51+
text: string,
52+
provenance?: ContentProvenance
53+
): string | undefined | Promise<string | undefined>
5054
}
5155

5256
/**
@@ -149,17 +153,17 @@ export function observeContentDelivery<T>(gate: ContentDeliveryGate, execute: ()
149153
/**
150154
* Text a use case is about to cut (a line window, a clip), redacted first by the surface's gate.
151155
* Egress redacts whole secrets only, so a cut through one would leave a fragment it cannot
152-
* recognize. Other callers, and surfaces that do not redact, get the text unchanged.
156+
* recognize. Other callers and explicit outside-model reads get the text unchanged.
153157
*/
154-
export function redactBeforeCutting(
158+
export async function redactBeforeCutting(
155159
principal: Principal | undefined,
156160
text: string,
157161
kind: ContentSourceKind
158-
): string {
162+
): Promise<string> {
159163
if (principal && !isCopilotPrincipal(principal)) return text
160164
const project = gates.getStore()?.project
161165
if (!project) return text
162-
const projected = project(text)
166+
const projected = await project(text, ledgers.getStore()?.admitted[0])
163167
if (projected === undefined) throw new ContentRefusedError(kind)
164168
return projected
165169
}

‎apps/sim/lib/execution/sandbox/__integration__/content-reads.integration.ts‎

Lines changed: 108 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import { execFile } from 'node:child_process'
2-
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
2+
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
33
import { tmpdir } from 'node:os'
44
import { join } from 'node:path'
55
import { promisify } from 'node:util'
66
import { db } from '@sim/db'
7-
import { auditLog, organization, user, workspace } from '@sim/db/schema'
7+
import { auditLog, organization, user, workspace, workspaceFiles } from '@sim/db/schema'
88
import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
99
import { toolsMock, toolsMockFns } from '@sim/testing/mocks/tools.mock'
1010
import { generateId } from '@sim/utils/id'
@@ -29,6 +29,7 @@ import { createCopilotChatFilePrincipal } from '@/lib/mothership/auth/file-deleg
2929
import { createRegistryContentGate } from '@/lib/mothership/content-delivery'
3030
import { executeFunctionExecute } from '@/lib/mothership/tools/handlers/function-execute'
3131
import { compileDoc } from '@/lib/mothership/tools/server/files/doc-compile'
32+
import { compileDocForWrite } from '@/lib/mothership/tools/server/files/workspace-file'
3233
import {
3334
updateWorkspaceFileContent,
3435
uploadWorkspaceFile,
@@ -179,6 +180,111 @@ describe('Chat admits the file revision used by a sandbox', () => {
179180
expect(projected.value).toContain('{{BROKER_TOKEN}}')
180181
}, 60_000)
181182

183+
it('admits broker inputs during legacy document-write validation before returning compiler errors', async () => {
184+
const principal = createCopilotChatFilePrincipal({
185+
userId: ids.aliceId,
186+
workspaceId: ids.workspaceId,
187+
chatId,
188+
})
189+
const validate = (source: string, trace: ResolvedSecretTraceRegistry) =>
190+
observeContentDelivery(
191+
createRegistryContentGate({
192+
registry: trace,
193+
workspaceId: ids.workspaceId,
194+
actorUserId: ids.aliceId,
195+
}),
196+
() =>
197+
compileDocForWrite({
198+
source,
199+
fileName: 'write-validation.pdf',
200+
workspaceId: ids.workspaceId,
201+
principal,
202+
ownerKey: `user:${ids.aliceId}`,
203+
fallbackMime: 'application/pdf',
204+
})
205+
)
206+
207+
await updateWorkspaceFileContent(
208+
ids.workspaceId,
209+
fileId,
210+
ids.aliceId,
211+
Buffer.from('unverified input'),
212+
undefined,
213+
{
214+
version: { source: 'api', authorUserId: ids.aliceId },
215+
secretProvenancePolicy: { mode: 'replace', provenance: { status: 'unknown' } },
216+
}
217+
)
218+
const refused = await validate(
219+
`const id = ${JSON.stringify(fileId)}; await getFileBase64(id); pdf.addPage();`,
220+
registry()
221+
)
222+
expect.soft(refused).toMatchObject({ ok: false })
223+
if (!refused.ok) expect(refused.message).toMatch(/provenance|read safely/i)
224+
225+
await updateWorkspaceFileContent(
226+
ids.workspaceId,
227+
fileId,
228+
ids.aliceId,
229+
Buffer.from(secret),
230+
undefined,
231+
{
232+
version: { source: 'api', authorUserId: ids.aliceId },
233+
secretProvenancePolicy: {
234+
mode: 'replace',
235+
provenance: {
236+
status: 'exact',
237+
entries: [
238+
{
239+
name: 'BROKER_TOKEN',
240+
sourceUserId: ids.aliceId,
241+
sourceWorkspaceId: ids.workspaceId,
242+
encryptedValue: (await encryptSecret(secret)).encrypted,
243+
},
244+
],
245+
},
246+
},
247+
}
248+
)
249+
const trace = registry()
250+
const failed = await validate(
251+
`const id = ${JSON.stringify(fileId)}; const uri = await getFileBase64(id); throw new Error(Buffer.from(uri.split(',')[1], 'base64').toString());`,
252+
trace
253+
)
254+
expect(failed.ok).toBe(false)
255+
if (failed.ok) throw new Error('Expected the document source to fail')
256+
expect(failed.message).toContain(secret)
257+
const projected = projectResolvedSecretModelContent(failed.message, trace)
258+
if (!projected.safe) throw new Error('Expected a safely projected compiler error')
259+
expect(projected.value).not.toContain(secret)
260+
expect(projected.value).toContain('{{BROKER_TOKEN}}')
261+
}, 60_000)
262+
263+
it('keeps successful legacy dynamic-input documents readable after their first compile', async () => {
264+
const references = Array.from({ length: 501 }, (_, index) => ({
265+
id: `wf_${generateId()}`,
266+
key: `workspace/${ids.workspaceId}/cache-reference-${generateId()}.txt`,
267+
originalName: `reference-${index}.txt`,
268+
userId: ids.aliceId,
269+
workspaceId: ids.workspaceId,
270+
context: 'workspace',
271+
contentType: 'text/plain',
272+
sizeBytes: 6,
273+
}))
274+
await db.insert(workspaceFiles).values(references)
275+
await mkdir(join(storage.root, 'workspace', ids.workspaceId), { recursive: true })
276+
for (const reference of references) {
277+
await writeFile(join(storage.root, reference.key), 'public')
278+
}
279+
const source = `const ids = ${JSON.stringify(references.map((file) => file.id))}; for (const id of ids) await getFileBase64(id); pdf.addPage().drawText('compiled all references');`
280+
const first = await compile(source, registry())
281+
expect(first.buffer.subarray(0, 4).toString()).toBe('%PDF')
282+
expect(first.contributingFiles).toHaveLength(references.length)
283+
const second = await compile(source, registry())
284+
expect(second.buffer.subarray(0, 4).toString()).toBe('%PDF')
285+
expect(second.contributingFiles).toHaveLength(references.length)
286+
}, 120_000)
287+
182288
it('does not run code with a stale local file after refusing its requested replacement', async () => {
183289
await updateWorkspaceFileContent(
184290
ids.workspaceId,

‎apps/sim/lib/knowledge/__integration__/content-delivery.integration.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,66 @@ describe('knowledge content delivery', () => {
273273
}
274274
})
275275

276+
it('streams chunk pages without reloading the tracked inline original for each page', async () => {
277+
const content = 'inline original content '.repeat(4096)
278+
const doc = await seedDocument({ content, provenance: { status: 'exact', entries: [] } })
279+
await db.insert(embedding).values(
280+
Array.from({ length: 100 }, (_, index) => {
281+
const text = `public exported chunk ${index + 1}`
282+
return {
283+
id: generateId(),
284+
documentId: doc.id,
285+
knowledgeBaseId: ids.knowledgeBaseId,
286+
chunkIndex: index + 1,
287+
...embeddingVectorValues(1536, [1, ...Array<number>(1535).fill(0)]),
288+
chunkHash: sha256Hex(text),
289+
content: text,
290+
contentLength: text.length,
291+
tokenCount: 4,
292+
startOffset: 0,
293+
endOffset: text.length,
294+
tag1: 'public',
295+
}
296+
})
297+
)
298+
const bundle = await inChat(() =>
299+
exportKnowledgeBase.execute({
300+
principal,
301+
input: { knowledgeBaseId: ids.knowledgeBaseId, vectors: true },
302+
})
303+
)
304+
const originalBytes = Buffer.byteLength(
305+
`data:text/plain;base64,${Buffer.from(content).toString('base64')}`
306+
)
307+
let loadedOriginalBytes = 0
308+
const execute = PostgresJsPreparedQuery.prototype.execute
309+
const measure = vi
310+
.spyOn(PostgresJsPreparedQuery.prototype, 'execute')
311+
.mockImplementation(async function (this: typeof PostgresJsPreparedQuery.prototype, values) {
312+
const rows = await execute.call(this, values)
313+
if (Array.isArray(rows)) {
314+
for (const row of rows) {
315+
if (!isRecordLike(row)) continue
316+
const value = row.fileUrl
317+
if (typeof value === 'string' && value.startsWith('data:'))
318+
loadedOriginalBytes += Buffer.byteLength(value)
319+
}
320+
}
321+
return rows
322+
})
323+
try {
324+
const exported: string[] = []
325+
for await (const chunk of bundle.chunks(doc.id)) exported.push(chunk.content)
326+
expect(exported).toHaveLength(101)
327+
expect(exported[0]).toBe(content)
328+
expect(exported.at(-1)).toBe('public exported chunk 100')
329+
expect(loadedOriginalBytes).toBeGreaterThan(0)
330+
expect(loadedOriginalBytes).toBeLessThanOrEqual(originalBytes)
331+
} finally {
332+
measure.mockRestore()
333+
}
334+
})
335+
276336
it('keeps stored search totals and continuation while withholding only unsafe documents', async () => {
277337
await seedDocument({ name: 'page-a.txt' })
278338
await seedDocument({ name: 'page-b.txt', provenance: { status: 'unknown' } })

0 commit comments

Comments
 (0)