Skip to content

Commit a5d2d82

Browse files
committed
fix(mothership): preserve file read semantics
1 parent 9b1a95d commit a5d2d82

9 files changed

Lines changed: 284 additions & 53 deletions

File tree

‎apps/sim/executor/utils/resolved-secret-content-projection.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ interface ProjectionState {
9393
}
9494

9595
export interface ResolvedSecretContentProjectionOptions {
96+
/** Keeps original logical lines when masking complete text before a file line window. */
97+
preserveLineBreaks?: boolean
9698
/** Values already materialized and verified by a boundary-specific projector. */
9799
isOpaqueSafeObject?: (value: object) => boolean
98100
/** Whether string-shaped secret literals should also match typed number/boolean/null values. */
@@ -304,13 +306,13 @@ function sanitizeProjectedString(
304306
? sanitizeCollisionProneInternalDiagnosticIdentifiers(value, matcher)
305307
: value
306308
sanitized = matcher
307-
? sanitizeResolvedSecretString(sanitized, matcher, maxBytes, options.onMatch)
309+
? sanitizeResolvedSecretString(sanitized, matcher, maxBytes, options.onMatch, options)
308310
: sanitized
309311
if (options.sanitizeInternalIdentifiers) {
310312
sanitized = sanitizeInternalDiagnosticIdentifiers(sanitized)
311313
}
312314
if (matcher && containsResolvedSecret(sanitized, matcher)) {
313-
sanitized = sanitizeResolvedSecretString(sanitized, matcher, maxBytes, options.onMatch)
315+
sanitized = sanitizeResolvedSecretString(sanitized, matcher, maxBytes, options.onMatch, options)
314316
if (containsResolvedSecret(sanitized, matcher)) {
315317
throw new ResolvedSecretContentProjectionError(
316318
'Secret replacement could not produce safe model content'

‎apps/sim/executor/utils/resolved-secret-matcher.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,31 @@ import {
1111
const PRESERVE_NAMED_PROVENANCE = { preserveNamedProvenanceLabels: true } as const
1212

1313
describe('resolved secret matcher', () => {
14+
it.each([
15+
['first-line\nsecond-line', '{{TOKEN}}\n{{TOKEN}}'],
16+
['first-line\r\nsecond-line', '{{TOKEN}}\r\n{{TOKEN}}'],
17+
['first-line\nsecond-line\n', '{{TOKEN}}\n{{TOKEN}}\n'],
18+
['first-line\n\nsecond-line', '{{TOKEN}}\n{{TOKEN}}\n{{TOKEN}}'],
19+
['\n'.repeat(8), '{{TOKEN}}\n'.repeat(8)],
20+
])(
21+
'preserves line boundaries for a complete literal %j when selecting file lines',
22+
(plaintext, expected) => {
23+
const matcher = createResolvedSecretMatcher([{ plaintext, replacement: '{{TOKEN}}' }])
24+
if (!matcher) throw new Error('Fixture literal did not produce a matcher')
25+
expect(
26+
sanitizeResolvedSecretString(plaintext, matcher, undefined, undefined, {
27+
preserveLineBreaks: true,
28+
})
29+
).toBe(expected)
30+
expect(sanitizeResolvedSecretString(plaintext, matcher)).toBe('{{TOKEN}}')
31+
expect(
32+
sanitizeResolvedSecretString(`before ${plaintext} after`, matcher, undefined, undefined, {
33+
preserveLineBreaks: true,
34+
})
35+
).toBe(`before ${expected} after`)
36+
}
37+
)
38+
1439
it('reports each matched literal once across large repeated content', () => {
1540
const matcher = createResolvedSecretMatcher([
1641
{ plaintext: 'xxxxxxxx', replacement: '{{SHORT}}' },

‎apps/sim/executor/utils/resolved-secret-matcher.ts‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,8 @@ export function sanitizeResolvedSecretString(
293293
value: string,
294294
matcher: ResolvedSecretMatcher,
295295
maxBytes = MAX_INLINE_MATERIALIZATION_BYTES,
296-
onMatch?: (plaintext: string) => void
296+
onMatch?: (plaintext: string) => void,
297+
options: { preserveLineBreaks?: boolean } = {}
297298
): string {
298299
/**
299300
* A substitution can leave a literal the previous pass could not act on: an empty replacement can
@@ -302,12 +303,24 @@ export function sanitizeResolvedSecretString(
302303
* pathological chain from looping, and the throw past it stays the fail-closed backstop callers
303304
* already handle by dropping the value.
304305
*/
305-
let sanitized = substituteResolvedSecrets(value, matcher, maxBytes, onMatch)
306+
let sanitized = substituteResolvedSecrets(
307+
value,
308+
matcher,
309+
maxBytes,
310+
onMatch,
311+
options.preserveLineBreaks
312+
)
306313
for (let pass = 1; containsResolvedSecret(sanitized, matcher); pass += 1) {
307314
if (pass >= MAX_SETTLE_PASSES) {
308315
throw new ResolvedSecretMatcherError('Sanitized content still contains an active secret')
309316
}
310-
sanitized = substituteResolvedSecrets(sanitized, matcher, maxBytes, onMatch)
317+
sanitized = substituteResolvedSecrets(
318+
sanitized,
319+
matcher,
320+
maxBytes,
321+
onMatch,
322+
options.preserveLineBreaks
323+
)
311324
}
312325
return sanitized
313326
}
@@ -316,7 +329,8 @@ function substituteResolvedSecrets(
316329
value: string,
317330
matcher: ResolvedSecretMatcher,
318331
maxBytes: number,
319-
onMatch?: (plaintext: string) => void
332+
onMatch?: (plaintext: string) => void,
333+
preserveLineBreaks = false
320334
): string {
321335
if (maxBytes < 0) {
322336
throw new ResolvedSecretMatcherError('Sanitized secret-bearing string exceeds the size limit')
@@ -356,7 +370,21 @@ function substituteResolvedSecrets(
356370
const slot = emitCursor % windowSize
357371
if (slotStarts[slot] === emitCursor && slotReplacements[slot] !== undefined) {
358372
append(value.slice(literalStart, emitCursor))
359-
append(slotReplacements[slot] ?? '')
373+
const replacement = slotReplacements[slot] ?? ''
374+
if (preserveLineBreaks) {
375+
const lineReplacement = replacement.replace(/[\r\n]/g, ' ') || ' '
376+
const matched = value.slice(emitCursor, slotEnds[slot])
377+
let position = 0
378+
while (position < matched.length) {
379+
append(lineReplacement)
380+
const newline = matched.indexOf('\n', position)
381+
if (newline < 0) break
382+
append(newline > position && matched[newline - 1] === '\r' ? '\r\n' : '\n')
383+
position = newline + 1
384+
}
385+
} else {
386+
append(replacement)
387+
}
360388
emitCursor = slotEnds[slot]
361389
literalStart = emitCursor
362390
} else {

‎apps/sim/lib/media/ffmpeg-content.integration.ts‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@ import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
1010
import { generateId } from '@sim/utils/id'
1111
import { eq, inArray } from 'drizzle-orm'
1212
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
13-
import { observeContentDelivery } from '@/lib/core/application/content-delivery'
13+
import {
14+
observeContentDelivery,
15+
observeContentOutsideModel,
16+
} from '@/lib/core/application/content-delivery'
1417
import { encryptSecret } from '@/lib/core/security/encryption'
1518
import {
1619
createKnowledgeAclFixtureIds,
@@ -117,6 +120,38 @@ afterAll(async () => {
117120
})
118121

119122
describe('media input version lineage', () => {
123+
it.each(['known', 'unknown'] as const)(
124+
'preserves the storage size-limit error for an admitted %s media source',
125+
async (classification) => {
126+
const file = await uploadWorkspaceFile(
127+
ids.workspaceId,
128+
ids.aliceId,
129+
Buffer.from(SECRET),
130+
`bounded-${generateId()}.bin`,
131+
'application/octet-stream',
132+
{
133+
secretProvenance: classification === 'known' ? known : { status: 'unknown' },
134+
notifyWorkspaceChange: false,
135+
}
136+
)
137+
await expect(
138+
observeContentOutsideModel(() =>
139+
readWorkspaceFileContent.execute({
140+
principal: createCopilotChatPrincipal(
141+
{ userId: ids.aliceId, workspaceId: ids.workspaceId },
142+
WORKSPACE_FILES_DELEGATION_AUDIENCE
143+
),
144+
input: { fileId: file.id, maxBytes: 8, includeSecretProvenance: true },
145+
})
146+
)
147+
).rejects.toMatchObject({
148+
name: 'PayloadSizeLimitError',
149+
maxBytes: 8,
150+
observedBytes: Buffer.byteLength(SECRET),
151+
})
152+
}
153+
)
154+
120155
it.each(['known', 'unknown'] as const)(
121156
'does not expose storage failures for %s input bytes before a read result exists',
122157
async (classification) => {

0 commit comments

Comments
 (0)