Skip to content

Commit dd8a373

Browse files
committed
fix(mothership): bound browser-run workflow logs and keep the pointer length-blind
run_workflow can complete in the browser, and that restoration projected the raw block logs, so a large browser-run workflow was still withheld. The block-log compaction now lives in the shared workflow-output module and applies to both the server handler and the client restoration when no `select` is given. `select` still reads full values. The omitted-output pointer no longer reports bytes. They were measured before secret projection and so disclosed a secret's length. It reports the value count and says "inspect with logs get" rather than promising the full value. One `measureModelContent` in the projection module now serves both the compaction and the withheld-size logging. It stops counting at the value cap and tolerates values JSON cannot encode. The byte cap is exported beside `MAX_CONTENT_NODES`.
1 parent e46e3cb commit dd8a373

8 files changed

Lines changed: 270 additions & 192 deletions

File tree

‎apps/sim/executor/utils/resolved-secret-content-projection.ts‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ export {
2525

2626
/** Values one model-content projection will walk before refusing the whole payload. */
2727
export const MAX_CONTENT_NODES = 100_000
28+
/** Encoded bytes a model-content projection accepts by default before refusing the payload. */
29+
export const MAX_MODEL_CONTENT_BYTES = MAX_INLINE_MATERIALIZATION_BYTES
2830
const MAX_CONTENT_DEPTH = 100
2931
const INTERNAL_DIAGNOSTIC_IDENTIFIER_PATTERN =
3032
/__var_[A-Za-z0-9_]+|__sim_code_\d+_(?:binding|input|runtime)_\d+[A-Za-z0-9_]*|__sim_placeholder_[a-f0-9]{64}__|__sim_runtime_[A-Za-z0-9_]+_\d+[A-Za-z0-9_]*|__SIM_RUNTIME_PAYLOAD_PATH/g
@@ -352,12 +354,41 @@ function projectContent(
352354
export function projectResolvedSecretContent(
353355
value: unknown,
354356
matcher: ResolvedSecretMatcher,
355-
maxBytes = MAX_INLINE_MATERIALIZATION_BYTES,
357+
maxBytes = MAX_MODEL_CONTENT_BYTES,
356358
options: ResolvedSecretContentProjectionOptions = {}
357359
): ResolvedSecretContentProjection {
358360
return projectContent(value, matcher, maxBytes, options)
359361
}
360362

363+
/** Size of a value as the model-content projection sees it, for budgeting and diagnostics. */
364+
export interface ModelContentMeasure {
365+
/** Values walked, stopping at `MAX_CONTENT_NODES + 1`: past the cap the exact count is moot. */
366+
values: number
367+
/** Encoded bytes; absent when counting stopped at the value cap. */
368+
bytes?: number
369+
}
370+
371+
class ModelContentMeasureLimit extends Error {}
372+
373+
/**
374+
* Measures a value in the units the projection caps, without walking past the value cap: a
375+
* payload that is already over it is reported as over rather than serialized in full. Returns
376+
* undefined for a value JSON cannot encode (a BigInt, a cycle), which the projection refuses too.
377+
*/
378+
export function measureModelContent(value: unknown): ModelContentMeasure | undefined {
379+
let values = 0
380+
try {
381+
const encoded = JSON.stringify(value, (_key, item: unknown) => {
382+
values += 1
383+
if (values > MAX_CONTENT_NODES) throw new ModelContentMeasureLimit()
384+
return item
385+
})
386+
return { values, bytes: encoded === undefined ? 0 : Buffer.byteLength(encoded, 'utf8') }
387+
} catch (error) {
388+
return error instanceof ModelContentMeasureLimit ? { values } : undefined
389+
}
390+
}
391+
361392
/** Returns the registry-revision-cached matcher used for all model-visible projection. */
362393
export function getResolvedSecretModelMatcher(
363394
registry: ResolvedSecretTraceRegistry | undefined
@@ -397,7 +428,7 @@ export function getResolvedSecretModelMatcher(
397428
export function projectResolvedSecretModelContent(
398429
value: unknown,
399430
registry: ResolvedSecretTraceRegistry | undefined,
400-
maxBytes = MAX_INLINE_MATERIALIZATION_BYTES,
431+
maxBytes = MAX_MODEL_CONTENT_BYTES,
401432
options: ResolvedSecretContentProjectionOptions = {}
402433
): ResolvedSecretContentProjection {
403434
const snapshot = getResolvedSecretModelMatcher(registry)
@@ -420,7 +451,7 @@ export function projectResolvedSecretModelContent(
420451
export function projectResolvedSecretModelJsonContent(
421452
value: unknown,
422453
registry: ResolvedSecretTraceRegistry | undefined,
423-
maxBytes = MAX_INLINE_MATERIALIZATION_BYTES,
454+
maxBytes = MAX_MODEL_CONTENT_BYTES,
424455
options: ResolvedSecretContentProjectionOptions = {}
425456
): ResolvedSecretContentProjection {
426457
const snapshot = getResolvedSecretModelMatcher(registry)
@@ -456,7 +487,7 @@ export function projectResolvedSecretModelJsonContent(
456487
export function projectResolvedSecretDiagnosticContent(
457488
value: unknown,
458489
registry: ResolvedSecretTraceRegistry | undefined,
459-
maxBytes = MAX_INLINE_MATERIALIZATION_BYTES
490+
maxBytes = MAX_MODEL_CONTENT_BYTES
460491
): ResolvedSecretContentProjection {
461492
return projectResolvedSecretModelContent(value, registry, maxBytes, {
462493
sanitizeInternalIdentifiers: true,
@@ -510,7 +541,7 @@ export function isResolvedSecretModelContentUnchanged(
510541
if (!snapshot.complete) return false
511542
if (!snapshot.matcher) return true
512543

513-
const projection = projectContent(value, snapshot.matcher, MAX_INLINE_MATERIALIZATION_BYTES, {
544+
const projection = projectContent(value, snapshot.matcher, MAX_MODEL_CONTENT_BYTES, {
514545
projectPrimitiveLiterals: true,
515546
rejectResolvedSecretLiterals: true,
516547
})
@@ -524,7 +555,7 @@ export function isResolvedSecretModelContentUnchanged(
524555
export function projectResolvedSecretModelJsonStrings(
525556
values: readonly (string | undefined)[],
526557
registry: ResolvedSecretTraceRegistry | undefined,
527-
maxBytes = MAX_INLINE_MATERIALIZATION_BYTES
558+
maxBytes = MAX_MODEL_CONTENT_BYTES
528559
): ResolvedSecretContentProjection {
529560
const snapshot = getResolvedSecretModelMatcher(registry)
530561
if (!snapshot.complete) return { safe: false }

‎apps/sim/lib/mothership/request/tools/client.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,51 @@ describe('workflow client tool completion', () => {
202202
)
203203
})
204204

205+
/**
206+
* A browser-run workflow reaches the model through this restoration, not the server handler, so
207+
* it needs the same block-log budget: a synthetic run whose block outputs exceed the projection's
208+
* traversal cap would otherwise be withheld whole once a secret is active.
209+
*/
210+
it('bounds bulky block-log outputs so a large browser run still projects', async () => {
211+
const rows = (count: number) =>
212+
Array.from({ length: count }, (_, index) => ({
213+
id: `row_${index}`,
214+
data: { a: 'x', b: 'y', c: 'z', d: 'w' },
215+
}))
216+
getTrustedWorkflowToolExecution.mockResolvedValue({
217+
...trustedExecution('execution-1'),
218+
blockLogs: [
219+
{ blockId: 'small', blockName: 'Small', output: { count: 1 } },
220+
...Array.from({ length: 4 }, (_, index) => ({
221+
blockId: `query-${index}`,
222+
blockName: `Query ${index}`,
223+
output: { rows: rows(5_000) },
224+
})),
225+
],
226+
})
227+
waitForToolConfirmation.mockResolvedValue({
228+
status: 'success',
229+
data: { workflowId: 'workflow-1', executionId: 'execution-1' },
230+
})
231+
232+
const completion = await waitForWorkflowToolCompletion({
233+
toolCallId: 'tool-1',
234+
workflowId: 'workflow-1',
235+
timeoutMs: 1_000,
236+
registry: createParentRegistry(),
237+
})
238+
239+
const data = completion?.data as Record<string, unknown>
240+
expect(data.output).toEqual({ value: 'child read {{PARENT_SECRET}} from execution-1' })
241+
const logs = data.logs as Array<Record<string, unknown>>
242+
expect(logs[0]?.output).toEqual({ count: 1 })
243+
expect(logs.some((log) => typeof log.output === 'string')).toBe(true)
244+
for (const log of logs.filter((entry) => typeof entry.output === 'string')) {
245+
expect(log.output).toContain('logs get execution-1 --trace')
246+
}
247+
expect(JSON.stringify(completion)).not.toContain('parent-secret-value')
248+
})
249+
205250
it('preserves the server-confirmed status while omitting unavailable execution content', async () => {
206251
const registry = createParentRegistry()
207252
waitForToolConfirmation.mockResolvedValue({

‎apps/sim/lib/mothership/request/tools/client.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import {
1515
unsealClientToolContext,
1616
} from '@/lib/mothership/request/tools/client-completion-seal.server'
1717
import { inspectToolResultForCopilot } from '@/lib/mothership/request/tools/resolved-secret-result'
18-
import { presentWorkflowLogs } from '@/lib/mothership/tools/workflow-output'
18+
import { compactBlockLogOutputs, presentWorkflowLogs } from '@/lib/mothership/tools/workflow-output'
1919
import {
2020
createStructuralWorkflowToolCompletionData,
2121
getWorkflowToolCompletionExecutionId,
@@ -376,7 +376,10 @@ export async function waitForWorkflowToolCompletion({
376376
...(Object.hasOwn(trustedExecution, 'finalOutput')
377377
? { output: trustedExecution.finalOutput }
378378
: {}),
379-
logs: trustedExecution.blockLogs,
379+
// `select` reads full values from these logs; only logs echoed whole are bounded.
380+
logs: select?.length
381+
? trustedExecution.blockLogs
382+
: compactBlockLogOutputs(trustedExecution.blockLogs, executionId),
380383
...(trustedExecution.error !== undefined ? { error: trustedExecution.error } : {}),
381384
...(status === MothershipStreamV1ToolOutcome.cancelled
382385
? { reason: 'user_cancelled', cancelledByUser: true }

‎apps/sim/lib/mothership/request/tools/resolved-secret-result.ts‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import type { ToolCallEffect, ToolExecutionResult } from '@/lib/mothership/tool-executor/types'
22
import { TOOL_EFFECT_PHASE } from '@/lib/mothership/tool-executor/types'
3-
import { projectResolvedSecretModelJsonContent } from '@/executor/utils/resolved-secret-content-projection'
3+
import {
4+
measureModelContent,
5+
projectResolvedSecretModelJsonContent,
6+
} from '@/executor/utils/resolved-secret-content-projection'
47
import type {
58
ResolvedSecretIncompletenessReason,
69
ResolvedSecretTraceRegistry,
@@ -259,26 +262,19 @@ export function projectToolErrorMessageForCopilot(
259262
* Sizes the content a withheld result would have carried, for the log line only.
260263
*
261264
* A complete registry can still refuse content by its encoded size or by the number of values the
262-
* projection must walk (its node cap is reached well before the byte cap by row-shaped payloads).
263-
* Both measures are reported so a `content-refused` line names which one it hit. Numbers only:
264-
* the content itself never reaches the log.
265+
* projection must walk (its value cap is reached well before the byte cap by row-shaped payloads).
266+
* Both measures are reported so a `content-refused` line names which one it hit; counting stops
267+
* at the value cap, so a huge payload is not serialized again just to be logged. Numbers only.
265268
*/
266269
export function measureWithheldContent(result: ToolExecutionResult): {
267270
resultBytes?: number
268271
resultValues?: number
269272
} {
270-
try {
271-
let values = 0
272-
const encoded = JSON.stringify(
273-
{ output: result.output, error: result.error },
274-
(_key, value) => {
275-
values += 1
276-
return value
277-
}
278-
)
279-
return { resultBytes: Buffer.byteLength(encoded, 'utf8'), resultValues: values }
280-
} catch {
281-
return {}
273+
const measure = measureModelContent({ output: result.output, error: result.error })
274+
if (!measure) return {}
275+
return {
276+
resultValues: measure.values,
277+
...(measure.bytes !== undefined ? { resultBytes: measure.bytes } : {}),
282278
}
283279
}
284280

‎apps/sim/lib/mothership/tools/handlers/function-execute-file-mounts.test.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,17 @@ describe('Mothership file mounts bind content and classification to the same rec
250250
safe: true,
251251
provenance: {
252252
status: 'exact',
253-
entries: kind === 'secret' ? [expect.objectContaining({})] : [],
253+
// A mounted file's secret crosses anonymously: its ciphertext binds it, not a name.
254+
entries:
255+
kind === 'secret'
256+
? [
257+
{
258+
encryptedValue: 'fixture-ciphertext',
259+
sourceUserId: 'reader',
260+
sourceWorkspaceId: 'workspace',
261+
},
262+
]
263+
: [],
254264
},
255265
})
256266
}

‎apps/sim/lib/mothership/tools/handlers/workflow/mutations.ts‎

Lines changed: 5 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ import { createLogger } from '@sim/logger'
22
import { filterUndefined, isPlainRecord, isRecordLike } from '@sim/utils/object'
33
import { createCopilotWorkspaceApiKey } from '@/lib/api-key/application/create-api-key'
44
import { PlatformEvents } from '@/lib/core/telemetry'
5-
import { MAX_INLINE_MATERIALIZATION_BYTES } from '@/lib/execution/payloads/limits'
65
import { messageForCopilotApplicationError } from '@/lib/mothership/application/error'
76
import { executeCopilotApiKeyUseCase } from '@/lib/mothership/application/execute-api-key-use-case'
87
import {
@@ -30,7 +29,11 @@ import type {
3029
VariableOperation,
3130
} from '@/lib/mothership/tools/handlers/param-types'
3231
import { requireCopilotWorkspace } from '@/lib/mothership/tools/server/workspace-scope'
33-
import { presentWorkflowLogs } from '@/lib/mothership/tools/workflow-output'
32+
import {
33+
compactBlockLogInputs,
34+
compactBlockLogOutputs,
35+
presentWorkflowLogs,
36+
} from '@/lib/mothership/tools/workflow-output'
3437
import { decodeVfsPathSegments, encodeVfsPathSegments } from '@/lib/mothership/vfs/path-utils'
3538
import { cancelWorkflowRun } from '@/lib/workflows/application/cancel-run'
3639
import { createWorkflow } from '@/lib/workflows/application/create-workflow'
@@ -48,101 +51,10 @@ import {
4851
} from '@/lib/workflows/application/update-workflow-content'
4952
import { sanitizeForCopilot } from '@/lib/workflows/sanitization/json-sanitizer'
5053
import { hasExecutionResult, readAttemptedExecutionId } from '@/executor/utils/errors'
51-
import { MAX_CONTENT_NODES } from '@/executor/utils/resolved-secret-content-projection'
5254
import type { WorkflowState } from '@/stores/workflows/workflow/types'
5355

5456
const logger = createLogger('WorkflowMutations')
5557

56-
/** Above this a Function block's `input.code` is echoed upstream JSON, not code worth reading. */
57-
const LOG_CODE_INPUT_MAX_CHARS = 240
58-
/** Any other echoed input string over this is data the caller already has, or can fetch. */
59-
const LOG_INPUT_STRING_MAX_CHARS = 2_000
60-
const LOG_INPUT_KEEP_CHARS = 200
61-
62-
/**
63-
* Compacts the block inputs echoed back in `logs`. A Function block's `input.code` embeds the
64-
* fully serialized upstream rows, so a seven-block run repeated the same rows several times
65-
* across ~14k chars of tool result. Outputs are bounded separately by {@link compactBlockLogOutputs},
66-
* and the full input stays one `logs get <executionId> --trace` away.
67-
*/
68-
function compactBlockLogInputs(logs: unknown, executionId: string | undefined): unknown {
69-
if (!Array.isArray(logs)) return logs
70-
const reference = executionId ?? '<executionId>'
71-
return logs.map((entry) => {
72-
if (!isPlainRecord(entry) || !isPlainRecord(entry.input)) return entry
73-
const input: Record<string, unknown> = {}
74-
for (const [key, value] of Object.entries(entry.input)) {
75-
const limit = key === 'code' ? LOG_CODE_INPUT_MAX_CHARS : LOG_INPUT_STRING_MAX_CHARS
76-
input[key] =
77-
typeof value === 'string' && value.length > limit
78-
? `${value.slice(0, LOG_INPUT_KEEP_CHARS)} …[${value.length} chars, see logs get ${reference} --trace]`
79-
: value
80-
}
81-
return { ...entry, input }
82-
})
83-
}
84-
85-
/**
86-
* Budgets for the block outputs echoed back in `logs`, a quarter of each cap the model-facing
87-
* projection enforces on the whole result. Reaching either cap withholds everything, including
88-
* the final output and error the run was for; those stay intact here and share the remaining
89-
* three quarters with the rest of the envelope. The value budget matters first: row-shaped
90-
* outputs reach the projection's traversal cap long before its byte cap.
91-
*/
92-
const LOG_OUTPUT_VALUE_BUDGET = Math.floor(MAX_CONTENT_NODES / 4)
93-
const LOG_OUTPUT_BYTE_BUDGET = Math.floor(MAX_INLINE_MATERIALIZATION_BYTES / 4)
94-
95-
/** Counts values the way the projection walks them, and the bytes they encode to. */
96-
function measureLogOutput(value: unknown): { values: number; bytes: number } {
97-
let values = 0
98-
const encoded = JSON.stringify(value, (_key, item) => {
99-
values += 1
100-
return item
101-
})
102-
return { values, bytes: encoded === undefined ? 0 : Buffer.byteLength(encoded, 'utf8') }
103-
}
104-
105-
/**
106-
* Replaces the bulkiest block outputs in `logs` with a pointer once they exceed the budgets
107-
* above, largest first, so the rest of the run still reaches the model. The full outputs stay in
108-
* the run's archived trace, one `logs get <executionId> --trace` away, matching how
109-
* {@link compactBlockLogInputs} treats oversized inputs.
110-
*/
111-
function compactBlockLogOutputs(logs: unknown, executionId: string | undefined): unknown {
112-
if (!Array.isArray(logs)) return logs
113-
const reference = executionId ?? '<executionId>'
114-
const sizes = logs.map((entry) =>
115-
isPlainRecord(entry) && entry.output !== undefined ? measureLogOutput(entry.output) : undefined
116-
)
117-
let values = 0
118-
let bytes = 0
119-
for (const size of sizes) {
120-
values += size?.values ?? 0
121-
bytes += size?.bytes ?? 0
122-
}
123-
if (values <= LOG_OUTPUT_VALUE_BUDGET && bytes <= LOG_OUTPUT_BYTE_BUDGET) return logs
124-
125-
const compacted = [...logs]
126-
const bulkiestFirst = sizes
127-
.map((size, index) => ({ size, index }))
128-
.filter((entry): entry is { size: { values: number; bytes: number }; index: number } =>
129-
Boolean(entry.size)
130-
)
131-
.sort(
132-
(left, right) => right.size.values - left.size.values || right.size.bytes - left.size.bytes
133-
)
134-
for (const { size, index } of bulkiestFirst) {
135-
if (values <= LOG_OUTPUT_VALUE_BUDGET && bytes <= LOG_OUTPUT_BYTE_BUDGET) break
136-
compacted[index] = {
137-
...(logs[index] as Record<string, unknown>),
138-
output: `…[output omitted: ${size.values} values, ${size.bytes} bytes; see logs get ${reference} --trace]`,
139-
}
140-
values -= size.values
141-
bytes -= size.bytes
142-
}
143-
return compacted
144-
}
145-
14658
function stripBinaryFields(value: unknown): unknown {
14759
if (value === null || value === undefined) return value
14860
if (typeof value !== 'object') return value

0 commit comments

Comments
 (0)