fix(mothership): preserve sharing and search semantics

Author: icecrasher321

apps/sim/lib/benchmarks/application/operations.ts

@@ -102,6 +102,7 @@ export const benchmarkOperations = {
 
 export const benchmarkSourceOperation = defineWorkspaceOperation({
   id: 'benchmarks.source.read',
+  contentDelivery: 'none',
   minimumRole: 'read',
   workspaceApiKey: 'deny',
   principalKinds: ['delegated'],

apps/sim/lib/mothership/__integration__/content-delivery.integration.ts

@@ -8,6 +8,7 @@ import { mkdtempSync } from 'node:fs'
 import { rm } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import path from 'node:path'
+import { createSessionPrincipal } from '@sim/testing/factories/principal.factory'
 import { featureFlagsMock, featureFlagsMockFns } from '@sim/testing/mocks/feature-flags.mock'
 import { rateLimiterMock } from '@sim/testing/mocks/rate-limiter.mock'
 import { generateId } from '@sim/utils/id'
@@ -108,13 +109,22 @@ import { createCopilotChatPrincipal } from '@/lib/mothership/auth/application-de
 import { createRegistryContentGate } from '@/lib/mothership/content-delivery/registry-gate'
 import { proxySandboxResourceRequest } from '@/lib/mothership/tools/sandbox-resource-transport'
 import { chatSandboxSessionKey } from '@/lib/mothership/tools/sandbox-session-key'
+import { resolveServableDoc } from '@/lib/mothership/tools/server/files/doc-compile'
+import { getShareForResource } from '@/lib/public-shares/share-manager'
 import {
   createOwnedWorkspaceFile,
+  getWorkspaceFile,
+  updateWorkspaceFileContent as replaceFixtureFileContent,
   uploadWorkspaceFile,
 } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
 import type { WorkspaceFileSecretProvenance } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
 import { WORKSPACE_FILES_DELEGATION_AUDIENCE } from '@/lib/workspace-files/application/authorization'
 import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content'
+import {
+  getWorkspaceFileShares,
+  updateWorkspaceFileShare,
+} from '@/lib/workspace-files/application/share-workspace-file'
+import { storeCompiledDoc } from '@/lib/workspace-files/compiled-document-store'
 import { indexWorkspaceFileForSearch } from '@/lib/workspace-files/search/indexing'
 import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
@@ -176,6 +186,8 @@ describe('content reaching Chat is admitted per item', () => {
 
   beforeAll(async () => {
     fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-content-delivery-'))
+    // Compile the lazy tool graph before request deadlines and per-case provider fixtures exist.
+    await matchV2Route('/api/v2/tools/file_get_content/execute')?.load()
     fixture = await seedContentFixture()
     connector = await seedKnowledgeConnector(fixture)
   }, 120_000)
@@ -578,6 +590,147 @@ describe('content reaching Chat is admitted per item', () => {
       expect((await share(fixture.files.entries)).status).toBe(403)
       expect((await share(fixture.files.clean)).status).toBe(200)
     })
+    it.each([
+      'clean',
+      'unrecorded',
+      'secret',
+      'unknown',
+      'old-secret',
+      'malformed',
+      'invalid',
+      'mislabeled',
+      'uploaded',
+      'legacy',
+    ] as const)(
+      'checks the published document inputs before exposing a share (%s)',
+      async (kind) => {
+        const entries = [
+          {
+            name: 'COMPILED_INPUT',
+            encryptedValue: (await encryptSecret(CANARY.secret)).encrypted,
+            sourceUserId: fixture.ids.aliceId,
+            sourceWorkspaceId: ws(),
+          },
+        ]
+        const inputContent = kind === 'clean' ? CANARY.clean : CANARY.secret
+        const input = await uploadWorkspaceFile(
+          ws(),
+          fixture.ids.aliceId,
+          Buffer.from(inputContent),
+          `compiled-input-${generateId()}.txt`,
+          'text/plain',
+          {
+            notifyWorkspaceChange: false,
+            secretProvenance:
+              kind === 'unknown'
+                ? { status: 'unknown' }
+                : kind === 'unrecorded'
+                  ? { status: 'unrecorded' }
+                  : { status: 'exact', entries: kind === 'clean' ? [] : entries },
+          }
+        )
+        const record = await getWorkspaceFile(ws(), input.id, { throwOnError: true })
+        if (!record) throw new Error('Missing compiled input fixture')
+        const source =
+          kind === 'uploaded'
+            ? '%PDF-1.7\nordinary uploaded binary'
+            : `data = open(input_path("${input.id}"), "rb").read()`
+        const document = await uploadWorkspaceFile(
+          ws(),
+          fixture.ids.aliceId,
+          Buffer.from(source),
+          `compiled-share-${generateId()}.pdf`,
+          kind === 'mislabeled' || kind === 'uploaded' ? 'application/pdf' : 'text/x-python-pdf',
+          { notifyWorkspaceChange: false, secretProvenance: { status: 'exact', entries: [] } }
+        )
+        const identity = JSON.stringify({
+          version: 1,
+          inputs: [
+            {
+              fileId: input.id,
+              key: record.key,
+              context: record.storageContext ?? 'workspace',
+              contentVersion: (record.contentUpdatedAt ?? record.updatedAt).toISOString(),
+              size: record.size,
+            },
+          ],
+        })
+        const artifact = Buffer.from(`%PDF-1.7\n${inputContent}`)
+        await storeCompiledDoc(
+          ws(),
+          source,
+          'pdf',
+          'application/pdf',
+          artifact,
+          kind === 'legacy'
+            ? undefined
+            : kind === 'malformed'
+              ? '{invalid'
+              : kind === 'invalid'
+                ? JSON.stringify({ version: 1, inputs: [{}] })
+                : identity
+        )
+        const published = await resolveServableDoc(ws(), Buffer.from(source), document.name)
+        expect(published.kind).toBe(kind === 'uploaded' ? 'passthrough' : 'artifact')
+        if (published.kind === 'artifact') expect(published.buffer).toEqual(artifact)
+        if (kind === 'old-secret') {
+          await replaceFixtureFileContent(
+            ws(),
+            input.id,
+            fixture.ids.aliceId,
+            Buffer.from('now clean'),
+            undefined,
+            {
+              version: { source: 'api', authorUserId: fixture.ids.aliceId },
+              secretProvenancePolicy: {
+                mode: 'replace',
+                provenance: { status: 'exact', entries: [] },
+              },
+            }
+          )
+        }
+        const allowed = ['clean', 'unrecorded', 'legacy', 'uploaded'].includes(kind)
+        const invalid = kind === 'malformed' || kind === 'invalid'
+        const activation = await cli('PATCH', `/api/v2/files/${document.id}/share`, {
+          workspaceId: ws(),
+          isActive: true,
+          authType: 'public',
+        })
+        expect.soft(activation.status).toBe(allowed ? 200 : invalid ? 500 : 403)
+        if (!allowed) expect.soft(await getShareForResource('file', document.id)).toBeNull()
+
+        const humanShare = await updateWorkspaceFileShare.execute({
+          principal: createSessionPrincipal({ userId: fixture.ids.aliceId }),
+          input: { fileId: document.id, isActive: true, authType: 'public' },
+        })
+        expect(humanShare.share.isActive).toBe(true)
+        for (const read of [cli, sandbox]) {
+          const link = await read('GET', `/api/v2/files/${document.id}/share?workspaceId=${ws()}`)
+          expect.soft(link.status).toBe(allowed ? 200 : invalid ? 500 : 403)
+          if (!allowed) expect.soft(link.body).not.toContain(humanShare.share.token)
+        }
+        if (!invalid) {
+          const result = await observeContentDelivery(
+            createRegistryContentGate({
+              registry: new ResolvedSecretTraceRegistry([]),
+              actorUserId: fixture.ids.aliceId,
+              workspaceId: ws(),
+            }),
+            () =>
+              getWorkspaceFileShares.execute({
+                principal: createCopilotChatPrincipal(
+                  { userId: fixture.ids.aliceId, workspaceId: ws() },
+                  WORKSPACE_FILES_DELEGATION_AUDIENCE
+                ),
+                input: { workspaceId: ws(), fileIds: [document.id, fixture.files.clean] },
+              })
+          )
+          expect(result.shares.has(document.id)).toBe(allowed)
+          expect(result.shares.has(fixture.files.clean)).toBe(true)
+        }
+      }
+    )
+
     it('reads one file directly: clean as is, secrets redacted, unknown withheld whole (#1, D7)', async () => {
       const download = (fileId: string) => cli('GET', `/api/v2/files/${fileId}?workspaceId=${ws()}`)
       const clean = await download(fixture.files.clean)

apps/sim/lib/mothership/agent-cli/engines/universal-grep.ts

@@ -461,7 +461,9 @@ function clip(line: string): string {
  */
 function redactForModel(text: string, egress: ResolvedSecretTraceRegistry | undefined) {
   if (!egress) return text
-  const projection = projectResolvedSecretModelContent(text, egress)
+  const projection = projectResolvedSecretModelContent(text, egress, undefined, {
+    preserveLineBreaks: true,
+  })
   return projection.safe && typeof projection.value === 'string' ? projection.value : null
 }
 

apps/sim/lib/mothership/agent-cli/file-provenance.test.ts

@@ -623,6 +623,8 @@ describe('file provenance at the actual CLI and model-result boundary', () => {
         'FIXTURE KEY|KEYLINE'
       )
       for (const fragment of fragments(keyBlock)) expect(projected).not.toContain(fragment)
+      const footer = await grepSecretFile(`config start\n${keyBlock}\nconfig end`, 'config end')
+      expect(footer).toContain(':6: config end')
     })
   })
 

apps/sim/lib/mothership/memory/application/read-scope.ts

@@ -11,6 +11,7 @@ import { requireOwnedMemorySpace } from '@/lib/mothership/memory/spaces'
 export const MEMORY_SCOPE_AUDIENCE = 'sim:copilot-memory'
 export const readMemoryScopeOperation = defineWorkspaceOperation({
   id: 'mothership.memory.read_scope',
+  contentDelivery: 'none',
   minimumRole: 'read',
   workspaceApiKey: 'deny',
   capability: 'copilot.use',
@@ -23,6 +24,7 @@ export const readMemoryScope = defineAuthorizedChatUseCase({
   operation: readMemoryScopeOperation,
   organizationOperation: defineOrganizationOperation({
     id: readMemoryScopeOperation.id,
+    contentDelivery: readMemoryScopeOperation.contentDelivery,
     minimumRole: 'member',
     capability: readMemoryScopeOperation.capability,
     principalKinds: ['organization_delegated'],

apps/sim/lib/mothership/tools/server/files/doc-compile.test.ts

@@ -29,7 +29,8 @@ vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', () =>
   readWorkspaceFileMetadata: { execute: readWorkspaceFileMetadataMock },
 }))
 vi.mock('@/lib/execution/sandbox/run-task', () => ({ runSandboxTask: runSandboxTaskMock }))
-vi.mock('./doc-compiled-store', () => ({
+vi.mock('@/lib/workspace-files/compiled-document-store', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@/lib/workspace-files/compiled-document-store')>()),
   loadCompiledDoc: loadCompiledDocMock,
   loadPublishedCompiledDoc: vi.fn(),
   publishCompiledDocArtifact: publishCompiledDocArtifactMock,

apps/sim/lib/mothership/tools/server/files/doc-compile.ts

@@ -11,18 +11,20 @@ import {
 } from '@/lib/execution/remote-sandbox'
 import { runSandboxTask } from '@/lib/execution/sandbox/run-task'
 import { DocCompileUserError } from '@/lib/mothership/tools/server/files/doc-compile-error'
+import { PPTX_SHIM_JS } from '@/lib/mothership/tools/server/files/pptx-shim'
+import type { WorkspaceFileSecretProvenanceIdentity } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
+import { admitWorkspaceFiles } from '@/lib/workspace-files/application/admit-file-content'
+import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content'
+import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata'
 import {
   type CompiledDocReadOptions,
   loadCompiledDoc,
   loadPublishedCompiledDoc,
+  COMPILED_PDF_MAGIC as PDF_MAGIC,
   publishCompiledDocArtifact,
   storeCompiledDoc,
-} from '@/lib/mothership/tools/server/files/doc-compiled-store'
-import { PPTX_SHIM_JS } from '@/lib/mothership/tools/server/files/pptx-shim'
-import type { WorkspaceFileSecretProvenanceIdentity } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
-import { admitWorkspaceFiles } from '@/lib/workspace-files/application/admit-file-content'
-import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content'
-import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata'
+  COMPILED_ZIP_MAGIC as ZIP_MAGIC,
+} from '@/lib/workspace-files/compiled-document-store'
 import { getContentType } from '@/app/api/files/utils'
 import type { SandboxTaskId } from '@/sandbox-tasks/registry'
 
@@ -664,9 +666,6 @@ interface CompilableFormat {
   contentType: string
 }
 
-const ZIP_MAGIC = Buffer.from([0x50, 0x4b, 0x03, 0x04])
-const PDF_MAGIC = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x2d])
-
 const COMPILABLE_FORMATS: Record<string, CompilableFormat> = {
   '.pptx': { magic: ZIP_MAGIC, taskId: 'pptx-generate', contentType: PPTX_MIME },
   '.docx': { magic: ZIP_MAGIC, taskId: 'docx-generate', contentType: DOCX_MIME },

apps/sim/lib/mothership/tools/server/files/doc-servable.test.ts

@@ -33,7 +33,8 @@ vi.mock('@/lib/workspace-files/application/read-workspace-file-content', () => (
 vi.mock('@/lib/workspace-files/application/read-workspace-file-metadata', () => ({
   readWorkspaceFileMetadata: { execute: mockReadWorkspaceFileMetadata },
 }))
-vi.mock('./doc-compiled-store', () => ({
+vi.mock('@/lib/workspace-files/compiled-document-store', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@/lib/workspace-files/compiled-document-store')>()),
   loadCompiledDoc: mockLoadCompiledDoc,
   loadPublishedCompiledDoc: mockLoadPublishedCompiledDoc,
   publishCompiledDocArtifact: mockPublishCompiledDocArtifact,
@@ -48,8 +49,12 @@ vi.mock('@/app/api/files/utils', () => ({
         : 'application/octet-stream',
 }))
 
+import {
+  compileDoc,
+  resolveServableDoc,
+  resolveServableDocBytes,
+} from '@/lib/mothership/tools/server/files/doc-compile'
 import { DocCompileUserError } from '@/lib/mothership/tools/server/files/doc-compile-error'
-import { compileDoc, resolveServableDoc, resolveServableDocBytes } from './doc-compile'
 
 const { mockExecuteInSandbox } = remoteSandboxMockFns
 

apps/sim/lib/workspace-files/application/share-workspace-file.ts

@@ -21,16 +21,22 @@ import {
   upsertFileShare,
 } from '@/lib/public-shares/share-manager'
 import {
+  findListedWorkspaceFiles,
   getWorkspaceFile,
   loadActiveWorkspaceContext,
 } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
 import {
+  getBoundWorkspaceFileSecretProvenanceByIdentity,
   getCurrentWorkspaceFileSecretProvenance,
   type WorkspaceFileSecretProvenance,
 } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
+import { downloadFileStream } from '@/lib/uploads/core/storage-service'
+import { getFileExtension, isRenderableDocumentName } from '@/lib/uploads/utils/file-utils'
+import { workspaceFileProvenanceIdentity } from '@/lib/workspace-files/application/admit-file-content'
 import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/application/authorized-workspace-file-use-case'
 import { fileOperations } from '@/lib/workspace-files/application/operations'
 import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/workspace-file-context'
+import { loadPublishedCompiledDocInputs } from '@/lib/workspace-files/compiled-document-store'
 import { MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS } from '@/lib/workspace-files/limits'
 import { validatePublicFileSharing } from '@/ee/access-control/utils/permission-check'
 
@@ -97,7 +103,28 @@ async function admitSharedFiles(
   await admitContent(principal, [])
   if (!isCopilotPrincipal(principal) || fileIds.length === 0) return new Set(fileIds)
   const provenance = await getCurrentWorkspaceFileSecretProvenance(workspaceId, fileIds)
-  return new Set(fileIds.filter((id) => isShareableByChat(provenance.get(id))))
+  const shareable = new Set(fileIds.filter((id) => isShareableByChat(provenance.get(id))))
+  if (shareable.size === 0) return shareable
+  const files = await findListedWorkspaceFiles(workspaceId, [...shareable])
+  for (const id of shareable) {
+    const file = files.get(id)
+    if (!file || file.id !== id || !isRenderableDocumentName(file.name)) continue
+    const source = await downloadFileStream({
+      key: file.key,
+      context: file.storageContext ?? 'workspace',
+    })
+    const contributors = await loadPublishedCompiledDocInputs(
+      workspaceId,
+      source,
+      getFileExtension(file.name)
+    )
+    const bound = await getBoundWorkspaceFileSecretProvenanceByIdentity(workspaceId, [
+      workspaceFileProvenanceIdentity(file),
+      ...contributors,
+    ])
+    if ([...bound.values()].some((value) => !isShareableByChat(value))) shareable.delete(id)
+  }
+  return shareable
 }
 
 export const getWorkspaceFileShare = defineAuthorizedWorkspaceFileUseCase({

…/server/files/doc-compiled-store.test.ts …ce-files/compiled-document-store.test.tsapps/sim/lib/mothership/tools/server/files/doc-compiled-store.test.ts renamed to apps/sim/lib/workspace-files/compiled-document-store.test.ts apps/sim/lib/mothership/tools/server/files/doc-compiled-store.test.ts renamed to apps/sim/lib/workspace-files/compiled-document-store.test.ts

@@ -4,12 +4,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
 vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock)
 
 import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
+import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
 import {
   loadCompiledDoc,
   loadPublishedCompiledDoc,
   storeCompiledDoc,
-} from '@/lib/mothership/tools/server/files/doc-compiled-store'
-import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
+} from '@/lib/workspace-files/compiled-document-store'
 
 const { mockDownloadFile, mockUploadFile } = storageServiceMockFns