|
| 1 | +/** Real PostgreSQL name allocation and name lookups for workspace files, plus the URL fetch path. */ |
| 2 | +import { mkdtempSync } from 'node:fs' |
| 3 | +import { rm } from 'node:fs/promises' |
| 4 | +import { tmpdir } from 'node:os' |
| 5 | +import path from 'node:path' |
| 6 | +import { db, dbFor } from '@sim/db' |
| 7 | +import { organization, user, workspace, workspaceFiles } from '@sim/db/schema' |
| 8 | +import { generateId } from '@sim/utils/id' |
| 9 | +import { and, eq, inArray, isNull, sql } from 'drizzle-orm' |
| 10 | +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' |
| 11 | + |
| 12 | +const fixtureStorage = vi.hoisted(() => ({ root: '' })) |
| 13 | +vi.mock('@/lib/uploads/core/setup.server', () => ({ |
| 14 | + get UPLOAD_DIR_SERVER() { |
| 15 | + return fixtureStorage.root |
| 16 | + }, |
| 17 | +})) |
| 18 | + |
| 19 | +import { fileParseBodySchema } from '@/lib/api/contracts/storage-transfer' |
| 20 | +import * as inputValidation from '@/lib/core/security/input-validation.server' |
| 21 | +import { executeFileParserOperation } from '@/lib/internal/file/parser' |
| 22 | +import { |
| 23 | + createKnowledgeAclFixtureIds, |
| 24 | + seedKnowledgeAclFixture, |
| 25 | +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' |
| 26 | +import { |
| 27 | + createWorkspaceFileFolder, |
| 28 | + fileNameExistsInWorkspaceFolder, |
| 29 | + workspaceFileNameFolderCondition, |
| 30 | +} from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager' |
| 31 | +import { |
| 32 | + getWorkspaceFileByName, |
| 33 | + uploadWorkspaceFile, |
| 34 | +} from '@/lib/uploads/contexts/workspace/workspace-file-manager' |
| 35 | +import { createWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal' |
| 36 | + |
| 37 | +describe('workspace file names in PostgreSQL', () => { |
| 38 | + const fixtures: ReturnType<typeof createKnowledgeAclFixtureIds>[] = [] |
| 39 | + |
| 40 | + beforeAll(() => { |
| 41 | + fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-file-names-')) |
| 42 | + }) |
| 43 | + |
| 44 | + afterAll(async () => { |
| 45 | + vi.restoreAllMocks() |
| 46 | + for (const ids of fixtures) { |
| 47 | + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) |
| 48 | + await db.delete(organization).where(eq(organization.id, ids.organizationId)) |
| 49 | + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) |
| 50 | + } |
| 51 | + await rm(fixtureStorage.root, { recursive: true, force: true }) |
| 52 | + await Promise.all([db.$client.end(), dbFor('cleanup').$client.end()]) |
| 53 | + }) |
| 54 | + |
| 55 | + async function seedWorkspace() { |
| 56 | + const ids = createKnowledgeAclFixtureIds() |
| 57 | + fixtures.push(ids) |
| 58 | + await seedKnowledgeAclFixture(ids) |
| 59 | + return ids |
| 60 | + } |
| 61 | + |
| 62 | + function upload(workspaceId: string, userId: string, name: string, folderId?: string | null) { |
| 63 | + return uploadWorkspaceFile(workspaceId, userId, Buffer.from(name), name, 'text/plain', { |
| 64 | + folderId, |
| 65 | + notifyWorkspaceChange: false, |
| 66 | + }) |
| 67 | + } |
| 68 | + |
| 69 | + async function parseExternalUrl(executionId?: string) { |
| 70 | + const fixture = await seedWorkspace() |
| 71 | + const url = 'https://example.com/page.txt' |
| 72 | + vi.spyOn(inputValidation, 'validateUrlWithDNS').mockResolvedValue({ |
| 73 | + isValid: true, |
| 74 | + resolvedIP: '203.0.113.10', |
| 75 | + originalHostname: new URL(url).hostname, |
| 76 | + }) |
| 77 | + vi.spyOn(inputValidation, 'secureFetchWithPinnedIP').mockImplementation(async () => { |
| 78 | + const response = new Response('fetched page body') |
| 79 | + return { |
| 80 | + ok: response.ok, |
| 81 | + status: response.status, |
| 82 | + statusText: response.statusText, |
| 83 | + headers: new inputValidation.SecureFetchHeaders({ 'content-type': 'text/plain' }), |
| 84 | + body: response.body, |
| 85 | + text: () => response.text(), |
| 86 | + json: () => response.json(), |
| 87 | + arrayBuffer: () => response.arrayBuffer(), |
| 88 | + } |
| 89 | + }) |
| 90 | + |
| 91 | + const response = await executeFileParserOperation( |
| 92 | + fileParseBodySchema.parse({ filePath: url, workspaceId: fixture.workspaceId }), |
| 93 | + { |
| 94 | + principal: createWorkspaceFileDelegatedPrincipal({ |
| 95 | + serviceId: 'executor', |
| 96 | + subjectUserId: fixture.aliceId, |
| 97 | + workspaceId: fixture.workspaceId, |
| 98 | + delegationId: generateId(), |
| 99 | + executionId, |
| 100 | + }), |
| 101 | + workspaceId: fixture.workspaceId, |
| 102 | + workflowId: generateId(), |
| 103 | + executionId, |
| 104 | + attributedUserId: fixture.aliceId, |
| 105 | + fileAccessUserId: fixture.aliceId, |
| 106 | + } |
| 107 | + ) |
| 108 | + const body = await response.json() |
| 109 | + |
| 110 | + expect(response.status).toBe(200) |
| 111 | + expect(body.output.content).toContain('fetched page body') |
| 112 | + return db |
| 113 | + .select({ context: workspaceFiles.context }) |
| 114 | + .from(workspaceFiles) |
| 115 | + .where(eq(workspaceFiles.workspaceId, fixture.workspaceId)) |
| 116 | + } |
| 117 | + |
| 118 | + it('parses an external URL without saving a copy to workspace Files', async () => { |
| 119 | + expect(await parseExternalUrl()).toEqual([]) |
| 120 | + }) |
| 121 | + |
| 122 | + it('keeps an external URL parsed during an execution as an execution file only', async () => { |
| 123 | + expect(await parseExternalUrl(generateId())).toEqual([{ context: 'execution' }]) |
| 124 | + }) |
| 125 | + |
| 126 | + it('scopes name lookups to root or folder through the unique name index', async () => { |
| 127 | + const fixture = await seedWorkspace() |
| 128 | + const folder = await createWorkspaceFileFolder({ |
| 129 | + workspaceId: fixture.workspaceId, |
| 130 | + userId: fixture.aliceId, |
| 131 | + name: 'Reports', |
| 132 | + }) |
| 133 | + const rootFile = await upload(fixture.workspaceId, fixture.aliceId, 'root.txt') |
| 134 | + const folderFile = await upload(fixture.workspaceId, fixture.aliceId, 'nested.txt', folder.id) |
| 135 | + |
| 136 | + expect(await fileNameExistsInWorkspaceFolder(fixture.workspaceId, 'root.txt', null)).toBe(true) |
| 137 | + expect(await fileNameExistsInWorkspaceFolder(fixture.workspaceId, 'root.txt', folder.id)).toBe( |
| 138 | + false |
| 139 | + ) |
| 140 | + expect(await fileNameExistsInWorkspaceFolder(fixture.workspaceId, 'nested.txt', null)).toBe( |
| 141 | + false |
| 142 | + ) |
| 143 | + expect( |
| 144 | + await fileNameExistsInWorkspaceFolder(fixture.workspaceId, 'nested.txt', folder.id) |
| 145 | + ).toBe(true) |
| 146 | + expect((await getWorkspaceFileByName(fixture.workspaceId, 'root.txt'))?.id).toBe(rootFile.id) |
| 147 | + expect( |
| 148 | + await getWorkspaceFileByName(fixture.workspaceId, 'root.txt', { folderId: folder.id }) |
| 149 | + ).toBeNull() |
| 150 | + expect( |
| 151 | + (await getWorkspaceFileByName(fixture.workspaceId, 'nested.txt', { folderId: folder.id }))?.id |
| 152 | + ).toBe(folderFile.id) |
| 153 | + expect(await getWorkspaceFileByName(fixture.workspaceId, 'nested.txt')).toBeNull() |
| 154 | + |
| 155 | + await db.execute(sql` |
| 156 | + INSERT INTO ${workspaceFiles} (id, key, user_id, workspace_id, folder_id, context, original_name, content_type) |
| 157 | + SELECT 'wf_pad_' || n || '_' || ${fixture.workspaceId}, 'pad/' || n || '/' || ${fixture.workspaceId}, |
| 158 | + ${fixture.aliceId}, ${fixture.workspaceId}, CASE WHEN n % 2 = 0 THEN ${folder.id} END, |
| 159 | + 'workspace', 'pad-' || n || '.txt', 'text/plain' |
| 160 | + FROM generate_series(1, 2000) AS n`) |
| 161 | + await db.execute(sql`ANALYZE ${workspaceFiles}`) |
| 162 | + |
| 163 | + for (const folderId of [null, folder.id]) { |
| 164 | + const plan = await db.execute( |
| 165 | + sql`EXPLAIN (FORMAT JSON) SELECT id FROM ${workspaceFiles} WHERE ${and( |
| 166 | + eq(workspaceFiles.workspaceId, fixture.workspaceId), |
| 167 | + eq(workspaceFiles.originalName, 'root.txt'), |
| 168 | + eq(workspaceFiles.context, 'workspace'), |
| 169 | + workspaceFileNameFolderCondition(folderId), |
| 170 | + isNull(workspaceFiles.deletedAt) |
| 171 | + )}` |
| 172 | + ) |
| 173 | + const scan = JSON.stringify(plan[0]['QUERY PLAN']) |
| 174 | + expect(scan).toContain('workspace_files_workspace_folder_name_active_unique') |
| 175 | + expect(scan).toMatch(/"Index Cond":"[^"]*COALESCE\(folder_id/) |
| 176 | + } |
| 177 | + }) |
| 178 | + |
| 179 | + it('falls back to a short-id suffix after 20 numbered copies, including under concurrency', async () => { |
| 180 | + const fixture = await seedWorkspace() |
| 181 | + await upload(fixture.workspaceId, fixture.aliceId, 'page.html') |
| 182 | + for (let n = 1; n <= 20; n++) { |
| 183 | + await upload(fixture.workspaceId, fixture.aliceId, `page (${n}).html`) |
| 184 | + } |
| 185 | + |
| 186 | + const next = await upload(fixture.workspaceId, fixture.aliceId, 'page.html') |
| 187 | + const concurrent = await Promise.all( |
| 188 | + Array.from({ length: 8 }, () => upload(fixture.workspaceId, fixture.aliceId, 'page.html')) |
| 189 | + ) |
| 190 | + |
| 191 | + const shortIdSuffixed = /^page \([A-Za-z0-9_-]{8}\)\.html$/ |
| 192 | + expect(next.name).toMatch(shortIdSuffixed) |
| 193 | + const names = concurrent.map((file) => file.name) |
| 194 | + for (const name of names) expect(name).toMatch(shortIdSuffixed) |
| 195 | + expect(new Set([next.name, ...names]).size).toBe(names.length + 1) |
| 196 | + }) |
| 197 | +}) |
0 commit comments