diff --git a/apps/media-server/src/__tests__/lib/media-edit.test.ts b/apps/media-server/src/__tests__/lib/media-edit.test.ts index a1836f5a7bd..12234a74db0 100644 --- a/apps/media-server/src/__tests__/lib/media-edit.test.ts +++ b/apps/media-server/src/__tests__/lib/media-edit.test.ts @@ -3,6 +3,7 @@ import { existsSync, rmSync } from "node:fs"; import { join } from "node:path"; import { buildStreamCopySegmentArgs, + buildTranscodeEditArgs, buildTranscodeSegmentArgs, normalizeEditRanges, renderEditedVideo, @@ -98,6 +99,41 @@ describe("media edit helpers", () => { ); expect(args).toContain("[a]"); }); + + test("builds a bounded multi-input transcode graph", () => { + const args = buildTranscodeEditArgs( + "/input.mp4", + [ + { start: 0, end: 1 }, + { start: 1.2, end: 2 }, + { start: 2.3, end: 3 }, + ], + "/output.mp4", + true, + 60, + ); + const filter = args[args.indexOf("-filter_complex") + 1]; + + expect(args.filter((value) => value === "-i")).toHaveLength(3); + expect(args.filter((value) => value === "-ss")).toHaveLength(3); + expect(filter).toContain("concat=n=3:v=1:a=1[v][a]"); + expect(filter).toContain("[1:v:0]fps=60,setpts=PTS-STARTPTS[v1]"); + expect(filter).toContain("[2:a:0]asetpts=PTS-STARTPTS[a2]"); + }); + + test("rejects unbounded transcode graphs", () => { + expect(() => + buildTranscodeEditArgs( + "/input.mp4", + Array.from({ length: 5 }, (_, index) => ({ + start: index, + end: index + 0.5, + })), + "/output.mp4", + true, + ), + ).toThrow("Transcode batches must contain 1-4 ranges"); + }); }); describe("renderEditedVideo integration tests", () => { @@ -108,8 +144,11 @@ describe("renderEditedVideo integration tests", () => { const editedFile = await renderEditedVideo({ inputPath: TEST_VIDEO_WITH_AUDIO, keepRanges: [ - { start: 0, end: 0.4 }, - { start: 0.55, end: 0.95 }, + { start: 0.08, end: 0.16 }, + { start: 0.24, end: 0.32 }, + { start: 0.4, end: 0.48 }, + { start: 0.56, end: 0.64 }, + { start: 0.72, end: 0.8 }, ], metadata, onProgress: (progress) => { diff --git a/apps/media-server/src/lib/media-edit.ts b/apps/media-server/src/lib/media-edit.ts index 35c698be0b5..c475a7c62e8 100644 --- a/apps/media-server/src/lib/media-edit.ts +++ b/apps/media-server/src/lib/media-edit.ts @@ -23,6 +23,7 @@ const MIN_RANGE_DURATION = 0.05; const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000; const MAX_TIMEOUT_MS = 60 * 60 * 1000; const DEFAULT_OUTPUT_FPS = 30; +const MAX_TRANSCODE_RANGES_PER_BATCH = 4; function roundTime(value: number) { return Math.round(value * 1000) / 1000; @@ -147,6 +148,60 @@ export function buildTranscodeSegmentArgs( ]; } +export function buildTranscodeEditArgs( + inputPath: string, + ranges: EditRange[], + outputPath: string, + hasAudio: boolean, + fps = DEFAULT_OUTPUT_FPS, +) { + if (ranges.length === 0 || ranges.length > MAX_TRANSCODE_RANGES_PER_BATCH) { + throw new Error( + `Transcode batches must contain 1-${MAX_TRANSCODE_RANGES_PER_BATCH} ranges`, + ); + } + + const filters = ranges.flatMap((_, index) => { + const videoFilter = `[${index}:v:0]fps=${getOutputFps(fps)},setpts=PTS-STARTPTS[v${index}]`; + if (!hasAudio) return [videoFilter]; + return [videoFilter, `[${index}:a:0]asetpts=PTS-STARTPTS[a${index}]`]; + }); + const inputs = ranges + .map((_, index) => `[v${index}]${hasAudio ? `[a${index}]` : ""}`) + .join(""); + const concat = `${inputs}concat=n=${ranges.length}:v=1:a=${hasAudio ? 1 : 0}[v]${hasAudio ? "[a]" : ""}`; + + return [ + "ffmpeg", + "-hide_banner", + "-y", + ...ranges.flatMap((range) => [ + "-ss", + formatTime(range.start), + "-t", + formatTime(getRangeDuration(range)), + "-i", + inputPath, + ]), + "-filter_complex", + [...filters, concat].join(";"), + "-map", + "[v]", + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "18", + "-pix_fmt", + "yuv420p", + ...(hasAudio ? ["-map", "[a]", "-c:a", "aac", "-b:a", "160k"] : ["-an"]), + "-movflags", + "+faststart", + outputPath, + ]; +} + function buildConcatArgs(listPath: string, outputPath: string) { return [ "ffmpeg", @@ -233,6 +288,8 @@ async function runFfmpegCommand( timeoutMs: number, abortSignal?: AbortSignal, ) { + abortSignal?.throwIfAborted(); + const proc = registerSubprocess( spawn({ cmd: args, @@ -313,49 +370,74 @@ async function concatSegments( } } -async function renderSegments({ - keepRanges, - timeoutMs, - buildArgs, - onProgress, - abortSignal, - progressStart, - progressEnd, -}: { - keepRanges: EditRange[]; - timeoutMs: number; - buildArgs: (range: EditRange, outputPath: string) => string[]; - onProgress?: ProgressCallback; - abortSignal?: AbortSignal; - progressStart: number; - progressEnd: number; -}) { - const segmentFiles: TempFileHandle[] = []; +function createTranscodeBatches(ranges: EditRange[]) { + const batches: EditRange[][] = []; + for ( + let index = 0; + index < ranges.length; + index += MAX_TRANSCODE_RANGES_PER_BATCH + ) { + batches.push(ranges.slice(index, index + MAX_TRANSCODE_RANGES_PER_BATCH)); + } + return batches; +} + +function getRemainingTimeoutMs(startedAt: number, timeoutMs: number) { + const remainingMs = Math.ceil(timeoutMs - (performance.now() - startedAt)); + if (remainingMs <= 0) { + throw new Error(`Operation timed out after ${timeoutMs}ms`); + } + return remainingMs; +} + +async function renderTranscodedEdit( + inputPath: string, + keepRanges: EditRange[], + hasAudio: boolean, + fps: number | undefined, + timeoutMs: number, + onProgress?: ProgressCallback, + abortSignal?: AbortSignal, +) { + const batches = createTranscodeBatches(keepRanges); + const batchFiles: TempFileHandle[] = []; + const startedAt = performance.now(); try { - for (const [index, range] of keepRanges.entries()) { - const segmentFile = await createTempFile(".mp4"); - segmentFiles.push(segmentFile); + onProgress?.(5, "Preparing edit..."); + for (const [index, batch] of batches.entries()) { + const batchFile = await createTempFile(".mp4"); + batchFiles.push(batchFile); await runFfmpegCommand( - buildArgs(range, segmentFile.path), - timeoutMs, + buildTranscodeEditArgs(inputPath, batch, batchFile.path, hasAudio, fps), + getRemainingTimeoutMs(startedAt, timeoutMs), abortSignal, ); - const progress = - progressStart + - ((index + 1) / keepRanges.length) * (progressEnd - progressStart); - onProgress?.(progress, "Preparing edit..."); + onProgress?.( + 5 + ((index + 1) / batches.length) * 65, + "Preparing edit...", + ); + } + + if (batchFiles.length === 1) { + const outputFile = batchFiles[0]; + if (!outputFile || (await file(outputFile.path).size) === 0) { + throw new Error("FFmpeg produced empty edited output"); + } + batchFiles.length = 0; + onProgress?.(75, "Edit prepared"); + return outputFile; } const outputFile = await concatSegments( - segmentFiles, - timeoutMs, + batchFiles, + getRemainingTimeoutMs(startedAt, timeoutMs), abortSignal, ); - onProgress?.(progressEnd, "Edit prepared"); + onProgress?.(75, "Edit prepared"); return outputFile; } finally { - await Promise.all(segmentFiles.map((segment) => segment.cleanup())); + await Promise.all(batchFiles.map((batchFile) => batchFile.cleanup())); } } @@ -373,20 +455,13 @@ export async function renderEditedVideo({ const timeoutMs = getTimeoutMs(normalizedRanges); - return await renderSegments({ - keepRanges: normalizedRanges, + return await renderTranscodedEdit( + inputPath, + normalizedRanges, + Boolean(metadata.audioCodec), + metadata.fps, timeoutMs, - buildArgs: (range, outputPath) => - buildTranscodeSegmentArgs( - inputPath, - range, - outputPath, - Boolean(metadata.audioCodec), - metadata.fps, - ), onProgress, abortSignal, - progressStart: 5, - progressEnd: 75, - }); + ); } diff --git a/apps/web/__tests__/fixtures/assemblyai-edit-response.ts b/apps/web/__tests__/fixtures/assemblyai-edit-response.ts new file mode 100644 index 00000000000..de441cfce90 --- /dev/null +++ b/apps/web/__tests__/fixtures/assemblyai-edit-response.ts @@ -0,0 +1,107 @@ +export const assemblyAIEditResponse = { + id: "transcript_edit_fixture", + status: "completed", + speech_model_used: "universal-3-5-pro", + language_code: "en", + words: [ + { + text: "Um,", + start: 120, + end: 380, + confidence: 0.98, + speaker: "A", + channel: null, + }, + { + text: "this", + start: 520, + end: 720, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "is", + start: 730, + end: 840, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "a", + start: 850, + end: 910, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "real", + start: 930, + end: 1130, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "example.", + start: 1140, + end: 1510, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "Erm", + start: 2300, + end: 2570, + confidence: 0.93, + speaker: "A", + channel: null, + }, + { + text: "nothing", + start: 2700, + end: 3070, + confidence: 0.98, + speaker: "A", + channel: null, + }, + { + text: "else.", + start: 3090, + end: 3400, + confidence: 0.98, + speaker: "A", + channel: null, + }, + ] as const, +} as const; + +export const assemblyAIRealFillerResponse = { + id: "sanitized_real_filler_response", + status: "completed", + speech_model_used: "universal-2", + language_code: "en", + words: [ + { text: "Hey,", start: 960, end: 1280, confidence: 0.98779297 }, + { text: "my", start: 1280, end: 1560, confidence: 0.9995117 }, + { text: "name's", start: 1560, end: 1880, confidence: 0.9082031 }, + { text: "Richie,", start: 1880, end: 2320, confidence: 0.9448242 }, + { text: "and", start: 2320, end: 2520, confidence: 0.99560547 }, + { text: "this", start: 2520, end: 2680, confidence: 0.9995117 }, + { text: "is", start: 2680, end: 2880, confidence: 0.9970703 }, + { text: "my,", start: 2880, end: 3200, confidence: 0.9995117 }, + { text: "um,", start: 3460, end: 3980, confidence: 0.9959265 }, + { text: "test", start: 3980, end: 4320, confidence: 0.99902344 }, + { text: "from", start: 4320, end: 4600, confidence: 0.99902344 }, + { text: "the", start: 4600, end: 4800, confidence: 0.99853516 }, + { text: "CAP", start: 4800, end: 5120, confidence: 0.95458984 }, + { text: "Mobile", start: 5120, end: 5680, confidence: 0.99658203 }, + { text: "app.", start: 5680, end: 5760, confidence: 0.6508789 }, + { text: "Uh,", start: 5760, end: 7240, confidence: 0.9857637 }, + { text: "say", start: 7240, end: 7360, confidence: 0.9975586 }, + { text: "hello,", start: 7360, end: 7640, confidence: 0.99975586 }, + ] as const, +} as const; diff --git a/apps/web/__tests__/integration/transcribe-workflow.test.ts b/apps/web/__tests__/integration/transcribe-workflow.test.ts new file mode 100644 index 00000000000..60a997ac06c --- /dev/null +++ b/apps/web/__tests__/integration/transcribe-workflow.test.ts @@ -0,0 +1,364 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { assemblyAIEditResponse } from "../fixtures/assemblyai-edit-response"; + +const mocks = vi.hoisted(() => ({ + transcribe: vi.fn(), + putObject: vi.fn(), + deleteObject: vi.fn(), + getInternalSignedObjectUrl: vi.fn(), + startAiGeneration: vi.fn(), + updates: [] as Record[], +})); + +const schemaMocks = vi.hoisted(() => ({ + videos: { + id: "videos.id", + metadata: "videos.metadata", + transcriptionStatus: "videos.transcriptionStatus", + updatedAt: "videos.updatedAt", + }, + organizations: { id: "organizations.id" }, + videoUploads: { videoId: "videoUploads.videoId" }, + videoEdits: { + videoId: "videoEdits.videoId", + editSpec: "videoEdits.editSpec", + }, +})); + +const videoRow = vi.hoisted(() => ({ + id: "video-123", + ownerId: "user-456", + duration: 4, + settings: null, + source: { type: "webMP4" }, + isScreenshot: false, + transcriptionStatus: "COMPLETE", + updatedAt: new Date("2026-07-30T00:00:00.000Z"), + metadata: { + editTranscriptBackfill: { + status: "processing", + requestId: "request-1", + requestedAt: "2026-07-30T00:00:00.000Z", + }, + }, +})); + +const state = vi.hoisted(() => ({ editRows: [] as unknown[] })); + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + ASSEMBLY_API_KEY: "test-assembly-api-key", + NEXTAUTH_SECRET: "test-secret-with-enough-entropy", + }), +})); + +vi.mock("@cap/database/schema", () => schemaMocks); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => { + const query = { + from: (table: unknown) => { + if (table === schemaMocks.videoUploads) { + return { + where: () => ({ limit: async () => [] }), + }; + } + if (table === schemaMocks.videoEdits) { + return { where: async () => state.editRows }; + } + return { + leftJoin: () => ({ + where: async () => [{ video: videoRow, orgSettings: null }], + }), + where: async () => [videoRow], + }; + }, + }; + return query; + }, + update: () => ({ + set: (values: Record) => { + mocks.updates.push(values); + return { where: async () => [{ affectedRows: 1 }] }; + }, + }), + }), +})); + +vi.mock("drizzle-orm", () => ({ + and: (...conditions: unknown[]) => ({ conditions }), + eq: (field: unknown, value: unknown) => ({ field, value }), + sql: (strings: TemplateStringsArray, ...values: unknown[]) => ({ + strings, + values, + }), +})); + +vi.mock("server-only", () => ({})); + +vi.mock("workflow", () => ({ + FatalError: class FatalError extends Error {}, +})); + +vi.mock("assemblyai", () => ({ + AssemblyAI: class { + transcripts = { transcribe: mocks.transcribe }; + }, +})); + +vi.mock("@cap/web-backend/src/Storage/index", () => ({ + Storage: { + getAccessForVideo: () => ({ + pipe: (runner: (value: unknown) => unknown) => + runner([ + { + putObject: mocks.putObject, + deleteObject: mocks.deleteObject, + getInternalSignedObjectUrl: mocks.getInternalSignedObjectUrl, + }, + ]), + }), + }, +})); + +vi.mock("@/lib/workflow-runtime", () => ({ + runWorkflowPromise: (value: unknown) => Promise.resolve(value), +})); + +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: (video: unknown) => video, +})); + +vi.mock("@/lib/media-client", () => ({ + isMediaServerConfigured: () => true, + probeVideoViaMediaServer: async () => ({ + audioCodec: "aac", + videoCodec: "h264", + duration: 4, + audioChannels: 2, + sampleRate: 48_000, + }), + extractAudioViaMediaServer: async () => Buffer.from("audio"), + checkHasAudioTrackViaMediaServer: async () => true, +})); + +vi.mock("@/lib/audio-extract", () => ({ + checkHasAudioTrack: async () => true, + extractAudioFromUrl: async () => ({ + filePath: "/tmp/audio.mp3", + cleanup: async () => {}, + }), +})); + +vi.mock("@/lib/audio-enhance", () => ({ + ENHANCED_AUDIO_CONTENT_TYPE: "audio/mpeg", + ENHANCED_AUDIO_EXTENSION: "mp3", + enhanceAudioFromUrl: async () => Buffer.from(""), +})); + +vi.mock("@/lib/generate-ai", () => ({ + startAiGeneration: mocks.startAiGeneration, +})); + +function pipeValue(value: unknown) { + return { pipe: (runner: (input: unknown) => unknown) => runner(value) }; +} + +describe("transcribeVideoWorkflow", () => { + beforeEach(() => { + mocks.updates.length = 0; + state.editRows = []; + mocks.transcribe.mockResolvedValue({ + ...assemblyAIEditResponse, + audio_duration: 4, + }); + mocks.putObject.mockImplementation(() => pipeValue(undefined)); + mocks.deleteObject.mockImplementation(() => pipeValue(undefined)); + mocks.getInternalSignedObjectUrl.mockImplementation(() => + pipeValue("https://storage.test/object"), + ); + mocks.startAiGeneration.mockResolvedValue({ success: true, message: "ok" }); + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + ok: true, + status: 200, + statusText: "OK", + arrayBuffer: async () => new ArrayBuffer(8), + })), + ); + }); + + it("persists captions and the word transcript from a single paid pass", async () => { + const { transcribeVideoWorkflow } = await import("@/workflows/transcribe"); + + const result = await transcribeVideoWorkflow({ + videoId: "video-123", + userId: "user-456", + aiGenerationEnabled: false, + }); + + expect(result.success).toBe(true); + expect(mocks.transcribe).toHaveBeenCalledTimes(1); + expect(mocks.transcribe.mock.calls[0]?.[0]).toMatchObject({ + disfluencies: true, + speech_models: ["universal-3-5-pro", "universal-2"], + }); + + const writes = new Map( + mocks.putObject.mock.calls.map((call) => [call[0] as string, call[1]]), + ); + expect( + [...writes.keys()].filter((key) => key.includes("transcription")), + ).toEqual([ + "user-456/video-123/transcription.vtt", + "user-456/video-123/transcription.edit.v3.json", + ]); + + const vtt = writes.get("user-456/video-123/transcription.vtt") as string; + expect(vtt.startsWith("WEBVTT")).toBe(true); + expect(vtt).toContain("this is a real example."); + expect(vtt).not.toContain("Um,"); + + const { parseEditTranscript } = await import("@/lib/edit-transcript"); + const { decryptEditTranscriptObject } = await import( + "@/lib/edit-transcript-storage" + ); + const encrypted = writes.get( + "user-456/video-123/transcription.edit.v3.json", + ) as string; + expect(encrypted).not.toContain("this is a real example"); + const stored = parseEditTranscript( + decryptEditTranscriptObject(encrypted, "user-456", "video-123") ?? "", + ); + expect(stored).toMatchObject({ version: 3, durationMs: 4_000 }); + // verbatim: fillers stay in the stored words even though captions drop them + expect(stored?.words.map((word) => word.text)).toContain("Um,"); + expect(stored?.words).toHaveLength(9); + + expect(mocks.updates.at(-1)).toEqual({ transcriptionStatus: "COMPLETE" }); + }); + + it("never overwrites the original-timeline transcript of an edited video", async () => { + state.editRows = [ + { + sourceKey: "user-456/video-123/original.mp4", + editSpec: { + version: 1, + sourceDuration: 12, + keepRanges: [{ start: 0, end: 4 }], + }, + }, + ]; + + const { transcribeVideoWorkflow } = await import("@/workflows/transcribe"); + await transcribeVideoWorkflow({ + videoId: "video-123", + userId: "user-456", + aiGenerationEnabled: false, + }); + + const writtenKeys = mocks.putObject.mock.calls.map((call) => call[0]); + expect(writtenKeys).toContain("user-456/video-123/transcription.vtt"); + expect(writtenKeys).not.toContain( + "user-456/video-123/transcription.edit.v3.json", + ); + }); +}); + +describe("backfillEditTranscriptWorkflow", () => { + beforeEach(() => { + state.editRows = []; + videoRow.metadata = { + editTranscriptBackfill: { + status: "processing", + requestId: "request-1", + requestedAt: "2026-07-30T00:00:00.000Z", + }, + }; + mocks.transcribe.mockResolvedValue({ + ...assemblyAIEditResponse, + audio_duration: 4, + }); + mocks.putObject.mockImplementation(() => pipeValue(undefined)); + mocks.deleteObject.mockImplementation(() => pipeValue(undefined)); + mocks.getInternalSignedObjectUrl.mockImplementation(() => + pipeValue("https://storage.test/object"), + ); + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + ok: true, + status: 200, + statusText: "OK", + arrayBuffer: async () => new ArrayBuffer(8), + })), + ); + }); + + it("transcribes the original media in the original timeline when edits exist", async () => { + state.editRows = [ + { + sourceKey: "user-456/video-123/original.mp4", + editSpec: { + version: 1, + sourceDuration: 12, + keepRanges: [{ start: 0, end: 4 }], + }, + }, + ]; + + const { backfillEditTranscriptWorkflow } = await import( + "@/workflows/transcribe" + ); + const result = await backfillEditTranscriptWorkflow({ + videoId: "video-123", + userId: "user-456", + requestId: "request-1", + }); + + expect(result.success).toBe(true); + expect(mocks.getInternalSignedObjectUrl.mock.calls[0]?.[0]).toBe( + "user-456/video-123/original.mp4", + ); + + const { parseEditTranscript } = await import("@/lib/edit-transcript"); + const { decryptEditTranscriptObject } = await import( + "@/lib/edit-transcript-storage" + ); + const write = mocks.putObject.mock.calls.find( + (call) => call[0] === "user-456/video-123/transcription.edit.v3.json", + ); + const decrypted = decryptEditTranscriptObject( + write?.[1] as string, + "user-456", + "video-123", + ); + expect(parseEditTranscript(decrypted ?? "")).toMatchObject({ + durationMs: 12_000, + }); + expect(mocks.deleteObject).not.toHaveBeenCalledWith( + "user-456/video-123/transcription.edit.v3.status.json", + ); + }); + + it("does not run a backfill after its database claim is replaced", async () => { + const { backfillEditTranscriptWorkflow } = await import( + "@/workflows/transcribe" + ); + const result = await backfillEditTranscriptWorkflow({ + videoId: "video-123", + userId: "user-456", + requestId: "expired-request", + }); + + expect(result.success).toBe(false); + expect(mocks.transcribe).not.toHaveBeenCalled(); + expect(mocks.putObject).not.toHaveBeenCalledWith( + "user-456/video-123/transcription.edit.v3.json", + expect.anything(), + expect.anything(), + ); + }); +}); diff --git a/apps/web/__tests__/unit/dev-reset-transcript.test.ts b/apps/web/__tests__/unit/dev-reset-transcript.test.ts new file mode 100644 index 00000000000..a3165ff68aa --- /dev/null +++ b/apps/web/__tests__/unit/dev-reset-transcript.test.ts @@ -0,0 +1,124 @@ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + and: vi.fn((...conditions: unknown[]) => ({ conditions })), + db: vi.fn(), + eq: vi.fn((field: unknown, value: unknown) => ({ field, value })), + getAccessForVideo: vi.fn(), + getCurrentUser: vi.fn(), + where: vi.fn(), +})); + +const schema = vi.hoisted(() => ({ + videos: { + id: "videos.id", + ownerId: "videos.ownerId", + }, + videoEdits: { + videoId: "videoEdits.videoId", + }, + videoUploads: { + videoId: "videoUploads.videoId", + }, +})); + +vi.mock("@cap/database", () => ({ + db: mocks.db, +})); + +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: mocks.getCurrentUser, +})); + +vi.mock("@cap/database/schema", () => schema); + +vi.mock("@cap/web-backend", () => ({ + Storage: { + getAccessForVideo: mocks.getAccessForVideo, + }, +})); + +vi.mock("drizzle-orm", () => ({ + and: mocks.and, + eq: mocks.eq, +})); + +vi.mock("@/lib/server", () => ({ + runPromise: vi.fn(), +})); + +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: vi.fn(), +})); + +beforeEach(() => { + vi.clearAllMocks(); + vi.stubEnv("NODE_ENV", "development"); + mocks.db.mockReturnValue({ + select: () => ({ + from: () => ({ + where: mocks.where, + }), + }), + }); + mocks.where.mockResolvedValue([]); +}); + +describe("development transcript reset", () => { + it("rejects cross-origin requests before authentication", async () => { + mocks.getCurrentUser.mockResolvedValue({ id: "owner-1" }); + const { POST } = await import("@/app/api/dev-reset-transcript/route"); + const request = new NextRequest( + "http://localhost:3000/api/dev-reset-transcript?videoId=video-1", + { + method: "POST", + headers: { origin: "https://attacker.example" }, + }, + ); + + const response = await POST(request); + + expect(response.status).toBe(403); + expect(mocks.getCurrentUser).not.toHaveBeenCalled(); + expect(mocks.db).not.toHaveBeenCalled(); + }); + + it("requires a signed-in user", async () => { + mocks.getCurrentUser.mockResolvedValue(null); + const { POST } = await import("@/app/api/dev-reset-transcript/route"); + const request = new NextRequest( + "http://localhost:3000/api/dev-reset-transcript?videoId=video-1", + { + method: "POST", + headers: { origin: "http://localhost:3000" }, + }, + ); + + const response = await POST(request); + + expect(response.status).toBe(401); + expect(mocks.db).not.toHaveBeenCalled(); + }); + + it("scopes the target video to the signed-in owner", async () => { + mocks.getCurrentUser.mockResolvedValue({ id: "owner-1" }); + const { POST } = await import("@/app/api/dev-reset-transcript/route"); + const request = new NextRequest( + "http://localhost:3000/api/dev-reset-transcript?videoId=video-1", + { + method: "POST", + headers: { origin: "http://localhost:3000" }, + }, + ); + + const response = await POST(request); + + expect(response.status).toBe(404); + expect(mocks.and).toHaveBeenCalledWith( + { field: schema.videos.id, value: "video-1" }, + { field: schema.videos.ownerId, value: "owner-1" }, + ); + expect(mocks.getAccessForVideo).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/edit-transcript-storage.test.ts b/apps/web/__tests__/unit/edit-transcript-storage.test.ts new file mode 100644 index 00000000000..60cb23b29d0 --- /dev/null +++ b/apps/web/__tests__/unit/edit-transcript-storage.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + NEXTAUTH_SECRET: "test-secret-with-enough-entropy", + }), +})); + +vi.mock("server-only", () => ({})); + +describe("edit transcript storage", () => { + it("encrypts and authenticates transcript objects for one video", async () => { + const { decryptEditTranscriptObject, encryptEditTranscriptObject } = + await import("@/lib/edit-transcript-storage"); + const plaintext = JSON.stringify({ + words: [{ text: "private deleted phrase" }], + }); + + const encrypted = encryptEditTranscriptObject( + plaintext, + "owner-1", + "video-1", + ); + + expect(encrypted).not.toContain("private deleted phrase"); + expect(decryptEditTranscriptObject(encrypted, "owner-1", "video-1")).toBe( + plaintext, + ); + expect( + decryptEditTranscriptObject(encrypted, "owner-1", "video-2"), + ).toBeNull(); + }); + + it("rejects modified and plaintext objects", async () => { + const { decryptEditTranscriptObject, encryptEditTranscriptObject } = + await import("@/lib/edit-transcript-storage"); + const encrypted = encryptEditTranscriptObject( + '{"words":[]}', + "owner-1", + "video-1", + ); + const parts = encrypted.split("."); + const ciphertext = parts[3] ?? ""; + parts[3] = `${ciphertext.startsWith("A") ? "B" : "A"}${ciphertext.slice(1)}`; + const tampered = parts.join("."); + + expect( + decryptEditTranscriptObject(tampered, "owner-1", "video-1"), + ).toBeNull(); + expect( + decryptEditTranscriptObject( + '{"words":[{"text":"plaintext"}]}', + "owner-1", + "video-1", + ), + ).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/edit-transcript.test.ts b/apps/web/__tests__/unit/edit-transcript.test.ts new file mode 100644 index 00000000000..d82d32940d4 --- /dev/null +++ b/apps/web/__tests__/unit/edit-transcript.test.ts @@ -0,0 +1,550 @@ +import type { VideoEditSpec } from "@cap/database/types"; +import { describe, expect, it } from "vitest"; +import { + createEditTranscript, + editTranscriptWordsToCaptionVtt, + findActiveTranscriptWordIndex, + getDeletedTranscriptWordIds, + groupEditTranscriptWords, + isFillerWord, + normalizeTranscriptSelection, + parseEditTranscript, + planFillerCuts, + planTranscriptCut, + remapEditTranscriptThroughSpec, + serializeEditTranscript, +} from "@/lib/edit-transcript"; +import { createIdentityEditSpec } from "@/lib/video-edits"; +import { + assemblyAIEditResponse, + assemblyAIRealFillerResponse, +} from "../fixtures/assemblyai-edit-response"; + +function editSpec( + sourceDuration: number, + keepRanges: { start: number; end: number }[], +): VideoEditSpec { + return { version: 1, sourceDuration, keepRanges }; +} + +describe("edit transcript", () => { + it("normalizes and preserves AssemblyAI word timing", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + + expect(transcript).toMatchObject({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 4_000, + languageCode: "en", + }); + expect(transcript.words).toHaveLength(9); + expect(transcript.words[0]).toMatchObject({ + id: "word-0-120-380", + text: "Um,", + startMs: 120, + endMs: 380, + confidence: 0.98, + speaker: "A", + channel: null, + }); + expect(parseEditTranscript(serializeEditTranscript(transcript))).toEqual( + transcript, + ); + }); + + it("sorts, clamps, and rejects malformed provider words", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-2", + words: [ + { text: "second", start: 800, end: 900 }, + { text: "first", start: -100, end: 250, confidence: 2 }, + { text: "", start: 300, end: 400 }, + { text: "invalid", start: Number.NaN, end: 600 }, + { text: "reverse", start: 700, end: 650, channel: 1 }, + { text: "outside", start: 1_100, end: 1_200 }, + ], + language_code: null, + }, + 1_000, + ); + + expect(transcript.words.map((word) => word.text)).toEqual([ + "first", + "reverse", + "second", + ]); + expect(transcript.words[0]?.confidence).toBe(1); + expect(transcript.words[1]?.channel).toBe("1"); + }); + + it("records whichever speech model the provider used", () => { + expect( + createEditTranscript( + { ...assemblyAIEditResponse, speech_model_used: "universal-2" }, + 4_000, + ).speechModelUsed, + ).toBe("universal-2"); + expect( + createEditTranscript( + { ...assemblyAIEditResponse, speech_model_used: undefined }, + 4_000, + ).speechModelUsed, + ).toBe("unknown"); + }); + + it("preserves fillers from the real accuracy regression response", () => { + const transcript = createEditTranscript( + assemblyAIRealFillerResponse, + 8_000, + ); + + expect( + transcript.words + .filter((word) => isFillerWord(word.text)) + .map(({ text, startMs, endMs, confidence }) => ({ + text, + startMs, + endMs, + confidence, + })), + ).toEqual([ + { + text: "um,", + startMs: 3_460, + endMs: 3_980, + confidence: 0.9959265, + }, + { + text: "Uh,", + startMs: 5_760, + endMs: 7_240, + confidence: 0.9857637, + }, + ]); + }); + + it("recognizes exact filler variants without substring false positives", () => { + for (const filler of [ + "um", + "Umm,", + "uh", + "uhhh!", + "erm", + "ermm", + "er", + "ah", + "ahh", + "hmm", + ]) { + expect(isFillerWord(filler), filler).toBe(true); + } + + for (const word of [ + "umbrella", + "term", + "human", + "ahead", + "hammer", + "m", + "mhm", + "huh", + "uh-huh", + ]) { + expect(isFillerWord(word), word).toBe(false); + } + }); + + it("normalizes forward and backward word selections", () => { + expect(normalizeTranscriptSelection(2, 5, 8)).toEqual({ + startIndex: 2, + endIndex: 5, + }); + expect(normalizeTranscriptSelection(5, 2, 8)).toEqual({ + startIndex: 2, + endIndex: 5, + }); + expect(normalizeTranscriptSelection(-1, 2, 8)).toBeNull(); + expect(normalizeTranscriptSelection(2, 8, 8)).toBeNull(); + }); + + it("uses bounded silence padding for a selected word", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const plan = planTranscriptCut(transcript.words, 6, 6, 4_000); + + expect(plan).toEqual({ + startMs: 2_220, + endMs: 2_635, + safe: true, + reason: null, + }); + }); + + it("marks a cut unsafe when unselected speech overlaps it", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-2", + words: [ + { text: "speaker one", start: 100, end: 500, speaker: "A" }, + { text: "um", start: 200, end: 350, speaker: "B" }, + { text: "continues", start: 520, end: 800, speaker: "A" }, + ], + }, + 1_000, + ); + + expect(planTranscriptCut(transcript.words, 1, 1, 1_000)).toEqual({ + startMs: 200, + endMs: 350, + safe: false, + reason: "overlapping-speech", + }); + expect(planFillerCuts(transcript.words, 1_000)).toEqual({ + ranges: [], + fillerCount: 1, + skippedCount: 1, + }); + }); + + it("plans batch filler cuts in chronological order", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const plan = planFillerCuts(transcript.words, 4_000); + + expect(plan).toEqual({ + ranges: [ + { + startMs: 60, + endMs: 450, + safe: true, + reason: null, + }, + { + startMs: 2_220, + endMs: 2_635, + safe: true, + reason: null, + }, + ], + fillerCount: 2, + skippedCount: 0, + }); + }); + + it("plans exact safe cuts for the real missing-filler response", () => { + const transcript = createEditTranscript( + assemblyAIRealFillerResponse, + 8_000, + ); + + expect(planFillerCuts(transcript.words, transcript.durationMs)).toEqual({ + ranges: [ + { + startMs: 3_380, + endMs: 3_980, + safe: true, + reason: null, + }, + { + startMs: 5_760, + endMs: 7_240, + safe: true, + reason: null, + }, + ], + fillerCount: 2, + skippedCount: 0, + }); + }); + + it("groups words linearly and finds the active word with binary search", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const groups = groupEditTranscriptWords(transcript.words, 4); + + expect(groups.map((group) => [group.startIndex, group.endIndex])).toEqual([ + [0, 3], + [4, 5], + [6, 8], + ]); + expect(findActiveTranscriptWordIndex(transcript.words, 750)).toBe(2); + expect(findActiveTranscriptWordIndex(transcript.words, 2_000)).toBe(-1); + }); + + it("finds an earlier active speaker beneath completed overlapping words", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-2", + words: [ + { text: "long", start: 0, end: 1_000, speaker: "A" }, + { text: "short", start: 100, end: 200, speaker: "B" }, + { text: "later", start: 300, end: 400, speaker: "B" }, + ], + }, + 1_200, + ); + + expect(findActiveTranscriptWordIndex(transcript.words, 500)).toBe(0); + }); + + it("derives deleted word presentation from timeline keep ranges", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const deleted = getDeletedTranscriptWordIds(transcript.words, [ + { start: 0.45, end: 2.22 }, + { start: 2.635, end: 4 }, + ]); + + expect([...deleted]).toEqual(["word-0-120-380", "word-6-2300-2570"]); + }); + + it("handles a 50,000 word transcript without quadratic grouping", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-2", + words: Array.from({ length: 50_000 }, (_, index) => ({ + text: index % 100 === 0 ? "um" : `word${index}`, + start: index * 300, + end: index * 300 + 200, + })), + }, + 15_000_000, + ); + + expect(groupEditTranscriptWords(transcript.words)).toHaveLength(1_042); + expect( + planFillerCuts(transcript.words, transcript.durationMs).fillerCount, + ).toBe(500); + expect(findActiveTranscriptWordIndex(transcript.words, 14_999_900)).toBe( + 49_999, + ); + }); + + it("rejects malformed stored sidecars", () => { + expect(parseEditTranscript("not json")).toBeNull(); + expect( + parseEditTranscript( + JSON.stringify({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 1000, + languageCode: "en", + words: [{ id: "bad", text: "bad", startMs: 900, endMs: 1100 }], + }), + ), + ).toBeNull(); + expect( + parseEditTranscript( + JSON.stringify({ + version: 3, + speechModelUsed: "", + durationMs: 1000, + languageCode: "en", + words: [], + }), + ), + ).toBeNull(); + }); + + it("rejects sidecars from previous schema versions and accepts any model", () => { + for (const version of [1, 2]) { + expect( + parseEditTranscript( + JSON.stringify({ + version, + speechModelUsed: "universal-2", + durationMs: 1000, + languageCode: "en", + words: [], + }), + ), + ).toBeNull(); + } + + expect( + parseEditTranscript( + JSON.stringify({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 1000, + languageCode: "en", + words: [], + }), + ), + ).toMatchObject({ version: 3, speechModelUsed: "universal-3-5-pro" }); + }); +}); + +describe("remapEditTranscriptThroughSpec", () => { + it("leaves word timing untouched for an identity spec", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const remapped = remapEditTranscriptThroughSpec( + transcript, + createIdentityEditSpec(4), + ); + + expect(remapped.durationMs).toBe(4_000); + expect(remapped.words).toEqual(transcript.words); + expect(remapped.speechModelUsed).toBe(transcript.speechModelUsed); + expect(remapped.languageCode).toBe(transcript.languageCode); + }); + + it("drops words whose midpoint falls inside a cut and shifts the rest", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(4, [ + { start: 0.45, end: 2.22 }, + { start: 2.635, end: 4 }, + ]), + ); + + expect(remapped.durationMs).toBe(3_135); + expect( + remapped.words.map((word) => [word.id, word.startMs, word.endMs]), + ).toEqual([ + ["word-1-520-720", 70, 270], + ["word-2-730-840", 280, 390], + ["word-3-850-910", 400, 460], + ["word-4-930-1130", 480, 680], + ["word-5-1140-1510", 690, 1_060], + ["word-7-2700-3070", 1_835, 2_205], + ["word-8-3090-3400", 2_225, 2_535], + ]); + }); + + it("clamps words that straddle a cut boundary instead of dropping them", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-3-5-pro", + words: [ + { text: "straddles", start: 800, end: 1_400 }, + { text: "inside", start: 2_000, end: 2_200 }, + ], + }, + 4_000, + ); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(4, [{ start: 1, end: 3 }]), + ); + + expect(remapped.durationMs).toBe(2_000); + expect(remapped.words).toHaveLength(2); + // midpoint 1.1s is kept, and the leading 200ms outside the range is clamped + expect(remapped.words[0]).toMatchObject({ + text: "straddles", + startMs: 0, + endMs: 400, + }); + expect(remapped.words[1]).toMatchObject({ + text: "inside", + startMs: 1_000, + endMs: 1_200, + }); + }); + + it("accumulates offsets across multiple keep ranges", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-3-5-pro", + words: [ + { text: "one", start: 100, end: 300 }, + { text: "two", start: 2_100, end: 2_400 }, + { text: "three", start: 4_100, end: 4_500 }, + { text: "cut", start: 5_200, end: 5_400 }, + ], + }, + 6_000, + ); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(6, [ + { start: 0, end: 1 }, + { start: 2, end: 3 }, + { start: 4, end: 5 }, + ]), + ); + + expect(remapped.durationMs).toBe(3_000); + expect( + remapped.words.map((word) => [word.text, word.startMs, word.endMs]), + ).toEqual([ + ["one", 100, 300], + ["two", 1_100, 1_400], + ["three", 2_100, 2_500], + ]); + }); + + it("preserves ids and word metadata while remapping", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(4, [{ start: 2, end: 4 }]), + ); + const original = transcript.words.find( + (word) => word.id === "word-7-2700-3070", + ); + const moved = remapped.words.find((word) => word.id === "word-7-2700-3070"); + + expect(moved).toMatchObject({ + text: original?.text, + confidence: original?.confidence, + speaker: original?.speaker, + channel: original?.channel, + startMs: 700, + endMs: 1_070, + }); + }); + + it("drops every word when the spec keeps nothing", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(4, []), + ); + + expect(remapped).toMatchObject({ durationMs: 0, words: [] }); + }); + + it("keeps remapped words sorted and parseable", () => { + const transcript = createEditTranscript( + assemblyAIRealFillerResponse, + 8_000, + ); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(8, [ + { start: 1, end: 3.3 }, + { start: 4, end: 8 }, + ]), + ); + + expect(remapped.durationMs).toBe(6_300); + for (let index = 1; index < remapped.words.length; index++) { + const previous = remapped.words[index - 1]; + const current = remapped.words[index]; + expect(current?.startMs).toBeGreaterThanOrEqual(previous?.startMs ?? 0); + expect(current?.endMs).toBeGreaterThan(current?.startMs ?? 0); + expect(current?.endMs).toBeLessThanOrEqual(remapped.durationMs); + } + expect(parseEditTranscript(serializeEditTranscript(remapped))).toEqual( + remapped, + ); + }); +}); + +describe("editTranscriptWordsToCaptionVtt", () => { + it("strips fillers from captions built off the verbatim words", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const vtt = editTranscriptWordsToCaptionVtt(transcript.words); + + expect(vtt.startsWith("WEBVTT\n\n")).toBe(true); + expect(vtt).not.toContain("Um,"); + expect(vtt).not.toContain("Erm"); + expect(vtt).toContain("this is a real example."); + expect(vtt).toContain("00:00:00.520 --> 00:00:01.510"); + expect(vtt).toContain("nothing else."); + }); + + it("returns a header-only VTT when there are no words", () => { + expect(editTranscriptWordsToCaptionVtt([])).toBe("WEBVTT\n\n"); + }); +}); diff --git a/apps/web/__tests__/unit/get-edit-transcript.test.ts b/apps/web/__tests__/unit/get-edit-transcript.test.ts new file mode 100644 index 00000000000..06e5d5916da --- /dev/null +++ b/apps/web/__tests__/unit/get-edit-transcript.test.ts @@ -0,0 +1,462 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { Option } from "effect"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + db: vi.fn(), + decodeStorageVideo: vi.fn(), + getAccessForVideo: vi.fn(), + getCurrentUser: vi.fn(), + runPromise: vi.fn(), + start: vi.fn(), +})); + +const schema = vi.hoisted(() => ({ + videos: { + id: "videos.id", + metadata: "videos.metadata", + updatedAt: "videos.updatedAt", + }, + videoEdits: { + videoId: "videoEdits.videoId", + editSpec: "videoEdits.editSpec", + }, +})); + +vi.mock("@cap/database", () => ({ + db: mocks.db, +})); + +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: mocks.getCurrentUser, +})); + +vi.mock("@cap/database/schema", () => schema); + +vi.mock("@cap/utils", () => ({ + userIsPro: () => true, +})); + +vi.mock("@cap/web-backend", () => ({ + Storage: { + getAccessForVideo: mocks.getAccessForVideo, + }, +})); + +vi.mock("drizzle-orm", () => ({ + and: vi.fn((...conditions: unknown[]) => ({ conditions })), + eq: vi.fn((left: unknown, right: unknown) => ({ left, right })), + sql: vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ + strings, + values, + })), +})); + +vi.mock("workflow/api", () => ({ + start: mocks.start, +})); + +vi.mock("@/lib/server", () => ({ + runPromise: mocks.runPromise, +})); + +vi.mock("@/lib/edit-transcript-storage", () => ({ + decryptEditTranscriptObject: (value: string) => value, +})); + +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: mocks.decodeStorageVideo, +})); + +vi.mock("@/workflows/transcribe", () => ({ + backfillEditTranscriptWorkflow: vi.fn(), +})); + +vi.mock("server-only", () => ({})); + +type PipeValue = { + pipe: (runner: (value: T) => unknown) => unknown; +}; + +function pipeValue(value: T): PipeValue { + return { + pipe: (runner) => runner(value), + }; +} + +function createDatabase( + video: Record, + edit?: Record, +) { + const currentVideo: Record = { + metadata: null, + updatedAt: new Date("2026-07-30T00:00:00.000Z"), + ...video, + }; + const transactionContext = new AsyncLocalStorage(); + const lockedSelect = { + select: vi.fn(), + from: vi.fn(), + where: vi.fn(), + for: vi.fn(), + }; + lockedSelect.select.mockReturnValue(lockedSelect); + lockedSelect.from.mockReturnValue(lockedSelect); + lockedSelect.where.mockReturnValue(lockedSelect); + lockedSelect.for.mockImplementation(async () => [currentVideo]); + + const transactionUpdate = { + set: vi.fn(), + where: vi.fn(), + }; + transactionUpdate.set.mockImplementation( + (values: Record) => { + if ( + typeof values.metadata === "object" && + values.metadata !== null && + "values" in values.metadata && + Array.isArray(values.metadata.values) + ) { + const encodedStatus = values.metadata.values.find( + (value) => + typeof value === "string" && value.startsWith('{"status":'), + ); + if (typeof encodedStatus === "string") { + currentVideo.metadata = { + editTranscriptBackfill: JSON.parse(encodedStatus), + }; + } + } + return transactionUpdate; + }, + ); + transactionUpdate.where.mockResolvedValue([{ affectedRows: 1 }]); + + const select = vi.fn(() => ({ + from: vi.fn((table: unknown) => ({ + where: vi + .fn() + .mockResolvedValue( + table === schema.videoEdits ? (edit ? [edit] : []) : [currentVideo], + ), + })), + })); + const update = vi.fn(() => ({ + set: vi.fn(() => ({ + where: vi.fn().mockResolvedValue([{ affectedRows: 1 }]), + })), + })); + + let transactionTail = Promise.resolve(); + const transaction = vi.fn( + ( + callback: ( + tx: typeof lockedSelect & { update: typeof update }, + ) => Promise, + ): Promise => { + const result = transactionTail.then(() => + transactionContext.run(true, () => + callback({ + ...lockedSelect, + update: vi.fn(() => transactionUpdate), + }), + ), + ); + transactionTail = result.then( + () => undefined, + () => undefined, + ); + return result; + }, + ); + + return { + database: { select, transaction, update }, + isTransactionActive: () => transactionContext.getStore() === true, + lockedSelect, + }; +} + +function createBucket( + objects: Map, + isTransactionActive = () => false, +) { + return { + getObject: vi.fn((key: string) => { + if (isTransactionActive()) { + throw new Error("Storage read occurred inside a database transaction"); + } + return pipeValue(Option.fromNullable(objects.get(key))); + }), + }; +} + +const video = { + id: "video-1", + ownerId: "user-1", + duration: 25, + source: { type: "webMP4" }, + isScreenshot: false, + transcriptionStatus: "COMPLETE", + updatedAt: new Date("2026-07-30T00:00:00.000Z"), +}; + +const TRANSCRIPT_KEY = "user-1/video-1/transcription.edit.v3.json"; + +function storedOriginalTranscript(durationMs = 25_000) { + return JSON.stringify({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs, + languageCode: "en", + words: [ + { + id: "word-0-1000-1500", + text: "before", + startMs: 1_000, + endMs: 1_500, + confidence: 0.9, + speaker: null, + channel: null, + }, + { + id: "word-1-6000-6500", + text: "kept", + startMs: 6_000, + endMs: 6_500, + confidence: 0.9, + speaker: null, + channel: null, + }, + { + id: "word-2-20000-20500", + text: "after", + startMs: 20_000, + endMs: 20_500, + confidence: 0.9, + speaker: null, + channel: null, + }, + ], + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.getCurrentUser.mockResolvedValue({ id: "user-1" }); + mocks.runPromise.mockImplementation((value) => Promise.resolve(value)); + mocks.start.mockResolvedValue({ runId: "run-1" }); +}); + +describe("requestEditTranscript", () => { + it("serializes concurrent requests and starts one paid transcript job", async () => { + const objects = new Map(); + const { database, isTransactionActive, lockedSelect } = + createDatabase(video); + const bucket = createBucket(objects, isTransactionActive); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + mocks.start.mockImplementation(async () => { + if (isTransactionActive()) { + throw new Error( + "Workflow start occurred inside a database transaction", + ); + } + return { runId: "run-1" }; + }); + + const { requestEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const [first, second] = await Promise.all([ + requestEditTranscript("video-1" as never), + requestEditTranscript("video-1" as never), + ]); + + expect(first).toEqual({ status: "processing" }); + expect(second).toEqual({ status: "processing" }); + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(lockedSelect.for).toHaveBeenCalledTimes(2); + expect(lockedSelect.for).toHaveBeenCalledWith("update"); + expect(mocks.start.mock.calls[0]?.[1]?.[0]).toMatchObject({ + videoId: "video-1", + userId: "user-1", + requestId: expect.any(String), + }); + }); + + it("returns a current cached sidecar without starting another job", async () => { + const objects = new Map([ + [ + TRANSCRIPT_KEY, + JSON.stringify({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 25_000, + languageCode: "en", + words: [], + }), + ], + ]); + const bucket = createBucket(objects); + const { database } = createDatabase(video); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { requestEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const response = await requestEditTranscript("video-1" as never); + + expect(response.status).toBe("ready"); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("remaps the stored original words through the video's edit spec", async () => { + const objects = new Map([ + [TRANSCRIPT_KEY, storedOriginalTranscript()], + ]); + const bucket = createBucket(objects); + const { database } = createDatabase( + { ...video, duration: 10 }, + { + editSpec: { + version: 1, + sourceDuration: 25, + keepRanges: [{ start: 5, end: 15 }], + }, + }, + ); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { requestEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const response = await requestEditTranscript("video-1" as never); + + if (response.status !== "ready") { + throw new Error(`Expected a ready transcript, got ${response.status}`); + } + expect(response.transcript.durationMs).toBe(10_000); + expect( + response.transcript.words.map((word) => [ + word.id, + word.startMs, + word.endMs, + ]), + ).toEqual([["word-1-6000-6500", 1_000, 1_500]]); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("does not expose workflow startup errors to the client", async () => { + const bucket = createBucket(new Map()); + const { database } = createDatabase(video); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + mocks.start.mockRejectedValue( + new Error("provider account and request identifiers"), + ); + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + + const { requestEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const response = await requestEditTranscript("video-1" as never); + + expect(response).toEqual({ + status: "error", + message: "Transcript preparation could not start. Try again.", + }); + expect(JSON.stringify(response)).not.toContain("provider account"); + consoleError.mockRestore(); + }); +}); + +describe("getEditTranscript", () => { + it("remaps stored words instead of reporting them as stale", async () => { + const objects = new Map([ + [TRANSCRIPT_KEY, storedOriginalTranscript()], + ]); + const bucket = createBucket(objects); + const { database } = createDatabase( + { ...video, duration: 10 }, + { + editSpec: { + version: 1, + sourceDuration: 25, + keepRanges: [{ start: 5, end: 15 }], + }, + }, + ); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { getEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const response = await getEditTranscript("video-1" as never); + + if (response.status !== "ready") { + throw new Error(`Expected a ready transcript, got ${response.status}`); + } + expect(response.transcript.words.map((word) => word.text)).toEqual([ + "kept", + ]); + }); + + it("reports missing when the sidecar no longer matches the original media", async () => { + const objects = new Map([ + [TRANSCRIPT_KEY, storedOriginalTranscript(18_000)], + ]); + const bucket = createBucket(objects); + const { database } = createDatabase( + { ...video, duration: 10 }, + { + editSpec: { + version: 1, + sourceDuration: 25, + keepRanges: [{ start: 5, end: 15 }], + }, + }, + ); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { getEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + + expect(await getEditTranscript("video-1" as never)).toEqual({ + status: "missing", + }); + }); + + it("reports only a generic message for a failed backfill", async () => { + const bucket = createBucket(new Map()); + const { database } = createDatabase({ + ...video, + metadata: { + editTranscriptBackfill: { + status: "error", + requestId: "request-1", + requestedAt: "2026-07-30T00:00:00.000Z", + message: "private provider detail", + }, + }, + }); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { getEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + + expect(await getEditTranscript("video-1" as never)).toEqual({ + status: "error", + message: "Transcript preparation failed. Try again.", + }); + }); +}); diff --git a/apps/web/__tests__/unit/storage-object-route.test.ts b/apps/web/__tests__/unit/storage-object-route.test.ts new file mode 100644 index 00000000000..431705af8fd --- /dev/null +++ b/apps/web/__tests__/unit/storage-object-route.test.ts @@ -0,0 +1,40 @@ +import { NextRequest } from "next/server"; +import { describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + verifyStorageObjectToken: vi.fn(), +})); + +vi.mock("@cap/web-backend", () => ({ + provideOptionalAuth: {}, + Storage: {}, + Videos: {}, + VideosRepo: {}, + verifyStorageObjectToken: mocks.verifyStorageObjectToken, +})); + +vi.mock("@/lib/server", () => ({ + runPromise: vi.fn(), +})); + +vi.mock("@/utils/helpers", () => ({ + CACHE_CONTROL_HEADERS: {}, +})); + +describe("storage object route", () => { + it.each([ + "owner-1/video-1/transcription.edit.v3.json", + "owner-1/video-1/transcription.edit.v3.status.json", + ])("never proxies the private transcript object %s", async (key) => { + const { GET } = await import("@/app/api/storage/object/route"); + const request = new NextRequest( + `http://localhost:3000/api/storage/object?videoId=video-1&key=${encodeURIComponent(key)}&token=attacker-token`, + ); + + const response = await GET(request); + + expect(response.status).toBe(404); + expect(await response.text()).toBe("Not found"); + expect(mocks.verifyStorageObjectToken).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/transcribe-language.test.ts b/apps/web/__tests__/unit/transcribe-language.test.ts index 505c4d1bfe9..c315acb732d 100644 --- a/apps/web/__tests__/unit/transcribe-language.test.ts +++ b/apps/web/__tests__/unit/transcribe-language.test.ts @@ -27,6 +27,7 @@ describe("AI generation language support", () => { speech_models: ["universal-3-5-pro", "universal-2"], language_detection: true, }); + expect(options.disfluencies).toBe(true); if (!("language_detection_options" in options)) { throw new Error("Expected automatic language detection options"); } @@ -39,6 +40,7 @@ describe("AI generation language support", () => { expect(getAssemblyAITranscriptionOptions("ro")).toMatchObject({ speech_models: ["universal-3-5-pro", "universal-2"], language_code: "ro", + disfluencies: true, }); expect(getAssemblyAITranscriptionOptions("zh")).toMatchObject({ speech_models: ["universal-3-5-pro", "universal-2"], diff --git a/apps/web/__tests__/unit/transcript-text.test.ts b/apps/web/__tests__/unit/transcript-text.test.ts new file mode 100644 index 00000000000..a86770a2802 --- /dev/null +++ b/apps/web/__tests__/unit/transcript-text.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { + formatTranscriptAsParagraphs, + type TranscriptTextEntry, +} from "@/lib/transcript-text"; + +function entry( + text: string, + startTime: number, + endTime: number, +): TranscriptTextEntry { + return { text, startTime, endTime }; +} + +describe("formatTranscriptAsParagraphs", () => { + it("returns an empty string for no entries", () => { + expect(formatTranscriptAsParagraphs([])).toBe(""); + }); + + it("joins caption fragments into one flowing paragraph", () => { + const result = formatTranscriptAsParagraphs([ + entry("This is my test to make sure that,", 0.9, 2.4), + entry("the video is working correctly.", 3.0, 4.3), + entry("I'm going to stop the recording now.", 4.4, 5.8), + ]); + expect(result).toBe( + "This is my test to make sure that, the video is working correctly. I'm going to stop the recording now.", + ); + }); + + it("breaks a paragraph at a sentence end followed by a long pause", () => { + const result = formatTranscriptAsParagraphs([ + entry("First topic ends here.", 0, 2), + entry("Second topic starts fresh.", 4.5, 6), + ]); + expect(result).toBe("First topic ends here.\n\nSecond topic starts fresh."); + }); + + it("does not break on a pause mid-sentence", () => { + const result = formatTranscriptAsParagraphs([ + entry("This sentence keeps", 0, 2), + entry("going after a pause.", 5, 6), + ]); + expect(result).toBe("This sentence keeps going after a pause."); + }); + + it("ignores pauses when the entry has no usable end time", () => { + const result = formatTranscriptAsParagraphs([ + entry("A full sentence.", 2, 2), + entry("Another sentence.", 10, 11), + ]); + expect(result).toBe("A full sentence. Another sentence."); + }); + + it("breaks long paragraphs at the next sentence boundary", () => { + const sentence = `${"word ".repeat(30).trim()}.`; + const entries = Array.from({ length: 6 }, (_, index) => + entry(sentence, index, index + 0.5), + ); + const result = formatTranscriptAsParagraphs(entries); + const paragraphs = result.split("\n\n"); + expect(paragraphs.length).toBeGreaterThan(1); + for (const paragraph of paragraphs) { + expect(paragraph.endsWith(".")).toBe(true); + } + }); + + it("hard-breaks transcripts that never end a sentence", () => { + const fragment = "no punctuation here just words".repeat(3); + const entries = Array.from({ length: 20 }, (_, index) => + entry(fragment, index, index + 0.5), + ); + const result = formatTranscriptAsParagraphs(entries); + expect(result.split("\n\n").length).toBeGreaterThan(1); + }); + + it("skips empty fragments", () => { + const result = formatTranscriptAsParagraphs([ + entry("Hello there.", 0, 1), + entry(" ", 1, 2), + entry("General greeting.", 2, 3), + ]); + expect(result).toBe("Hello there. General greeting."); + }); + + it("respects sentence enders followed by closing quotes", () => { + const result = formatTranscriptAsParagraphs([ + entry('He said "stop."', 0, 1), + entry("Then we did.", 4, 5), + ]); + expect(result).toBe('He said "stop."\n\nThen we did.'); + }); +}); diff --git a/apps/web/__tests__/unit/use-active-transcript-word-index.test.ts b/apps/web/__tests__/unit/use-active-transcript-word-index.test.ts new file mode 100644 index 00000000000..214234e55b8 --- /dev/null +++ b/apps/web/__tests__/unit/use-active-transcript-word-index.test.ts @@ -0,0 +1,199 @@ +// @vitest-environment jsdom + +import { act, createElement, type RefObject } from "react"; +import { createRoot } from "react-dom/client"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; +import { useActiveTranscriptWordIndex } from "@/app/s/[videoId]/edit/use-active-transcript-word-index"; +import type { EditTranscriptWord } from "@/lib/edit-transcript"; + +const words: EditTranscriptWord[] = [ + { + id: "word-0", + text: "One", + startMs: 100, + endMs: 300, + confidence: 1, + speaker: null, + channel: null, + }, + { + id: "word-1", + text: "two", + startMs: 600, + endMs: 900, + confidence: 1, + speaker: null, + channel: null, + }, + { + id: "word-2", + text: "three", + startMs: 1_000, + endMs: 1_200, + confidence: 1, + speaker: null, + channel: null, + }, +]; + +const actEnvironment = globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; +}; + +function createVideo(currentTime: number, paused: boolean) { + const video = document.createElement("video"); + Object.defineProperties(video, { + currentTime: { + configurable: true, + writable: true, + value: currentTime, + }, + ended: { + configurable: true, + writable: true, + value: false, + }, + paused: { + configurable: true, + writable: true, + value: paused, + }, + }); + return video; +} + +describe("useActiveTranscriptWordIndex", () => { + beforeAll(() => { + actEnvironment.IS_REACT_ACT_ENVIRONMENT = true; + }); + + beforeEach(() => { + vi.stubGlobal( + "requestAnimationFrame", + vi.fn(() => 1), + ); + vi.stubGlobal("cancelAnimationFrame", vi.fn()); + }); + + afterEach(() => { + document.body.replaceChildren(); + vi.unstubAllGlobals(); + }); + + afterAll(() => { + delete actEnvironment.IS_REACT_ACT_ENVIRONMENT; + }); + + it("binds after the video mounts and follows presented video frames", async () => { + const videoRef: RefObject = { current: null }; + let presentedFrame: VideoFrameRequestCallback | null = null; + const requestVideoFrameCallback = vi.fn( + (callback: VideoFrameRequestCallback) => { + presentedFrame = callback; + return 1; + }, + ); + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + const Harness = () => + createElement( + "output", + null, + useActiveTranscriptWordIndex(videoRef, words), + ); + + await act(async () => { + root.render(createElement(Harness)); + }); + expect(container.textContent).toBe("-1"); + + const video = createVideo(0.15, true); + Object.defineProperties(video, { + cancelVideoFrameCallback: { + configurable: true, + value: vi.fn(), + }, + requestVideoFrameCallback: { + configurable: true, + value: requestVideoFrameCallback, + }, + }); + await act(async () => { + videoRef.current = video; + document.body.append(video); + await Promise.resolve(); + }); + expect(container.textContent).toBe("0"); + + Object.defineProperty(video, "paused", { + configurable: true, + writable: true, + value: false, + }); + await act(async () => { + video.dispatchEvent(new Event("play")); + }); + expect(requestVideoFrameCallback).toHaveBeenCalledOnce(); + + await act(async () => { + presentedFrame?.(0, { + mediaTime: 0.65, + } as VideoFrameCallbackMetadata); + }); + expect(container.textContent).toBe("1"); + + await act(async () => { + root.unmount(); + }); + }); + + it("rebinds when the player replaces its video element", async () => { + const firstVideo = createVideo(0.15, true); + const videoRef: RefObject = { + current: firstVideo, + }; + document.body.append(firstVideo); + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + const Harness = () => + createElement( + "output", + null, + useActiveTranscriptWordIndex(videoRef, words), + ); + + await act(async () => { + root.render(createElement(Harness)); + }); + expect(container.textContent).toBe("0"); + + const secondVideo = createVideo(1.1, true); + await act(async () => { + videoRef.current = secondVideo; + firstVideo.replaceWith(secondVideo); + await Promise.resolve(); + }); + expect(container.textContent).toBe("2"); + + await act(async () => { + firstVideo.currentTime = 0.65; + firstVideo.dispatchEvent(new Event("timeupdate")); + }); + expect(container.textContent).toBe("2"); + + await act(async () => { + root.unmount(); + }); + }); +}); diff --git a/apps/web/__tests__/unit/video-edits.test.ts b/apps/web/__tests__/unit/video-edits.test.ts index 3fe7a6de49e..ff3fb5cbf3e 100644 --- a/apps/web/__tests__/unit/video-edits.test.ts +++ b/apps/web/__tests__/unit/video-edits.test.ts @@ -7,8 +7,11 @@ import { createTimelineHistory, createTimelineState, deleteSelectedTimelineSegment, + deleteTimelineRanges, dragTimelineDisplaySplitPoint, findNextPlayableTime, + findNextPlayableTimeInRanges, + findPlayableRangeIndex, getTimelineDisplayDuration, getTimelineDisplaySegments, getTimelineDisplaySplitPoints, @@ -126,6 +129,46 @@ describe("video edit specs", () => { }); describe("timeline editing", () => { + it("deletes transcript ranges as one normalized timeline change", () => { + const nextState = deleteTimelineRanges(createTimelineState(10), [ + { start: 1.02, end: 1.35 }, + { start: 4.1, end: 4.4 }, + { start: 4.35, end: 4.7 }, + ]); + + expect(getTimelineKeepRanges(nextState)).toEqual([ + { start: 0, end: 1.02 }, + { start: 1.35, end: 4.1 }, + { start: 4.7, end: 10 }, + ]); + expect(nextState.splitPoints).toEqual([1.02, 1.35, 4.1, 4.7]); + }); + + it("does not allow transcript deletion to remove all playable video", () => { + const state = createTimelineState(10); + expect(deleteTimelineRanges(state, [{ start: 0, end: 10 }])).toEqual(state); + }); + + it("stores a batch transcript deletion as one undoable history entry", () => { + const initialState = createTimelineState(10); + const history = createTimelineHistory(initialState); + const deleted = deleteTimelineRanges(initialState, [ + { start: 1, end: 1.3 }, + { start: 3, end: 3.4 }, + ]); + const changedHistory = pushTimelineHistory(history, deleted); + + expect(changedHistory.entries).toHaveLength(2); + expect( + getTimelineKeepRanges(changedHistory.entries[1] ?? initialState), + ).toEqual([ + { start: 0, end: 1 }, + { start: 1.3, end: 3 }, + { start: 3.4, end: 10 }, + ]); + expect(undoTimelineHistory(changedHistory).index).toBe(0); + }); + it("splits, selects, and deletes a segment", () => { const splitState = splitTimelineAt(createTimelineState(10), 4); const firstSegment = getTimelineSegments(splitState)[0]; @@ -434,4 +477,18 @@ describe("timeline editing", () => { expect(findNextPlayableTime(3, spec)).toBe(5); expect(findNextPlayableTime(8.1, spec)).toBeNull(); }); + + it("finds preview skip targets in normalized ranges without rescanning them", () => { + const ranges = Array.from({ length: 1000 }, (_, index) => ({ + start: index * 2, + end: index * 2 + 1, + })); + + expect(findNextPlayableTimeInRanges(998.5, ranges)).toBe(998.5); + expect(findNextPlayableTimeInRanges(999, ranges)).toBe(1000); + expect(findNextPlayableTimeInRanges(1999, ranges)).toBeNull(); + expect(findNextPlayableTimeInRanges(Number.NaN, ranges)).toBeNull(); + expect(findPlayableRangeIndex(998.5, ranges)).toBe(499); + expect(findPlayableRangeIndex(999, ranges)).toBe(-1); + }); }); diff --git a/apps/web/__tests__/unit/video-frame-thumbnail.test.ts b/apps/web/__tests__/unit/video-frame-thumbnail.test.ts index 8bec6942a60..8644b8c7138 100644 --- a/apps/web/__tests__/unit/video-frame-thumbnail.test.ts +++ b/apps/web/__tests__/unit/video-frame-thumbnail.test.ts @@ -31,8 +31,12 @@ function createMockCanvas(options?: { }; } -function createMockVideo(readyState = 3): HTMLVideoElement { - return { readyState } as unknown as HTMLVideoElement; +function createMockVideo( + readyState = 3, + videoWidth = 224, + videoHeight = 128, +): HTMLVideoElement { + return { readyState, videoWidth, videoHeight } as unknown as HTMLVideoElement; } describe("captureVideoFrameDataUrl", () => { @@ -83,13 +87,23 @@ describe("captureVideoFrameDataUrl", () => { createCanvas: () => canvas, }); expect(result).toBe("data:image/jpeg;base64,abc123"); - expect(ctx.drawImage).toHaveBeenCalledWith(video, 0, 0, 224, 128); + expect(ctx.drawImage).toHaveBeenCalledWith( + video, + 0, + 0, + 224, + 128, + 0, + 0, + 224, + 128, + ); expect(canvas.toDataURL).toHaveBeenCalledWith("image/jpeg", 0.8); }); it("respects custom width and height", () => { const { canvas, ctx } = createMockCanvas(); - const video = createMockVideo(); + const video = createMockVideo(3, 320, 180); captureVideoFrameDataUrl({ video, createCanvas: () => canvas, @@ -98,7 +112,61 @@ describe("captureVideoFrameDataUrl", () => { }); expect(canvas.width).toBe(320); expect(canvas.height).toBe(180); - expect(ctx.drawImage).toHaveBeenCalledWith(video, 0, 0, 320, 180); + expect(ctx.drawImage).toHaveBeenCalledWith( + video, + 0, + 0, + 320, + 180, + 0, + 0, + 320, + 180, + ); + }); + + it("center-crops a taller source to the output aspect ratio", () => { + const { canvas, ctx } = createMockCanvas(); + const video = createMockVideo(3, 1280, 960); + captureVideoFrameDataUrl({ + video, + createCanvas: () => canvas, + width: 160, + height: 90, + }); + expect(ctx.drawImage).toHaveBeenCalledWith( + video, + 0, + 120, + 1280, + 720, + 0, + 0, + 160, + 90, + ); + }); + + it("center-crops a wider source to the output aspect ratio", () => { + const { canvas, ctx } = createMockCanvas(); + const video = createMockVideo(3, 2560, 1080); + captureVideoFrameDataUrl({ + video, + createCanvas: () => canvas, + width: 160, + height: 90, + }); + expect(ctx.drawImage).toHaveBeenCalledWith( + video, + 320, + 0, + 1920, + 1080, + 0, + 0, + 160, + 90, + ); }); it("respects custom JPEG quality", () => { diff --git a/apps/web/actions/videos/get-edit-transcript.ts b/apps/web/actions/videos/get-edit-transcript.ts new file mode 100644 index 00000000000..7cb1b5c2147 --- /dev/null +++ b/apps/web/actions/videos/get-edit-transcript.ts @@ -0,0 +1,312 @@ +"use server"; + +import { randomUUID } from "node:crypto"; +import { db } from "@cap/database"; +import { getCurrentUser } from "@cap/database/auth/session"; +import { videoEdits, videos } from "@cap/database/schema"; +import { userIsPro } from "@cap/utils"; +import { Storage } from "@cap/web-backend"; +import type { Video } from "@cap/web-domain"; +import { and, eq, sql } from "drizzle-orm"; +import { Option } from "effect"; +import { start } from "workflow/api"; +import { + type EditTranscript, + type EditTranscriptBackfillStatus, + getEditTranscriptBackfillStatus, + getEditTranscriptObjectKey, + parseEditTranscript, + remapEditTranscriptThroughSpec, +} from "@/lib/edit-transcript"; +import { decryptEditTranscriptObject } from "@/lib/edit-transcript-storage"; +import { runPromise } from "@/lib/server"; +import { decodeStorageVideo } from "@/lib/video-storage"; +import { backfillEditTranscriptWorkflow } from "@/workflows/transcribe"; + +type EditTranscriptResponse = + | { status: "ready"; transcript: EditTranscript } + | { status: "missing" } + | { status: "processing" } + | { status: "error"; message: string }; + +const BACKFILL_STALE_AFTER_MS = 60 * 60 * 1000; +const PUBLIC_ERROR_MESSAGES = new Set([ + "Unauthorized", + "Cap Pro is required", + "Video not found", + "Transcript editing is not available for this video", + "Transcript is not ready", +]); + +function isMp4BackedVideo(source: typeof videos.$inferSelect.source) { + return source.type === "desktopMP4" || source.type === "webMP4"; +} + +function isEditableTranscriptVideo(video: typeof videos.$inferSelect) { + return ( + video.transcriptionStatus === "COMPLETE" && + !video.isScreenshot && + isMp4BackedVideo(video.source) && + Boolean(video.duration && video.duration > 0) + ); +} + +function getPublicErrorMessage(error: unknown, fallback: string) { + return isPublicError(error) ? error.message : fallback; +} + +function isPublicError(error: unknown): error is Error { + return error instanceof Error && PUBLIC_ERROR_MESSAGES.has(error.message); +} + +async function loadEditableTranscriptVideo(videoId: Video.VideoId) { + const user = await getCurrentUser(); + if (!user) throw new Error("Unauthorized"); + if (!userIsPro(user)) throw new Error("Cap Pro is required"); + + const [video] = await db() + .select() + .from(videos) + .where(eq(videos.id, videoId)); + if (!video || video.ownerId !== user.id) throw new Error("Video not found"); + if (!isEditableTranscriptVideo(video)) { + if (video.transcriptionStatus !== "COMPLETE") { + throw new Error("Transcript is not ready"); + } + throw new Error("Transcript editing is not available for this video"); + } + + const [bucket] = await Storage.getAccessForVideo( + decodeStorageVideo(video), + ).pipe(runPromise); + + return { user, video, bucket }; +} + +async function getOptionalObject( + bucket: Awaited>["bucket"], + key: string, +) { + const object = await bucket.getObject(key).pipe(runPromise); + return Option.isSome(object) ? object.value : null; +} + +function isFreshProcessingStatus(status: EditTranscriptBackfillStatus) { + if (status.status !== "processing") return false; + const requestedAt = Date.parse(status.requestedAt); + return ( + Number.isFinite(requestedAt) && + Date.now() - requestedAt < BACKFILL_STALE_AFTER_MS + ); +} + +function isDurationCurrent(durationMs: number, expectedSeconds: number) { + return Math.abs(durationMs - Math.round(expectedSeconds * 1000)) <= 250; +} + +/** + * Reads the stored word transcript. Stored words always describe the ORIGINAL + * media, so for edited videos they are checked against the edit spec's source + * duration and remapped onto the current output timeline before being returned. + */ +async function readCurrentEditTranscript( + bucket: Awaited>["bucket"], + video: typeof videos.$inferSelect, +): Promise { + const content = await getOptionalObject( + bucket, + getEditTranscriptObjectKey(video.ownerId, video.id), + ); + const decrypted = content + ? decryptEditTranscriptObject(content, video.ownerId, video.id) + : null; + const transcript = decrypted ? parseEditTranscript(decrypted) : null; + if (!transcript) return null; + + const [edit] = await db() + .select({ editSpec: videoEdits.editSpec }) + .from(videoEdits) + .where(eq(videoEdits.videoId, video.id)); + + if (!edit?.editSpec) { + return isDurationCurrent(transcript.durationMs, video.duration ?? 0) + ? transcript + : null; + } + + return isDurationCurrent(transcript.durationMs, edit.editSpec.sourceDuration) + ? remapEditTranscriptThroughSpec(transcript, edit.editSpec) + : null; +} + +export async function getEditTranscript( + videoId: Video.VideoId, +): Promise { + try { + const { video, bucket } = await loadEditableTranscriptVideo(videoId); + const transcript = await readCurrentEditTranscript(bucket, video); + + if (transcript) { + return { status: "ready", transcript }; + } + + const status = getEditTranscriptBackfillStatus(video.metadata); + if (status && isFreshProcessingStatus(status)) { + return { status: "processing" }; + } + if (status?.status === "processing") { + return { + status: "error", + message: "Transcript preparation stalled. Try again.", + }; + } + if (status?.status === "error") { + return { + status: "error", + message: "Transcript preparation failed. Try again.", + }; + } + return { status: "missing" }; + } catch (error) { + if (!isPublicError(error)) { + console.error( + `[editTranscript] Failed to load transcript for ${videoId}`, + error, + ); + } + return { + status: "error", + message: getPublicErrorMessage(error, "Transcript unavailable"), + }; + } +} + +async function claimEditTranscriptBackfill( + videoId: Video.VideoId, + userId: string, +) { + return db().transaction(async (tx) => { + const [video] = await tx + .select() + .from(videos) + .where(eq(videos.id, videoId)) + .for("update"); + if (!video || video.ownerId !== userId) throw new Error("Video not found"); + if (!isEditableTranscriptVideo(video)) { + throw new Error("Transcript editing is not available for this video"); + } + + const existing = getEditTranscriptBackfillStatus(video.metadata); + if (existing && isFreshProcessingStatus(existing)) { + return { claimed: false as const }; + } + + const requestId = randomUUID(); + const status = { + status: "processing", + requestId, + requestedAt: new Date().toISOString(), + }; + await tx + .update(videos) + .set({ + metadata: sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.editTranscriptBackfill', CAST(${JSON.stringify(status)} AS JSON))`, + updatedAt: video.updatedAt, + }) + .where(eq(videos.id, videoId)); + + return { claimed: true as const, requestId }; + }); +} + +async function setEditTranscriptBackfillError( + videoId: Video.VideoId, + requestId: string, +) { + await db() + .update(videos) + .set({ + metadata: sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.editTranscriptBackfill.status', 'error')`, + updatedAt: sql`${videos.updatedAt}`, + }) + .where( + and( + eq(videos.id, videoId), + sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.editTranscriptBackfill.requestId')) = ${requestId}`, + ), + ); +} + +async function clearEditTranscriptBackfill( + videoId: Video.VideoId, + requestId: string, +) { + await db() + .update(videos) + .set({ + metadata: sql`JSON_REMOVE(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.editTranscriptBackfill')`, + updatedAt: sql`${videos.updatedAt}`, + }) + .where( + and( + eq(videos.id, videoId), + sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.editTranscriptBackfill.requestId')) = ${requestId}`, + ), + ); +} + +export async function requestEditTranscript( + videoId: Video.VideoId, +): Promise { + try { + const { user, video, bucket } = await loadEditableTranscriptVideo(videoId); + const transcript = await readCurrentEditTranscript(bucket, video); + if (transcript) { + return { status: "ready", transcript }; + } + + const claim = await claimEditTranscriptBackfill(videoId, user.id); + if (!claim.claimed) { + return { status: "processing" }; + } + + const transcriptAfterClaim = await readCurrentEditTranscript(bucket, video); + if (transcriptAfterClaim) { + await clearEditTranscriptBackfill(videoId, claim.requestId); + return { status: "ready", transcript: transcriptAfterClaim }; + } + + try { + await start(backfillEditTranscriptWorkflow, [ + { + videoId, + userId: user.id, + requestId: claim.requestId, + }, + ]); + } catch (error) { + console.error( + `[editTranscript] Failed to start backfill for ${videoId}`, + error, + ); + await setEditTranscriptBackfillError(videoId, claim.requestId); + return { + status: "error", + message: "Transcript preparation could not start. Try again.", + }; + } + + return { status: "processing" }; + } catch (error) { + if (!isPublicError(error)) { + console.error( + `[editTranscript] Failed to request transcript for ${videoId}`, + error, + ); + } + return { + status: "error", + message: getPublicErrorMessage(error, "Transcript unavailable"), + }; + } +} diff --git a/apps/web/app/api/dev-reset-transcript/route.ts b/apps/web/app/api/dev-reset-transcript/route.ts new file mode 100644 index 00000000000..c8a1d4ff9f3 --- /dev/null +++ b/apps/web/app/api/dev-reset-transcript/route.ts @@ -0,0 +1,100 @@ +import { db } from "@cap/database"; +import { getCurrentUser } from "@cap/database/auth/session"; +import { videoEdits, videos, videoUploads } from "@cap/database/schema"; +import { Storage } from "@cap/web-backend"; +import type { Video } from "@cap/web-domain"; +import { and, eq } from "drizzle-orm"; +import type { NextRequest } from "next/server"; +import { runPromise } from "@/lib/server"; +import { decodeStorageVideo } from "@/lib/video-storage"; + +export async function POST(request: NextRequest) { + if (process.env.NODE_ENV !== "development") { + return Response.json({ error: "dev only" }, { status: 403 }); + } + if (request.headers.get("origin") !== request.nextUrl.origin) { + return Response.json({ error: "same origin required" }, { status: 403 }); + } + const user = await getCurrentUser(); + if (!user) { + return Response.json({ error: "unauthorized" }, { status: 401 }); + } + const videoIdParam = request.nextUrl.searchParams.get("videoId"); + if (!videoIdParam) { + return Response.json({ error: "videoId required" }, { status: 400 }); + } + const videoId = videoIdParam as Video.VideoId; + + const [video] = await db() + .select() + .from(videos) + .where(and(eq(videos.id, videoId), eq(videos.ownerId, user.id))); + if (!video) { + return Response.json({ error: "video not found" }, { status: 404 }); + } + + const [bucket] = await Storage.getAccessForVideo( + decodeStorageVideo(video), + ).pipe(runPromise); + + const deleted: string[] = []; + for (const prefix of [ + `${video.ownerId}/${video.id}/transcription`, + `${video.ownerId}/${video.id}/audio`, + ]) { + const listed = await bucket.listObjects({ prefix }).pipe(runPromise); + const objects = (listed.Contents ?? []) + .map((object) => ({ Key: object.Key })) + .filter((object): object is { Key: string } => Boolean(object.Key)); + if (objects.length > 0) { + await bucket.deleteObjects(objects).pipe(runPromise); + deleted.push(...objects.map((object) => object.Key)); + } + } + + const [edit] = await db() + .select() + .from(videoEdits) + .where(eq(videoEdits.videoId, videoId)); + let restoredOriginal = false; + let restoredDuration: number | null = null; + if (edit) { + await bucket + .copyObject( + `${bucket.bucketName}/${edit.sourceKey}`, + `${video.ownerId}/${video.id}/result.mp4`, + ) + .pipe(runPromise); + await bucket.deleteObject(edit.sourceKey).pipe(runPromise); + deleted.push(edit.sourceKey); + restoredOriginal = true; + restoredDuration = edit.editSpec.sourceDuration; + await db().delete(videoEdits).where(eq(videoEdits.videoId, videoId)); + } + + const metadata = { + ...((video.metadata as Record | null) ?? {}), + }; + delete metadata.summary; + delete metadata.chapters; + delete metadata.aiGenerationStatus; + delete metadata.editTranscriptBackfill; + + await db() + .update(videos) + .set({ + transcriptionStatus: null, + metadata, + ...(restoredDuration === null ? {} : { duration: restoredDuration }), + }) + .where(eq(videos.id, videoId)); + await db().delete(videoUploads).where(eq(videoUploads.videoId, videoId)); + + return Response.json({ + ok: true, + resolvedBucket: bucket.bucketName, + deleted, + restoredOriginal, + restoredDuration, + }); +} diff --git a/apps/web/app/api/storage/object/route.ts b/apps/web/app/api/storage/object/route.ts index 8ec8f88f4c5..0a1d70e50d0 100644 --- a/apps/web/app/api/storage/object/route.ts +++ b/apps/web/app/api/storage/object/route.ts @@ -8,6 +8,7 @@ import { import { Storage as StorageDomain, Video } from "@cap/web-domain"; import { Effect, Option } from "effect"; import type { NextRequest } from "next/server"; +import { isPrivateEditTranscriptObjectKey } from "@/lib/edit-transcript"; import { runPromise } from "@/lib/server"; import { CACHE_CONTROL_HEADERS } from "@/utils/helpers"; @@ -106,6 +107,9 @@ export async function GET(request: NextRequest) { if (!videoIdParam || !key) { return new Response("Missing videoId or key", { status: 400 }); } + if (isPrivateEditTranscriptObjectKey(key)) { + return new Response("Not found", { status: 404 }); + } const effect = Effect.gen(function* () { const tokenPayload = token ? verifyStorageObjectToken(token) : null; diff --git a/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx b/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx index 49578e5f563..efdf063dc72 100644 --- a/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx +++ b/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx @@ -94,6 +94,7 @@ interface Props { disableCommentStamps?: boolean; disableReactionStamps?: boolean; disablePreviewGif?: boolean; + disablePlaybackSpeedDial?: boolean; comments?: Array<{ id: string; timestamp: number | null; @@ -134,6 +135,7 @@ export function CapVideoPlayer({ disableCommentStamps = false, disableReactionStamps = false, disablePreviewGif = false, + disablePlaybackSpeedDial = false, onSeek, enhancedAudioUrl: _enhancedAudioUrl, enhancedAudioStatus: _enhancedAudioStatus, @@ -767,7 +769,8 @@ export function CapVideoPlayer({ videoLoaded && !hasActiveProgress && !showUploadFailureOverlay && - !showPlaybackResolutionError && ( + !showPlaybackResolutionError && + !disablePlaybackSpeedDial && ( { @@ -77,9 +79,12 @@ const parseVTT = (vttContent: string): TranscriptEntry[] => { ); if (totalSeconds === null) return null; + const fractionPart = secondsWithMs.split(".")[1]; + const fraction = fractionPart ? Number(`0.${fractionPart}`) : 0; + return { mm_ss: `${minutesStr}:${secondsPart}`, - totalSeconds, + totalSeconds: totalSeconds + (Number.isFinite(fraction) ? fraction : 0), }; }; @@ -101,11 +106,13 @@ const parseVTT = (vttContent: string): TranscriptEntry[] => { if (!startTimeStr || !endTimeStr) continue; const startTimestamp = parseTimestamp(startTimeStr); + const endTimestamp = parseTimestamp(endTimeStr); if (startTimestamp) { currentEntry = { id: currentId, timestamp: startTimestamp.mm_ss, startTime: startTimestamp.totalSeconds, + endTime: endTimestamp?.totalSeconds ?? startTimestamp.totalSeconds, }; } continue; @@ -148,7 +155,11 @@ export const Transcript: React.FC = ({ data, onSeek }) => { const [copyPressed, setCopyPressed] = useState(false); const [downloadPressed, setDownloadPressed] = useState(false); const [showLanguageMenu, setShowLanguageMenu] = useState(false); + const [showDownloadMenu, setShowDownloadMenu] = useState(false); + const [showCopyMenu, setShowCopyMenu] = useState(false); const languageMenuRef = useRef(null); + const downloadMenuRef = useRef(null); + const copyMenuRef = useRef(null); const selectedLanguage = captionContext.selectedLanguage === "off" @@ -176,6 +187,44 @@ export const Transcript: React.FC = ({ data, onSeek }) => { }; }, [showLanguageMenu]); + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + downloadMenuRef.current && + !downloadMenuRef.current.contains(event.target as Node) + ) { + setShowDownloadMenu(false); + } + }; + + if (showDownloadMenu) { + document.addEventListener("mousedown", handleClickOutside); + } + + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, [showDownloadMenu]); + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + copyMenuRef.current && + !copyMenuRef.current.contains(event.target as Node) + ) { + setShowCopyMenu(false); + } + }; + + if (showCopyMenu) { + document.addEventListener("mousedown", handleClickOutside); + } + + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, [showCopyMenu]); + const { data: transcriptContent, isLoading: isTranscriptLoading, @@ -361,13 +410,12 @@ export const Transcript: React.FC = ({ data, onSeek }) => { return vttHeader + vttEntries.join("\n"); }; - const copyTranscriptToClipboard = async () => { - if (transcriptData.length === 0) return; + const copyTranscriptText = async (content: string) => { + if (!content) return; setIsCopying(true); try { - const formattedTranscript = formatTranscriptForClipboard(transcriptData); - await navigator.clipboard.writeText(formattedTranscript); + await navigator.clipboard.writeText(content); setCopyPressed(true); setTimeout(() => { setCopyPressed(false); @@ -379,18 +427,29 @@ export const Transcript: React.FC = ({ data, onSeek }) => { } }; - const downloadTranscriptFile = () => { + const copyFormattedTranscript = () => { + if (transcriptData.length === 0) return; + void copyTranscriptText(formatTranscriptAsParagraphs(transcriptData)); + }; + + const copyTimestampedTranscript = () => { if (transcriptData.length === 0) return; + void copyTranscriptText(formatTranscriptForClipboard(transcriptData)); + }; - const vttContent = formatTranscriptAsVTT(transcriptData); - const blob = new Blob([vttContent], { type: "text/vtt" }); + const triggerTranscriptDownload = ( + extension: string, + content: string, + mimeType: string, + ) => { + const blob = new Blob([content], { type: mimeType }); const url = URL.createObjectURL(blob); const langSuffix = selectedLanguage === "original" ? "" : `.${selectedLanguage}`; const link = document.createElement("a"); link.href = url; - link.download = `transcript-${data.id}${langSuffix}.vtt`; + link.download = `transcript-${data.id}${langSuffix}.${extension}`; document.body.appendChild(link); link.click(); document.body.removeChild(link); @@ -403,38 +462,35 @@ export const Transcript: React.FC = ({ data, onSeek }) => { }, 2000); }; + const downloadTranscriptFile = () => { + if (transcriptData.length === 0) return; + triggerTranscriptDownload( + "vtt", + formatTranscriptAsVTT(transcriptData), + "text/vtt", + ); + }; + + const downloadFormattedTranscript = () => { + if (transcriptData.length === 0) return; + const content = formatTranscriptAsParagraphs(transcriptData); + if (!content) return; + triggerTranscriptDownload("txt", `${content}\n`, "text/plain"); + }; + const canEdit = user?.id === data.owner.id && selectedLanguage === "original"; if (isTranscriptionProcessing && !hasTimedOut) { return ( -
+
-
- -
-

Transcription in progress...

+ +

+ Transcribing audio +

+

+ The transcript will appear here shortly. +

); @@ -442,43 +498,23 @@ export const Transcript: React.FC = ({ data, onSeek }) => { if (isQueryLoading) { return ( -
- +
+
); } if (data.transcriptionStatus === "NO_AUDIO") { return ( -
+
- -

+

+ +
+

No audio track detected

-

+

This video doesn't contain audio for transcription

{canEdit && ( @@ -512,13 +548,15 @@ export const Transcript: React.FC = ({ data, onSeek }) => { if (data.transcriptionStatus === "SKIPPED") { return ( -
+
- -

+

+ +
+

Transcription disabled

-

+

Transcription has been disabled for this video

@@ -536,10 +574,12 @@ export const Transcript: React.FC = ({ data, onSeek }) => { if (showRetryButton) { return ( -
+
- -

+

+ +
+

{data.transcriptionStatus === "ERROR" ? "Transcript not available" : "No transcript available"} @@ -573,118 +613,165 @@ export const Transcript: React.FC = ({ data, onSeek }) => { return (

-
-
-
- + {showLanguageMenu && ( +
+ +
+ {( + Object.entries(SUPPORTED_LANGUAGES) as [LanguageCode, string][] + ).map(([code, name]) => ( + + ))} +
+ )} +
+ +
+
+ - + + Formatted text + + + Clean paragraphs for docs + + + +
+ )}
-
+
- {showLanguageMenu && ( -
+ {showDownloadMenu && ( +
+ -
- {( - Object.entries(SUPPORTED_LANGUAGES) as [ - LanguageCode, - string, - ][] - ).map(([code, name]) => ( - - ))}
)}
@@ -693,118 +780,94 @@ export const Transcript: React.FC = ({ data, onSeek }) => {
{isTranslating && ( -
+
- -

- Translating transcript... + +

+ Translating transcript…

)} -
+
{transcriptData.map((entry) => (
-
-
- {entry.timestamp} -
- {canEdit && editingEntry !== entry.id && ( - - )} -
- {editingEntry === entry.id ? ( -
-
-