From d7f8e139dd6540643e078bed823ca62cdc1c3c8e Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:16 +0100 Subject: [PATCH 01/20] perf(media-server): render edits in a single ffmpeg filter graph --- .../src/__tests__/lib/media-edit.test.ts | 25 ++++ apps/media-server/src/lib/media-edit.ts | 139 +++++++----------- 2 files changed, 80 insertions(+), 84 deletions(-) diff --git a/apps/media-server/src/__tests__/lib/media-edit.test.ts b/apps/media-server/src/__tests__/lib/media-edit.test.ts index a1836f5a7b..8f79c93b61 100644 --- a/apps/media-server/src/__tests__/lib/media-edit.test.ts +++ b/apps/media-server/src/__tests__/lib/media-edit.test.ts @@ -3,6 +3,7 @@ import { existsSync, rmSync } from "node:fs"; import { join } from "node:path"; import { buildStreamCopySegmentArgs, + buildTranscodeEditArgs, buildTranscodeSegmentArgs, normalizeEditRanges, renderEditedVideo, @@ -98,6 +99,30 @@ describe("media edit helpers", () => { ); expect(args).toContain("[a]"); }); + + test("builds one transcode graph for many edit ranges", () => { + const args = buildTranscodeEditArgs( + "/input.mp4", + [ + { start: 0, end: 1 }, + { start: 1.2, end: 2 }, + { start: 2.3, end: 3 }, + ], + "/output.mp4", + true, + 60, + ); + const filter = args[args.indexOf("-filter_complex") + 1]; + + expect(args.filter((value) => value === "-i")).toHaveLength(1); + expect(filter).toContain("concat=n=3:v=1:a=1[v][a]"); + expect(filter).toContain( + "[0:v:0]fps=60,trim=start=1.200:end=2.000,setpts=PTS-STARTPTS[v1]", + ); + expect(filter).toContain( + "[0:a:0]atrim=start=2.300:end=3.000,asetpts=PTS-STARTPTS[a2]", + ); + }); }); describe("renderEditedVideo integration tests", () => { diff --git a/apps/media-server/src/lib/media-edit.ts b/apps/media-server/src/lib/media-edit.ts index 35c698be0b..eb36625abc 100644 --- a/apps/media-server/src/lib/media-edit.ts +++ b/apps/media-server/src/lib/media-edit.ts @@ -1,8 +1,7 @@ -import { writeFile } from "node:fs/promises"; import { file, spawn } from "bun"; import type { VideoMetadata } from "./job-manager"; import { registerSubprocess, terminateProcess } from "./subprocess"; -import { createTempFile, type TempFileHandle } from "./temp-files"; +import { createTempFile } from "./temp-files"; export type EditRange = { start: number; @@ -147,21 +146,45 @@ export function buildTranscodeSegmentArgs( ]; } -function buildConcatArgs(listPath: string, outputPath: string) { +export function buildTranscodeEditArgs( + inputPath: string, + ranges: EditRange[], + outputPath: string, + hasAudio: boolean, + fps = DEFAULT_OUTPUT_FPS, +) { + const filters = ranges.flatMap((range, index) => { + const videoFilter = `[0:v:0]fps=${getOutputFps(fps)},trim=start=${formatTime(range.start)}:end=${formatTime(range.end)},setpts=PTS-STARTPTS[v${index}]`; + if (!hasAudio) return [videoFilter]; + return [ + videoFilter, + `[0:a:0]atrim=start=${formatTime(range.start)}:end=${formatTime(range.end)},asetpts=PTS-STARTPTS[a${index}]`, + ]; + }); + const inputs = ranges + .map((_, index) => `[v${index}]${hasAudio ? `[a${index}]` : ""}`) + .join(""); + const concat = `${inputs}concat=n=${ranges.length}:v=1:a=${hasAudio ? 1 : 0}[v]${hasAudio ? "[a]" : ""}`; + return [ "ffmpeg", "-hide_banner", "-y", - "-f", - "concat", - "-safe", - "0", "-i", - listPath, + inputPath, + "-filter_complex", + [...filters, concat].join(";"), "-map", - "0", - "-c", - "copy", + "[v]", + "-c:v", + "libx264", + "-preset", + "fast", + "-crf", + "18", + "-pix_fmt", + "yuv420p", + ...(hasAudio ? ["-map", "[a]", "-c:a", "aac", "-b:a", "160k"] : ["-an"]), "-movflags", "+faststart", outputPath, @@ -276,25 +299,27 @@ async function runFfmpegCommand( } } -function concatFileLine(path: string) { - return `file '${path.replaceAll("'", "'\\''")}'`; -} - -async function concatSegments( - segmentFiles: TempFileHandle[], +async function renderTranscodedEdit( + inputPath: string, + keepRanges: EditRange[], + hasAudio: boolean, + fps: number | undefined, timeoutMs: number, + onProgress?: ProgressCallback, abortSignal?: AbortSignal, ) { - const concatList = await createTempFile(".txt"); const outputFile = await createTempFile(".mp4"); try { - await writeFile( - concatList.path, - `${segmentFiles.map((segment) => concatFileLine(segment.path)).join("\n")}\n`, - ); + onProgress?.(5, "Preparing edit..."); await runFfmpegCommand( - buildConcatArgs(concatList.path, outputFile.path), + buildTranscodeEditArgs( + inputPath, + keepRanges, + outputFile.path, + hasAudio, + fps, + ), timeoutMs, abortSignal, ); @@ -304,58 +329,11 @@ async function concatSegments( throw new Error("FFmpeg produced empty edited output"); } + onProgress?.(75, "Edit prepared"); return outputFile; } catch (error) { await outputFile.cleanup(); throw error; - } finally { - await concatList.cleanup(); - } -} - -async function renderSegments({ - keepRanges, - timeoutMs, - buildArgs, - onProgress, - abortSignal, - progressStart, - progressEnd, -}: { - keepRanges: EditRange[]; - timeoutMs: number; - buildArgs: (range: EditRange, outputPath: string) => string[]; - onProgress?: ProgressCallback; - abortSignal?: AbortSignal; - progressStart: number; - progressEnd: number; -}) { - const segmentFiles: TempFileHandle[] = []; - - try { - for (const [index, range] of keepRanges.entries()) { - const segmentFile = await createTempFile(".mp4"); - segmentFiles.push(segmentFile); - await runFfmpegCommand( - buildArgs(range, segmentFile.path), - timeoutMs, - abortSignal, - ); - const progress = - progressStart + - ((index + 1) / keepRanges.length) * (progressEnd - progressStart); - onProgress?.(progress, "Preparing edit..."); - } - - const outputFile = await concatSegments( - segmentFiles, - timeoutMs, - abortSignal, - ); - onProgress?.(progressEnd, "Edit prepared"); - return outputFile; - } finally { - await Promise.all(segmentFiles.map((segment) => segment.cleanup())); } } @@ -373,20 +351,13 @@ export async function renderEditedVideo({ const timeoutMs = getTimeoutMs(normalizedRanges); - return await renderSegments({ - keepRanges: normalizedRanges, + return await renderTranscodedEdit( + inputPath, + normalizedRanges, + Boolean(metadata.audioCodec), + metadata.fps, timeoutMs, - buildArgs: (range, outputPath) => - buildTranscodeSegmentArgs( - inputPath, - range, - outputPath, - Boolean(metadata.audioCodec), - metadata.fps, - ), onProgress, abortSignal, - progressStart: 5, - progressEnd: 75, - }); + ); } From b179ed3b42fd5f272b09c52c02882023c70ec1a5 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:16 +0100 Subject: [PATCH 02/20] feat(web): add transcript paragraph formatting helper --- .../__tests__/unit/transcript-text.test.ts | 93 +++++++++++++++++++ apps/web/lib/transcript-text.ts | 59 ++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 apps/web/__tests__/unit/transcript-text.test.ts create mode 100644 apps/web/lib/transcript-text.ts diff --git a/apps/web/__tests__/unit/transcript-text.test.ts b/apps/web/__tests__/unit/transcript-text.test.ts new file mode 100644 index 0000000000..a86770a280 --- /dev/null +++ b/apps/web/__tests__/unit/transcript-text.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { + formatTranscriptAsParagraphs, + type TranscriptTextEntry, +} from "@/lib/transcript-text"; + +function entry( + text: string, + startTime: number, + endTime: number, +): TranscriptTextEntry { + return { text, startTime, endTime }; +} + +describe("formatTranscriptAsParagraphs", () => { + it("returns an empty string for no entries", () => { + expect(formatTranscriptAsParagraphs([])).toBe(""); + }); + + it("joins caption fragments into one flowing paragraph", () => { + const result = formatTranscriptAsParagraphs([ + entry("This is my test to make sure that,", 0.9, 2.4), + entry("the video is working correctly.", 3.0, 4.3), + entry("I'm going to stop the recording now.", 4.4, 5.8), + ]); + expect(result).toBe( + "This is my test to make sure that, the video is working correctly. I'm going to stop the recording now.", + ); + }); + + it("breaks a paragraph at a sentence end followed by a long pause", () => { + const result = formatTranscriptAsParagraphs([ + entry("First topic ends here.", 0, 2), + entry("Second topic starts fresh.", 4.5, 6), + ]); + expect(result).toBe("First topic ends here.\n\nSecond topic starts fresh."); + }); + + it("does not break on a pause mid-sentence", () => { + const result = formatTranscriptAsParagraphs([ + entry("This sentence keeps", 0, 2), + entry("going after a pause.", 5, 6), + ]); + expect(result).toBe("This sentence keeps going after a pause."); + }); + + it("ignores pauses when the entry has no usable end time", () => { + const result = formatTranscriptAsParagraphs([ + entry("A full sentence.", 2, 2), + entry("Another sentence.", 10, 11), + ]); + expect(result).toBe("A full sentence. Another sentence."); + }); + + it("breaks long paragraphs at the next sentence boundary", () => { + const sentence = `${"word ".repeat(30).trim()}.`; + const entries = Array.from({ length: 6 }, (_, index) => + entry(sentence, index, index + 0.5), + ); + const result = formatTranscriptAsParagraphs(entries); + const paragraphs = result.split("\n\n"); + expect(paragraphs.length).toBeGreaterThan(1); + for (const paragraph of paragraphs) { + expect(paragraph.endsWith(".")).toBe(true); + } + }); + + it("hard-breaks transcripts that never end a sentence", () => { + const fragment = "no punctuation here just words".repeat(3); + const entries = Array.from({ length: 20 }, (_, index) => + entry(fragment, index, index + 0.5), + ); + const result = formatTranscriptAsParagraphs(entries); + expect(result.split("\n\n").length).toBeGreaterThan(1); + }); + + it("skips empty fragments", () => { + const result = formatTranscriptAsParagraphs([ + entry("Hello there.", 0, 1), + entry(" ", 1, 2), + entry("General greeting.", 2, 3), + ]); + expect(result).toBe("Hello there. General greeting."); + }); + + it("respects sentence enders followed by closing quotes", () => { + const result = formatTranscriptAsParagraphs([ + entry('He said "stop."', 0, 1), + entry("Then we did.", 4, 5), + ]); + expect(result).toBe('He said "stop."\n\nThen we did.'); + }); +}); diff --git a/apps/web/lib/transcript-text.ts b/apps/web/lib/transcript-text.ts new file mode 100644 index 0000000000..3f982e35bd --- /dev/null +++ b/apps/web/lib/transcript-text.ts @@ -0,0 +1,59 @@ +export type TranscriptTextEntry = { + text: string; + startTime: number; + endTime: number; +}; + +const PARAGRAPH_GAP_SECONDS = 1.75; +const SOFT_PARAGRAPH_CHAR_LIMIT = 500; +const HARD_PARAGRAPH_CHAR_LIMIT = 1200; + +function endsSentence(text: string) { + return /[.!?]["')\]]?$/.test(text); +} + +/** + * Joins caption-sized fragments into flowing paragraphs for use in documents. + * Paragraphs break at sentence boundaries when the speaker pauses, or when a + * paragraph grows past a comfortable reading length; a hard limit guards + * against transcripts with no sentence punctuation at all. + */ +export function formatTranscriptAsParagraphs( + entries: readonly TranscriptTextEntry[], +): string { + const paragraphs: string[] = []; + let current = ""; + + for (let index = 0; index < entries.length; index++) { + const entry = entries[index]; + if (!entry) continue; + const text = entry.text.trim(); + if (!text) continue; + + current = current ? `${current} ${text}` : text; + + const next = entries[index + 1]; + if (!next) break; + + if (current.length >= HARD_PARAGRAPH_CHAR_LIMIT) { + paragraphs.push(current); + current = ""; + continue; + } + + if (!endsSentence(text)) continue; + + const gapSeconds = + entry.endTime > entry.startTime ? next.startTime - entry.endTime : 0; + if ( + gapSeconds >= PARAGRAPH_GAP_SECONDS || + current.length >= SOFT_PARAGRAPH_CHAR_LIMIT + ) { + paragraphs.push(current); + current = ""; + } + } + + if (current) paragraphs.push(current); + return paragraphs.join("\n\n"); +} From 3826f3da582a0e59b7497c2ba6f8666da4d44965 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:16 +0100 Subject: [PATCH 03/20] test(web): add AssemblyAI edit transcript fixture --- .../fixtures/assemblyai-edit-response.ts | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 apps/web/__tests__/fixtures/assemblyai-edit-response.ts diff --git a/apps/web/__tests__/fixtures/assemblyai-edit-response.ts b/apps/web/__tests__/fixtures/assemblyai-edit-response.ts new file mode 100644 index 0000000000..de441cfce9 --- /dev/null +++ b/apps/web/__tests__/fixtures/assemblyai-edit-response.ts @@ -0,0 +1,107 @@ +export const assemblyAIEditResponse = { + id: "transcript_edit_fixture", + status: "completed", + speech_model_used: "universal-3-5-pro", + language_code: "en", + words: [ + { + text: "Um,", + start: 120, + end: 380, + confidence: 0.98, + speaker: "A", + channel: null, + }, + { + text: "this", + start: 520, + end: 720, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "is", + start: 730, + end: 840, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "a", + start: 850, + end: 910, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "real", + start: 930, + end: 1130, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "example.", + start: 1140, + end: 1510, + confidence: 0.99, + speaker: "A", + channel: null, + }, + { + text: "Erm", + start: 2300, + end: 2570, + confidence: 0.93, + speaker: "A", + channel: null, + }, + { + text: "nothing", + start: 2700, + end: 3070, + confidence: 0.98, + speaker: "A", + channel: null, + }, + { + text: "else.", + start: 3090, + end: 3400, + confidence: 0.98, + speaker: "A", + channel: null, + }, + ] as const, +} as const; + +export const assemblyAIRealFillerResponse = { + id: "sanitized_real_filler_response", + status: "completed", + speech_model_used: "universal-2", + language_code: "en", + words: [ + { text: "Hey,", start: 960, end: 1280, confidence: 0.98779297 }, + { text: "my", start: 1280, end: 1560, confidence: 0.9995117 }, + { text: "name's", start: 1560, end: 1880, confidence: 0.9082031 }, + { text: "Richie,", start: 1880, end: 2320, confidence: 0.9448242 }, + { text: "and", start: 2320, end: 2520, confidence: 0.99560547 }, + { text: "this", start: 2520, end: 2680, confidence: 0.9995117 }, + { text: "is", start: 2680, end: 2880, confidence: 0.9970703 }, + { text: "my,", start: 2880, end: 3200, confidence: 0.9995117 }, + { text: "um,", start: 3460, end: 3980, confidence: 0.9959265 }, + { text: "test", start: 3980, end: 4320, confidence: 0.99902344 }, + { text: "from", start: 4320, end: 4600, confidence: 0.99902344 }, + { text: "the", start: 4600, end: 4800, confidence: 0.99853516 }, + { text: "CAP", start: 4800, end: 5120, confidence: 0.95458984 }, + { text: "Mobile", start: 5120, end: 5680, confidence: 0.99658203 }, + { text: "app.", start: 5680, end: 5760, confidence: 0.6508789 }, + { text: "Uh,", start: 5760, end: 7240, confidence: 0.9857637 }, + { text: "say", start: 7240, end: 7360, confidence: 0.9975586 }, + { text: "hello,", start: 7360, end: 7640, confidence: 0.99975586 }, + ] as const, +} as const; From a4be5e43fa22624593512cac32928b6de51d9e77 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:16 +0100 Subject: [PATCH 04/20] feat(web): add edit transcript model and remapping helpers --- .../__tests__/unit/edit-transcript.test.ts | 550 ++++++++++++++ apps/web/lib/edit-transcript.ts | 678 ++++++++++++++++++ 2 files changed, 1228 insertions(+) create mode 100644 apps/web/__tests__/unit/edit-transcript.test.ts create mode 100644 apps/web/lib/edit-transcript.ts diff --git a/apps/web/__tests__/unit/edit-transcript.test.ts b/apps/web/__tests__/unit/edit-transcript.test.ts new file mode 100644 index 0000000000..d82d32940d --- /dev/null +++ b/apps/web/__tests__/unit/edit-transcript.test.ts @@ -0,0 +1,550 @@ +import type { VideoEditSpec } from "@cap/database/types"; +import { describe, expect, it } from "vitest"; +import { + createEditTranscript, + editTranscriptWordsToCaptionVtt, + findActiveTranscriptWordIndex, + getDeletedTranscriptWordIds, + groupEditTranscriptWords, + isFillerWord, + normalizeTranscriptSelection, + parseEditTranscript, + planFillerCuts, + planTranscriptCut, + remapEditTranscriptThroughSpec, + serializeEditTranscript, +} from "@/lib/edit-transcript"; +import { createIdentityEditSpec } from "@/lib/video-edits"; +import { + assemblyAIEditResponse, + assemblyAIRealFillerResponse, +} from "../fixtures/assemblyai-edit-response"; + +function editSpec( + sourceDuration: number, + keepRanges: { start: number; end: number }[], +): VideoEditSpec { + return { version: 1, sourceDuration, keepRanges }; +} + +describe("edit transcript", () => { + it("normalizes and preserves AssemblyAI word timing", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + + expect(transcript).toMatchObject({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 4_000, + languageCode: "en", + }); + expect(transcript.words).toHaveLength(9); + expect(transcript.words[0]).toMatchObject({ + id: "word-0-120-380", + text: "Um,", + startMs: 120, + endMs: 380, + confidence: 0.98, + speaker: "A", + channel: null, + }); + expect(parseEditTranscript(serializeEditTranscript(transcript))).toEqual( + transcript, + ); + }); + + it("sorts, clamps, and rejects malformed provider words", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-2", + words: [ + { text: "second", start: 800, end: 900 }, + { text: "first", start: -100, end: 250, confidence: 2 }, + { text: "", start: 300, end: 400 }, + { text: "invalid", start: Number.NaN, end: 600 }, + { text: "reverse", start: 700, end: 650, channel: 1 }, + { text: "outside", start: 1_100, end: 1_200 }, + ], + language_code: null, + }, + 1_000, + ); + + expect(transcript.words.map((word) => word.text)).toEqual([ + "first", + "reverse", + "second", + ]); + expect(transcript.words[0]?.confidence).toBe(1); + expect(transcript.words[1]?.channel).toBe("1"); + }); + + it("records whichever speech model the provider used", () => { + expect( + createEditTranscript( + { ...assemblyAIEditResponse, speech_model_used: "universal-2" }, + 4_000, + ).speechModelUsed, + ).toBe("universal-2"); + expect( + createEditTranscript( + { ...assemblyAIEditResponse, speech_model_used: undefined }, + 4_000, + ).speechModelUsed, + ).toBe("unknown"); + }); + + it("preserves fillers from the real accuracy regression response", () => { + const transcript = createEditTranscript( + assemblyAIRealFillerResponse, + 8_000, + ); + + expect( + transcript.words + .filter((word) => isFillerWord(word.text)) + .map(({ text, startMs, endMs, confidence }) => ({ + text, + startMs, + endMs, + confidence, + })), + ).toEqual([ + { + text: "um,", + startMs: 3_460, + endMs: 3_980, + confidence: 0.9959265, + }, + { + text: "Uh,", + startMs: 5_760, + endMs: 7_240, + confidence: 0.9857637, + }, + ]); + }); + + it("recognizes exact filler variants without substring false positives", () => { + for (const filler of [ + "um", + "Umm,", + "uh", + "uhhh!", + "erm", + "ermm", + "er", + "ah", + "ahh", + "hmm", + ]) { + expect(isFillerWord(filler), filler).toBe(true); + } + + for (const word of [ + "umbrella", + "term", + "human", + "ahead", + "hammer", + "m", + "mhm", + "huh", + "uh-huh", + ]) { + expect(isFillerWord(word), word).toBe(false); + } + }); + + it("normalizes forward and backward word selections", () => { + expect(normalizeTranscriptSelection(2, 5, 8)).toEqual({ + startIndex: 2, + endIndex: 5, + }); + expect(normalizeTranscriptSelection(5, 2, 8)).toEqual({ + startIndex: 2, + endIndex: 5, + }); + expect(normalizeTranscriptSelection(-1, 2, 8)).toBeNull(); + expect(normalizeTranscriptSelection(2, 8, 8)).toBeNull(); + }); + + it("uses bounded silence padding for a selected word", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const plan = planTranscriptCut(transcript.words, 6, 6, 4_000); + + expect(plan).toEqual({ + startMs: 2_220, + endMs: 2_635, + safe: true, + reason: null, + }); + }); + + it("marks a cut unsafe when unselected speech overlaps it", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-2", + words: [ + { text: "speaker one", start: 100, end: 500, speaker: "A" }, + { text: "um", start: 200, end: 350, speaker: "B" }, + { text: "continues", start: 520, end: 800, speaker: "A" }, + ], + }, + 1_000, + ); + + expect(planTranscriptCut(transcript.words, 1, 1, 1_000)).toEqual({ + startMs: 200, + endMs: 350, + safe: false, + reason: "overlapping-speech", + }); + expect(planFillerCuts(transcript.words, 1_000)).toEqual({ + ranges: [], + fillerCount: 1, + skippedCount: 1, + }); + }); + + it("plans batch filler cuts in chronological order", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const plan = planFillerCuts(transcript.words, 4_000); + + expect(plan).toEqual({ + ranges: [ + { + startMs: 60, + endMs: 450, + safe: true, + reason: null, + }, + { + startMs: 2_220, + endMs: 2_635, + safe: true, + reason: null, + }, + ], + fillerCount: 2, + skippedCount: 0, + }); + }); + + it("plans exact safe cuts for the real missing-filler response", () => { + const transcript = createEditTranscript( + assemblyAIRealFillerResponse, + 8_000, + ); + + expect(planFillerCuts(transcript.words, transcript.durationMs)).toEqual({ + ranges: [ + { + startMs: 3_380, + endMs: 3_980, + safe: true, + reason: null, + }, + { + startMs: 5_760, + endMs: 7_240, + safe: true, + reason: null, + }, + ], + fillerCount: 2, + skippedCount: 0, + }); + }); + + it("groups words linearly and finds the active word with binary search", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const groups = groupEditTranscriptWords(transcript.words, 4); + + expect(groups.map((group) => [group.startIndex, group.endIndex])).toEqual([ + [0, 3], + [4, 5], + [6, 8], + ]); + expect(findActiveTranscriptWordIndex(transcript.words, 750)).toBe(2); + expect(findActiveTranscriptWordIndex(transcript.words, 2_000)).toBe(-1); + }); + + it("finds an earlier active speaker beneath completed overlapping words", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-2", + words: [ + { text: "long", start: 0, end: 1_000, speaker: "A" }, + { text: "short", start: 100, end: 200, speaker: "B" }, + { text: "later", start: 300, end: 400, speaker: "B" }, + ], + }, + 1_200, + ); + + expect(findActiveTranscriptWordIndex(transcript.words, 500)).toBe(0); + }); + + it("derives deleted word presentation from timeline keep ranges", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const deleted = getDeletedTranscriptWordIds(transcript.words, [ + { start: 0.45, end: 2.22 }, + { start: 2.635, end: 4 }, + ]); + + expect([...deleted]).toEqual(["word-0-120-380", "word-6-2300-2570"]); + }); + + it("handles a 50,000 word transcript without quadratic grouping", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-2", + words: Array.from({ length: 50_000 }, (_, index) => ({ + text: index % 100 === 0 ? "um" : `word${index}`, + start: index * 300, + end: index * 300 + 200, + })), + }, + 15_000_000, + ); + + expect(groupEditTranscriptWords(transcript.words)).toHaveLength(1_042); + expect( + planFillerCuts(transcript.words, transcript.durationMs).fillerCount, + ).toBe(500); + expect(findActiveTranscriptWordIndex(transcript.words, 14_999_900)).toBe( + 49_999, + ); + }); + + it("rejects malformed stored sidecars", () => { + expect(parseEditTranscript("not json")).toBeNull(); + expect( + parseEditTranscript( + JSON.stringify({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 1000, + languageCode: "en", + words: [{ id: "bad", text: "bad", startMs: 900, endMs: 1100 }], + }), + ), + ).toBeNull(); + expect( + parseEditTranscript( + JSON.stringify({ + version: 3, + speechModelUsed: "", + durationMs: 1000, + languageCode: "en", + words: [], + }), + ), + ).toBeNull(); + }); + + it("rejects sidecars from previous schema versions and accepts any model", () => { + for (const version of [1, 2]) { + expect( + parseEditTranscript( + JSON.stringify({ + version, + speechModelUsed: "universal-2", + durationMs: 1000, + languageCode: "en", + words: [], + }), + ), + ).toBeNull(); + } + + expect( + parseEditTranscript( + JSON.stringify({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 1000, + languageCode: "en", + words: [], + }), + ), + ).toMatchObject({ version: 3, speechModelUsed: "universal-3-5-pro" }); + }); +}); + +describe("remapEditTranscriptThroughSpec", () => { + it("leaves word timing untouched for an identity spec", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const remapped = remapEditTranscriptThroughSpec( + transcript, + createIdentityEditSpec(4), + ); + + expect(remapped.durationMs).toBe(4_000); + expect(remapped.words).toEqual(transcript.words); + expect(remapped.speechModelUsed).toBe(transcript.speechModelUsed); + expect(remapped.languageCode).toBe(transcript.languageCode); + }); + + it("drops words whose midpoint falls inside a cut and shifts the rest", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(4, [ + { start: 0.45, end: 2.22 }, + { start: 2.635, end: 4 }, + ]), + ); + + expect(remapped.durationMs).toBe(3_135); + expect( + remapped.words.map((word) => [word.id, word.startMs, word.endMs]), + ).toEqual([ + ["word-1-520-720", 70, 270], + ["word-2-730-840", 280, 390], + ["word-3-850-910", 400, 460], + ["word-4-930-1130", 480, 680], + ["word-5-1140-1510", 690, 1_060], + ["word-7-2700-3070", 1_835, 2_205], + ["word-8-3090-3400", 2_225, 2_535], + ]); + }); + + it("clamps words that straddle a cut boundary instead of dropping them", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-3-5-pro", + words: [ + { text: "straddles", start: 800, end: 1_400 }, + { text: "inside", start: 2_000, end: 2_200 }, + ], + }, + 4_000, + ); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(4, [{ start: 1, end: 3 }]), + ); + + expect(remapped.durationMs).toBe(2_000); + expect(remapped.words).toHaveLength(2); + // midpoint 1.1s is kept, and the leading 200ms outside the range is clamped + expect(remapped.words[0]).toMatchObject({ + text: "straddles", + startMs: 0, + endMs: 400, + }); + expect(remapped.words[1]).toMatchObject({ + text: "inside", + startMs: 1_000, + endMs: 1_200, + }); + }); + + it("accumulates offsets across multiple keep ranges", () => { + const transcript = createEditTranscript( + { + speech_model_used: "universal-3-5-pro", + words: [ + { text: "one", start: 100, end: 300 }, + { text: "two", start: 2_100, end: 2_400 }, + { text: "three", start: 4_100, end: 4_500 }, + { text: "cut", start: 5_200, end: 5_400 }, + ], + }, + 6_000, + ); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(6, [ + { start: 0, end: 1 }, + { start: 2, end: 3 }, + { start: 4, end: 5 }, + ]), + ); + + expect(remapped.durationMs).toBe(3_000); + expect( + remapped.words.map((word) => [word.text, word.startMs, word.endMs]), + ).toEqual([ + ["one", 100, 300], + ["two", 1_100, 1_400], + ["three", 2_100, 2_500], + ]); + }); + + it("preserves ids and word metadata while remapping", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(4, [{ start: 2, end: 4 }]), + ); + const original = transcript.words.find( + (word) => word.id === "word-7-2700-3070", + ); + const moved = remapped.words.find((word) => word.id === "word-7-2700-3070"); + + expect(moved).toMatchObject({ + text: original?.text, + confidence: original?.confidence, + speaker: original?.speaker, + channel: original?.channel, + startMs: 700, + endMs: 1_070, + }); + }); + + it("drops every word when the spec keeps nothing", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(4, []), + ); + + expect(remapped).toMatchObject({ durationMs: 0, words: [] }); + }); + + it("keeps remapped words sorted and parseable", () => { + const transcript = createEditTranscript( + assemblyAIRealFillerResponse, + 8_000, + ); + const remapped = remapEditTranscriptThroughSpec( + transcript, + editSpec(8, [ + { start: 1, end: 3.3 }, + { start: 4, end: 8 }, + ]), + ); + + expect(remapped.durationMs).toBe(6_300); + for (let index = 1; index < remapped.words.length; index++) { + const previous = remapped.words[index - 1]; + const current = remapped.words[index]; + expect(current?.startMs).toBeGreaterThanOrEqual(previous?.startMs ?? 0); + expect(current?.endMs).toBeGreaterThan(current?.startMs ?? 0); + expect(current?.endMs).toBeLessThanOrEqual(remapped.durationMs); + } + expect(parseEditTranscript(serializeEditTranscript(remapped))).toEqual( + remapped, + ); + }); +}); + +describe("editTranscriptWordsToCaptionVtt", () => { + it("strips fillers from captions built off the verbatim words", () => { + const transcript = createEditTranscript(assemblyAIEditResponse, 4_000); + const vtt = editTranscriptWordsToCaptionVtt(transcript.words); + + expect(vtt.startsWith("WEBVTT\n\n")).toBe(true); + expect(vtt).not.toContain("Um,"); + expect(vtt).not.toContain("Erm"); + expect(vtt).toContain("this is a real example."); + expect(vtt).toContain("00:00:00.520 --> 00:00:01.510"); + expect(vtt).toContain("nothing else."); + }); + + it("returns a header-only VTT when there are no words", () => { + expect(editTranscriptWordsToCaptionVtt([])).toBe("WEBVTT\n\n"); + }); +}); diff --git a/apps/web/lib/edit-transcript.ts b/apps/web/lib/edit-transcript.ts new file mode 100644 index 0000000000..cf06b9ae4c --- /dev/null +++ b/apps/web/lib/edit-transcript.ts @@ -0,0 +1,678 @@ +import type { VideoEditRange, VideoEditSpec } from "@cap/database/types"; +import { formatToWebVTT } from "@/lib/transcribe-utils"; +import { + getEditSpecOutputDuration, + mapSourceTimeToOutputTime, + normalizeKeepRanges, +} from "@/lib/video-edits"; + +/** + * Stored word transcripts are always expressed in the ORIGINAL media timeline + * (the mp4 preserved at `videoEdits.sourceKey`) and are immutable across edits. + * Everything shown for the current output is derived by remapping the stored + * words through the video's cumulative edit spec. + */ +export const EDIT_TRANSCRIPT_VERSION = 3; +export const EDIT_TRANSCRIPT_KEY_SUFFIX = "transcription.edit.v3.json"; +export const EDIT_TRANSCRIPT_STATUS_KEY_SUFFIX = + "transcription.edit.v3.status.json"; + +export type EditTranscriptBackfillStatus = { + status: "processing" | "error"; + requestId: string; + requestedAt: string; +}; + +const MAX_CUT_PADDING_MS = 80; +const MIN_PLAYABLE_DURATION_MS = 50; +const UNKNOWN_SPEECH_MODEL = "unknown"; + +export type EditTranscriptWord = { + id: string; + text: string; + startMs: number; + endMs: number; + confidence: number | null; + speaker: string | null; + channel: string | null; +}; + +export type EditTranscript = { + version: typeof EDIT_TRANSCRIPT_VERSION; + speechModelUsed: string; + durationMs: number; + languageCode: string | null; + words: EditTranscriptWord[]; +}; + +export type EditTranscriptGroup = { + id: string; + startIndex: number; + endIndex: number; + startMs: number; + endMs: number; +}; + +export type TranscriptCutPlan = { + startMs: number; + endMs: number; + safe: boolean; + reason: "overlapping-speech" | "no-playable-output" | null; +}; + +export type FillerCutPlan = { + ranges: TranscriptCutPlan[]; + fillerCount: number; + skippedCount: number; +}; + +export type AssemblyAIEditWord = { + text?: unknown; + start?: unknown; + end?: unknown; + confidence?: unknown; + speaker?: unknown; + channel?: unknown; +}; + +export type AssemblyAIEditResult = { + words?: readonly AssemblyAIEditWord[] | null; + language_code?: unknown; + speech_model_used?: unknown; +}; + +const wordMaxEndIndexes = new WeakMap< + readonly EditTranscriptWord[], + { endMs: number; index: number }[] +>(); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function finiteNumber(value: unknown) { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function nullableString(value: unknown) { + if (typeof value === "string" && value.length > 0) return value; + if (typeof value === "number" && Number.isFinite(value)) return `${value}`; + return null; +} + +function clampMilliseconds(value: number, durationMs: number) { + return Math.min(Math.max(Math.round(value), 0), durationMs); +} + +function normalizeToken(text: string) { + return text + .trim() + .toLocaleLowerCase() + .replace(/^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu, ""); +} + +function isExactFillerToken(token: string) { + return ( + /^u+m+$/.test(token) || + /^u+h+$/.test(token) || + /^e+rm+$/.test(token) || + token === "er" || + /^a+h+$/.test(token) || + /^h+m+$/.test(token) + ); +} + +export function isFillerWord(text: string) { + return isExactFillerToken(normalizeToken(text)); +} + +export function createEditTranscript( + result: AssemblyAIEditResult, + durationMs: number, +): EditTranscript { + const safeDurationMs = + Number.isFinite(durationMs) && durationMs > 0 ? Math.round(durationMs) : 0; + const candidates = (result.words ?? []) + .map((word, originalIndex) => { + const text = typeof word.text === "string" ? word.text.trim() : ""; + const rawStart = finiteNumber(word.start); + const rawEnd = finiteNumber(word.end); + if (!text || rawStart === null || rawEnd === null) return null; + + const startMs = clampMilliseconds( + Math.min(rawStart, rawEnd), + safeDurationMs, + ); + const endMs = clampMilliseconds( + Math.max(rawStart, rawEnd), + safeDurationMs, + ); + if (endMs <= startMs) return null; + + const rawConfidence = finiteNumber(word.confidence); + return { + originalIndex, + text, + startMs, + endMs, + confidence: + rawConfidence === null + ? null + : Math.min(Math.max(rawConfidence, 0), 1), + speaker: nullableString(word.speaker), + channel: nullableString(word.channel), + }; + }) + .filter((word): word is NonNullable => word !== null) + .sort( + (left, right) => + left.startMs - right.startMs || + left.endMs - right.endMs || + left.originalIndex - right.originalIndex, + ); + + return { + version: EDIT_TRANSCRIPT_VERSION, + speechModelUsed: + typeof result.speech_model_used === "string" && + result.speech_model_used.length > 0 + ? result.speech_model_used + : UNKNOWN_SPEECH_MODEL, + durationMs: safeDurationMs, + languageCode: nullableString(result.language_code), + words: candidates.map( + ({ originalIndex: _originalIndex, ...word }, index) => ({ + ...word, + id: `word-${index}-${word.startMs}-${word.endMs}`, + }), + ), + }; +} + +export function serializeEditTranscript(transcript: EditTranscript) { + return JSON.stringify(transcript); +} + +export function parseEditTranscript(value: string): EditTranscript | null { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return null; + } + + if ( + !isRecord(parsed) || + parsed.version !== EDIT_TRANSCRIPT_VERSION || + typeof parsed.speechModelUsed !== "string" || + parsed.speechModelUsed.length === 0 || + typeof parsed.durationMs !== "number" || + !Number.isFinite(parsed.durationMs) || + parsed.durationMs <= 0 || + !Array.isArray(parsed.words) + ) { + return null; + } + + const languageCode = + parsed.languageCode === null || typeof parsed.languageCode === "string" + ? parsed.languageCode + : null; + const words: EditTranscriptWord[] = []; + + for (const value of parsed.words) { + if ( + !isRecord(value) || + typeof value.id !== "string" || + typeof value.text !== "string" || + typeof value.startMs !== "number" || + typeof value.endMs !== "number" || + !Number.isFinite(value.startMs) || + !Number.isFinite(value.endMs) || + value.startMs < 0 || + value.endMs <= value.startMs || + value.endMs > parsed.durationMs + ) { + return null; + } + + words.push({ + id: value.id, + text: value.text, + startMs: value.startMs, + endMs: value.endMs, + confidence: + typeof value.confidence === "number" && + Number.isFinite(value.confidence) + ? Math.min(Math.max(value.confidence, 0), 1) + : null, + speaker: nullableString(value.speaker), + channel: nullableString(value.channel), + }); + } + + for (let index = 1; index < words.length; index++) { + const previous = words[index - 1]; + const current = words[index]; + if ( + !previous || + !current || + current.startMs < previous.startMs || + (current.startMs === previous.startMs && current.endMs < previous.endMs) + ) { + return null; + } + } + + return { + version: EDIT_TRANSCRIPT_VERSION, + speechModelUsed: parsed.speechModelUsed, + durationMs: Math.round(parsed.durationMs), + languageCode, + words, + }; +} + +/** + * Projects a transcript stored in the ORIGINAL media timeline onto the output + * timeline produced by `editSpec`. Words whose midpoint falls inside a cut are + * dropped (same rule as {@link getDeletedTranscriptWordIds}); words that + * straddle a cut boundary are clamped into the keep range they belong to. + */ +export function remapEditTranscriptThroughSpec( + transcript: EditTranscript, + editSpec: VideoEditSpec, +): EditTranscript { + const spec = normalizeKeepRanges( + editSpec.keepRanges, + editSpec.sourceDuration, + ); + const durationMs = Math.round(getEditSpecOutputDuration(spec) * 1000); + const keepRanges = spec.keepRanges; + const words: EditTranscriptWord[] = []; + let rangeIndex = 0; + + for (const word of transcript.words) { + const midpointSeconds = (word.startMs + word.endMs) / 2000; + while ( + rangeIndex < keepRanges.length && + (keepRanges[rangeIndex]?.end ?? 0) < midpointSeconds + ) { + rangeIndex++; + } + + const range = keepRanges[rangeIndex]; + if ( + !range || + midpointSeconds < range.start || + midpointSeconds > range.end + ) { + continue; + } + + const clampedStartSeconds = Math.min( + Math.max(word.startMs / 1000, range.start), + range.end, + ); + const clampedEndSeconds = Math.min( + Math.max(word.endMs / 1000, range.start), + range.end, + ); + const outputStartSeconds = mapSourceTimeToOutputTime( + clampedStartSeconds, + spec, + ); + if (outputStartSeconds === null) continue; + + const startMs = clampMilliseconds(outputStartSeconds * 1000, durationMs); + const endMs = Math.max( + clampMilliseconds( + outputStartSeconds * 1000 + + (clampedEndSeconds - clampedStartSeconds) * 1000, + durationMs, + ), + startMs + 1, + ); + if (endMs > durationMs) continue; + + words.push({ ...word, startMs, endMs }); + } + + words.sort( + (left, right) => left.startMs - right.startMs || left.endMs - right.endMs, + ); + + return { + version: EDIT_TRANSCRIPT_VERSION, + speechModelUsed: transcript.speechModelUsed, + durationMs, + languageCode: transcript.languageCode, + words, + }; +} + +/** + * Caption VTT derived from the stored words. Fillers are stripped so captions + * stay clean even though the stored transcript is verbatim. + */ +export function editTranscriptWordsToCaptionVtt( + words: readonly EditTranscriptWord[], +): string { + return formatToWebVTT({ + words: words + .filter((word) => !isFillerWord(word.text)) + .map((word) => ({ + text: word.text, + start: word.startMs, + end: word.endMs, + })), + }); +} + +export function getEditTranscriptObjectKey(ownerId: string, videoId: string) { + return `${ownerId}/${videoId}/${EDIT_TRANSCRIPT_KEY_SUFFIX}`; +} + +export function getEditTranscriptStatusObjectKey( + ownerId: string, + videoId: string, +) { + return `${ownerId}/${videoId}/${EDIT_TRANSCRIPT_STATUS_KEY_SUFFIX}`; +} + +export function isPrivateEditTranscriptObjectKey(key: string) { + return ( + key.endsWith(`/${EDIT_TRANSCRIPT_KEY_SUFFIX}`) || + key.endsWith(`/${EDIT_TRANSCRIPT_STATUS_KEY_SUFFIX}`) + ); +} + +export function getEditTranscriptBackfillStatus( + metadata: unknown, +): EditTranscriptBackfillStatus | null { + if (!isRecord(metadata)) return null; + const status = metadata.editTranscriptBackfill; + if ( + !isRecord(status) || + (status.status !== "processing" && status.status !== "error") || + typeof status.requestId !== "string" || + status.requestId.length === 0 || + typeof status.requestedAt !== "string" || + status.requestedAt.length === 0 + ) { + return null; + } + + return { + status: status.status, + requestId: status.requestId, + requestedAt: status.requestedAt, + }; +} + +export function groupEditTranscriptWords( + words: readonly EditTranscriptWord[], + maxWords = 48, +): EditTranscriptGroup[] { + if (words.length === 0) return []; + + const groups: EditTranscriptGroup[] = []; + let startIndex = 0; + + for (let index = 0; index < words.length; index++) { + const word = words[index]; + const nextWord = words[index + 1]; + if (!word) continue; + + const reachedLimit = index - startIndex + 1 >= maxWords; + const punctuationBreak = /[.!?]$/.test(word.text); + const silenceBreak = nextWord ? nextWord.startMs - word.endMs >= 800 : true; + const speakerBreak = + nextWord !== undefined && + word.speaker !== null && + nextWord.speaker !== null && + word.speaker !== nextWord.speaker; + + if ( + nextWord && + !reachedLimit && + !punctuationBreak && + !silenceBreak && + !speakerBreak + ) { + continue; + } + + groups.push({ + id: `group-${groups.length}-${word.startMs}`, + startIndex, + endIndex: index, + startMs: words[startIndex]?.startMs ?? word.startMs, + endMs: word.endMs, + }); + startIndex = index + 1; + } + + return groups; +} + +export function normalizeTranscriptSelection( + firstIndex: number, + secondIndex: number, + wordCount: number, +) { + if ( + !Number.isInteger(firstIndex) || + !Number.isInteger(secondIndex) || + wordCount <= 0 + ) { + return null; + } + + const startIndex = Math.min(firstIndex, secondIndex); + const endIndex = Math.max(firstIndex, secondIndex); + if (startIndex < 0 || endIndex >= wordCount) return null; + return { startIndex, endIndex }; +} + +export function findActiveTranscriptWordIndex( + words: readonly EditTranscriptWord[], + timeMs: number, +) { + if (!Number.isFinite(timeMs) || words.length === 0) return -1; + + let low = 0; + let high = words.length - 1; + let candidate = -1; + + while (low <= high) { + const middle = Math.floor((low + high) / 2); + const word = words[middle]; + if (!word) break; + if (word.startMs <= timeMs) { + candidate = middle; + low = middle + 1; + } else { + high = middle - 1; + } + } + + if (candidate < 0) return -1; + + const candidateWord = words[candidate]; + if (candidateWord && candidateWord.endMs >= timeMs) return candidate; + + let maxEndIndexes = wordMaxEndIndexes.get(words); + if (!maxEndIndexes) { + maxEndIndexes = []; + let maximum = { endMs: Number.NEGATIVE_INFINITY, index: -1 }; + for (let index = 0; index < words.length; index++) { + const word = words[index]; + if (word && word.endMs > maximum.endMs) { + maximum = { endMs: word.endMs, index }; + } + maxEndIndexes.push(maximum); + } + wordMaxEndIndexes.set(words, maxEndIndexes); + } + + const active = maxEndIndexes[candidate]; + return active && active.endMs >= timeMs ? active.index : -1; +} + +function getCutPadding(gapMs: number) { + return Math.max(0, Math.min(MAX_CUT_PADDING_MS, Math.floor(gapMs / 2))); +} + +function overlaps(startMs: number, endMs: number, word: EditTranscriptWord) { + return word.startMs < endMs && word.endMs > startMs; +} + +export function planTranscriptCut( + words: readonly EditTranscriptWord[], + firstIndex: number, + secondIndex: number, + durationMs: number, +): TranscriptCutPlan | null { + const selection = normalizeTranscriptSelection( + firstIndex, + secondIndex, + words.length, + ); + if (!selection || !Number.isFinite(durationMs) || durationMs <= 0) + return null; + + const firstWord = words[selection.startIndex]; + const lastWord = words[selection.endIndex]; + if (!firstWord || !lastWord) return null; + + const hasOutsideOverlap = words.some( + (word, index) => + (index < selection.startIndex || index > selection.endIndex) && + overlaps(firstWord.startMs, lastWord.endMs, word), + ); + if (hasOutsideOverlap) { + return { + startMs: firstWord.startMs, + endMs: lastWord.endMs, + safe: false, + reason: "overlapping-speech", + }; + } + + const previousWord = words[selection.startIndex - 1]; + const nextWord = words[selection.endIndex + 1]; + const startPadding = getCutPadding( + firstWord.startMs - (previousWord?.endMs ?? 0), + ); + const endPadding = getCutPadding( + (nextWord?.startMs ?? durationMs) - lastWord.endMs, + ); + const startMs = Math.max(0, firstWord.startMs - startPadding); + const endMs = Math.min(durationMs, lastWord.endMs + endPadding); + const leavesPlayableOutput = + startMs >= MIN_PLAYABLE_DURATION_MS || + durationMs - endMs >= MIN_PLAYABLE_DURATION_MS; + + return { + startMs, + endMs, + safe: leavesPlayableOutput, + reason: leavesPlayableOutput ? null : "no-playable-output", + }; +} + +export function planFillerCuts( + words: readonly EditTranscriptWord[], + durationMs: number, +): FillerCutPlan { + const ranges: TranscriptCutPlan[] = []; + let skippedCount = 0; + let previousMaxEnd = 0; + + for (let index = 0; index < words.length; index++) { + const word = words[index]; + if (!word) continue; + + if (isFillerWord(word.text)) { + const nextWord = words[index + 1]; + const overlappingSpeech = + previousMaxEnd > word.startMs || + (nextWord !== undefined && nextWord.startMs < word.endMs); + + if (overlappingSpeech) { + skippedCount++; + } else { + const startPadding = getCutPadding(word.startMs - previousMaxEnd); + const endPadding = getCutPadding( + (nextWord?.startMs ?? durationMs) - word.endMs, + ); + ranges.push({ + startMs: Math.max(0, word.startMs - startPadding), + endMs: Math.min(durationMs, word.endMs + endPadding), + safe: true, + reason: null, + }); + } + } + + previousMaxEnd = Math.max(previousMaxEnd, word.endMs); + } + + const mergedRanges: TranscriptCutPlan[] = []; + for (const range of ranges) { + const previous = mergedRanges.at(-1); + if (previous && range.startMs <= previous.endMs) { + previous.endMs = Math.max(previous.endMs, range.endMs); + } else { + mergedRanges.push({ ...range }); + } + } + + const totalDeletedMs = mergedRanges.reduce( + (total, range) => total + range.endMs - range.startMs, + 0, + ); + if (durationMs - totalDeletedMs < MIN_PLAYABLE_DURATION_MS) { + return { + ranges: [], + fillerCount: ranges.length + skippedCount, + skippedCount: ranges.length + skippedCount, + }; + } + + return { + ranges: mergedRanges, + fillerCount: ranges.length + skippedCount, + skippedCount, + }; +} + +export function getDeletedTranscriptWordIds( + words: readonly EditTranscriptWord[], + keepRanges: readonly VideoEditRange[], +) { + const deletedIds = new Set(); + let rangeIndex = 0; + + for (const word of words) { + const midpointSeconds = (word.startMs + word.endMs) / 2000; + while ( + rangeIndex < keepRanges.length && + (keepRanges[rangeIndex]?.end ?? 0) < midpointSeconds + ) { + rangeIndex++; + } + + const range = keepRanges[rangeIndex]; + if ( + !range || + midpointSeconds < range.start || + midpointSeconds > range.end + ) { + deletedIds.add(word.id); + } + } + + return deletedIds; +} From 66a4cc212983bac09af90307c9e9bc38e436808d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:16 +0100 Subject: [PATCH 05/20] feat(web): encrypt stored edit transcript objects --- .../unit/edit-transcript-storage.test.ts | 58 ++++++++++++++ apps/web/lib/edit-transcript-storage.ts | 80 +++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 apps/web/__tests__/unit/edit-transcript-storage.test.ts create mode 100644 apps/web/lib/edit-transcript-storage.ts diff --git a/apps/web/__tests__/unit/edit-transcript-storage.test.ts b/apps/web/__tests__/unit/edit-transcript-storage.test.ts new file mode 100644 index 0000000000..60cb23b29d --- /dev/null +++ b/apps/web/__tests__/unit/edit-transcript-storage.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + NEXTAUTH_SECRET: "test-secret-with-enough-entropy", + }), +})); + +vi.mock("server-only", () => ({})); + +describe("edit transcript storage", () => { + it("encrypts and authenticates transcript objects for one video", async () => { + const { decryptEditTranscriptObject, encryptEditTranscriptObject } = + await import("@/lib/edit-transcript-storage"); + const plaintext = JSON.stringify({ + words: [{ text: "private deleted phrase" }], + }); + + const encrypted = encryptEditTranscriptObject( + plaintext, + "owner-1", + "video-1", + ); + + expect(encrypted).not.toContain("private deleted phrase"); + expect(decryptEditTranscriptObject(encrypted, "owner-1", "video-1")).toBe( + plaintext, + ); + expect( + decryptEditTranscriptObject(encrypted, "owner-1", "video-2"), + ).toBeNull(); + }); + + it("rejects modified and plaintext objects", async () => { + const { decryptEditTranscriptObject, encryptEditTranscriptObject } = + await import("@/lib/edit-transcript-storage"); + const encrypted = encryptEditTranscriptObject( + '{"words":[]}', + "owner-1", + "video-1", + ); + const parts = encrypted.split("."); + const ciphertext = parts[3] ?? ""; + parts[3] = `${ciphertext.startsWith("A") ? "B" : "A"}${ciphertext.slice(1)}`; + const tampered = parts.join("."); + + expect( + decryptEditTranscriptObject(tampered, "owner-1", "video-1"), + ).toBeNull(); + expect( + decryptEditTranscriptObject( + '{"words":[{"text":"plaintext"}]}', + "owner-1", + "video-1", + ), + ).toBeNull(); + }); +}); diff --git a/apps/web/lib/edit-transcript-storage.ts b/apps/web/lib/edit-transcript-storage.ts new file mode 100644 index 0000000000..cc62b839bd --- /dev/null +++ b/apps/web/lib/edit-transcript-storage.ts @@ -0,0 +1,80 @@ +import "server-only"; + +import { + createCipheriv, + createDecipheriv, + createHmac, + randomBytes, +} from "node:crypto"; +import { serverEnv } from "@cap/env"; + +const FORMAT_VERSION = "cap-edit-transcript-v1"; +const IV_BYTES = 12; +const AUTH_TAG_BYTES = 16; + +function getEncryptionKey() { + return createHmac("sha256", serverEnv().NEXTAUTH_SECRET) + .update(FORMAT_VERSION) + .digest(); +} + +function getAdditionalData(ownerId: string, videoId: string) { + return Buffer.from(`${FORMAT_VERSION}:${ownerId}:${videoId}`); +} + +export function encryptEditTranscriptObject( + value: string, + ownerId: string, + videoId: string, +) { + const iv = randomBytes(IV_BYTES); + const cipher = createCipheriv("aes-256-gcm", getEncryptionKey(), iv); + cipher.setAAD(getAdditionalData(ownerId, videoId)); + const ciphertext = Buffer.concat([ + cipher.update(value, "utf8"), + cipher.final(), + ]); + + return [ + FORMAT_VERSION, + iv.toString("base64url"), + cipher.getAuthTag().toString("base64url"), + ciphertext.toString("base64url"), + ].join("."); +} + +export function decryptEditTranscriptObject( + value: string, + ownerId: string, + videoId: string, +) { + const [version, encodedIv, encodedTag, encodedCiphertext, extra] = + value.split("."); + if ( + version !== FORMAT_VERSION || + !encodedIv || + !encodedTag || + !encodedCiphertext || + extra !== undefined + ) { + return null; + } + + try { + const iv = Buffer.from(encodedIv, "base64url"); + const authTag = Buffer.from(encodedTag, "base64url"); + if (iv.length !== IV_BYTES || authTag.length !== AUTH_TAG_BYTES) { + return null; + } + + const decipher = createDecipheriv("aes-256-gcm", getEncryptionKey(), iv); + decipher.setAAD(getAdditionalData(ownerId, videoId)); + decipher.setAuthTag(authTag); + return Buffer.concat([ + decipher.update(Buffer.from(encodedCiphertext, "base64url")), + decipher.final(), + ]).toString("utf8"); + } catch { + return null; + } +} From a61a8f6171b30741a68fc52389d8179d1c687268 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:16 +0100 Subject: [PATCH 06/20] feat(web): enable AssemblyAI disfluencies in transcription options --- apps/web/__tests__/unit/transcribe-language.test.ts | 2 ++ apps/web/lib/assemblyai.ts | 3 +++ 2 files changed, 5 insertions(+) diff --git a/apps/web/__tests__/unit/transcribe-language.test.ts b/apps/web/__tests__/unit/transcribe-language.test.ts index 505c4d1bfe..c315acb732 100644 --- a/apps/web/__tests__/unit/transcribe-language.test.ts +++ b/apps/web/__tests__/unit/transcribe-language.test.ts @@ -27,6 +27,7 @@ describe("AI generation language support", () => { speech_models: ["universal-3-5-pro", "universal-2"], language_detection: true, }); + expect(options.disfluencies).toBe(true); if (!("language_detection_options" in options)) { throw new Error("Expected automatic language detection options"); } @@ -39,6 +40,7 @@ describe("AI generation language support", () => { expect(getAssemblyAITranscriptionOptions("ro")).toMatchObject({ speech_models: ["universal-3-5-pro", "universal-2"], language_code: "ro", + disfluencies: true, }); expect(getAssemblyAITranscriptionOptions("zh")).toMatchObject({ speech_models: ["universal-3-5-pro", "universal-2"], diff --git a/apps/web/lib/assemblyai.ts b/apps/web/lib/assemblyai.ts index 0a00797ece..0c743dc0f3 100644 --- a/apps/web/lib/assemblyai.ts +++ b/apps/web/lib/assemblyai.ts @@ -44,6 +44,9 @@ export function getAssemblyAITranscriptionOptions( speech_models: [...ASSEMBLYAI_SPEECH_MODELS], format_text: true, punctuate: true, + // Verbatim words: the single transcription pass feeds both the word-level + // edit transcript and the caption VTT (which strips fillers itself). + disfluencies: true, }; if (language === AI_GENERATION_LANGUAGE_AUTO) { From 6bf461c5bd0b4fa90f8322f6137e6c0978b562f3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:16 +0100 Subject: [PATCH 07/20] feat(web): add batch timeline range deletion helpers --- apps/web/__tests__/unit/video-edits.test.ts | 57 +++++++++++++++ apps/web/lib/video-edits.ts | 80 ++++++++++++++++++--- 2 files changed, 128 insertions(+), 9 deletions(-) diff --git a/apps/web/__tests__/unit/video-edits.test.ts b/apps/web/__tests__/unit/video-edits.test.ts index 3fe7a6de49..ff3fb5cbf3 100644 --- a/apps/web/__tests__/unit/video-edits.test.ts +++ b/apps/web/__tests__/unit/video-edits.test.ts @@ -7,8 +7,11 @@ import { createTimelineHistory, createTimelineState, deleteSelectedTimelineSegment, + deleteTimelineRanges, dragTimelineDisplaySplitPoint, findNextPlayableTime, + findNextPlayableTimeInRanges, + findPlayableRangeIndex, getTimelineDisplayDuration, getTimelineDisplaySegments, getTimelineDisplaySplitPoints, @@ -126,6 +129,46 @@ describe("video edit specs", () => { }); describe("timeline editing", () => { + it("deletes transcript ranges as one normalized timeline change", () => { + const nextState = deleteTimelineRanges(createTimelineState(10), [ + { start: 1.02, end: 1.35 }, + { start: 4.1, end: 4.4 }, + { start: 4.35, end: 4.7 }, + ]); + + expect(getTimelineKeepRanges(nextState)).toEqual([ + { start: 0, end: 1.02 }, + { start: 1.35, end: 4.1 }, + { start: 4.7, end: 10 }, + ]); + expect(nextState.splitPoints).toEqual([1.02, 1.35, 4.1, 4.7]); + }); + + it("does not allow transcript deletion to remove all playable video", () => { + const state = createTimelineState(10); + expect(deleteTimelineRanges(state, [{ start: 0, end: 10 }])).toEqual(state); + }); + + it("stores a batch transcript deletion as one undoable history entry", () => { + const initialState = createTimelineState(10); + const history = createTimelineHistory(initialState); + const deleted = deleteTimelineRanges(initialState, [ + { start: 1, end: 1.3 }, + { start: 3, end: 3.4 }, + ]); + const changedHistory = pushTimelineHistory(history, deleted); + + expect(changedHistory.entries).toHaveLength(2); + expect( + getTimelineKeepRanges(changedHistory.entries[1] ?? initialState), + ).toEqual([ + { start: 0, end: 1 }, + { start: 1.3, end: 3 }, + { start: 3.4, end: 10 }, + ]); + expect(undoTimelineHistory(changedHistory).index).toBe(0); + }); + it("splits, selects, and deletes a segment", () => { const splitState = splitTimelineAt(createTimelineState(10), 4); const firstSegment = getTimelineSegments(splitState)[0]; @@ -434,4 +477,18 @@ describe("timeline editing", () => { expect(findNextPlayableTime(3, spec)).toBe(5); expect(findNextPlayableTime(8.1, spec)).toBeNull(); }); + + it("finds preview skip targets in normalized ranges without rescanning them", () => { + const ranges = Array.from({ length: 1000 }, (_, index) => ({ + start: index * 2, + end: index * 2 + 1, + })); + + expect(findNextPlayableTimeInRanges(998.5, ranges)).toBe(998.5); + expect(findNextPlayableTimeInRanges(999, ranges)).toBe(1000); + expect(findNextPlayableTimeInRanges(1999, ranges)).toBeNull(); + expect(findNextPlayableTimeInRanges(Number.NaN, ranges)).toBeNull(); + expect(findPlayableRangeIndex(998.5, ranges)).toBe(499); + expect(findPlayableRangeIndex(999, ranges)).toBe(-1); + }); }); diff --git a/apps/web/lib/video-edits.ts b/apps/web/lib/video-edits.ts index 04a9607531..94ce746eea 100644 --- a/apps/web/lib/video-edits.ts +++ b/apps/web/lib/video-edits.ts @@ -763,6 +763,37 @@ export function deleteSelectedTimelineSegment( }); } +export function deleteTimelineRanges( + state: VideoTimelineState, + ranges: readonly VideoEditRange[], +): VideoTimelineState { + const normalized = normalizeTimelineState(state); + const deletionRanges = normalizeKeepRanges( + ranges.map((range) => ({ + start: clampEditTime( + range.start, + normalized.trimStart, + normalized.trimEnd, + ), + end: clampEditTime(range.end, normalized.trimStart, normalized.trimEnd), + })), + normalized.duration, + ).keepRanges; + if (deletionRanges.length === 0) return normalized; + + const nextState = normalizeTimelineState({ + ...normalized, + splitPoints: [ + ...normalized.splitPoints, + ...deletionRanges.flatMap((range) => [range.start, range.end]), + ], + deletedRanges: [...normalized.deletedRanges, ...deletionRanges], + selectedSegmentId: null, + }); + + return getTimelineKeepRanges(nextState).length > 0 ? nextState : normalized; +} + export function setTimelineTrim( state: VideoTimelineState, start: number, @@ -967,19 +998,50 @@ export function findNextPlayableTime( editSpec.keepRanges, editSpec.sourceDuration, ); - if (normalized.keepRanges.length === 0) return null; + return findNextPlayableTimeInRanges(currentTime, normalized.keepRanges); +} - for (const range of normalized.keepRanges) { - if (currentTime < range.start - EPSILON) return range.start; - if ( - currentTime >= range.start - EPSILON && - currentTime < range.end - EPSILON - ) { - return currentTime; +export function findNextPlayableTimeInRanges( + currentTime: number, + keepRanges: readonly VideoEditRange[], +) { + const rangeIndex = findNextPlayableRangeIndex(currentTime, keepRanges); + if (rangeIndex < 0) return null; + const range = keepRanges[rangeIndex]; + if (!range) return null; + return currentTime < range.start - EPSILON ? range.start : currentTime; +} + +function findNextPlayableRangeIndex( + currentTime: number, + keepRanges: readonly VideoEditRange[], +) { + if (!isFiniteNumber(currentTime) || keepRanges.length === 0) return -1; + + let low = 0; + let high = keepRanges.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + const range = keepRanges[middle]; + if (range && currentTime < range.end - EPSILON) { + high = middle; + } else { + low = middle + 1; } } - return null; + return low < keepRanges.length ? low : -1; +} + +export function findPlayableRangeIndex( + currentTime: number, + keepRanges: readonly VideoEditRange[], +) { + const rangeIndex = findNextPlayableRangeIndex(currentTime, keepRanges); + if (rangeIndex < 0) return -1; + const range = keepRanges[rangeIndex]; + if (!range || currentTime < range.start - EPSILON) return -1; + return rangeIndex; } export function createTimelineHistory( From 30600893d12de9e783eb7281d9ecb398881d2f15 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:22 +0100 Subject: [PATCH 08/20] feat(web): persist immutable word transcripts on transcription --- apps/web/workflows/transcribe.ts | 366 +++++++++++++++++++++++++++++-- 1 file changed, 352 insertions(+), 14 deletions(-) diff --git a/apps/web/workflows/transcribe.ts b/apps/web/workflows/transcribe.ts index 65736fcc7e..761860cd65 100644 --- a/apps/web/workflows/transcribe.ts +++ b/apps/web/workflows/transcribe.ts @@ -1,7 +1,12 @@ import { promises as fs } from "node:fs"; import { db } from "@cap/database"; -import { organizations, videos, videoUploads } from "@cap/database/schema"; -import type { VideoMetadata } from "@cap/database/types"; +import { + organizations, + videoEdits, + videos, + videoUploads, +} from "@cap/database/schema"; +import type { VideoEditSpec, VideoMetadata } from "@cap/database/types"; import { serverEnv } from "@cap/env"; import { Storage } from "@cap/web-backend/src/Storage/index"; import { @@ -10,7 +15,7 @@ import { type Video, } from "@cap/web-domain"; import { AssemblyAI } from "assemblyai"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { FatalError } from "workflow"; import { getAssemblyAITranscriptionOptions } from "@/lib/assemblyai"; import { @@ -19,6 +24,14 @@ import { enhanceAudioFromUrl, } from "@/lib/audio-enhance"; import { checkHasAudioTrack, extractAudioFromUrl } from "@/lib/audio-extract"; +import { + createEditTranscript, + editTranscriptWordsToCaptionVtt, + getEditTranscriptBackfillStatus, + getEditTranscriptObjectKey, + serializeEditTranscript, +} from "@/lib/edit-transcript"; +import { encryptEditTranscriptObject } from "@/lib/edit-transcript-storage"; import { startAiGeneration } from "@/lib/generate-ai"; import { checkHasAudioTrackViaMediaServer, @@ -26,7 +39,6 @@ import { isMediaServerConfigured, probeVideoViaMediaServer, } from "@/lib/media-client"; -import { formatToWebVTT } from "@/lib/transcribe-utils"; import { decodeStorageVideo } from "@/lib/video-storage"; import { runWorkflowPromise } from "@/lib/workflow-runtime"; @@ -42,6 +54,19 @@ interface VideoData { aiGenerationLanguage: AiGenerationLanguage; } +interface BackfillEditTranscriptPayload { + videoId: string; + userId: string; + requestId: string; +} + +const EDIT_TRANSCRIPT_TEMP_AUDIO_FILENAME = "audio-edit-transcript-v3-temp.mp3"; + +interface TranscriptionArtifacts { + vtt: string; + editTranscript: string; +} + export async function transcribeVideoWorkflow( payload: TranscribeWorkflowPayload, ) { @@ -73,9 +98,11 @@ export async function transcribeVideoWorkflow( }; } - const [transcription] = await Promise.all([ - transcribeWithAssemblyAI(audioUrl, videoData.aiGenerationLanguage), - ]); + const transcription = await transcribeWithAssemblyAI( + audioUrl, + videoData.aiGenerationLanguage, + Math.max(0, (videoData.video.duration ?? 0) * 1000), + ); await saveTranscription(videoId, userId, videoData.video, transcription); } catch (error) { @@ -93,6 +120,68 @@ export async function transcribeVideoWorkflow( return { success: true, message: "Transcription completed successfully" }; } +export async function backfillEditTranscriptWorkflow( + payload: BackfillEditTranscriptPayload, +) { + "use workflow"; + + const { videoId, userId, requestId } = payload; + let video: typeof videos.$inferSelect; + + try { + video = await validateEditTranscriptBackfill(videoId, userId, requestId); + // Legacy videos only: the transcript must describe the ORIGINAL media, so + // edited videos are transcribed from their preserved source mp4. + const videoEdit = await loadVideoEditForBackfill(videoId); + const audioUrl = await extractAudio( + videoId, + userId, + video, + EDIT_TRANSCRIPT_TEMP_AUDIO_FILENAME, + videoEdit?.sourceKey, + ); + if (!audioUrl) { + await saveEditTranscriptBackfillError(videoId, requestId, video); + return { success: false }; + } + + const editTranscript = await transcribeEditTranscriptWithAssemblyAI( + audioUrl, + videoEdit ? videoEdit.editSpec.sourceDuration : (video.duration ?? 0), + ); + await saveEditTranscriptBackfill( + videoId, + userId, + requestId, + video, + editTranscript, + ); + await cleanupTempAudio( + videoId, + userId, + video, + EDIT_TRANSCRIPT_TEMP_AUDIO_FILENAME, + ); + return { success: true }; + } catch (error) { + console.error( + `[transcribe] Editable transcript backfill failed for ${videoId}`, + error, + ); + const loadedVideo = await loadVideoForEditTranscriptBackfill(videoId); + if (loadedVideo?.ownerId === userId) { + await saveEditTranscriptBackfillError(videoId, requestId, loadedVideo); + await cleanupTempAudio( + videoId, + userId, + loadedVideo, + EDIT_TRANSCRIPT_TEMP_AUDIO_FILENAME, + ); + } + return { success: false }; + } +} + async function validateVideo(videoId: string): Promise { "use step"; @@ -138,6 +227,67 @@ async function validateVideo(videoId: string): Promise { }; } +async function loadVideoForEditTranscriptBackfill(videoId: string) { + "use step"; + + const [video] = await db() + .select() + .from(videos) + .where(eq(videos.id, videoId as Video.VideoId)); + return video ?? null; +} + +async function loadVideoEditForBackfill( + videoId: string, +): Promise<{ sourceKey: string; editSpec: VideoEditSpec } | null> { + "use step"; + + const [edit] = await db() + .select({ + sourceKey: videoEdits.sourceKey, + editSpec: videoEdits.editSpec, + }) + .from(videoEdits) + .where(eq(videoEdits.videoId, videoId as Video.VideoId)); + + return edit ?? null; +} + +async function validateEditTranscriptBackfill( + videoId: string, + userId: string, + requestId: string, +) { + "use step"; + + if (!serverEnv().ASSEMBLY_API_KEY) { + throw new FatalError("Missing ASSEMBLY_API_KEY"); + } + + const [video] = await db() + .select() + .from(videos) + .where(eq(videos.id, videoId as Video.VideoId)); + if (!video || video.ownerId !== userId) { + throw new FatalError("Video does not exist"); + } + if ( + video.transcriptionStatus !== "COMPLETE" || + video.isScreenshot || + (video.source.type !== "desktopMP4" && video.source.type !== "webMP4") || + !video.duration || + video.duration <= 0 + ) { + throw new FatalError("Transcript editing is not available"); + } + const status = getEditTranscriptBackfillStatus(video.metadata); + if (status?.status !== "processing" || status.requestId !== requestId) { + throw new FatalError("Transcript request expired"); + } + + return video; +} + async function markSkipped(videoId: string): Promise { "use step"; @@ -174,6 +324,8 @@ async function extractAudio( videoId: string, userId: string, video: typeof videos.$inferSelect, + tempAudioFilename = "audio-temp.mp3", + sourceKeyOverride?: string, ): Promise { "use step"; @@ -181,7 +333,12 @@ async function extractAudio( decodeStorageVideo(video), ).pipe(runWorkflowPromise); - const videoUrl = await resolveVideoSourceUrl(videoId, userId, video); + const videoUrl = await resolveVideoSourceUrl( + videoId, + userId, + video, + sourceKeyOverride, + ); const useMediaServer = isMediaServerConfigured(); console.log( @@ -239,7 +396,7 @@ async function extractAudio( `[transcribe] Extracted audio for ${videoId}: ${audioBuffer.length} bytes`, ); - const audioKey = `${userId}/${videoId}/audio-temp.mp3`; + const audioKey = `${userId}/${videoId}/${tempAudioFilename}`; await bucket .putObject(audioKey, audioBuffer, { @@ -258,6 +415,7 @@ async function resolveVideoSourceUrl( videoId: string, userId: string, video: typeof videos.$inferSelect, + sourceKeyOverride?: string, ): Promise { const [resolvedBucket] = await Storage.getAccessForVideo( decodeStorageVideo(video), @@ -270,6 +428,7 @@ async function resolveVideoSourceUrl( .limit(1); const candidateKeys = [ + sourceKeyOverride, `${userId}/${videoId}/result.mp4`, upload[0]?.rawFileKey, ].filter( @@ -298,7 +457,8 @@ async function resolveVideoSourceUrl( async function transcribeWithAssemblyAI( audioUrl: string, language: AiGenerationLanguage, -): Promise { + videoDurationMs: number, +): Promise { "use step"; const audioResponse = await fetch(audioUrl); @@ -316,6 +476,7 @@ async function transcribeWithAssemblyAI( const transcript = await client.transcripts.transcribe({ audio: audioBuffer, ...getAssemblyAITranscriptionOptions(language), + disfluencies: true, }); console.log( @@ -328,14 +489,65 @@ async function transcribeWithAssemblyAI( ); } - return formatToWebVTT(transcript); + // One paid pass produces both artifacts: the immutable word transcript (in + // the original media timeline) and the caption VTT derived from those words. + const durationMs = + videoDurationMs > 0 + ? videoDurationMs + : (transcript.audio_duration ?? 0) * 1000; + const editTranscript = createEditTranscript(transcript, durationMs); + + return { + vtt: editTranscriptWordsToCaptionVtt(editTranscript.words), + editTranscript: serializeEditTranscript(editTranscript), + }; +} + +async function transcribeEditTranscriptWithAssemblyAI( + audioUrl: string, + videoDurationSeconds: number, +): Promise { + "use step"; + + const audioResponse = await fetch(audioUrl); + if (!audioResponse.ok) { + throw new Error( + `Audio URL not accessible: ${audioResponse.status} ${audioResponse.statusText}`, + ); + } + + const audioBuffer = Buffer.from(await audioResponse.arrayBuffer()); + const client = new AssemblyAI({ + apiKey: serverEnv().ASSEMBLY_API_KEY as string, + }); + const transcript = await client.transcripts.transcribe({ + audio: audioBuffer, + ...getAssemblyAITranscriptionOptions("auto"), + disfluencies: true, + }); + + console.log( + `[transcribe] AssemblyAI editable transcript ${transcript.id} finished with status=${transcript.status}, model=${transcript.speech_model_used ?? "unknown"}`, + ); + + if (transcript.status === "error") { + throw new Error( + `AssemblyAI editable transcription failed (id=${transcript.id}): ${transcript.error ?? "Unknown error"}`, + ); + } + + const durationMs = + videoDurationSeconds > 0 + ? videoDurationSeconds * 1000 + : (transcript.audio_duration ?? 0) * 1000; + return serializeEditTranscript(createEditTranscript(transcript, durationMs)); } async function saveTranscription( videoId: string, userId: string, video: typeof videos.$inferSelect, - transcription: string, + transcription: TranscriptionArtifacts, ): Promise { "use step"; @@ -344,25 +556,151 @@ async function saveTranscription( ).pipe(runWorkflowPromise); await bucket - .putObject(`${userId}/${videoId}/transcription.vtt`, transcription, { + .putObject(`${userId}/${videoId}/transcription.vtt`, transcription.vtt, { contentType: "text/vtt", }) .pipe(runWorkflowPromise); + // The word transcript must always describe the ORIGINAL media. This pass ran + // against whatever media is current, so it may only be persisted while the + // video is un-edited; edited videos keep (or backfill) their own transcript. + const [edit] = await db() + .select({ videoId: videoEdits.videoId }) + .from(videoEdits) + .where(eq(videoEdits.videoId, videoId as Video.VideoId)); + + if (edit) { + console.log( + `[transcribe] Skipping edit transcript write for edited video ${videoId}`, + ); + } else { + await bucket + .putObject( + getEditTranscriptObjectKey(userId, videoId), + encryptEditTranscriptObject( + transcription.editTranscript, + userId, + videoId, + ), + { contentType: "application/octet-stream" }, + ) + .pipe(runWorkflowPromise); + } + await db() .update(videos) .set({ transcriptionStatus: "COMPLETE" }) .where(eq(videos.id, videoId as Video.VideoId)); } +async function saveEditTranscriptBackfill( + videoId: string, + userId: string, + requestId: string, + video: typeof videos.$inferSelect, + editTranscript: string, +) { + "use step"; + + const [bucket] = await Storage.getAccessForVideo( + decodeStorageVideo(video), + ).pipe(runWorkflowPromise); + + if (!(await canPersistEditTranscriptBackfill(videoId, requestId, video))) + return; + + await bucket + .putObject( + getEditTranscriptObjectKey(userId, videoId), + encryptEditTranscriptObject(editTranscript, userId, videoId), + { + contentType: "application/octet-stream", + }, + ) + .pipe(runWorkflowPromise); + await clearEditTranscriptBackfill(videoId, requestId); +} + +async function saveEditTranscriptBackfillError( + videoId: string, + requestId: string, + video: typeof videos.$inferSelect, +) { + "use step"; + + if (!(await canPersistEditTranscriptBackfill(videoId, requestId, video))) + return; + + await db() + .update(videos) + .set({ + metadata: sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.editTranscriptBackfill.status', 'error')`, + updatedAt: sql`${videos.updatedAt}`, + }) + .where( + and( + eq(videos.id, videoId as Video.VideoId), + sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.editTranscriptBackfill.requestId')) = ${requestId}`, + ), + ); +} + +async function canPersistEditTranscriptBackfill( + videoId: string, + requestId: string, + video: typeof videos.$inferSelect, +) { + const [[currentVideo], [activeUpload]] = await Promise.all([ + db() + .select({ + duration: videos.duration, + metadata: videos.metadata, + transcriptionStatus: videos.transcriptionStatus, + updatedAt: videos.updatedAt, + }) + .from(videos) + .where(eq(videos.id, videoId as Video.VideoId)), + db() + .select({ phase: videoUploads.phase }) + .from(videoUploads) + .where(eq(videoUploads.videoId, videoId as Video.VideoId)) + .limit(1), + ]); + + return ( + currentVideo?.transcriptionStatus === "COMPLETE" && + currentVideo.duration === video.duration && + currentVideo.updatedAt.getTime() === video.updatedAt.getTime() && + getEditTranscriptBackfillStatus(currentVideo.metadata)?.requestId === + requestId && + !activeUpload + ); +} + +async function clearEditTranscriptBackfill(videoId: string, requestId: string) { + await db() + .update(videos) + .set({ + metadata: sql`JSON_REMOVE(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.editTranscriptBackfill')`, + updatedAt: sql`${videos.updatedAt}`, + }) + .where( + and( + eq(videos.id, videoId as Video.VideoId), + sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.editTranscriptBackfill.requestId')) = ${requestId}`, + ), + ); +} + async function cleanupTempAudio( videoId: string, userId: string, video: typeof videos.$inferSelect, + tempAudioFilename = "audio-temp.mp3", ): Promise { "use step"; - const audioKey = `${userId}/${videoId}/audio-temp.mp3`; + const audioKey = `${userId}/${videoId}/${tempAudioFilename}`; try { const [bucket] = await Storage.getAccessForVideo( From 6dab3a6b314a7ca221a83963a6396e510c428f74 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:22 +0100 Subject: [PATCH 09/20] test(web): cover edit transcript transcription workflows --- .../integration/transcribe-workflow.test.ts | 364 ++++++++++++++++++ 1 file changed, 364 insertions(+) create mode 100644 apps/web/__tests__/integration/transcribe-workflow.test.ts diff --git a/apps/web/__tests__/integration/transcribe-workflow.test.ts b/apps/web/__tests__/integration/transcribe-workflow.test.ts new file mode 100644 index 0000000000..60a997ac06 --- /dev/null +++ b/apps/web/__tests__/integration/transcribe-workflow.test.ts @@ -0,0 +1,364 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { assemblyAIEditResponse } from "../fixtures/assemblyai-edit-response"; + +const mocks = vi.hoisted(() => ({ + transcribe: vi.fn(), + putObject: vi.fn(), + deleteObject: vi.fn(), + getInternalSignedObjectUrl: vi.fn(), + startAiGeneration: vi.fn(), + updates: [] as Record[], +})); + +const schemaMocks = vi.hoisted(() => ({ + videos: { + id: "videos.id", + metadata: "videos.metadata", + transcriptionStatus: "videos.transcriptionStatus", + updatedAt: "videos.updatedAt", + }, + organizations: { id: "organizations.id" }, + videoUploads: { videoId: "videoUploads.videoId" }, + videoEdits: { + videoId: "videoEdits.videoId", + editSpec: "videoEdits.editSpec", + }, +})); + +const videoRow = vi.hoisted(() => ({ + id: "video-123", + ownerId: "user-456", + duration: 4, + settings: null, + source: { type: "webMP4" }, + isScreenshot: false, + transcriptionStatus: "COMPLETE", + updatedAt: new Date("2026-07-30T00:00:00.000Z"), + metadata: { + editTranscriptBackfill: { + status: "processing", + requestId: "request-1", + requestedAt: "2026-07-30T00:00:00.000Z", + }, + }, +})); + +const state = vi.hoisted(() => ({ editRows: [] as unknown[] })); + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + ASSEMBLY_API_KEY: "test-assembly-api-key", + NEXTAUTH_SECRET: "test-secret-with-enough-entropy", + }), +})); + +vi.mock("@cap/database/schema", () => schemaMocks); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => { + const query = { + from: (table: unknown) => { + if (table === schemaMocks.videoUploads) { + return { + where: () => ({ limit: async () => [] }), + }; + } + if (table === schemaMocks.videoEdits) { + return { where: async () => state.editRows }; + } + return { + leftJoin: () => ({ + where: async () => [{ video: videoRow, orgSettings: null }], + }), + where: async () => [videoRow], + }; + }, + }; + return query; + }, + update: () => ({ + set: (values: Record) => { + mocks.updates.push(values); + return { where: async () => [{ affectedRows: 1 }] }; + }, + }), + }), +})); + +vi.mock("drizzle-orm", () => ({ + and: (...conditions: unknown[]) => ({ conditions }), + eq: (field: unknown, value: unknown) => ({ field, value }), + sql: (strings: TemplateStringsArray, ...values: unknown[]) => ({ + strings, + values, + }), +})); + +vi.mock("server-only", () => ({})); + +vi.mock("workflow", () => ({ + FatalError: class FatalError extends Error {}, +})); + +vi.mock("assemblyai", () => ({ + AssemblyAI: class { + transcripts = { transcribe: mocks.transcribe }; + }, +})); + +vi.mock("@cap/web-backend/src/Storage/index", () => ({ + Storage: { + getAccessForVideo: () => ({ + pipe: (runner: (value: unknown) => unknown) => + runner([ + { + putObject: mocks.putObject, + deleteObject: mocks.deleteObject, + getInternalSignedObjectUrl: mocks.getInternalSignedObjectUrl, + }, + ]), + }), + }, +})); + +vi.mock("@/lib/workflow-runtime", () => ({ + runWorkflowPromise: (value: unknown) => Promise.resolve(value), +})); + +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: (video: unknown) => video, +})); + +vi.mock("@/lib/media-client", () => ({ + isMediaServerConfigured: () => true, + probeVideoViaMediaServer: async () => ({ + audioCodec: "aac", + videoCodec: "h264", + duration: 4, + audioChannels: 2, + sampleRate: 48_000, + }), + extractAudioViaMediaServer: async () => Buffer.from("audio"), + checkHasAudioTrackViaMediaServer: async () => true, +})); + +vi.mock("@/lib/audio-extract", () => ({ + checkHasAudioTrack: async () => true, + extractAudioFromUrl: async () => ({ + filePath: "/tmp/audio.mp3", + cleanup: async () => {}, + }), +})); + +vi.mock("@/lib/audio-enhance", () => ({ + ENHANCED_AUDIO_CONTENT_TYPE: "audio/mpeg", + ENHANCED_AUDIO_EXTENSION: "mp3", + enhanceAudioFromUrl: async () => Buffer.from(""), +})); + +vi.mock("@/lib/generate-ai", () => ({ + startAiGeneration: mocks.startAiGeneration, +})); + +function pipeValue(value: unknown) { + return { pipe: (runner: (input: unknown) => unknown) => runner(value) }; +} + +describe("transcribeVideoWorkflow", () => { + beforeEach(() => { + mocks.updates.length = 0; + state.editRows = []; + mocks.transcribe.mockResolvedValue({ + ...assemblyAIEditResponse, + audio_duration: 4, + }); + mocks.putObject.mockImplementation(() => pipeValue(undefined)); + mocks.deleteObject.mockImplementation(() => pipeValue(undefined)); + mocks.getInternalSignedObjectUrl.mockImplementation(() => + pipeValue("https://storage.test/object"), + ); + mocks.startAiGeneration.mockResolvedValue({ success: true, message: "ok" }); + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + ok: true, + status: 200, + statusText: "OK", + arrayBuffer: async () => new ArrayBuffer(8), + })), + ); + }); + + it("persists captions and the word transcript from a single paid pass", async () => { + const { transcribeVideoWorkflow } = await import("@/workflows/transcribe"); + + const result = await transcribeVideoWorkflow({ + videoId: "video-123", + userId: "user-456", + aiGenerationEnabled: false, + }); + + expect(result.success).toBe(true); + expect(mocks.transcribe).toHaveBeenCalledTimes(1); + expect(mocks.transcribe.mock.calls[0]?.[0]).toMatchObject({ + disfluencies: true, + speech_models: ["universal-3-5-pro", "universal-2"], + }); + + const writes = new Map( + mocks.putObject.mock.calls.map((call) => [call[0] as string, call[1]]), + ); + expect( + [...writes.keys()].filter((key) => key.includes("transcription")), + ).toEqual([ + "user-456/video-123/transcription.vtt", + "user-456/video-123/transcription.edit.v3.json", + ]); + + const vtt = writes.get("user-456/video-123/transcription.vtt") as string; + expect(vtt.startsWith("WEBVTT")).toBe(true); + expect(vtt).toContain("this is a real example."); + expect(vtt).not.toContain("Um,"); + + const { parseEditTranscript } = await import("@/lib/edit-transcript"); + const { decryptEditTranscriptObject } = await import( + "@/lib/edit-transcript-storage" + ); + const encrypted = writes.get( + "user-456/video-123/transcription.edit.v3.json", + ) as string; + expect(encrypted).not.toContain("this is a real example"); + const stored = parseEditTranscript( + decryptEditTranscriptObject(encrypted, "user-456", "video-123") ?? "", + ); + expect(stored).toMatchObject({ version: 3, durationMs: 4_000 }); + // verbatim: fillers stay in the stored words even though captions drop them + expect(stored?.words.map((word) => word.text)).toContain("Um,"); + expect(stored?.words).toHaveLength(9); + + expect(mocks.updates.at(-1)).toEqual({ transcriptionStatus: "COMPLETE" }); + }); + + it("never overwrites the original-timeline transcript of an edited video", async () => { + state.editRows = [ + { + sourceKey: "user-456/video-123/original.mp4", + editSpec: { + version: 1, + sourceDuration: 12, + keepRanges: [{ start: 0, end: 4 }], + }, + }, + ]; + + const { transcribeVideoWorkflow } = await import("@/workflows/transcribe"); + await transcribeVideoWorkflow({ + videoId: "video-123", + userId: "user-456", + aiGenerationEnabled: false, + }); + + const writtenKeys = mocks.putObject.mock.calls.map((call) => call[0]); + expect(writtenKeys).toContain("user-456/video-123/transcription.vtt"); + expect(writtenKeys).not.toContain( + "user-456/video-123/transcription.edit.v3.json", + ); + }); +}); + +describe("backfillEditTranscriptWorkflow", () => { + beforeEach(() => { + state.editRows = []; + videoRow.metadata = { + editTranscriptBackfill: { + status: "processing", + requestId: "request-1", + requestedAt: "2026-07-30T00:00:00.000Z", + }, + }; + mocks.transcribe.mockResolvedValue({ + ...assemblyAIEditResponse, + audio_duration: 4, + }); + mocks.putObject.mockImplementation(() => pipeValue(undefined)); + mocks.deleteObject.mockImplementation(() => pipeValue(undefined)); + mocks.getInternalSignedObjectUrl.mockImplementation(() => + pipeValue("https://storage.test/object"), + ); + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + ok: true, + status: 200, + statusText: "OK", + arrayBuffer: async () => new ArrayBuffer(8), + })), + ); + }); + + it("transcribes the original media in the original timeline when edits exist", async () => { + state.editRows = [ + { + sourceKey: "user-456/video-123/original.mp4", + editSpec: { + version: 1, + sourceDuration: 12, + keepRanges: [{ start: 0, end: 4 }], + }, + }, + ]; + + const { backfillEditTranscriptWorkflow } = await import( + "@/workflows/transcribe" + ); + const result = await backfillEditTranscriptWorkflow({ + videoId: "video-123", + userId: "user-456", + requestId: "request-1", + }); + + expect(result.success).toBe(true); + expect(mocks.getInternalSignedObjectUrl.mock.calls[0]?.[0]).toBe( + "user-456/video-123/original.mp4", + ); + + const { parseEditTranscript } = await import("@/lib/edit-transcript"); + const { decryptEditTranscriptObject } = await import( + "@/lib/edit-transcript-storage" + ); + const write = mocks.putObject.mock.calls.find( + (call) => call[0] === "user-456/video-123/transcription.edit.v3.json", + ); + const decrypted = decryptEditTranscriptObject( + write?.[1] as string, + "user-456", + "video-123", + ); + expect(parseEditTranscript(decrypted ?? "")).toMatchObject({ + durationMs: 12_000, + }); + expect(mocks.deleteObject).not.toHaveBeenCalledWith( + "user-456/video-123/transcription.edit.v3.status.json", + ); + }); + + it("does not run a backfill after its database claim is replaced", async () => { + const { backfillEditTranscriptWorkflow } = await import( + "@/workflows/transcribe" + ); + const result = await backfillEditTranscriptWorkflow({ + videoId: "video-123", + userId: "user-456", + requestId: "expired-request", + }); + + expect(result.success).toBe(false); + expect(mocks.transcribe).not.toHaveBeenCalled(); + expect(mocks.putObject).not.toHaveBeenCalledWith( + "user-456/video-123/transcription.edit.v3.json", + expect.anything(), + expect.anything(), + ); + }); +}); From 5e14b9d10482fad59125d94a04a18d92ee70eea2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:22 +0100 Subject: [PATCH 10/20] feat(web): remap captions from stored transcripts after edits --- apps/web/workflows/edit-video.ts | 153 ++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 5 deletions(-) diff --git a/apps/web/workflows/edit-video.ts b/apps/web/workflows/edit-video.ts index 019d2f970d..acf88004e7 100644 --- a/apps/web/workflows/edit-video.ts +++ b/apps/web/workflows/edit-video.ts @@ -20,8 +20,17 @@ import { AwsCredentials } from "@cap/web-backend/src/Aws"; import { Storage } from "@cap/web-backend/src/Storage/index"; import { Video } from "@cap/web-domain"; import { and, eq } from "drizzle-orm"; -import { Effect } from "effect"; +import { Effect, Option } from "effect"; import { FatalError } from "workflow"; +import { + type EditTranscript, + editTranscriptWordsToCaptionVtt, + getEditTranscriptObjectKey, + parseEditTranscript, + remapEditTranscriptThroughSpec, +} from "@/lib/edit-transcript"; +import { decryptEditTranscriptObject } from "@/lib/edit-transcript-storage"; +import { startAiGeneration } from "@/lib/generate-ai"; import { transcribeVideo } from "@/lib/transcribe"; import { getEditSpecOutputDuration, @@ -57,6 +66,7 @@ const MEDIA_SERVER_PRESIGNED_GET_EXPIRES_SECONDS = 3 * 60 * 60; const MEDIA_SERVER_PRESIGNED_PUT_EXPIRES_SECONDS = 3 * 60 * 60; const MEDIA_SERVER_OUTPUT_VERIFICATION_MAX_ATTEMPTS = 4; const MEDIA_SERVER_OUTPUT_VERIFICATION_RETRY_MS = 1000; +const EDIT_TRANSCRIPT_CURRENCY_TOLERANCE_MS = 250; function isPositiveNumber(value: number | null): value is number { return typeof value === "number" && Number.isFinite(value) && value > 0; @@ -102,14 +112,28 @@ export async function editVideoWorkflow( const result = await renderVideoEditOnMediaServer(payload); await verifyRenderedEditOutput(videoId, userId, editSpec, result.metadata); await invalidateEditedVideoCache(videoId, editSpec); - await saveEditResultAndComplete( + const { transcriptRemapped } = await saveEditResultAndComplete( videoId, sourceKey, previousSpec, editSpec, result.metadata, ); - await queueTranscriptionRegeneration(videoId, userId, aiGenerationEnabled); + + if (transcriptRemapped) { + // Captions were derived from the immutable word transcript, so the + // transcription stays COMPLETE and no new paid pass is needed. + if (aiGenerationEnabled) { + await queueAiGeneration(videoId, userId); + } + } else { + await queueTranscriptionRegeneration( + videoId, + userId, + aiGenerationEnabled, + ); + } + return result; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); @@ -495,6 +519,26 @@ function clearAiMetadata(metadata: VideoMetadata | null): VideoMetadata { return nextMetadata; } +async function queueAiGeneration( + videoId: string, + userId: string, +): Promise { + "use step"; + + try { + const result = await startAiGeneration(videoId as Video.VideoId, userId); + + if (!result.success) { + console.warn("[editVideoWorkflow] Failed to queue AI generation", { + videoId, + message: result.message, + }); + } + } catch (error) { + console.warn("[editVideoWorkflow] Failed to queue AI generation", error); + } +} + async function queueTranscriptionRegeneration( videoId: string, userId: string, @@ -520,6 +564,72 @@ async function queueTranscriptionRegeneration( } } +/** + * Loads the immutable word transcript (stored in the ORIGINAL media timeline) + * and returns it only when it still describes the original media. + */ +async function loadOriginalEditTranscript( + video: typeof videos.$inferSelect, + editSpec: VideoEditSpec, +) { + const [bucket] = await Storage.getAccessForVideo( + decodeStorageVideo(video), + ).pipe(runWorkflowPromise); + const stored = await bucket + .getObject(getEditTranscriptObjectKey(video.ownerId, video.id)) + .pipe(runWorkflowPromise); + + const decrypted = Option.isSome(stored) + ? decryptEditTranscriptObject(stored.value, video.ownerId, video.id) + : null; + const transcript = decrypted ? parseEditTranscript(decrypted) : null; + if (!transcript) return null; + + const expectedDurationMs = Math.round(editSpec.sourceDuration * 1000); + return Math.abs(transcript.durationMs - expectedDurationMs) <= + EDIT_TRANSCRIPT_CURRENCY_TOLERANCE_MS + ? transcript + : null; +} + +/** + * Replaces every derived transcript object (captions plus stale translations + * and status markers) with captions remapped through the new edit spec. The + * word transcript itself is never touched — it is the single source of truth. + */ +async function rewriteTranscriptObjectsForEdit( + video: typeof videos.$inferSelect, + transcript: EditTranscript, + editSpec: VideoEditSpec, +) { + const [bucket] = await Storage.getAccessForVideo( + decodeStorageVideo(video), + ).pipe(runWorkflowPromise); + const transcriptKey = getEditTranscriptObjectKey(video.ownerId, video.id); + const prefix = `${video.ownerId}/${video.id}/transcription`; + const listed = await bucket.listObjects({ prefix }).pipe(runWorkflowPromise); + const objects = (listed.Contents ?? []) + .map((object) => ({ Key: object.Key })) + .filter( + (object): object is { Key: string } => + Boolean(object.Key) && object.Key !== transcriptKey, + ); + + if (objects.length > 0) { + await bucket.deleteObjects(objects).pipe(runWorkflowPromise); + } + + await bucket + .putObject( + `${video.ownerId}/${video.id}/transcription.vtt`, + editTranscriptWordsToCaptionVtt( + remapEditTranscriptThroughSpec(transcript, editSpec).words, + ), + { contentType: "text/vtt" }, + ) + .pipe(runWorkflowPromise); +} + async function clearTranscriptObjects(video: typeof videos.$inferSelect) { const [bucket] = await Storage.getAccessForVideo( decodeStorageVideo(video), @@ -669,7 +779,7 @@ async function saveEditResultAndComplete( previousSpec: VideoEditSpec, editSpec: VideoEditSpec, metadata: { duration: number; width: number; height: number; fps: number }, -): Promise { +): Promise<{ transcriptRemapped: boolean }> { "use step"; const duration = getValidDuration(metadata.duration); @@ -683,6 +793,15 @@ async function saveEditResultAndComplete( } const nextMetadata = clearAiMetadata(video.metadata as VideoMetadata | null); + let originalTranscript: EditTranscript | null = null; + try { + originalTranscript = await loadOriginalEditTranscript(video, editSpec); + } catch (error) { + console.warn( + "[editVideoWorkflow] Failed to load stored edit transcript", + error, + ); + } await db().transaction(async (tx) => { await tx @@ -692,7 +811,9 @@ async function saveEditResultAndComplete( height: metadata.height, fps: metadata.fps, metadata: nextMetadata, - transcriptionStatus: null, + // Derivable captions keep the transcription COMPLETE; only legacy + // videos without a stored word transcript get re-transcribed. + ...(originalTranscript ? {} : { transcriptionStatus: null }), ...(duration === undefined ? {} : { duration }), }) .where(eq(videos.id, videoId as Video.VideoId)); @@ -746,6 +867,26 @@ async function saveEditResultAndComplete( ); }); + if (originalTranscript) { + try { + await rewriteTranscriptObjectsForEdit( + video, + originalTranscript, + editSpec, + ); + return { transcriptRemapped: true }; + } catch (error) { + console.warn( + "[editVideoWorkflow] Failed to remap transcript objects", + error, + ); + await db() + .update(videos) + .set({ transcriptionStatus: null }) + .where(eq(videos.id, videoId as Video.VideoId)); + } + } + try { await clearTranscriptObjects(video); } catch (error) { @@ -754,6 +895,8 @@ async function saveEditResultAndComplete( error, ); } + + return { transcriptRemapped: false }; } async function clearEditProcessingState( From eeb81ba9e18d4cb80da09850656983df058c8f7a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:22 +0100 Subject: [PATCH 11/20] feat(web): add get-edit-transcript action with backfill --- .../unit/get-edit-transcript.test.ts | 462 ++++++++++++++++++ .../web/actions/videos/get-edit-transcript.ts | 312 ++++++++++++ 2 files changed, 774 insertions(+) create mode 100644 apps/web/__tests__/unit/get-edit-transcript.test.ts create mode 100644 apps/web/actions/videos/get-edit-transcript.ts diff --git a/apps/web/__tests__/unit/get-edit-transcript.test.ts b/apps/web/__tests__/unit/get-edit-transcript.test.ts new file mode 100644 index 0000000000..06e5d5916d --- /dev/null +++ b/apps/web/__tests__/unit/get-edit-transcript.test.ts @@ -0,0 +1,462 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { Option } from "effect"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + db: vi.fn(), + decodeStorageVideo: vi.fn(), + getAccessForVideo: vi.fn(), + getCurrentUser: vi.fn(), + runPromise: vi.fn(), + start: vi.fn(), +})); + +const schema = vi.hoisted(() => ({ + videos: { + id: "videos.id", + metadata: "videos.metadata", + updatedAt: "videos.updatedAt", + }, + videoEdits: { + videoId: "videoEdits.videoId", + editSpec: "videoEdits.editSpec", + }, +})); + +vi.mock("@cap/database", () => ({ + db: mocks.db, +})); + +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: mocks.getCurrentUser, +})); + +vi.mock("@cap/database/schema", () => schema); + +vi.mock("@cap/utils", () => ({ + userIsPro: () => true, +})); + +vi.mock("@cap/web-backend", () => ({ + Storage: { + getAccessForVideo: mocks.getAccessForVideo, + }, +})); + +vi.mock("drizzle-orm", () => ({ + and: vi.fn((...conditions: unknown[]) => ({ conditions })), + eq: vi.fn((left: unknown, right: unknown) => ({ left, right })), + sql: vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ + strings, + values, + })), +})); + +vi.mock("workflow/api", () => ({ + start: mocks.start, +})); + +vi.mock("@/lib/server", () => ({ + runPromise: mocks.runPromise, +})); + +vi.mock("@/lib/edit-transcript-storage", () => ({ + decryptEditTranscriptObject: (value: string) => value, +})); + +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: mocks.decodeStorageVideo, +})); + +vi.mock("@/workflows/transcribe", () => ({ + backfillEditTranscriptWorkflow: vi.fn(), +})); + +vi.mock("server-only", () => ({})); + +type PipeValue = { + pipe: (runner: (value: T) => unknown) => unknown; +}; + +function pipeValue(value: T): PipeValue { + return { + pipe: (runner) => runner(value), + }; +} + +function createDatabase( + video: Record, + edit?: Record, +) { + const currentVideo: Record = { + metadata: null, + updatedAt: new Date("2026-07-30T00:00:00.000Z"), + ...video, + }; + const transactionContext = new AsyncLocalStorage(); + const lockedSelect = { + select: vi.fn(), + from: vi.fn(), + where: vi.fn(), + for: vi.fn(), + }; + lockedSelect.select.mockReturnValue(lockedSelect); + lockedSelect.from.mockReturnValue(lockedSelect); + lockedSelect.where.mockReturnValue(lockedSelect); + lockedSelect.for.mockImplementation(async () => [currentVideo]); + + const transactionUpdate = { + set: vi.fn(), + where: vi.fn(), + }; + transactionUpdate.set.mockImplementation( + (values: Record) => { + if ( + typeof values.metadata === "object" && + values.metadata !== null && + "values" in values.metadata && + Array.isArray(values.metadata.values) + ) { + const encodedStatus = values.metadata.values.find( + (value) => + typeof value === "string" && value.startsWith('{"status":'), + ); + if (typeof encodedStatus === "string") { + currentVideo.metadata = { + editTranscriptBackfill: JSON.parse(encodedStatus), + }; + } + } + return transactionUpdate; + }, + ); + transactionUpdate.where.mockResolvedValue([{ affectedRows: 1 }]); + + const select = vi.fn(() => ({ + from: vi.fn((table: unknown) => ({ + where: vi + .fn() + .mockResolvedValue( + table === schema.videoEdits ? (edit ? [edit] : []) : [currentVideo], + ), + })), + })); + const update = vi.fn(() => ({ + set: vi.fn(() => ({ + where: vi.fn().mockResolvedValue([{ affectedRows: 1 }]), + })), + })); + + let transactionTail = Promise.resolve(); + const transaction = vi.fn( + ( + callback: ( + tx: typeof lockedSelect & { update: typeof update }, + ) => Promise, + ): Promise => { + const result = transactionTail.then(() => + transactionContext.run(true, () => + callback({ + ...lockedSelect, + update: vi.fn(() => transactionUpdate), + }), + ), + ); + transactionTail = result.then( + () => undefined, + () => undefined, + ); + return result; + }, + ); + + return { + database: { select, transaction, update }, + isTransactionActive: () => transactionContext.getStore() === true, + lockedSelect, + }; +} + +function createBucket( + objects: Map, + isTransactionActive = () => false, +) { + return { + getObject: vi.fn((key: string) => { + if (isTransactionActive()) { + throw new Error("Storage read occurred inside a database transaction"); + } + return pipeValue(Option.fromNullable(objects.get(key))); + }), + }; +} + +const video = { + id: "video-1", + ownerId: "user-1", + duration: 25, + source: { type: "webMP4" }, + isScreenshot: false, + transcriptionStatus: "COMPLETE", + updatedAt: new Date("2026-07-30T00:00:00.000Z"), +}; + +const TRANSCRIPT_KEY = "user-1/video-1/transcription.edit.v3.json"; + +function storedOriginalTranscript(durationMs = 25_000) { + return JSON.stringify({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs, + languageCode: "en", + words: [ + { + id: "word-0-1000-1500", + text: "before", + startMs: 1_000, + endMs: 1_500, + confidence: 0.9, + speaker: null, + channel: null, + }, + { + id: "word-1-6000-6500", + text: "kept", + startMs: 6_000, + endMs: 6_500, + confidence: 0.9, + speaker: null, + channel: null, + }, + { + id: "word-2-20000-20500", + text: "after", + startMs: 20_000, + endMs: 20_500, + confidence: 0.9, + speaker: null, + channel: null, + }, + ], + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.getCurrentUser.mockResolvedValue({ id: "user-1" }); + mocks.runPromise.mockImplementation((value) => Promise.resolve(value)); + mocks.start.mockResolvedValue({ runId: "run-1" }); +}); + +describe("requestEditTranscript", () => { + it("serializes concurrent requests and starts one paid transcript job", async () => { + const objects = new Map(); + const { database, isTransactionActive, lockedSelect } = + createDatabase(video); + const bucket = createBucket(objects, isTransactionActive); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + mocks.start.mockImplementation(async () => { + if (isTransactionActive()) { + throw new Error( + "Workflow start occurred inside a database transaction", + ); + } + return { runId: "run-1" }; + }); + + const { requestEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const [first, second] = await Promise.all([ + requestEditTranscript("video-1" as never), + requestEditTranscript("video-1" as never), + ]); + + expect(first).toEqual({ status: "processing" }); + expect(second).toEqual({ status: "processing" }); + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(lockedSelect.for).toHaveBeenCalledTimes(2); + expect(lockedSelect.for).toHaveBeenCalledWith("update"); + expect(mocks.start.mock.calls[0]?.[1]?.[0]).toMatchObject({ + videoId: "video-1", + userId: "user-1", + requestId: expect.any(String), + }); + }); + + it("returns a current cached sidecar without starting another job", async () => { + const objects = new Map([ + [ + TRANSCRIPT_KEY, + JSON.stringify({ + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 25_000, + languageCode: "en", + words: [], + }), + ], + ]); + const bucket = createBucket(objects); + const { database } = createDatabase(video); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { requestEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const response = await requestEditTranscript("video-1" as never); + + expect(response.status).toBe("ready"); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("remaps the stored original words through the video's edit spec", async () => { + const objects = new Map([ + [TRANSCRIPT_KEY, storedOriginalTranscript()], + ]); + const bucket = createBucket(objects); + const { database } = createDatabase( + { ...video, duration: 10 }, + { + editSpec: { + version: 1, + sourceDuration: 25, + keepRanges: [{ start: 5, end: 15 }], + }, + }, + ); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { requestEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const response = await requestEditTranscript("video-1" as never); + + if (response.status !== "ready") { + throw new Error(`Expected a ready transcript, got ${response.status}`); + } + expect(response.transcript.durationMs).toBe(10_000); + expect( + response.transcript.words.map((word) => [ + word.id, + word.startMs, + word.endMs, + ]), + ).toEqual([["word-1-6000-6500", 1_000, 1_500]]); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("does not expose workflow startup errors to the client", async () => { + const bucket = createBucket(new Map()); + const { database } = createDatabase(video); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + mocks.start.mockRejectedValue( + new Error("provider account and request identifiers"), + ); + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + + const { requestEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const response = await requestEditTranscript("video-1" as never); + + expect(response).toEqual({ + status: "error", + message: "Transcript preparation could not start. Try again.", + }); + expect(JSON.stringify(response)).not.toContain("provider account"); + consoleError.mockRestore(); + }); +}); + +describe("getEditTranscript", () => { + it("remaps stored words instead of reporting them as stale", async () => { + const objects = new Map([ + [TRANSCRIPT_KEY, storedOriginalTranscript()], + ]); + const bucket = createBucket(objects); + const { database } = createDatabase( + { ...video, duration: 10 }, + { + editSpec: { + version: 1, + sourceDuration: 25, + keepRanges: [{ start: 5, end: 15 }], + }, + }, + ); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { getEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + const response = await getEditTranscript("video-1" as never); + + if (response.status !== "ready") { + throw new Error(`Expected a ready transcript, got ${response.status}`); + } + expect(response.transcript.words.map((word) => word.text)).toEqual([ + "kept", + ]); + }); + + it("reports missing when the sidecar no longer matches the original media", async () => { + const objects = new Map([ + [TRANSCRIPT_KEY, storedOriginalTranscript(18_000)], + ]); + const bucket = createBucket(objects); + const { database } = createDatabase( + { ...video, duration: 10 }, + { + editSpec: { + version: 1, + sourceDuration: 25, + keepRanges: [{ start: 5, end: 15 }], + }, + }, + ); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { getEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + + expect(await getEditTranscript("video-1" as never)).toEqual({ + status: "missing", + }); + }); + + it("reports only a generic message for a failed backfill", async () => { + const bucket = createBucket(new Map()); + const { database } = createDatabase({ + ...video, + metadata: { + editTranscriptBackfill: { + status: "error", + requestId: "request-1", + requestedAt: "2026-07-30T00:00:00.000Z", + message: "private provider detail", + }, + }, + }); + mocks.db.mockReturnValue(database); + mocks.getAccessForVideo.mockReturnValue(pipeValue([bucket])); + + const { getEditTranscript } = await import( + "@/actions/videos/get-edit-transcript" + ); + + expect(await getEditTranscript("video-1" as never)).toEqual({ + status: "error", + message: "Transcript preparation failed. Try again.", + }); + }); +}); diff --git a/apps/web/actions/videos/get-edit-transcript.ts b/apps/web/actions/videos/get-edit-transcript.ts new file mode 100644 index 0000000000..7cb1b5c214 --- /dev/null +++ b/apps/web/actions/videos/get-edit-transcript.ts @@ -0,0 +1,312 @@ +"use server"; + +import { randomUUID } from "node:crypto"; +import { db } from "@cap/database"; +import { getCurrentUser } from "@cap/database/auth/session"; +import { videoEdits, videos } from "@cap/database/schema"; +import { userIsPro } from "@cap/utils"; +import { Storage } from "@cap/web-backend"; +import type { Video } from "@cap/web-domain"; +import { and, eq, sql } from "drizzle-orm"; +import { Option } from "effect"; +import { start } from "workflow/api"; +import { + type EditTranscript, + type EditTranscriptBackfillStatus, + getEditTranscriptBackfillStatus, + getEditTranscriptObjectKey, + parseEditTranscript, + remapEditTranscriptThroughSpec, +} from "@/lib/edit-transcript"; +import { decryptEditTranscriptObject } from "@/lib/edit-transcript-storage"; +import { runPromise } from "@/lib/server"; +import { decodeStorageVideo } from "@/lib/video-storage"; +import { backfillEditTranscriptWorkflow } from "@/workflows/transcribe"; + +type EditTranscriptResponse = + | { status: "ready"; transcript: EditTranscript } + | { status: "missing" } + | { status: "processing" } + | { status: "error"; message: string }; + +const BACKFILL_STALE_AFTER_MS = 60 * 60 * 1000; +const PUBLIC_ERROR_MESSAGES = new Set([ + "Unauthorized", + "Cap Pro is required", + "Video not found", + "Transcript editing is not available for this video", + "Transcript is not ready", +]); + +function isMp4BackedVideo(source: typeof videos.$inferSelect.source) { + return source.type === "desktopMP4" || source.type === "webMP4"; +} + +function isEditableTranscriptVideo(video: typeof videos.$inferSelect) { + return ( + video.transcriptionStatus === "COMPLETE" && + !video.isScreenshot && + isMp4BackedVideo(video.source) && + Boolean(video.duration && video.duration > 0) + ); +} + +function getPublicErrorMessage(error: unknown, fallback: string) { + return isPublicError(error) ? error.message : fallback; +} + +function isPublicError(error: unknown): error is Error { + return error instanceof Error && PUBLIC_ERROR_MESSAGES.has(error.message); +} + +async function loadEditableTranscriptVideo(videoId: Video.VideoId) { + const user = await getCurrentUser(); + if (!user) throw new Error("Unauthorized"); + if (!userIsPro(user)) throw new Error("Cap Pro is required"); + + const [video] = await db() + .select() + .from(videos) + .where(eq(videos.id, videoId)); + if (!video || video.ownerId !== user.id) throw new Error("Video not found"); + if (!isEditableTranscriptVideo(video)) { + if (video.transcriptionStatus !== "COMPLETE") { + throw new Error("Transcript is not ready"); + } + throw new Error("Transcript editing is not available for this video"); + } + + const [bucket] = await Storage.getAccessForVideo( + decodeStorageVideo(video), + ).pipe(runPromise); + + return { user, video, bucket }; +} + +async function getOptionalObject( + bucket: Awaited>["bucket"], + key: string, +) { + const object = await bucket.getObject(key).pipe(runPromise); + return Option.isSome(object) ? object.value : null; +} + +function isFreshProcessingStatus(status: EditTranscriptBackfillStatus) { + if (status.status !== "processing") return false; + const requestedAt = Date.parse(status.requestedAt); + return ( + Number.isFinite(requestedAt) && + Date.now() - requestedAt < BACKFILL_STALE_AFTER_MS + ); +} + +function isDurationCurrent(durationMs: number, expectedSeconds: number) { + return Math.abs(durationMs - Math.round(expectedSeconds * 1000)) <= 250; +} + +/** + * Reads the stored word transcript. Stored words always describe the ORIGINAL + * media, so for edited videos they are checked against the edit spec's source + * duration and remapped onto the current output timeline before being returned. + */ +async function readCurrentEditTranscript( + bucket: Awaited>["bucket"], + video: typeof videos.$inferSelect, +): Promise { + const content = await getOptionalObject( + bucket, + getEditTranscriptObjectKey(video.ownerId, video.id), + ); + const decrypted = content + ? decryptEditTranscriptObject(content, video.ownerId, video.id) + : null; + const transcript = decrypted ? parseEditTranscript(decrypted) : null; + if (!transcript) return null; + + const [edit] = await db() + .select({ editSpec: videoEdits.editSpec }) + .from(videoEdits) + .where(eq(videoEdits.videoId, video.id)); + + if (!edit?.editSpec) { + return isDurationCurrent(transcript.durationMs, video.duration ?? 0) + ? transcript + : null; + } + + return isDurationCurrent(transcript.durationMs, edit.editSpec.sourceDuration) + ? remapEditTranscriptThroughSpec(transcript, edit.editSpec) + : null; +} + +export async function getEditTranscript( + videoId: Video.VideoId, +): Promise { + try { + const { video, bucket } = await loadEditableTranscriptVideo(videoId); + const transcript = await readCurrentEditTranscript(bucket, video); + + if (transcript) { + return { status: "ready", transcript }; + } + + const status = getEditTranscriptBackfillStatus(video.metadata); + if (status && isFreshProcessingStatus(status)) { + return { status: "processing" }; + } + if (status?.status === "processing") { + return { + status: "error", + message: "Transcript preparation stalled. Try again.", + }; + } + if (status?.status === "error") { + return { + status: "error", + message: "Transcript preparation failed. Try again.", + }; + } + return { status: "missing" }; + } catch (error) { + if (!isPublicError(error)) { + console.error( + `[editTranscript] Failed to load transcript for ${videoId}`, + error, + ); + } + return { + status: "error", + message: getPublicErrorMessage(error, "Transcript unavailable"), + }; + } +} + +async function claimEditTranscriptBackfill( + videoId: Video.VideoId, + userId: string, +) { + return db().transaction(async (tx) => { + const [video] = await tx + .select() + .from(videos) + .where(eq(videos.id, videoId)) + .for("update"); + if (!video || video.ownerId !== userId) throw new Error("Video not found"); + if (!isEditableTranscriptVideo(video)) { + throw new Error("Transcript editing is not available for this video"); + } + + const existing = getEditTranscriptBackfillStatus(video.metadata); + if (existing && isFreshProcessingStatus(existing)) { + return { claimed: false as const }; + } + + const requestId = randomUUID(); + const status = { + status: "processing", + requestId, + requestedAt: new Date().toISOString(), + }; + await tx + .update(videos) + .set({ + metadata: sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.editTranscriptBackfill', CAST(${JSON.stringify(status)} AS JSON))`, + updatedAt: video.updatedAt, + }) + .where(eq(videos.id, videoId)); + + return { claimed: true as const, requestId }; + }); +} + +async function setEditTranscriptBackfillError( + videoId: Video.VideoId, + requestId: string, +) { + await db() + .update(videos) + .set({ + metadata: sql`JSON_SET(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.editTranscriptBackfill.status', 'error')`, + updatedAt: sql`${videos.updatedAt}`, + }) + .where( + and( + eq(videos.id, videoId), + sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.editTranscriptBackfill.requestId')) = ${requestId}`, + ), + ); +} + +async function clearEditTranscriptBackfill( + videoId: Video.VideoId, + requestId: string, +) { + await db() + .update(videos) + .set({ + metadata: sql`JSON_REMOVE(COALESCE(${videos.metadata}, JSON_OBJECT()), '$.editTranscriptBackfill')`, + updatedAt: sql`${videos.updatedAt}`, + }) + .where( + and( + eq(videos.id, videoId), + sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.metadata}, '$.editTranscriptBackfill.requestId')) = ${requestId}`, + ), + ); +} + +export async function requestEditTranscript( + videoId: Video.VideoId, +): Promise { + try { + const { user, video, bucket } = await loadEditableTranscriptVideo(videoId); + const transcript = await readCurrentEditTranscript(bucket, video); + if (transcript) { + return { status: "ready", transcript }; + } + + const claim = await claimEditTranscriptBackfill(videoId, user.id); + if (!claim.claimed) { + return { status: "processing" }; + } + + const transcriptAfterClaim = await readCurrentEditTranscript(bucket, video); + if (transcriptAfterClaim) { + await clearEditTranscriptBackfill(videoId, claim.requestId); + return { status: "ready", transcript: transcriptAfterClaim }; + } + + try { + await start(backfillEditTranscriptWorkflow, [ + { + videoId, + userId: user.id, + requestId: claim.requestId, + }, + ]); + } catch (error) { + console.error( + `[editTranscript] Failed to start backfill for ${videoId}`, + error, + ); + await setEditTranscriptBackfillError(videoId, claim.requestId); + return { + status: "error", + message: "Transcript preparation could not start. Try again.", + }; + } + + return { status: "processing" }; + } catch (error) { + if (!isPublicError(error)) { + console.error( + `[editTranscript] Failed to request transcript for ${videoId}`, + error, + ); + } + return { + status: "error", + message: getPublicErrorMessage(error, "Transcript unavailable"), + }; + } +} From 235783f4b0c8176d8b1eb06954713cbb2d89646e Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:22 +0100 Subject: [PATCH 12/20] fix(web): block private edit transcript storage object access --- .../unit/storage-object-route.test.ts | 40 +++++++++++++++++++ apps/web/app/api/storage/object/route.ts | 4 ++ 2 files changed, 44 insertions(+) create mode 100644 apps/web/__tests__/unit/storage-object-route.test.ts diff --git a/apps/web/__tests__/unit/storage-object-route.test.ts b/apps/web/__tests__/unit/storage-object-route.test.ts new file mode 100644 index 0000000000..431705af8f --- /dev/null +++ b/apps/web/__tests__/unit/storage-object-route.test.ts @@ -0,0 +1,40 @@ +import { NextRequest } from "next/server"; +import { describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + verifyStorageObjectToken: vi.fn(), +})); + +vi.mock("@cap/web-backend", () => ({ + provideOptionalAuth: {}, + Storage: {}, + Videos: {}, + VideosRepo: {}, + verifyStorageObjectToken: mocks.verifyStorageObjectToken, +})); + +vi.mock("@/lib/server", () => ({ + runPromise: vi.fn(), +})); + +vi.mock("@/utils/helpers", () => ({ + CACHE_CONTROL_HEADERS: {}, +})); + +describe("storage object route", () => { + it.each([ + "owner-1/video-1/transcription.edit.v3.json", + "owner-1/video-1/transcription.edit.v3.status.json", + ])("never proxies the private transcript object %s", async (key) => { + const { GET } = await import("@/app/api/storage/object/route"); + const request = new NextRequest( + `http://localhost:3000/api/storage/object?videoId=video-1&key=${encodeURIComponent(key)}&token=attacker-token`, + ); + + const response = await GET(request); + + expect(response.status).toBe(404); + expect(await response.text()).toBe("Not found"); + expect(mocks.verifyStorageObjectToken).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/app/api/storage/object/route.ts b/apps/web/app/api/storage/object/route.ts index 8ec8f88f4c..0a1d70e50d 100644 --- a/apps/web/app/api/storage/object/route.ts +++ b/apps/web/app/api/storage/object/route.ts @@ -8,6 +8,7 @@ import { import { Storage as StorageDomain, Video } from "@cap/web-domain"; import { Effect, Option } from "effect"; import type { NextRequest } from "next/server"; +import { isPrivateEditTranscriptObjectKey } from "@/lib/edit-transcript"; import { runPromise } from "@/lib/server"; import { CACHE_CONTROL_HEADERS } from "@/utils/helpers"; @@ -106,6 +107,9 @@ export async function GET(request: NextRequest) { if (!videoIdParam || !key) { return new Response("Missing videoId or key", { status: 400 }); } + if (isPrivateEditTranscriptObjectKey(key)) { + return new Response("Not found", { status: 404 }); + } const effect = Effect.gen(function* () { const tokenPayload = token ? verifyStorageObjectToken(token) : null; From 849901e353b065b769ecce2fa674d105c9172ceb Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:22 +0100 Subject: [PATCH 13/20] feat(web): add development transcript reset endpoint --- .../unit/dev-reset-transcript.test.ts | 124 ++++++++++++++++++ .../web/app/api/dev-reset-transcript/route.ts | 100 ++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 apps/web/__tests__/unit/dev-reset-transcript.test.ts create mode 100644 apps/web/app/api/dev-reset-transcript/route.ts diff --git a/apps/web/__tests__/unit/dev-reset-transcript.test.ts b/apps/web/__tests__/unit/dev-reset-transcript.test.ts new file mode 100644 index 0000000000..a3165ff68a --- /dev/null +++ b/apps/web/__tests__/unit/dev-reset-transcript.test.ts @@ -0,0 +1,124 @@ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + and: vi.fn((...conditions: unknown[]) => ({ conditions })), + db: vi.fn(), + eq: vi.fn((field: unknown, value: unknown) => ({ field, value })), + getAccessForVideo: vi.fn(), + getCurrentUser: vi.fn(), + where: vi.fn(), +})); + +const schema = vi.hoisted(() => ({ + videos: { + id: "videos.id", + ownerId: "videos.ownerId", + }, + videoEdits: { + videoId: "videoEdits.videoId", + }, + videoUploads: { + videoId: "videoUploads.videoId", + }, +})); + +vi.mock("@cap/database", () => ({ + db: mocks.db, +})); + +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: mocks.getCurrentUser, +})); + +vi.mock("@cap/database/schema", () => schema); + +vi.mock("@cap/web-backend", () => ({ + Storage: { + getAccessForVideo: mocks.getAccessForVideo, + }, +})); + +vi.mock("drizzle-orm", () => ({ + and: mocks.and, + eq: mocks.eq, +})); + +vi.mock("@/lib/server", () => ({ + runPromise: vi.fn(), +})); + +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: vi.fn(), +})); + +beforeEach(() => { + vi.clearAllMocks(); + vi.stubEnv("NODE_ENV", "development"); + mocks.db.mockReturnValue({ + select: () => ({ + from: () => ({ + where: mocks.where, + }), + }), + }); + mocks.where.mockResolvedValue([]); +}); + +describe("development transcript reset", () => { + it("rejects cross-origin requests before authentication", async () => { + mocks.getCurrentUser.mockResolvedValue({ id: "owner-1" }); + const { POST } = await import("@/app/api/dev-reset-transcript/route"); + const request = new NextRequest( + "http://localhost:3000/api/dev-reset-transcript?videoId=video-1", + { + method: "POST", + headers: { origin: "https://attacker.example" }, + }, + ); + + const response = await POST(request); + + expect(response.status).toBe(403); + expect(mocks.getCurrentUser).not.toHaveBeenCalled(); + expect(mocks.db).not.toHaveBeenCalled(); + }); + + it("requires a signed-in user", async () => { + mocks.getCurrentUser.mockResolvedValue(null); + const { POST } = await import("@/app/api/dev-reset-transcript/route"); + const request = new NextRequest( + "http://localhost:3000/api/dev-reset-transcript?videoId=video-1", + { + method: "POST", + headers: { origin: "http://localhost:3000" }, + }, + ); + + const response = await POST(request); + + expect(response.status).toBe(401); + expect(mocks.db).not.toHaveBeenCalled(); + }); + + it("scopes the target video to the signed-in owner", async () => { + mocks.getCurrentUser.mockResolvedValue({ id: "owner-1" }); + const { POST } = await import("@/app/api/dev-reset-transcript/route"); + const request = new NextRequest( + "http://localhost:3000/api/dev-reset-transcript?videoId=video-1", + { + method: "POST", + headers: { origin: "http://localhost:3000" }, + }, + ); + + const response = await POST(request); + + expect(response.status).toBe(404); + expect(mocks.and).toHaveBeenCalledWith( + { field: schema.videos.id, value: "video-1" }, + { field: schema.videos.ownerId, value: "owner-1" }, + ); + expect(mocks.getAccessForVideo).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/app/api/dev-reset-transcript/route.ts b/apps/web/app/api/dev-reset-transcript/route.ts new file mode 100644 index 0000000000..c8a1d4ff9f --- /dev/null +++ b/apps/web/app/api/dev-reset-transcript/route.ts @@ -0,0 +1,100 @@ +import { db } from "@cap/database"; +import { getCurrentUser } from "@cap/database/auth/session"; +import { videoEdits, videos, videoUploads } from "@cap/database/schema"; +import { Storage } from "@cap/web-backend"; +import type { Video } from "@cap/web-domain"; +import { and, eq } from "drizzle-orm"; +import type { NextRequest } from "next/server"; +import { runPromise } from "@/lib/server"; +import { decodeStorageVideo } from "@/lib/video-storage"; + +export async function POST(request: NextRequest) { + if (process.env.NODE_ENV !== "development") { + return Response.json({ error: "dev only" }, { status: 403 }); + } + if (request.headers.get("origin") !== request.nextUrl.origin) { + return Response.json({ error: "same origin required" }, { status: 403 }); + } + const user = await getCurrentUser(); + if (!user) { + return Response.json({ error: "unauthorized" }, { status: 401 }); + } + const videoIdParam = request.nextUrl.searchParams.get("videoId"); + if (!videoIdParam) { + return Response.json({ error: "videoId required" }, { status: 400 }); + } + const videoId = videoIdParam as Video.VideoId; + + const [video] = await db() + .select() + .from(videos) + .where(and(eq(videos.id, videoId), eq(videos.ownerId, user.id))); + if (!video) { + return Response.json({ error: "video not found" }, { status: 404 }); + } + + const [bucket] = await Storage.getAccessForVideo( + decodeStorageVideo(video), + ).pipe(runPromise); + + const deleted: string[] = []; + for (const prefix of [ + `${video.ownerId}/${video.id}/transcription`, + `${video.ownerId}/${video.id}/audio`, + ]) { + const listed = await bucket.listObjects({ prefix }).pipe(runPromise); + const objects = (listed.Contents ?? []) + .map((object) => ({ Key: object.Key })) + .filter((object): object is { Key: string } => Boolean(object.Key)); + if (objects.length > 0) { + await bucket.deleteObjects(objects).pipe(runPromise); + deleted.push(...objects.map((object) => object.Key)); + } + } + + const [edit] = await db() + .select() + .from(videoEdits) + .where(eq(videoEdits.videoId, videoId)); + let restoredOriginal = false; + let restoredDuration: number | null = null; + if (edit) { + await bucket + .copyObject( + `${bucket.bucketName}/${edit.sourceKey}`, + `${video.ownerId}/${video.id}/result.mp4`, + ) + .pipe(runPromise); + await bucket.deleteObject(edit.sourceKey).pipe(runPromise); + deleted.push(edit.sourceKey); + restoredOriginal = true; + restoredDuration = edit.editSpec.sourceDuration; + await db().delete(videoEdits).where(eq(videoEdits.videoId, videoId)); + } + + const metadata = { + ...((video.metadata as Record | null) ?? {}), + }; + delete metadata.summary; + delete metadata.chapters; + delete metadata.aiGenerationStatus; + delete metadata.editTranscriptBackfill; + + await db() + .update(videos) + .set({ + transcriptionStatus: null, + metadata, + ...(restoredDuration === null ? {} : { duration: restoredDuration }), + }) + .where(eq(videos.id, videoId)); + await db().delete(videoUploads).where(eq(videoUploads.videoId, videoId)); + + return Response.json({ + ok: true, + resolvedBucket: bucket.bucketName, + deleted, + restoredOriginal, + restoredDuration, + }); +} From 208b2bf2d48641b1fb7b1aa1f05718feada92aba Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:22 +0100 Subject: [PATCH 14/20] feat(web): add active transcript word index hook --- .../use-active-transcript-word-index.test.ts | 199 ++++++++++++++++++ .../edit/use-active-transcript-word-index.ts | 113 ++++++++++ 2 files changed, 312 insertions(+) create mode 100644 apps/web/__tests__/unit/use-active-transcript-word-index.test.ts create mode 100644 apps/web/app/s/[videoId]/edit/use-active-transcript-word-index.ts diff --git a/apps/web/__tests__/unit/use-active-transcript-word-index.test.ts b/apps/web/__tests__/unit/use-active-transcript-word-index.test.ts new file mode 100644 index 0000000000..214234e55b --- /dev/null +++ b/apps/web/__tests__/unit/use-active-transcript-word-index.test.ts @@ -0,0 +1,199 @@ +// @vitest-environment jsdom + +import { act, createElement, type RefObject } from "react"; +import { createRoot } from "react-dom/client"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; +import { useActiveTranscriptWordIndex } from "@/app/s/[videoId]/edit/use-active-transcript-word-index"; +import type { EditTranscriptWord } from "@/lib/edit-transcript"; + +const words: EditTranscriptWord[] = [ + { + id: "word-0", + text: "One", + startMs: 100, + endMs: 300, + confidence: 1, + speaker: null, + channel: null, + }, + { + id: "word-1", + text: "two", + startMs: 600, + endMs: 900, + confidence: 1, + speaker: null, + channel: null, + }, + { + id: "word-2", + text: "three", + startMs: 1_000, + endMs: 1_200, + confidence: 1, + speaker: null, + channel: null, + }, +]; + +const actEnvironment = globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; +}; + +function createVideo(currentTime: number, paused: boolean) { + const video = document.createElement("video"); + Object.defineProperties(video, { + currentTime: { + configurable: true, + writable: true, + value: currentTime, + }, + ended: { + configurable: true, + writable: true, + value: false, + }, + paused: { + configurable: true, + writable: true, + value: paused, + }, + }); + return video; +} + +describe("useActiveTranscriptWordIndex", () => { + beforeAll(() => { + actEnvironment.IS_REACT_ACT_ENVIRONMENT = true; + }); + + beforeEach(() => { + vi.stubGlobal( + "requestAnimationFrame", + vi.fn(() => 1), + ); + vi.stubGlobal("cancelAnimationFrame", vi.fn()); + }); + + afterEach(() => { + document.body.replaceChildren(); + vi.unstubAllGlobals(); + }); + + afterAll(() => { + delete actEnvironment.IS_REACT_ACT_ENVIRONMENT; + }); + + it("binds after the video mounts and follows presented video frames", async () => { + const videoRef: RefObject = { current: null }; + let presentedFrame: VideoFrameRequestCallback | null = null; + const requestVideoFrameCallback = vi.fn( + (callback: VideoFrameRequestCallback) => { + presentedFrame = callback; + return 1; + }, + ); + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + const Harness = () => + createElement( + "output", + null, + useActiveTranscriptWordIndex(videoRef, words), + ); + + await act(async () => { + root.render(createElement(Harness)); + }); + expect(container.textContent).toBe("-1"); + + const video = createVideo(0.15, true); + Object.defineProperties(video, { + cancelVideoFrameCallback: { + configurable: true, + value: vi.fn(), + }, + requestVideoFrameCallback: { + configurable: true, + value: requestVideoFrameCallback, + }, + }); + await act(async () => { + videoRef.current = video; + document.body.append(video); + await Promise.resolve(); + }); + expect(container.textContent).toBe("0"); + + Object.defineProperty(video, "paused", { + configurable: true, + writable: true, + value: false, + }); + await act(async () => { + video.dispatchEvent(new Event("play")); + }); + expect(requestVideoFrameCallback).toHaveBeenCalledOnce(); + + await act(async () => { + presentedFrame?.(0, { + mediaTime: 0.65, + } as VideoFrameCallbackMetadata); + }); + expect(container.textContent).toBe("1"); + + await act(async () => { + root.unmount(); + }); + }); + + it("rebinds when the player replaces its video element", async () => { + const firstVideo = createVideo(0.15, true); + const videoRef: RefObject = { + current: firstVideo, + }; + document.body.append(firstVideo); + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + const Harness = () => + createElement( + "output", + null, + useActiveTranscriptWordIndex(videoRef, words), + ); + + await act(async () => { + root.render(createElement(Harness)); + }); + expect(container.textContent).toBe("0"); + + const secondVideo = createVideo(1.1, true); + await act(async () => { + videoRef.current = secondVideo; + firstVideo.replaceWith(secondVideo); + await Promise.resolve(); + }); + expect(container.textContent).toBe("2"); + + await act(async () => { + firstVideo.currentTime = 0.65; + firstVideo.dispatchEvent(new Event("timeupdate")); + }); + expect(container.textContent).toBe("2"); + + await act(async () => { + root.unmount(); + }); + }); +}); diff --git a/apps/web/app/s/[videoId]/edit/use-active-transcript-word-index.ts b/apps/web/app/s/[videoId]/edit/use-active-transcript-word-index.ts new file mode 100644 index 0000000000..fa038e24c5 --- /dev/null +++ b/apps/web/app/s/[videoId]/edit/use-active-transcript-word-index.ts @@ -0,0 +1,113 @@ +import { type RefObject, useEffect, useRef, useState } from "react"; +import { + type EditTranscriptWord, + findActiveTranscriptWordIndex, +} from "@/lib/edit-transcript"; + +export function useActiveTranscriptWordIndex( + videoRef: RefObject, + words: readonly EditTranscriptWord[] | null, +) { + const [activeWordIndex, setActiveWordIndex] = useState(-1); + const activeWordIndexRef = useRef(-1); + + useEffect(() => { + if (!words) { + if (activeWordIndexRef.current !== -1) { + activeWordIndexRef.current = -1; + setActiveWordIndex(-1); + } + return; + } + + let boundVideo: HTMLVideoElement | null = null; + let detachVideoListeners: (() => void) | null = null; + + const attachVideoListeners = () => { + const video = videoRef.current; + if (video === boundVideo) return; + + detachVideoListeners?.(); + detachVideoListeners = null; + boundVideo = video; + if (!video) return; + + let animationFrameId = 0; + let videoFrameId = 0; + const updateActiveWord = (timeSeconds = video.currentTime) => { + const nextIndex = findActiveTranscriptWordIndex( + words, + timeSeconds * 1000, + ); + if (activeWordIndexRef.current === nextIndex) return; + activeWordIndexRef.current = nextIndex; + setActiveWordIndex(nextIndex); + }; + const stopPlaybackFrames = () => { + cancelAnimationFrame(animationFrameId); + animationFrameId = 0; + if (videoFrameId) { + video.cancelVideoFrameCallback(videoFrameId); + videoFrameId = 0; + } + }; + const schedulePlaybackFrame = () => { + if (typeof video.requestVideoFrameCallback === "function") { + videoFrameId = video.requestVideoFrameCallback((_now, metadata) => { + videoFrameId = 0; + updateActiveWord(metadata.mediaTime); + if (!video.paused && !video.ended) { + schedulePlaybackFrame(); + } + }); + return; + } + animationFrameId = requestAnimationFrame(() => { + animationFrameId = 0; + updateActiveWord(); + if (!video.paused && !video.ended) { + schedulePlaybackFrame(); + } + }); + }; + const handlePlay = () => { + stopPlaybackFrames(); + schedulePlaybackFrame(); + }; + const handlePause = () => { + stopPlaybackFrames(); + updateActiveWord(); + }; + const handleTimeChange = () => updateActiveWord(); + + updateActiveWord(); + video.addEventListener("play", handlePlay); + video.addEventListener("pause", handlePause); + video.addEventListener("seeked", handleTimeChange); + video.addEventListener("timeupdate", handleTimeChange); + if (!video.paused) handlePlay(); + + detachVideoListeners = () => { + stopPlaybackFrames(); + video.removeEventListener("play", handlePlay); + video.removeEventListener("pause", handlePause); + video.removeEventListener("seeked", handleTimeChange); + video.removeEventListener("timeupdate", handleTimeChange); + }; + }; + + const videoObserver = new MutationObserver(attachVideoListeners); + videoObserver.observe(document.body, { + childList: true, + subtree: true, + }); + attachVideoListeners(); + + return () => { + videoObserver.disconnect(); + detachVideoListeners?.(); + }; + }, [videoRef, words]); + + return activeWordIndex; +} From 796b8afe21d3aa156b5d652a0e2a73f5943c1ff0 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:22 +0100 Subject: [PATCH 15/20] feat(web): add transcript sidebar for video editor --- .../s/[videoId]/edit/TranscriptSidebar.tsx | 816 ++++++++++++++++++ 1 file changed, 816 insertions(+) create mode 100644 apps/web/app/s/[videoId]/edit/TranscriptSidebar.tsx diff --git a/apps/web/app/s/[videoId]/edit/TranscriptSidebar.tsx b/apps/web/app/s/[videoId]/edit/TranscriptSidebar.tsx new file mode 100644 index 0000000000..dd72e7b398 --- /dev/null +++ b/apps/web/app/s/[videoId]/edit/TranscriptSidebar.tsx @@ -0,0 +1,816 @@ +"use client"; + +import type { VideoEditRange } from "@cap/database/types"; +import type { Video } from "@cap/web-domain"; +import { useVirtualizer } from "@virtual-grid/react"; +import { + ChevronDown, + ChevronUp, + FileText, + LoaderCircle, + Search, + Trash2, + X, +} from "lucide-react"; +import { + memo, + type PointerEvent as ReactPointerEvent, + type RefObject, + useCallback, + useDeferredValue, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { toast } from "sonner"; +import { + getEditTranscript, + requestEditTranscript, +} from "@/actions/videos/get-edit-transcript"; +import { + type EditTranscriptGroup, + type EditTranscriptWord, + getDeletedTranscriptWordIds, + groupEditTranscriptWords, + isFillerWord, + normalizeTranscriptSelection, + planFillerCuts, + planTranscriptCut, +} from "@/lib/edit-transcript"; +import { useActiveTranscriptWordIndex } from "./use-active-transcript-word-index"; + +type TranscriptResponse = Awaited>; + +type TranscriptSidebarProps = { + videoId: Video.VideoId; + videoRef: RefObject; + keepRanges: VideoEditRange[]; + onDeleteRanges: (ranges: VideoEditRange[]) => void; +}; + +const SIDEBAR_CLASS_NAME = + "mx-3 mb-4 flex min-h-[36rem] flex-col overflow-hidden rounded-2xl border border-gray-4 bg-gray-1 shadow-[0_16px_44px_-32px_rgba(15,23,42,0.28)] sm:mx-5 xl:fixed xl:top-20 xl:right-5 xl:mx-0 xl:mb-0 xl:h-[calc(100vh-6rem)] xl:w-[360px] min-[1540px]:right-[calc((100vw-1500px)/2+20px)]"; + +const SKELETON_LINE_WIDTHS = [ + ["w-full", "w-4/5"], + ["w-11/12", "w-2/3"], + ["w-full", "w-3/5", "w-2/3"], + ["w-5/6", "w-1/3"], + ["w-full", "w-3/4"], + ["w-10/12", "w-1/2"], +]; + +function formatTimestamp(milliseconds: number) { + const totalSeconds = Math.max(0, Math.floor(milliseconds / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${minutes}:${String(seconds).padStart(2, "0")}`; +} + +function isTimeKept(timeMs: number, keepRanges: readonly VideoEditRange[]) { + const timeSeconds = timeMs / 1000; + return keepRanges.some( + (range) => timeSeconds >= range.start && timeSeconds <= range.end, + ); +} + +function toVideoRanges( + ranges: readonly { startMs: number; endMs: number }[], +): VideoEditRange[] { + return ranges.map((range) => ({ + start: range.startMs / 1000, + end: range.endMs / 1000, + })); +} + +function TranscriptSkeleton() { + return ( +
+ {SKELETON_LINE_WIDTHS.map((lineWidths, rowIndex) => ( +
+
+
+ {lineWidths.map((width) => ( +
+ ))} +
+
+ ))} +
+ ); +} + +const TranscriptGroupRow = memo(function TranscriptGroupRow({ + group, + words, + activeWordIndex, + selectionStartIndex, + selectionEndIndex, + deletedWordIds, + searchMatchSet, + previewFillers, + wordElementsRef, + onSeek, + onWordPointerDown, + onWordPointerEnter, +}: { + group: EditTranscriptGroup; + words: readonly EditTranscriptWord[]; + activeWordIndex: number; + selectionStartIndex: number; + selectionEndIndex: number; + deletedWordIds: ReadonlySet; + searchMatchSet: ReadonlySet; + previewFillers: boolean; + wordElementsRef: RefObject>; + onSeek: (index: number) => void; + onWordPointerDown: ( + index: number, + event: ReactPointerEvent, + ) => void; + onWordPointerEnter: ( + index: number, + event: ReactPointerEvent, + ) => void; +}) { + return ( +
+ +

+ {words + .slice(group.startIndex, group.endIndex + 1) + .map((word, groupWordIndex) => { + const index = group.startIndex + groupWordIndex; + const isSelected = + index >= selectionStartIndex && index <= selectionEndIndex; + const isSelectionStart = + isSelected && index === selectionStartIndex; + const isSelectionEnd = isSelected && index === selectionEndIndex; + const isActive = activeWordIndex === index; + const isDeleted = deletedWordIds.has(word.id); + const isFiller = isFillerWord(word.text); + const isSearchMatch = searchMatchSet.has(index); + const willRemoveAsFiller = previewFillers && isFiller && !isDeleted; + + let stateClassName: string; + if (isSelected) { + stateClassName = [ + "bg-blue-500 text-white", + isSelectionStart ? "rounded-l-[5px]" : "", + isSelectionEnd ? "rounded-r-[5px]" : "", + ].join(" "); + } else if (isActive) { + stateClassName = "rounded-[5px] bg-blue-500/15"; + } else if (isSearchMatch) { + stateClassName = isDeleted + ? "rounded-[5px] bg-amber-100/60" + : "rounded-[5px] bg-amber-100 text-amber-900"; + } else if (willRemoveAsFiller) { + stateClassName = + "rounded-[5px] bg-red-50 text-red-400 line-through decoration-red-300/60"; + } else { + stateClassName = "rounded-[5px] hover:bg-gray-3"; + } + + return ( + + ); + })} +

+
+ ); +}); + +export function TranscriptSidebar({ + videoId, + videoRef, + keepRanges, + onDeleteRanges, +}: TranscriptSidebarProps) { + const [response, setResponse] = useState(null); + const [isRequesting, setIsRequesting] = useState(false); + const [selection, setSelection] = useState<{ + startIndex: number; + endIndex: number; + } | null>(null); + const [searchQuery, setSearchQuery] = useState(""); + const [searchMatchIndex, setSearchMatchIndex] = useState(0); + const [previewFillers, setPreviewFillers] = useState(false); + const deferredSearchQuery = useDeferredValue(searchQuery); + const dragAnchorRef = useRef(null); + const selectionRef = useRef(selection); + const transcriptScrollRef = useRef(null); + const wordElementsRef = useRef(new Map()); + const transcript = response?.status === "ready" ? response.transcript : null; + const activeWordIndex = useActiveTranscriptWordIndex( + videoRef, + transcript?.words ?? null, + ); + + const loadTranscript = useCallback(async () => { + setResponse(await getEditTranscript(videoId)); + }, [videoId]); + + useEffect(() => { + void loadTranscript(); + }, [loadTranscript]); + + useEffect(() => { + if (response?.status !== "processing") return; + let cancelled = false; + let timeout: number; + const poll = async () => { + try { + const nextResponse = await getEditTranscript(videoId); + if (cancelled) return; + setResponse(nextResponse); + if (nextResponse.status === "processing") { + timeout = window.setTimeout(() => void poll(), 2_000); + } + } catch { + if (!cancelled) { + timeout = window.setTimeout(() => void poll(), 2_000); + } + } + }; + timeout = window.setTimeout(() => void poll(), 2_000); + return () => { + cancelled = true; + window.clearTimeout(timeout); + }; + }, [response?.status, videoId]); + + useEffect(() => { + const handlePointerUp = () => { + dragAnchorRef.current = null; + }; + window.addEventListener("pointerup", handlePointerUp); + window.addEventListener("pointercancel", handlePointerUp); + return () => { + window.removeEventListener("pointerup", handlePointerUp); + window.removeEventListener("pointercancel", handlePointerUp); + }; + }, []); + + const groups = useMemo( + () => (transcript ? groupEditTranscriptWords(transcript.words) : []), + [transcript], + ); + const deletedWordIds = useMemo( + () => + transcript + ? getDeletedTranscriptWordIds(transcript.words, keepRanges) + : new Set(), + [keepRanges, transcript], + ); + const availableFillerCount = useMemo( + () => + transcript?.words.filter( + (word) => isFillerWord(word.text) && !deletedWordIds.has(word.id), + ).length ?? 0, + [deletedWordIds, transcript], + ); + const searchMatches = useMemo(() => { + const query = deferredSearchQuery.trim().toLocaleLowerCase(); + if (!transcript || !query) return []; + return transcript.words.flatMap((word, index) => + word.text.toLocaleLowerCase().includes(query) ? [index] : [], + ); + }, [deferredSearchQuery, transcript]); + const searchMatchSet = useMemo(() => new Set(searchMatches), [searchMatches]); + const groupVirtualizer = useVirtualizer({ + count: groups.length, + getScrollElement: () => transcriptScrollRef.current, + estimateSize: () => 116, + getItemKey: (index) => groups[index]?.id ?? index, + overscan: 6, + }); + + const seekToWord = useCallback( + (index: number) => { + const word = transcript?.words[index]; + const video = videoRef.current; + if (!word || !video) return; + video.currentTime = word.startMs / 1000; + }, + [transcript, videoRef], + ); + + const updateSelection = useCallback( + (firstIndex: number, secondIndex: number) => { + if (!transcript) return; + const nextSelection = normalizeTranscriptSelection( + firstIndex, + secondIndex, + transcript.words.length, + ); + selectionRef.current = nextSelection; + setSelection(nextSelection); + }, + [transcript], + ); + + const clearSelection = useCallback(() => { + selectionRef.current = null; + setSelection(null); + }, []); + + const handleWordPointerDown = useCallback( + (index: number, event: ReactPointerEvent) => { + if (event.button !== 0) return; + event.stopPropagation(); + + const currentSelection = selectionRef.current; + const anchor = + event.shiftKey && currentSelection + ? currentSelection.startIndex + : index; + dragAnchorRef.current = anchor; + updateSelection(anchor, index); + seekToWord(index); + }, + [seekToWord, updateSelection], + ); + + const handleWordPointerEnter = useCallback( + (index: number, event: ReactPointerEvent) => { + if (dragAnchorRef.current === null || event.buttons !== 1) return; + updateSelection(dragAnchorRef.current, index); + }, + [updateSelection], + ); + + const deleteSelection = useCallback(() => { + if (!transcript || !selection) return; + const plan = planTranscriptCut( + transcript.words, + selection.startIndex, + selection.endIndex, + transcript.durationMs, + ); + if (!plan) return; + if (!plan.safe) { + toast.error( + plan.reason === "overlapping-speech" + ? "This selection overlaps other speech and cannot be cut safely." + : "An edit must keep some playable video.", + ); + return; + } + + onDeleteRanges(toVideoRanges([plan])); + clearSelection(); + const nextWord = transcript.words[selection.endIndex + 1]; + if (nextWord) seekToWord(selection.endIndex + 1); + }, [clearSelection, onDeleteRanges, seekToWord, selection, transcript]); + + const deleteFillers = useCallback(() => { + if (!transcript) return; + const plan = planFillerCuts(transcript.words, transcript.durationMs); + const activeRanges = plan.ranges.filter((range) => + isTimeKept((range.startMs + range.endMs) / 2, keepRanges), + ); + if (activeRanges.length === 0) { + toast.info("No safe filler words remain."); + return; + } + + onDeleteRanges(toVideoRanges(activeRanges)); + let activeRangeIndex = 0; + let removedCount = 0; + for (const word of transcript.words) { + if (!isFillerWord(word.text) || deletedWordIds.has(word.id)) continue; + const midpointMs = (word.startMs + word.endMs) / 2; + while ( + activeRangeIndex < activeRanges.length && + (activeRanges[activeRangeIndex]?.endMs ?? 0) < midpointMs + ) { + activeRangeIndex++; + } + const activeRange = activeRanges[activeRangeIndex]; + if ( + activeRange && + midpointMs >= activeRange.startMs && + midpointMs <= activeRange.endMs + ) { + removedCount++; + } + } + const skippedCount = availableFillerCount - removedCount; + clearSelection(); + if (skippedCount > 0) { + toast.info( + `Removed ${removedCount} fillers and skipped ${skippedCount} unsafe cuts.`, + ); + } else { + toast.success( + `Removed ${removedCount} filler${removedCount === 1 ? "" : "s"}.`, + ); + } + }, [ + availableFillerCount, + clearSelection, + deletedWordIds, + keepRanges, + onDeleteRanges, + transcript, + ]); + + const moveSearchMatch = useCallback( + (direction: -1 | 1) => { + if (searchMatches.length === 0) return; + const next = + (searchMatchIndex + direction + searchMatches.length) % + searchMatches.length; + setSearchMatchIndex(next); + const wordIndex = searchMatches[next]; + if (wordIndex === undefined) return; + const groupIndex = groups.findIndex( + (group) => wordIndex >= group.startIndex && wordIndex <= group.endIndex, + ); + if (groupIndex >= 0) { + groupVirtualizer.scrollToIndex(groupIndex, { + align: "center", + behavior: "smooth", + }); + } + window.requestAnimationFrame(() => { + wordElementsRef.current + .get(wordIndex) + ?.scrollIntoView({ block: "center", behavior: "smooth" }); + }); + updateSelection(wordIndex, wordIndex); + seekToWord(wordIndex); + }, + [ + groupVirtualizer, + groups, + searchMatchIndex, + searchMatches, + seekToWord, + updateSelection, + ], + ); + + const handleRequest = useCallback(async () => { + setIsRequesting(true); + const nextResponse = await requestEditTranscript(videoId); + setResponse(nextResponse); + setIsRequesting(false); + }, [videoId]); + + // The verbatim pass only exists to serve this editor, so kick it off the + // moment we learn it hasn't run yet instead of asking the user to. + const autoRequestedRef = useRef(false); + useEffect(() => { + if (response?.status !== "missing" || autoRequestedRef.current) return; + autoRequestedRef.current = true; + void handleRequest(); + }, [response?.status, handleRequest]); + + if (response === null) { + return ( + + ); + } + + if (response.status === "missing" || response.status === "processing") { + return ( + + ); + } + + if (response.status === "error") { + return ( + + ); + } + + if (!transcript) return null; + + if (transcript.words.length === 0) { + return ( + + ); + } + + const selectedWordCount = selection + ? selection.endIndex - selection.startIndex + 1 + : 0; + + return ( + + ); +} From 398b9953ff5d7d5c2eb8a4bbc3074623cf349eae Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:25 +0100 Subject: [PATCH 16/20] feat(web): allow disabling CapVideoPlayer playback speed dial --- apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx b/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx index 49578e5f56..efdf063dc7 100644 --- a/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx +++ b/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx @@ -94,6 +94,7 @@ interface Props { disableCommentStamps?: boolean; disableReactionStamps?: boolean; disablePreviewGif?: boolean; + disablePlaybackSpeedDial?: boolean; comments?: Array<{ id: string; timestamp: number | null; @@ -134,6 +135,7 @@ export function CapVideoPlayer({ disableCommentStamps = false, disableReactionStamps = false, disablePreviewGif = false, + disablePlaybackSpeedDial = false, onSeek, enhancedAudioUrl: _enhancedAudioUrl, enhancedAudioStatus: _enhancedAudioStatus, @@ -767,7 +769,8 @@ export function CapVideoPlayer({ videoLoaded && !hasActiveProgress && !showUploadFailureOverlay && - !showPlaybackResolutionError && ( + !showPlaybackResolutionError && + !disablePlaybackSpeedDial && ( Date: Thu, 30 Jul 2026 22:11:25 +0100 Subject: [PATCH 17/20] fix(web): center-crop timeline video frame thumbnails --- .../unit/video-frame-thumbnail.test.ts | 78 +++++++++++++++++-- .../_components/video-frame-thumbnail.ts | 29 ++++++- 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/apps/web/__tests__/unit/video-frame-thumbnail.test.ts b/apps/web/__tests__/unit/video-frame-thumbnail.test.ts index 8bec6942a6..8644b8c713 100644 --- a/apps/web/__tests__/unit/video-frame-thumbnail.test.ts +++ b/apps/web/__tests__/unit/video-frame-thumbnail.test.ts @@ -31,8 +31,12 @@ function createMockCanvas(options?: { }; } -function createMockVideo(readyState = 3): HTMLVideoElement { - return { readyState } as unknown as HTMLVideoElement; +function createMockVideo( + readyState = 3, + videoWidth = 224, + videoHeight = 128, +): HTMLVideoElement { + return { readyState, videoWidth, videoHeight } as unknown as HTMLVideoElement; } describe("captureVideoFrameDataUrl", () => { @@ -83,13 +87,23 @@ describe("captureVideoFrameDataUrl", () => { createCanvas: () => canvas, }); expect(result).toBe("data:image/jpeg;base64,abc123"); - expect(ctx.drawImage).toHaveBeenCalledWith(video, 0, 0, 224, 128); + expect(ctx.drawImage).toHaveBeenCalledWith( + video, + 0, + 0, + 224, + 128, + 0, + 0, + 224, + 128, + ); expect(canvas.toDataURL).toHaveBeenCalledWith("image/jpeg", 0.8); }); it("respects custom width and height", () => { const { canvas, ctx } = createMockCanvas(); - const video = createMockVideo(); + const video = createMockVideo(3, 320, 180); captureVideoFrameDataUrl({ video, createCanvas: () => canvas, @@ -98,7 +112,61 @@ describe("captureVideoFrameDataUrl", () => { }); expect(canvas.width).toBe(320); expect(canvas.height).toBe(180); - expect(ctx.drawImage).toHaveBeenCalledWith(video, 0, 0, 320, 180); + expect(ctx.drawImage).toHaveBeenCalledWith( + video, + 0, + 0, + 320, + 180, + 0, + 0, + 320, + 180, + ); + }); + + it("center-crops a taller source to the output aspect ratio", () => { + const { canvas, ctx } = createMockCanvas(); + const video = createMockVideo(3, 1280, 960); + captureVideoFrameDataUrl({ + video, + createCanvas: () => canvas, + width: 160, + height: 90, + }); + expect(ctx.drawImage).toHaveBeenCalledWith( + video, + 0, + 120, + 1280, + 720, + 0, + 0, + 160, + 90, + ); + }); + + it("center-crops a wider source to the output aspect ratio", () => { + const { canvas, ctx } = createMockCanvas(); + const video = createMockVideo(3, 2560, 1080); + captureVideoFrameDataUrl({ + video, + createCanvas: () => canvas, + width: 160, + height: 90, + }); + expect(ctx.drawImage).toHaveBeenCalledWith( + video, + 320, + 0, + 1920, + 1080, + 0, + 0, + 160, + 90, + ); }); it("respects custom JPEG quality", () => { diff --git a/apps/web/app/s/[videoId]/_components/video-frame-thumbnail.ts b/apps/web/app/s/[videoId]/_components/video-frame-thumbnail.ts index 06495e03ea..886eae0162 100644 --- a/apps/web/app/s/[videoId]/_components/video-frame-thumbnail.ts +++ b/apps/web/app/s/[videoId]/_components/video-frame-thumbnail.ts @@ -20,7 +20,34 @@ export function captureVideoFrameDataUrl( const ctx = canvas.getContext("2d"); if (!ctx) return undefined; try { - ctx.drawImage(video, 0, 0, width, height); + const sourceWidth = video.videoWidth || width; + const sourceHeight = video.videoHeight || height; + const sourceAspectRatio = sourceWidth / sourceHeight; + const outputAspectRatio = width / height; + let sourceX = 0; + let sourceY = 0; + let croppedWidth = sourceWidth; + let croppedHeight = sourceHeight; + + if (sourceAspectRatio > outputAspectRatio) { + croppedWidth = sourceHeight * outputAspectRatio; + sourceX = (sourceWidth - croppedWidth) / 2; + } else if (sourceAspectRatio < outputAspectRatio) { + croppedHeight = sourceWidth / outputAspectRatio; + sourceY = (sourceHeight - croppedHeight) / 2; + } + + ctx.drawImage( + video, + sourceX, + sourceY, + croppedWidth, + croppedHeight, + 0, + 0, + width, + height, + ); return canvas.toDataURL("image/jpeg", quality); } catch { return undefined; From 44ac99fd0c6455878c4c68129e79fe3f321cbe32 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:25 +0100 Subject: [PATCH 18/20] feat(web): wire transcript editing into video editor --- .../app/s/[videoId]/edit/EditVideoClient.tsx | 147 +++++++++++++++--- apps/web/app/s/[videoId]/edit/page.tsx | 2 + 2 files changed, 129 insertions(+), 20 deletions(-) diff --git a/apps/web/app/s/[videoId]/edit/EditVideoClient.tsx b/apps/web/app/s/[videoId]/edit/EditVideoClient.tsx index 9b404623cb..e10a886529 100644 --- a/apps/web/app/s/[videoId]/edit/EditVideoClient.tsx +++ b/apps/web/app/s/[videoId]/edit/EditVideoClient.tsx @@ -51,7 +51,10 @@ import { createTimelineHistory, createTimelineState, deleteSelectedTimelineSegment, + deleteTimelineRanges, findNextPlayableTime, + findNextPlayableTimeInRanges, + findPlayableRangeIndex, getEditSpecOutputDuration, getTimelineDisplayDuration, getTimelineDisplaySegments, @@ -76,6 +79,7 @@ import { navigateWithTransition } from "@/utils/view-transition"; import { CapVideoPlayer } from "../_components/CapVideoPlayer"; import { VideoDownloadMenu } from "../_components/VideoDownloadMenu"; import { captureVideoFrameDataUrl } from "../_components/video-frame-thumbnail"; +import { TranscriptSidebar } from "./TranscriptSidebar"; type EditableVideo = { id: Video.VideoId; @@ -84,12 +88,14 @@ type EditableVideo = { duration: number; width: number | null; height: number | null; + transcriptionStatus: string | null; }; type DragHandle = "start" | "end"; const MAX_TIMELINE_THUMBNAILS = 48; const MAX_VISIBLE_THUMBNAIL_GENERATION = 16; +const PREVIEW_CUT_MUTE_LEAD_SECONDS = 0.03; const TIMELINE_THUMBNAIL_WIDTH = 160; const TIMELINE_THUMBNAIL_HEIGHT = 90; const MIN_ZOOM = 1; @@ -930,6 +936,21 @@ export function EditVideoClient({ setHistory(redoTimelineHistory); }, []); + const handleTranscriptDelete = useCallback( + (ranges: { start: number; end: number }[]) => { + const nextState = deleteTimelineRanges(stateRef.current, ranges); + const nextEditSpec = getTimelineEditSpec(nextState); + const nextPlayableTime = + findNextPlayableTime(playheadRef.current, nextEditSpec) ?? + nextEditSpec.keepRanges.at(-1)?.end ?? + 0; + commitState(nextState); + setPlayheadOnFrame(nextPlayableTime, true); + setVideoTimeOnFrame(nextPlayableTime, true); + }, + [commitState, setPlayheadOnFrame, setVideoTimeOnFrame], + ); + // Loom-style: cut the clip at the current playhead into two independent clips. const handleSplit = useCallback(() => { commitState(splitTimelineAt(stateRef.current, playheadRef.current)); @@ -1239,6 +1260,7 @@ export function EditVideoClient({ useEffect(() => { let frameId = 0; + let playbackFrameId = 0; let detachVideoListeners: (() => void) | null = null; const attachVideoListeners = () => { @@ -1248,19 +1270,46 @@ export function EditVideoClient({ return; } - const syncPlayhead = () => { + let restoreAudioAfterCut = false; + const muteForCut = () => { + if (restoreAudioAfterCut || videoElement.muted) return; + restoreAudioAfterCut = true; + videoElement.muted = true; + }; + const restoreCutAudio = () => { + if (!restoreAudioAfterCut) return; + restoreAudioAfterCut = false; + videoElement.muted = false; + }; + const syncPlayhead = (updatePlayhead: boolean) => { if (previewWithoutPlayheadRef.current) { // Trimming a clip edge: scrub the preview but keep the cursor put. return; } if (dragDraftRef.current !== null) { - setPlayheadOnFrame(videoElement.currentTime, true); + if (updatePlayhead) { + setPlayheadOnFrame(videoElement.currentTime, true); + } return; } - const nextTime = findNextPlayableTime( - videoElement.currentTime, - editSpec, + const currentTime = videoElement.currentTime; + const playableRangeIndex = findPlayableRangeIndex( + currentTime, + keepRanges, ); + const playableRange = keepRanges[playableRangeIndex]; + if ( + !videoElement.paused && + playableRange && + playableRangeIndex < keepRanges.length - 1 && + playableRange.end - currentTime <= PREVIEW_CUT_MUTE_LEAD_SECONDS + ) { + muteForCut(); + } + const nextTime = + playableRangeIndex >= 0 + ? currentTime + : findNextPlayableTimeInRanges(currentTime, keepRanges); if (nextTime === null) { videoElement.pause(); @@ -1269,32 +1318,73 @@ export function EditVideoClient({ return; } - if (Math.abs(nextTime - videoElement.currentTime) > 0.04) { + if (Math.abs(nextTime - currentTime) > 0.04) { + muteForCut(); videoElement.currentTime = nextTime; setPlayheadOnFrame(nextTime, true); return; } - setPlayheadOnFrame(videoElement.currentTime, true); + if (updatePlayhead) { + setPlayheadOnFrame(currentTime, true); + } }; + const handleMediaTimeChange = () => syncPlayhead(true); - const handlePlay = () => setIsPlaying(true); - const handlePause = () => setIsPlaying(false); + const stopPlaybackFrames = () => { + cancelAnimationFrame(playbackFrameId); + playbackFrameId = 0; + }; + const followPlayback = () => { + playbackFrameId = 0; + syncPlayhead(false); + if (!videoElement.paused && !videoElement.ended) { + playbackFrameId = requestAnimationFrame(followPlayback); + } + }; + const handlePlay = () => { + setIsPlaying(true); + stopPlaybackFrames(); + playbackFrameId = requestAnimationFrame(followPlayback); + }; + const handlePause = () => { + stopPlaybackFrames(); + setIsPlaying(false); + if (!videoElement.seeking) { + restoreCutAudio(); + } + syncPlayhead(true); + }; + const handleSeeked = () => { + restoreCutAudio(); + syncPlayhead(true); + }; - videoElement.addEventListener("timeupdate", syncPlayhead); - videoElement.addEventListener("seeking", syncPlayhead); - videoElement.addEventListener("loadedmetadata", syncPlayhead); + videoElement.addEventListener("timeupdate", handleMediaTimeChange); + videoElement.addEventListener("seeking", handleMediaTimeChange); + videoElement.addEventListener("seeked", handleSeeked); + videoElement.addEventListener("loadedmetadata", handleMediaTimeChange); videoElement.addEventListener("play", handlePlay); videoElement.addEventListener("pause", handlePause); - setIsPlaying(!videoElement.paused); - syncPlayhead(); + syncPlayhead(true); + if (videoElement.paused) { + setIsPlaying(false); + } else { + handlePlay(); + } detachVideoListeners = () => { - videoElement.removeEventListener("timeupdate", syncPlayhead); - videoElement.removeEventListener("seeking", syncPlayhead); - videoElement.removeEventListener("loadedmetadata", syncPlayhead); + stopPlaybackFrames(); + videoElement.removeEventListener("timeupdate", handleMediaTimeChange); + videoElement.removeEventListener("seeking", handleMediaTimeChange); + videoElement.removeEventListener("seeked", handleSeeked); + videoElement.removeEventListener( + "loadedmetadata", + handleMediaTimeChange, + ); videoElement.removeEventListener("play", handlePlay); videoElement.removeEventListener("pause", handlePause); + restoreCutAudio(); }; }; @@ -1304,7 +1394,7 @@ export function EditVideoClient({ cancelAnimationFrame(frameId); detachVideoListeners?.(); }; - }, [editSpec, keepRanges, setPlayheadOnFrame]); + }, [keepRanges, setPlayheadOnFrame]); useEffect(() => { const videoElement = videoRef.current; @@ -1467,7 +1557,7 @@ export function EditVideoClient({ return (
-
+
-
+
@@ -1889,6 +1987,15 @@ export function EditVideoClient({
+ {video.transcriptionStatus === "COMPLETE" && ( + + )} + { diff --git a/apps/web/app/s/[videoId]/edit/page.tsx b/apps/web/app/s/[videoId]/edit/page.tsx index 5ec3a60421..3ee65368a8 100644 --- a/apps/web/app/s/[videoId]/edit/page.tsx +++ b/apps/web/app/s/[videoId]/edit/page.tsx @@ -38,6 +38,7 @@ export default async function EditVideoPage(props: { height: videos.height, source: videos.source, isScreenshot: videos.isScreenshot, + transcriptionStatus: videos.transcriptionStatus, uploadPhase: videoUploads.phase, }) .from(videos) @@ -90,6 +91,7 @@ export default async function EditVideoPage(props: { duration: video.duration, width: video.width, height: video.height, + transcriptionStatus: video.transcriptionStatus, }} /> ); From df4ba6276c4c6491fa6f8fa4610e48e8a701dbcc Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:11:25 +0100 Subject: [PATCH 19/20] feat(web): expand transcript tab copy and download menus --- .../[videoId]/_components/tabs/Transcript.tsx | 567 ++++++++++-------- 1 file changed, 315 insertions(+), 252 deletions(-) diff --git a/apps/web/app/s/[videoId]/_components/tabs/Transcript.tsx b/apps/web/app/s/[videoId]/_components/tabs/Transcript.tsx index 05bee5768f..a198215cac 100644 --- a/apps/web/app/s/[videoId]/_components/tabs/Transcript.tsx +++ b/apps/web/app/s/[videoId]/_components/tabs/Transcript.tsx @@ -10,8 +10,8 @@ import { Download, Edit3, Globe, + LoaderCircle, MessageSquare, - X, } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { editTranscriptEntry } from "@/actions/videos/edit-transcript"; @@ -20,6 +20,7 @@ import { SUPPORTED_LANGUAGES, } from "@/actions/videos/translation-languages"; import { useCurrentUser } from "@/app/Layout/AuthContext"; +import { formatTranscriptAsParagraphs } from "@/lib/transcript-text"; import { normalizeTranscriptCueText } from "@/lib/transcript-vtt"; import type { VideoData } from "../../types"; import { type CaptionLanguage, useCaptionContext } from "../CaptionContext"; @@ -35,6 +36,7 @@ interface TranscriptEntry { timestamp: string; text: string; startTime: number; + endTime: number; } const parseVTT = (vttContent: string): TranscriptEntry[] => { @@ -77,9 +79,12 @@ const parseVTT = (vttContent: string): TranscriptEntry[] => { ); if (totalSeconds === null) return null; + const fractionPart = secondsWithMs.split(".")[1]; + const fraction = fractionPart ? Number(`0.${fractionPart}`) : 0; + return { mm_ss: `${minutesStr}:${secondsPart}`, - totalSeconds, + totalSeconds: totalSeconds + (Number.isFinite(fraction) ? fraction : 0), }; }; @@ -101,11 +106,13 @@ const parseVTT = (vttContent: string): TranscriptEntry[] => { if (!startTimeStr || !endTimeStr) continue; const startTimestamp = parseTimestamp(startTimeStr); + const endTimestamp = parseTimestamp(endTimeStr); if (startTimestamp) { currentEntry = { id: currentId, timestamp: startTimestamp.mm_ss, startTime: startTimestamp.totalSeconds, + endTime: endTimestamp?.totalSeconds ?? startTimestamp.totalSeconds, }; } continue; @@ -148,7 +155,11 @@ export const Transcript: React.FC = ({ data, onSeek }) => { const [copyPressed, setCopyPressed] = useState(false); const [downloadPressed, setDownloadPressed] = useState(false); const [showLanguageMenu, setShowLanguageMenu] = useState(false); + const [showDownloadMenu, setShowDownloadMenu] = useState(false); + const [showCopyMenu, setShowCopyMenu] = useState(false); const languageMenuRef = useRef(null); + const downloadMenuRef = useRef(null); + const copyMenuRef = useRef(null); const selectedLanguage = captionContext.selectedLanguage === "off" @@ -176,6 +187,44 @@ export const Transcript: React.FC = ({ data, onSeek }) => { }; }, [showLanguageMenu]); + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + downloadMenuRef.current && + !downloadMenuRef.current.contains(event.target as Node) + ) { + setShowDownloadMenu(false); + } + }; + + if (showDownloadMenu) { + document.addEventListener("mousedown", handleClickOutside); + } + + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, [showDownloadMenu]); + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if ( + copyMenuRef.current && + !copyMenuRef.current.contains(event.target as Node) + ) { + setShowCopyMenu(false); + } + }; + + if (showCopyMenu) { + document.addEventListener("mousedown", handleClickOutside); + } + + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, [showCopyMenu]); + const { data: transcriptContent, isLoading: isTranscriptLoading, @@ -361,13 +410,12 @@ export const Transcript: React.FC = ({ data, onSeek }) => { return vttHeader + vttEntries.join("\n"); }; - const copyTranscriptToClipboard = async () => { - if (transcriptData.length === 0) return; + const copyTranscriptText = async (content: string) => { + if (!content) return; setIsCopying(true); try { - const formattedTranscript = formatTranscriptForClipboard(transcriptData); - await navigator.clipboard.writeText(formattedTranscript); + await navigator.clipboard.writeText(content); setCopyPressed(true); setTimeout(() => { setCopyPressed(false); @@ -379,18 +427,29 @@ export const Transcript: React.FC = ({ data, onSeek }) => { } }; - const downloadTranscriptFile = () => { + const copyFormattedTranscript = () => { + if (transcriptData.length === 0) return; + void copyTranscriptText(formatTranscriptAsParagraphs(transcriptData)); + }; + + const copyTimestampedTranscript = () => { if (transcriptData.length === 0) return; + void copyTranscriptText(formatTranscriptForClipboard(transcriptData)); + }; - const vttContent = formatTranscriptAsVTT(transcriptData); - const blob = new Blob([vttContent], { type: "text/vtt" }); + const triggerTranscriptDownload = ( + extension: string, + content: string, + mimeType: string, + ) => { + const blob = new Blob([content], { type: mimeType }); const url = URL.createObjectURL(blob); const langSuffix = selectedLanguage === "original" ? "" : `.${selectedLanguage}`; const link = document.createElement("a"); link.href = url; - link.download = `transcript-${data.id}${langSuffix}.vtt`; + link.download = `transcript-${data.id}${langSuffix}.${extension}`; document.body.appendChild(link); link.click(); document.body.removeChild(link); @@ -403,38 +462,35 @@ export const Transcript: React.FC = ({ data, onSeek }) => { }, 2000); }; + const downloadTranscriptFile = () => { + if (transcriptData.length === 0) return; + triggerTranscriptDownload( + "vtt", + formatTranscriptAsVTT(transcriptData), + "text/vtt", + ); + }; + + const downloadFormattedTranscript = () => { + if (transcriptData.length === 0) return; + const content = formatTranscriptAsParagraphs(transcriptData); + if (!content) return; + triggerTranscriptDownload("txt", `${content}\n`, "text/plain"); + }; + const canEdit = user?.id === data.owner.id && selectedLanguage === "original"; if (isTranscriptionProcessing && !hasTimedOut) { return ( -
+
-
- -
-

Transcription in progress...

+ +

+ Transcribing audio +

+

+ The transcript will appear here shortly. +

); @@ -442,43 +498,23 @@ export const Transcript: React.FC = ({ data, onSeek }) => { if (isQueryLoading) { return ( -
- +
+
); } if (data.transcriptionStatus === "NO_AUDIO") { return ( -
+
- -

+

+ +
+

No audio track detected

-

+

This video doesn't contain audio for transcription

{canEdit && ( @@ -512,13 +548,15 @@ export const Transcript: React.FC = ({ data, onSeek }) => { if (data.transcriptionStatus === "SKIPPED") { return ( -
+
- -

+

+ +
+

Transcription disabled

-

+

Transcription has been disabled for this video

@@ -536,10 +574,12 @@ export const Transcript: React.FC = ({ data, onSeek }) => { if (showRetryButton) { return ( -
+
- -

+

+ +
+

{data.transcriptionStatus === "ERROR" ? "Transcript not available" : "No transcript available"} @@ -573,118 +613,165 @@ export const Transcript: React.FC = ({ data, onSeek }) => { return (

-
-
-
- + {showLanguageMenu && ( +
+ +
+ {( + Object.entries(SUPPORTED_LANGUAGES) as [LanguageCode, string][] + ).map(([code, name]) => ( + + ))} +
+ )} +
+ +
+
+ - + + Formatted text + + + Clean paragraphs for docs + + + +
+ )}
-
+
- {showLanguageMenu && ( -
+ {showDownloadMenu && ( +
+ -
- {( - Object.entries(SUPPORTED_LANGUAGES) as [ - LanguageCode, - string, - ][] - ).map(([code, name]) => ( - - ))}
)}
@@ -693,118 +780,94 @@ export const Transcript: React.FC = ({ data, onSeek }) => {
{isTranslating && ( -
+
- -

- Translating transcript... + +

+ Translating transcript…

)} -
+
{transcriptData.map((entry) => (
-
-
- {entry.timestamp} -
- {canEdit && editingEntry !== entry.id && ( - - )} -
- {editingEntry === entry.id ? ( -
-
-