Edit videos by transcript

apps/media-server/src/__tests__/lib/media-edit.test.ts

@@ -3,6 +3,7 @@ import { existsSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import {
 	buildStreamCopySegmentArgs,
+	buildTranscodeEditArgs,
 	buildTranscodeSegmentArgs,
 	normalizeEditRanges,
 	renderEditedVideo,
@@ -98,6 +99,41 @@ describe("media edit helpers", () => {
 		);
 		expect(args).toContain("[a]");
 	});
+
+	test("builds a bounded multi-input transcode graph", () => {
+		const args = buildTranscodeEditArgs(
+			"/input.mp4",
+			[
+				{ start: 0, end: 1 },
+				{ start: 1.2, end: 2 },
+				{ start: 2.3, end: 3 },
+			],
+			"/output.mp4",
+			true,
+			60,
+		);
+		const filter = args[args.indexOf("-filter_complex") + 1];
+
+		expect(args.filter((value) => value === "-i")).toHaveLength(3);
+		expect(args.filter((value) => value === "-ss")).toHaveLength(3);
+		expect(filter).toContain("concat=n=3:v=1:a=1[v][a]");
+		expect(filter).toContain("[1:v:0]fps=60,setpts=PTS-STARTPTS[v1]");
+		expect(filter).toContain("[2:a:0]asetpts=PTS-STARTPTS[a2]");
+	});
+
+	test("rejects unbounded transcode graphs", () => {
+		expect(() =>
+			buildTranscodeEditArgs(
+				"/input.mp4",
+				Array.from({ length: 5 }, (_, index) => ({
+					start: index,
+					end: index + 0.5,
+				})),
+				"/output.mp4",
+				true,
+			),
+		).toThrow("Transcode batches must contain 1-4 ranges");
+	});
 });
 
 describe("renderEditedVideo integration tests", () => {
@@ -108,8 +144,11 @@ describe("renderEditedVideo integration tests", () => {
 		const editedFile = await renderEditedVideo({
 			inputPath: TEST_VIDEO_WITH_AUDIO,
 			keepRanges: [
-				{ start: 0, end: 0.4 },
-				{ start: 0.55, end: 0.95 },
+				{ start: 0.08, end: 0.16 },
+				{ start: 0.24, end: 0.32 },
+				{ start: 0.4, end: 0.48 },
+				{ start: 0.56, end: 0.64 },
+				{ start: 0.72, end: 0.8 },
 			],
 			metadata,
 			onProgress: (progress) => {

apps/media-server/src/lib/media-edit.ts

@@ -23,6 +23,7 @@ const MIN_RANGE_DURATION = 0.05;
 const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000;
 const MAX_TIMEOUT_MS = 60 * 60 * 1000;
 const DEFAULT_OUTPUT_FPS = 30;
+const MAX_TRANSCODE_RANGES_PER_BATCH = 4;
 
 function roundTime(value: number) {
 	return Math.round(value * 1000) / 1000;
@@ -147,6 +148,60 @@ export function buildTranscodeSegmentArgs(
 	];
 }
 
+export function buildTranscodeEditArgs(
+	inputPath: string,
+	ranges: EditRange[],
+	outputPath: string,
+	hasAudio: boolean,
+	fps = DEFAULT_OUTPUT_FPS,
+) {
+	if (ranges.length === 0 || ranges.length > MAX_TRANSCODE_RANGES_PER_BATCH) {
+		throw new Error(
+			`Transcode batches must contain 1-${MAX_TRANSCODE_RANGES_PER_BATCH} ranges`,
+		);
+	}
+
+	const filters = ranges.flatMap((_, index) => {
+		const videoFilter = `[${index}:v:0]fps=${getOutputFps(fps)},setpts=PTS-STARTPTS[v${index}]`;
+		if (!hasAudio) return [videoFilter];
+		return [videoFilter, `[${index}:a:0]asetpts=PTS-STARTPTS[a${index}]`];
+	});
+	const inputs = ranges
+		.map((_, index) => `[v${index}]${hasAudio ? `[a${index}]` : ""}`)
+		.join("");
+	const concat = `${inputs}concat=n=${ranges.length}:v=1:a=${hasAudio ? 1 : 0}[v]${hasAudio ? "[a]" : ""}`;
+
+	return [
+		"ffmpeg",
+		"-hide_banner",
+		"-y",
+		...ranges.flatMap((range) => [
+			"-ss",
+			formatTime(range.start),
+			"-t",
+			formatTime(getRangeDuration(range)),
+			"-i",
+			inputPath,
+		]),
+		"-filter_complex",
+		[...filters, concat].join(";"),
+		"-map",
+		"[v]",
+		"-c:v",
+		"libx264",
+		"-preset",
+		"fast",
+		"-crf",
+		"18",
+		"-pix_fmt",
+		"yuv420p",
+		...(hasAudio ? ["-map", "[a]", "-c:a", "aac", "-b:a", "160k"] : ["-an"]),
+		"-movflags",
+		"+faststart",
+		outputPath,
+	];
+}
+
 function buildConcatArgs(listPath: string, outputPath: string) {
 	return [
 		"ffmpeg",
@@ -233,6 +288,8 @@ async function runFfmpegCommand(
 	timeoutMs: number,
 	abortSignal?: AbortSignal,
 ) {
+	abortSignal?.throwIfAborted();
+
 	const proc = registerSubprocess(
 		spawn({
 			cmd: args,
@@ -313,49 +370,74 @@ async function concatSegments(
 	}
 }
 
-async function renderSegments({
-	keepRanges,
-	timeoutMs,
-	buildArgs,
-	onProgress,
-	abortSignal,
-	progressStart,
-	progressEnd,
-}: {
-	keepRanges: EditRange[];
-	timeoutMs: number;
-	buildArgs: (range: EditRange, outputPath: string) => string[];
-	onProgress?: ProgressCallback;
-	abortSignal?: AbortSignal;
-	progressStart: number;
-	progressEnd: number;
-}) {
-	const segmentFiles: TempFileHandle[] = [];
+function createTranscodeBatches(ranges: EditRange[]) {
+	const batches: EditRange[][] = [];
+	for (
+		let index = 0;
+		index < ranges.length;
+		index += MAX_TRANSCODE_RANGES_PER_BATCH
+	) {
+		batches.push(ranges.slice(index, index + MAX_TRANSCODE_RANGES_PER_BATCH));
+	}
+	return batches;
+}
+
+function getRemainingTimeoutMs(startedAt: number, timeoutMs: number) {
+	const remainingMs = Math.ceil(timeoutMs - (performance.now() - startedAt));
+	if (remainingMs <= 0) {
+		throw new Error(`Operation timed out after ${timeoutMs}ms`);
+	}
+	return remainingMs;
+}
+
+async function renderTranscodedEdit(
+	inputPath: string,
+	keepRanges: EditRange[],
+	hasAudio: boolean,
+	fps: number | undefined,
+	timeoutMs: number,
+	onProgress?: ProgressCallback,
+	abortSignal?: AbortSignal,
+) {
+	const batches = createTranscodeBatches(keepRanges);
+	const batchFiles: TempFileHandle[] = [];
+	const startedAt = performance.now();
 
 	try {
-		for (const [index, range] of keepRanges.entries()) {
-			const segmentFile = await createTempFile(".mp4");
-			segmentFiles.push(segmentFile);
+		onProgress?.(5, "Preparing edit...");
+		for (const [index, batch] of batches.entries()) {
+			const batchFile = await createTempFile(".mp4");
+			batchFiles.push(batchFile);
 			await runFfmpegCommand(
-				buildArgs(range, segmentFile.path),
-				timeoutMs,
+				buildTranscodeEditArgs(inputPath, batch, batchFile.path, hasAudio, fps),
+				getRemainingTimeoutMs(startedAt, timeoutMs),
 				abortSignal,
 			);
-			const progress =
-				progressStart +
-				((index + 1) / keepRanges.length) * (progressEnd - progressStart);
-			onProgress?.(progress, "Preparing edit...");
+			onProgress?.(
+				5 + ((index + 1) / batches.length) * 65,
+				"Preparing edit...",
+			);
+		}
+
+		if (batchFiles.length === 1) {
+			const outputFile = batchFiles[0];
+			if (!outputFile || (await file(outputFile.path).size) === 0) {
+				throw new Error("FFmpeg produced empty edited output");
+			}
+			batchFiles.length = 0;
+			onProgress?.(75, "Edit prepared");
+			return outputFile;
 		}
 
 		const outputFile = await concatSegments(
-			segmentFiles,
-			timeoutMs,
+			batchFiles,
+			getRemainingTimeoutMs(startedAt, timeoutMs),
 			abortSignal,
 		);
-		onProgress?.(progressEnd, "Edit prepared");
+		onProgress?.(75, "Edit prepared");
 		return outputFile;
 	} finally {
-		await Promise.all(segmentFiles.map((segment) => segment.cleanup()));
+		await Promise.all(batchFiles.map((batchFile) => batchFile.cleanup()));
 	}
 }
 
@@ -373,20 +455,13 @@ export async function renderEditedVideo({
 
 	const timeoutMs = getTimeoutMs(normalizedRanges);
 
-	return await renderSegments({
-		keepRanges: normalizedRanges,
+	return await renderTranscodedEdit(
+		inputPath,
+		normalizedRanges,
+		Boolean(metadata.audioCodec),
+		metadata.fps,
 		timeoutMs,
-		buildArgs: (range, outputPath) =>
-			buildTranscodeSegmentArgs(
-				inputPath,
-				range,
-				outputPath,
-				Boolean(metadata.audioCodec),
-				metadata.fps,
-			),
 		onProgress,
 		abortSignal,
-		progressStart: 5,
-		progressEnd: 75,
-	});
+	);
 }

apps/web/__tests__/fixtures/assemblyai-edit-response.ts

@@ -0,0 +1,107 @@
+export const assemblyAIEditResponse = {
+	id: "transcript_edit_fixture",
+	status: "completed",
+	speech_model_used: "universal-3-5-pro",
+	language_code: "en",
+	words: [
+		{
+			text: "Um,",
+			start: 120,
+			end: 380,
+			confidence: 0.98,
+			speaker: "A",
+			channel: null,
+		},
+		{
+			text: "this",
+			start: 520,
+			end: 720,
+			confidence: 0.99,
+			speaker: "A",
+			channel: null,
+		},
+		{
+			text: "is",
+			start: 730,
+			end: 840,
+			confidence: 0.99,
+			speaker: "A",
+			channel: null,
+		},
+		{
+			text: "a",
+			start: 850,
+			end: 910,
+			confidence: 0.99,
+			speaker: "A",
+			channel: null,
+		},
+		{
+			text: "real",
+			start: 930,
+			end: 1130,
+			confidence: 0.99,
+			speaker: "A",
+			channel: null,
+		},
+		{
+			text: "example.",
+			start: 1140,
+			end: 1510,
+			confidence: 0.99,
+			speaker: "A",
+			channel: null,
+		},
+		{
+			text: "Erm",
+			start: 2300,
+			end: 2570,
+			confidence: 0.93,
+			speaker: "A",
+			channel: null,
+		},
+		{
+			text: "nothing",
+			start: 2700,
+			end: 3070,
+			confidence: 0.98,
+			speaker: "A",
+			channel: null,
+		},
+		{
+			text: "else.",
+			start: 3090,
+			end: 3400,
+			confidence: 0.98,
+			speaker: "A",
+			channel: null,
+		},
+	] as const,
+} as const;
+
+export const assemblyAIRealFillerResponse = {
+	id: "sanitized_real_filler_response",
+	status: "completed",
+	speech_model_used: "universal-2",
+	language_code: "en",
+	words: [
+		{ text: "Hey,", start: 960, end: 1280, confidence: 0.98779297 },
+		{ text: "my", start: 1280, end: 1560, confidence: 0.9995117 },
+		{ text: "name's", start: 1560, end: 1880, confidence: 0.9082031 },
+		{ text: "Richie,", start: 1880, end: 2320, confidence: 0.9448242 },
+		{ text: "and", start: 2320, end: 2520, confidence: 0.99560547 },
+		{ text: "this", start: 2520, end: 2680, confidence: 0.9995117 },
+		{ text: "is", start: 2680, end: 2880, confidence: 0.9970703 },
+		{ text: "my,", start: 2880, end: 3200, confidence: 0.9995117 },
+		{ text: "um,", start: 3460, end: 3980, confidence: 0.9959265 },
+		{ text: "test", start: 3980, end: 4320, confidence: 0.99902344 },
+		{ text: "from", start: 4320, end: 4600, confidence: 0.99902344 },
+		{ text: "the", start: 4600, end: 4800, confidence: 0.99853516 },
+		{ text: "CAP", start: 4800, end: 5120, confidence: 0.95458984 },
+		{ text: "Mobile", start: 5120, end: 5680, confidence: 0.99658203 },
+		{ text: "app.", start: 5680, end: 5760, confidence: 0.6508789 },
+		{ text: "Uh,", start: 5760, end: 7240, confidence: 0.9857637 },
+		{ text: "say", start: 7240, end: 7360, confidence: 0.9975586 },
+		{ text: "hello,", start: 7360, end: 7640, confidence: 0.99975586 },
+	] as const,
+} as const;