From 72d879f26b3831dfbcf9819872f19e9e0c864b31 Mon Sep 17 00:00:00 2001 From: Miao Yang Date: Sat, 14 Nov 2026 16:05:19 +0800 Subject: [PATCH] feat(cli): measure trail, box, rhythm, objects, track, flow and continuity Seven measure ops written for reverse-any-video were left out when the skill landed (#6754), because the skill did not call them yet. They answer what cuts, frames and faces do not: how a title enters, where a graphic sits and how it eases, whether an element moves in the render as it does in the original, and where a rebuild has seams the original lacks. - trail: a window's frames as one picture: onion skin, the moving part's path and a heat map of change - box: one graphic found by its colour: box, ink, colours, text lines and cap height; over a range, when it is on screen, and with --track each move's ease and the holds - rhythm: moving share, holds, update rate, cuts and each shot's camera, alone or as ratios against a render (--compare) - objects: the separate elements at one time and their arrangement (ring, grid, row, column, stack) - track: one textured thing followed by template matching until a cut or a lost match, its moves fitted to eases - flow: one element's moves from optical flow, or compared with a render's on one clock - continuity: jumps, flicker and extra or missing shapes a render has and the original does not, with a sheet per event They run on main's shared measure helpers, so they get its ffprobe reasons and odd-sized box reads; window flags also take --from/--to (trail's --start/--end) and folder flags -o, as the other ops do. curve exports Row, EASES and fitMoves again for box, track and flow. continuity's Gaussian and box blurs share one separable pass. The optical-flow tests load opencv.js with loadOpenCvFirst, as diff's do. docs/packages/cli.mdx lists every measure op, ink included. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NqeWUzwnWFt2hjCRT7p6ux --- docs/packages/cli.mdx | 8 + packages/cli/src/commands/measure.ts | 13 +- packages/cli/src/commands/measure/box.ts | 75 +++ .../cli/src/commands/measure/continuity.ts | 53 +++ packages/cli/src/commands/measure/flow.ts | 91 ++++ packages/cli/src/commands/measure/objects.ts | 38 ++ packages/cli/src/commands/measure/rhythm.ts | 69 +++ packages/cli/src/commands/measure/track.ts | 96 ++++ packages/cli/src/commands/measure/trail.ts | 269 +++++++++++ packages/cli/src/measure/box.test.ts | 49 ++ packages/cli/src/measure/box.ts | 431 ++++++++++++++++++ packages/cli/src/measure/continuity.test.ts | 50 ++ packages/cli/src/measure/continuity.ts | 329 +++++++++++++ packages/cli/src/measure/curve.ts | 6 +- packages/cli/src/measure/flow.ts | 285 ++++++++++++ packages/cli/src/measure/objects.test.ts | 54 +++ packages/cli/src/measure/objects.ts | 198 ++++++++ packages/cli/src/measure/opencv.test.ts | 79 ++++ packages/cli/src/measure/rhythm.test.ts | 32 ++ packages/cli/src/measure/rhythm.ts | 219 +++++++++ packages/cli/src/measure/track.ts | 123 +++++ 21 files changed, 2563 insertions(+), 4 deletions(-) create mode 100644 packages/cli/src/commands/measure/box.ts create mode 100644 packages/cli/src/commands/measure/continuity.ts create mode 100644 packages/cli/src/commands/measure/flow.ts create mode 100644 packages/cli/src/commands/measure/objects.ts create mode 100644 packages/cli/src/commands/measure/rhythm.ts create mode 100644 packages/cli/src/commands/measure/track.ts create mode 100644 packages/cli/src/commands/measure/trail.ts create mode 100644 packages/cli/src/measure/box.test.ts create mode 100644 packages/cli/src/measure/box.ts create mode 100644 packages/cli/src/measure/continuity.test.ts create mode 100644 packages/cli/src/measure/continuity.ts create mode 100644 packages/cli/src/measure/flow.ts create mode 100644 packages/cli/src/measure/objects.test.ts create mode 100644 packages/cli/src/measure/objects.ts create mode 100644 packages/cli/src/measure/opencv.test.ts create mode 100644 packages/cli/src/measure/rhythm.test.ts create mode 100644 packages/cli/src/measure/rhythm.ts create mode 100644 packages/cli/src/measure/track.ts diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index 092ea058ce5..47be4d4a658 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -1649,12 +1649,20 @@ npx hyperframes measure face ref.mp4 --start 1 --end 2.5 | `cuts` | Hard cuts from the frame-to-frame difference, and the shots between them | | `events` | Cuts, flashes, appears, disappears and holds; `--map` adds the sound's hits | | `frames` | Frames as pictures: chosen moments, an overview sheet, a range, or strips around events | +| `trail` | A window's motion as one picture: onion skin, the moving part's path, a heat map | | `extract` | A still or a clip cut from the video, cropped to a region, as project media | +| `box` | One graphic found by its colour: box, ink, colours, text lines; `--track` adds moves | | `curve` | Each move of one element: start, duration, overshoot and the closest GSAP ease | | `energy` | How much two clips move, and whether at the same moments | +| `ink` | The bounds, size and lines of the text or marks in a region of one frame | | `look` | Brightness, contrast, saturation and warmth, alone or against another video | +| `rhythm` | Moving share, holds, update rate (ones, twos), cuts, and each shot's zoom or pan | +| `objects` | Separate elements at one time: count, sizes, and a ring, grid, row, column or stack | +| `track` | One textured thing (a cursor, a hand) followed from its box until a cut or lost match | +| `flow` | One element's moves from optical flow as tweens, or against your render's | | `face` | The face's size and position over a window | | `sync` | The lag between mouth motion and the voice | +| `continuity` | Seams your render has and the original lacks: jumps, flicker, extra or missing shapes | ### `diff` diff --git a/packages/cli/src/commands/measure.ts b/packages/cli/src/commands/measure.ts index 387bded5e80..74df4c69c91 100644 --- a/packages/cli/src/commands/measure.ts +++ b/packages/cli/src/commands/measure.ts @@ -11,6 +11,10 @@ export const examples: Example[] = [ "hyperframes measure events ref.mp4 --map", ], ["The frames around every event, as strips", "hyperframes measure frames ref.mp4 --events"], + [ + "One element's box, colours, text lines and moves", + "hyperframes measure box ref.mp4 --start 1 --end 2.5 --roi 200,300,600,200 --track", + ], ]; const subCommands = { @@ -18,13 +22,20 @@ const subCommands = { cuts: () => import("./measure/cuts.js").then((m) => m.default), events: () => import("./measure/events.js").then((m) => m.default), frames: () => import("./measure/frames.js").then((m) => m.default), + trail: () => import("./measure/trail.js").then((m) => m.default), extract: () => import("./measure/extract.js").then((m) => m.default), + box: () => import("./measure/box.js").then((m) => m.default), curve: () => import("./measure/curve.js").then((m) => m.default), energy: () => import("./measure/energy.js").then((m) => m.default), ink: () => import("./measure/ink.js").then((m) => m.default), look: () => import("./measure/look.js").then((m) => m.default), + rhythm: () => import("./measure/rhythm.js").then((m) => m.default), + objects: () => import("./measure/objects.js").then((m) => m.default), + track: () => import("./measure/track.js").then((m) => m.default), + flow: () => import("./measure/flow.js").then((m) => m.default), face: () => import("./measure/face.js").then((m) => m.default), sync: () => import("./measure/sync.js").then((m) => m.default), + continuity: () => import("./measure/continuity.js").then((m) => m.default), }; export const MEASURE_OPS = Object.keys(subCommands); @@ -33,7 +44,7 @@ export default defineCommand({ meta: { name: "measure", description: - "Measure a video file as JSON: clock, cuts, events, frames, crops, eases, motion energy, ink, look, faces, lip sync", + "Measure a video file as JSON: clock, cuts, events, frames, motion trails, crops, element boxes, eases, motion energy, ink, look, rhythm, objects, tracking, optical flow, faces, lip sync, continuity", }, subCommands, }); diff --git a/packages/cli/src/commands/measure/box.ts b/packages/cli/src/commands/measure/box.ts new file mode 100644 index 00000000000..6d02362d4c0 --- /dev/null +++ b/packages/cli/src/commands/measure/box.ts @@ -0,0 +1,75 @@ +import { defineCommand } from "../../utils/defineCommand.js"; +import type { Example } from "../_examples.js"; +import { + num, + jsonArg, + outArg, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { boxAt, boxRange } from "../../measure/box.js"; +import { Fix, outDir } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "One graphic's box, colours, text lines and cap height at one time", + "hyperframes measure box ref.mp4 --at 1 --roi 240,280,600,540", + ], + [ + "When it is on screen over a range, and each move it makes with its ease", + "hyperframes measure box ref.mp4 --start 3.5 --end 4.5 --roi 240,280,600,540 --track", + ], +]; + +export default defineCommand({ + meta: { + name: "box", + description: + "Where one graphic sits, found by its colour: box, ink, colours, text lines, cap height; over a range its moves", + }, + args: { + video: videoArg, + at: { type: "string", description: "One time in seconds" }, + start: { type: "string", alias: ["from"], description: "Range start in seconds" }, + end: { type: "string", alias: ["to"], description: "Range end in seconds" }, + roi: { + ...roiArg, + description: "Region round the graphic with some of its ground (default: whole frame)", + }, + color: { + type: "string", + description: + "Its pixels: auto (the commonest colour unlike the ROI border), white, dark, alpha or #rrggbb (default auto)", + default: "auto", + }, + tol: { type: "string", description: "RGB distance that still matches the colour (40)" }, + track: { + type: "boolean", + description: "Over a range: each move of its centre, size and ink, with the ease", + default: false, + }, + out: outArg, + json: jsonArg, + }, + run: ({ args }) => + runMeasure(() => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const opts = { + video: info.path, + info, + roi, + color: args.color ?? "auto", + tol: num(args.tol, "--tol", 40), + dir: outDir(args.out, "measure/box"), + }; + if (args.at !== undefined) + return { video: info.path, roi, ...boxAt(opts, num(args.at, "--at", 0)) }; + if (args.start === undefined || args.end === undefined) + throw new Fix("give --at t, or --start and --end", "--start 3.5 --end 4.5"); + const [start, end] = window(info, args.start, args.end); + return { video: info.path, roi, ...boxRange(opts, start, end, args.track === true) }; + }), +}); diff --git a/packages/cli/src/commands/measure/continuity.ts b/packages/cli/src/commands/measure/continuity.ts new file mode 100644 index 00000000000..b83bcd12ec8 --- /dev/null +++ b/packages/cli/src/commands/measure/continuity.ts @@ -0,0 +1,53 @@ +import { resolve } from "node:path"; +import { defineCommand } from "../../utils/defineCommand.js"; +import type { Example } from "../_examples.js"; +import { jsonArg, num, outArg, runMeasure } from "../../measure/command.js"; +import { continuity } from "../../measure/continuity.js"; +import { outDir } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "Seams a chained shot has that the original does not", + "hyperframes measure continuity ref.mp4 renders/final.mp4 --start 4 --end 12", + ], +]; + +export default defineCommand({ + meta: { + name: "continuity", + description: + "Seams yours has and the original does not: jumps, background flicker, and extra or missing shapes, on the original's clock", + }, + args: { + reference: { type: "positional", description: "The original", required: true }, + yours: { type: "positional", description: "The rebuild's render", required: true }, + start: { type: "string", alias: ["from"], description: "First second (default 0)" }, + end: { type: "string", alias: ["to"], description: "Last second (default: the end)" }, + out: outArg, + json: jsonArg, + }, + async run({ args }) { + await runMeasure(() => { + const r = continuity(resolve(String(args.reference)), resolve(String(args.yours)), { + start: num(args.start, "--start"), + end: num(args.end, "--end"), + outDir: outDir(args.out, "measure/continuity"), + }); + const count = (kind: string) => r.events.filter((e) => e.kind === kind).length; + return { + reference: resolve(String(args.reference)), + yours: resolve(String(args.yours)), + frames: r.frames, + counts: { jump: count("jump"), flicker: count("flicker"), structure: count("structure") }, + events: r.events, + thresholds: r.thresholds, + reading: + "jump: a whole-frame change the original lacks; flicker: a change over background the original keeps still; structure: an extra, missing or misplaced shape at `at` (source pixels); open each sheet (original over yours, peak marked) before acting", + limitations: [ + "frames pair by time on the original's clock, so a retimed or offset rebuild reads as structure everywhere", + "events are candidates to look at, not verdicts", + ], + }; + }); + }, +}); diff --git a/packages/cli/src/commands/measure/flow.ts b/packages/cli/src/commands/measure/flow.ts new file mode 100644 index 00000000000..d451b448b24 --- /dev/null +++ b/packages/cli/src/commands/measure/flow.ts @@ -0,0 +1,91 @@ +import { defineCommand } from "../../utils/defineCommand.js"; +import type { Example } from "../_examples.js"; +import { + jsonArg, + num, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { compareFlow, flowMoves, flowSeries } from "../../measure/flow.js"; +import { parseRoi, probeVideo, type Roi } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "A card's moves from optical flow, as tweens", + "hyperframes measure flow ref.mp4 --start 1 --end 2.4 --roi 300,200,640,400", + ], + [ + "Does the rebuild's element move like the original's", + "hyperframes measure flow ref.mp4 --start 1 --end 2.4 --roi 300,200,640,400 --compare renders/v1.mp4", + ], +]; + +export default defineCommand({ + meta: { + name: "flow", + description: + "One element's motion from optical flow inside its box: its moves as tweens, or compared with another video's (energy, timing, path, scale, rotation)", + }, + args: { + video: videoArg, + start: { type: "string", alias: ["from"], description: "First second", required: true }, + end: { type: "string", alias: ["to"], description: "Last second", required: true }, + roi: roiArg, + compare: { + type: "string", + description: "Another video over the same window, on the same clock", + }, + "compare-start": { + type: "string", + description: "Second the window starts in it (default: --start)", + }, + "compare-roi": { type: "string", description: "Its box (default: --roi scaled by the widths)" }, + json: jsonArg, + }, + async run({ args }) { + await runMeasure(async () => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const [a, b] = window(info, args.start, args.end); + if (args.compare === undefined) { + const rows = await flowSeries(info, a, b, roi, 320); + return { + video: info.path, + start: a, + end: b, + roi, + moves: flowMoves(rows, info.fps, roi, a), + }; + } + const other = probeVideo(String(args.compare)); + // Each axis scales on its own: a generated shot is rarely the original's exact aspect (768x1344 for 9:16). + const [kx, ky] = [other.width / info.width, other.height / info.height]; + const scaled: Roi = [ + Math.round(roi[0] * kx), + Math.round(roi[1] * ky), + Math.round(roi[2] * kx), + Math.round(roi[3] * ky), + ]; + const otherRoi = + args["compare-roi"] !== undefined + ? parseRoi(args["compare-roi"], other.width, other.height) + : scaled; + const oa = num(args["compare-start"], "--compare-start", a); + const [ref, yours] = await Promise.all([ + flowSeries(info, a, b, roi, 160), + flowSeries(other, oa, oa + (b - a), otherRoi, 160), + ]); + return { + video: info.path, + compare: other.path, + start: a, + end: b, + ...compareFlow(ref, yours), + reading: + "energy_ratio near 1 moves as much; motion_corr near 1 at lag_frames 0 moves at the same moments; path, scale and rotation errors are per frame", + }; + }); + }, +}); diff --git a/packages/cli/src/commands/measure/objects.ts b/packages/cli/src/commands/measure/objects.ts new file mode 100644 index 00000000000..fac3944a637 --- /dev/null +++ b/packages/cli/src/commands/measure/objects.ts @@ -0,0 +1,38 @@ +import { defineCommand } from "../../utils/defineCommand.js"; +import type { Example } from "../_examples.js"; +import { jsonArg, num, roiArg, runMeasure, videoAndRoi, videoArg } from "../../measure/command.js"; +import { objects } from "../../measure/objects.js"; + +export const examples: Example[] = [ + [ + "How many elements are on screen and how they are arranged (ring, grid, row, stack)", + "hyperframes measure objects ref.mp4 --at 2.4 --roi 300,200,1320,680", + ], +]; + +export default defineCommand({ + meta: { + name: "objects", + description: + "Separate elements at one time: count, sizes and arrangement (ring, grid, row, column, stack) as shares of the frame", + }, + args: { + video: videoArg, + at: { type: "string", description: "Time in seconds", required: true }, + roi: roiArg, + tol: { + type: "string", + description: "Distance from the background that counts as an element (40)", + }, + json: jsonArg, + }, + run: ({ args }) => + runMeasure(() => { + const { info, roi } = videoAndRoi(args.video, args.roi); + return { + video: info.path, + roi, + ...objects(info.path, info, num(args.at, "--at", 0), roi, num(args.tol, "--tol", 40)), + }; + }), +}); diff --git a/packages/cli/src/commands/measure/rhythm.ts b/packages/cli/src/commands/measure/rhythm.ts new file mode 100644 index 00000000000..fabd4c9cb2f --- /dev/null +++ b/packages/cli/src/commands/measure/rhythm.ts @@ -0,0 +1,69 @@ +import { defineCommand } from "../../utils/defineCommand.js"; +import type { Example } from "../_examples.js"; +import { + compareWindowArgs, + jsonArg, + num, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { parseRoi, probeVideo } from "../../measure/media.js"; +import { compareRhythm, rhythm } from "../../measure/rhythm.js"; + +export const examples: Example[] = [ + [ + "How the film moves: holds, update rate, cuts and camera per shot", + "hyperframes measure rhythm ref.mp4", + ], + [ + "The same numbers for your render, as ratios to the original's", + "hyperframes measure rhythm ref.mp4 --start 4 --end 9 --compare renders/final.mp4", + ], +]; + +export default defineCommand({ + meta: { + name: "rhythm", + description: + "How a film moves: moving share, holds, update rate (ones, twos), cuts, and each shot's zoom, pan or handheld", + }, + args: { + video: videoArg, + start: { type: "string", alias: ["from"], description: "Window start in seconds (default 0)" }, + end: { type: "string", alias: ["to"], description: "Window end in seconds (default: the end)" }, + roi: roiArg, + // fallow-ignore-next-line code-duplication + compare: { + type: "string", + description: "A second video (your render) measured at the original's still level", + }, + ...compareWindowArgs, + json: jsonArg, + }, + run: ({ args }) => + runMeasure(() => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const [start, end] = window(info, args.start, args.end); + const region = args.roi === undefined ? undefined : roi; + const ref = rhythm(info.path, start, end - start, region); + if (!args.compare) return { video: info.path, ...ref }; + const other = probeVideo(args.compare); + const yoursRoi = args["compare-roi"] + ? parseRoi(args["compare-roi"], other.width, other.height) + : region; + const from = num(args["compare-start"], "--compare-start", start); + const yours = rhythm(other.path, from, end - start, yoursRoi, ref.still_threshold); + const { shots: refShots, ...refSummary } = ref; + const { shots: yoursShots, ...yoursSummary } = yours; + return { + reference: refSummary, + yours: yoursSummary, + ratio: compareRhythm(ref, yours), + reference_shots: refShots, + yours_shots: yoursShots, + }; + }), +}); diff --git a/packages/cli/src/commands/measure/track.ts b/packages/cli/src/commands/measure/track.ts new file mode 100644 index 00000000000..61896753efa --- /dev/null +++ b/packages/cli/src/commands/measure/track.ts @@ -0,0 +1,96 @@ +import { join } from "node:path"; +import { defineCommand } from "../../utils/defineCommand.js"; +import type { Example } from "../_examples.js"; +import { jsonArg, num, outArg, runMeasure, videoArg } from "../../measure/command.js"; +import { Fix, outDir, parseRoi, probeVideo, round, runFfmpeg } from "../../measure/media.js"; +import { trackTemplate } from "../../measure/track.js"; + +export const examples: Example[] = [ + [ + "Follow the cursor from 2.0 s to 5.5 s, starting from its box at 2.0 s", + "hyperframes measure track ref.mp4 --at 2 --to 5.5 --roi 840,410,36,48", + ], +]; + +const PATH_POINTS = 24; + +export default defineCommand({ + meta: { + name: "track", + description: + "Follow one textured thing (a cursor, a hand, a product) from its box at one moment, frame by frame, until a cut or a lost match", + }, + args: { + video: videoArg, + at: { type: "string", description: "The second its box is taken at", required: true }, + roi: { + type: "string", + description: "Its box at --at, x,y,w,h in the video's pixels", + required: true, + }, + to: { type: "string", description: "Last second to follow it to (default: the end)" }, + radius: { + type: "string", + description: "How far it can move between frames, in pixels (default 48)", + }, + "min-score": { type: "string", description: "Weakest match kept, 0–1 (default 0.6)" }, + out: outArg, + json: jsonArg, + }, + async run({ args }) { + await runMeasure(async () => { + const info = probeVideo(String(args.video)); + const at = num(args.at, "--at", 0); + const to = Math.min(info.duration, num(args.to, "--to", info.duration)); + if (to <= at + 1 / info.fps) throw new Fix("--to is not after --at", "--at 2 --to 5.5"); + const box = parseRoi(args.roi, info.width, info.height); + const r = await trackTemplate({ + info, + at, + box, + to, + radius: num(args.radius, "--radius"), + minScore: num(args["min-score"], "--min-score"), + }); + const step = Math.max(1, Math.ceil(r.rows.length / PATH_POINTS)); + const path = r.rows.filter((_, i) => i % step === 0 || i === r.rows.length - 1); + // The path drawn on the first frame, so one look shows where it went. + const dir = outDir(args.out, "measure/track"); + const picture = join(dir, `track-${at}-${round(to, 2)}s.png`); + const draw = path.flatMap((p, i) => + i + ? [ + `drawbox=x=${Math.round(p.x! - 3)}:y=${Math.round(p.y! - 3)}:w=6:h=6:color=red@0.9:t=fill`, + ] + : [], + ); + runFfmpeg([ + ...["-y", "-ss", String(at), "-i", info.path, "-frames:v", "1"], + ...[ + "-vf", + [ + `drawbox=x=${box[0]}:y=${box[1]}:w=${box[2]}:h=${box[3]}:color=yellow:t=3`, + ...draw, + ].join(","), + picture, + ], + ]); + const scores = r.scores.slice(1); + return { + video: info.path, + template: { at, box }, + followed_s: [round(at, 3), r.rows.at(-1)!.t], + frames: r.rows.length, + lost: r.lost, + mean_score: scores.length + ? round(scores.reduce((s, v) => s + v, 0) / scores.length, 3) + : null, + path: path.map((p) => ({ t: p.t, x: Math.round(p.x!), y: Math.round(p.y!) })), + moves: r.moves, + picture, + reading: + "x and y are the box centre in video pixels; moves are its eases; a lost track needs a new --at and --roi where it reappears", + }; + }); + }, +}); diff --git a/packages/cli/src/commands/measure/trail.ts b/packages/cli/src/commands/measure/trail.ts new file mode 100644 index 00000000000..485309445da --- /dev/null +++ b/packages/cli/src/commands/measure/trail.ts @@ -0,0 +1,269 @@ +// Motion as one picture, for an agent that reads still images: the frames of a window laid over each other as an +// onion skin (earliest faintest), the moving part's path with numbered points, and a heat map of where the picture +// changed. One look tells a slide from a scale from a mask reveal; `keyframes --shot` does the same for a project. +import { join } from "node:path"; +import { defineCommand } from "../../utils/defineCommand.js"; +import type { Example } from "../_examples.js"; +import { + jsonArg, + num, + outArg, + roiArg, + runMeasure, + videoAndRoi, + videoArg, + window, +} from "../../measure/command.js"; +import { Fix, grey, outDir, readClip, round, videoClip, writeImage } from "../../measure/media.js"; + +export const examples: Example[] = [ + [ + "How a title enters between 2.1 s and 2.9 s", + "hyperframes measure trail ref.mp4 --from 2.1 --to 2.9", + ], + [ + "One card's path, 10 ghosts", + "hyperframes measure trail ref.mp4 --from 4 --to 5.2 --roi 300,200,900,600 --ghosts 10", + ], +]; + +const WORK_W = 640; +const MOVING = 24; +// 3x5 digits, drawn without a font so the picture comes out the same on every machine. +const DIGITS = [ + "111101101101111", + "010110010010111", + "111001111100111", + "111001111001111", + "101101111001001", + "111100111001111", + "111100111101111", + "111001001001001", + "111101111101111", + "111101111001111", +]; + +function put(px: Uint8Array, w: number, h: number, x: number, y: number, rgb: number[]) { + if (x < 0 || y < 0 || x >= w || y >= h) return; + px.set(rgb, (y * w + x) * 3); +} + +function dot( + px: Uint8Array, + w: number, + h: number, + cx: number, + cy: number, + r: number, + rgb: number[], +) { + for (let y = -r; y <= r; y++) + for (let x = -r; x <= r; x++) + if (x * x + y * y <= r * r) put(px, w, h, Math.round(cx + x), Math.round(cy + y), rgb); +} + +function line( + px: Uint8Array, + w: number, + h: number, + [x0, y0]: number[], + [x1, y1]: number[], + rgb: number[], +) { + const n = Math.max(1, Math.ceil(Math.hypot(x1! - x0!, y1! - y0!))); + for (let i = 0; i <= n; i++) + dot(px, w, h, x0! + ((x1! - x0!) * i) / n, y0! + ((y1! - y0!) * i) / n, 1, rgb); +} + +function digits( + px: Uint8Array, + w: number, + h: number, + text: string, + x: number, + y: number, + s: number, +) { + [...text].forEach((ch, i) => { + const g = DIGITS[Number(ch)] ?? ""; + for (let k = 0; k < g.length; k++) + if (g[k] === "1") + for (let yy = 0; yy < s; yy++) + for (let xx = 0; xx < s; xx++) + put( + px, + w, + h, + x + (i * 4 + (k % 3)) * s + xx, + y + Math.floor(k / 3) * s + yy, + [255, 255, 255], + ); + }); +} + +/** Per pixel, the median of the frames' grey: the still background the moving part passes over. */ +function medianGrey(gs: Uint8Array[]): Uint8Array { + const out = new Uint8Array(gs[0]!.length); + const v = new Array(gs.length); + for (let i = 0; i < out.length; i++) { + for (let k = 0; k < gs.length; k++) v[k] = gs[k]![i]!; + v.sort((a, b) => a - b); + out[i] = v[gs.length >> 1]!; + } + return out; +} + +const heat = (u: number): number[] => + u < 0.5 + ? [Math.round(510 * u), 0, 0] + : [255, Math.round(510 * (u - 0.5)), Math.round(Math.max(0, 255 * (u - 0.8) * 5))]; + +export default defineCommand({ + meta: { + name: "trail", + description: + "Motion as one picture: an onion skin of a window's frames, the moving part's path, and a heat map of where it changed", + }, + args: { + video: videoArg, + from: { type: "string", alias: ["start"], description: "First second", required: true }, + to: { type: "string", alias: ["end"], description: "Last second", required: true }, + ghosts: { type: "string", description: "Frames laid over each other (default 8)" }, + roi: roiArg, + out: outArg, + json: jsonArg, + }, + async run({ args }) { + // fallow-ignore-next-line complexity + await runMeasure(() => { + const { info, roi } = videoAndRoi(args.video, args.roi); + const [a, b] = window(info, args.from, args.to, ["--from", "--to"]); + const count = Math.max(2, num(args.ghosts, "--ghosts", 8)); + const { frames, w, h } = readClip(videoClip(info.path, a, b - a, info), { + roi, + width: Math.min(WORK_W, roi[2]), + }); + if (frames.length < 2) + throw new Fix(`under 2 frames between ${a}s and ${b}s`, "widen --from/--to"); + const picks = [ + ...new Set( + Array.from({ length: count }, (_, i) => + Math.round((i * (frames.length - 1)) / (count - 1)), + ), + ), + ]; + const gs = frames.map(grey); + const base = medianGrey(picks.map((i) => gs[i]!)); + + // The onion skin: the background dimmed, each ghost's moving pixels over it, later ones stronger. + const skin = new Uint8Array(w * h * 3); + for (let i = 0; i < w * h; i++) skin.fill(Math.round(base[i]! * 0.45), i * 3, i * 3 + 3); + const [kx, ky] = [roi[2] / w, roi[3] / h]; + const path = picks.map((fi, order) => { + const alpha = 0.3 + (0.7 * order) / Math.max(1, picks.length - 1); + const [f, g] = [frames[fi]!, gs[fi]!]; + let [sx, sy, n] = [0, 0, 0]; + for (let i = 0; i < w * h; i++) { + if (Math.abs(g[i]! - base[i]!) <= MOVING) continue; + for (let c = 0; c < 3; c++) + skin[i * 3 + c] = Math.round(skin[i * 3 + c]! * (1 - alpha) + f[i * 3 + c]! * alpha); + sx += i % w; + sy += Math.floor(i / w); + n++; + } + const t = round(a + fi / info.fps, 3); + return n + ? { t, x: sx / n, y: sy / n, share: n / (w * h) } + : { t, x: null, y: null, share: 0 }; + }); + const seen = path.flatMap((p, i) => + p.x !== null && p.y !== null ? [{ ...p, x: p.x, y: p.y, i }] : [], + ); + for (let i = 1; i < seen.length; i++) + line( + skin, + w, + h, + [seen[i - 1]!.x, seen[i - 1]!.y], + [seen[i]!.x, seen[i]!.y], + [255, 200, 40], + ); + const scale = Math.max(1, Math.round(w / 320)); + for (const p of seen) { + dot(skin, w, h, p.x, p.y, 2 + scale, [255, 60, 60]); + digits( + skin, + w, + h, + String(p.i + 1), + Math.round(p.x) + 4 + scale, + Math.round(p.y) - 6 * scale, + scale, + ); + } + + // The heat map: every frame's change from the one before, summed, over the dimmed first frame. + const sum = new Float32Array(w * h); + for (let k = 1; k < gs.length; k++) + for (let i = 0; i < w * h; i++) sum[i]! += Math.abs(gs[k]![i]! - gs[k - 1]![i]!); + const top = sum.reduce((m, v) => Math.max(m, v), 0) || 1; + const map = new Uint8Array(w * h * 3); + let [x0, y0, x1, y1] = [w, h, -1, -1]; + for (let i = 0; i < w * h; i++) { + const u = Math.min(1, sum[i]! / top); + const [r, g, bl] = heat(u); + const under = gs[0]![i]! * 0.35; + map.set( + [Math.max(r!, under), Math.max(g!, under), Math.max(bl!, under)].map(Math.round), + i * 3, + ); + if (u > 0.15) { + const [x, y] = [i % w, Math.floor(i / w)]; + [x0, y0, x1, y1] = [Math.min(x0, x), Math.min(y0, y), Math.max(x1, x), Math.max(y1, y)]; + } + } + + const dir = outDir(args.out, "measure/trail"); + const name = `trail-${a}-${b}s`; + const first = seen[0]; + const final = seen.at(-1); + return { + video: info.path, + from: a, + to: b, + frames: frames.length, + ghosts: picks.length, + trail: writeImage(join(dir, `${name}.png`), skin, w, h), + heat: writeImage(join(dir, `${name}-heat.png`), map, w, h), + path: path.map((p, i) => ({ + n: i + 1, + t: p.t, + ...(p.x !== null && p.y !== null + ? { + x: Math.round(roi[0] + p.x * kx), + y: Math.round(roi[1] + p.y * ky), + moving_share: round(p.share, 3), + } + : { moving_share: 0 }), + })), + ...(first && final + ? { + net_move: { + dx: Math.round((final.x - first.x) * kx), + dy: Math.round((final.y - first.y) * ky), + }, + size_ratio: round(final.share / (first.share || 1e-9), 2), + } + : {}), + changed_box: + x1 >= 0 + ? [roi[0] + x0 * kx, roi[1] + y0 * ky, (x1 - x0 + 1) * kx, (y1 - y0 + 1) * ky].map( + Math.round, + ) + : null, + reading: + "ghosts fade from first (faint) to last (solid); numbered red points are the moving part's centre; in the heat map white-yellow changed most", + }; + }); + }, +}); diff --git a/packages/cli/src/measure/box.test.ts b/packages/cli/src/measure/box.test.ts new file mode 100644 index 00000000000..76730c89e05 --- /dev/null +++ b/packages/cli/src/measure/box.test.ts @@ -0,0 +1,49 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; +import { boxAt, boxRange } from "./box.js"; +import { probeVideo, type Roi } from "./media.js"; + +const ffmpeg = findFFmpeg(); +const dir = mkdtempSync(join(tmpdir(), "hf-box-test-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +function clip(name: string, overlayX: string): string { + const out = join(dir, name); + execFileSync(ffmpeg!, [ + ...["-nostdin", "-v", "error", "-y"], + ...["-f", "lavfi", "-i", "color=0x101820:s=320x180:d=1.2:r=30"], + ...["-f", "lavfi", "-i", "color=0xff8000:s=60x30:d=1.2:r=30"], + ...["-filter_complex", `[0][1]overlay=x='${overlayX}':y=40`], + ...["-pix_fmt", "yuv444p", "-c:v", "libx264", "-crf", "0", out], + ]); + return out; +} + +const options = (video: string) => { + const roi: Roi = [0, 0, 320, 180]; + return { video, info: probeVideo(video), roi, color: "auto", tol: 40, dir }; +}; + +describe.skipIf(!ffmpeg || !findFFprobe())("box", () => { + it("finds a coloured box by its colour at one time", () => { + const r = boxAt(options(clip("still.mp4", "100")), 0.5); + expect(r.color).toMatch(/^#f[0-9a-f]8/); + const want = { x: 100, y: 40, w: 60, h: 30 }; + for (const k of ["x", "y", "w", "h"] as const) + expect(Math.abs(r.box[k] - want[k])).toBeLessThanOrEqual(2); + }); + + it("tracks a linear slide as one x move", () => { + const r = boxRange(options(clip("slide.mp4", "20+400*min(max(t-0.3,0),0.5)")), 0, 1.2, true); + expect(r.first_seen_s).toBe(0); + const moves = "moves" in r ? r.moves : []; + const xs = moves.filter((m) => m.prop === "x"); + expect(xs).toHaveLength(1); + expect(moves.filter((m) => m.prop === "w" || m.prop === "h")).toHaveLength(0); + expect(xs[0]!.start_s).toBeCloseTo(0.3, 1); + }); +}); diff --git a/packages/cli/src/measure/box.ts b/packages/cli/src/measure/box.ts new file mode 100644 index 00000000000..fb4ddac262a --- /dev/null +++ b/packages/cli/src/measure/box.ts @@ -0,0 +1,431 @@ +// Where one graphic sits, found by its colour rather than by what moves, so a presenter who never stops does not +// pull the box (ported from the Studio's box.mjs): at one time its box, ink, dominant colours, text line bands and +// cap height; over a range when it is on screen; with track, each move of its centre, size and ink, fitted to the +// closest GSAP ease. +import { join } from "node:path"; +import { fitMoves, printable, type Row } from "./curve.js"; +import { + Fix, + frameAt, + readFrameAt, + readFrames, + round, + runFfmpeg, + type Roi, + type VideoInfo, +} from "./media.js"; + +// A range is read at this width (a 1080 px frame at 3 px a step); one time is read at full size. +const RANGE_W = 360; +const BLOB = 0.02; +const MIN_BLOB = 4; +// Over a range a frame shows the graphic when it holds this share of the range's most ink; less is noise. +const PRESENT = 0.05; +const BORDER = 2; +const HOLD_S = 0.3; + +export interface Frame { + px: Uint8Array; + w: number; + h: number; +} +export type Match = (r: number, g: number, b: number) => boolean; +export interface Blob { + n: number; + x0: number; + y0: number; + x1: number; + y1: number; +} +interface Cluster { + rgb: number[]; + n: number; +} + +const hexOf = (c: number[]) => + `#${c.map((v) => Math.round(v).toString(16).padStart(2, "0")).join("")}`; +const dist2 = (a: number[], b: number[]) => + (a[0]! - b[0]!) ** 2 + (a[1]! - b[1]!) ** 2 + (a[2]! - b[2]!) ** 2; + +/** Colour clusters of the pixels in `rect` (x0,y0,x1,y1 inclusive) that `keep` passes: 16-level bins, each merged + * into the first larger cluster within `radius`, largest first. */ +// fallow-ignore-next-line complexity +function clusters( + { px, w }: Frame, + [x0, y0, x1, y1]: number[], + keep: (x: number, y: number, o: number) => boolean, + radius: number, +): Cluster[] { + const bins = new Map(); + for (let y = y0!; y <= y1!; y++) + for (let x = x0!; x <= x1!; x++) { + const o = (y * w + x) * 3; + if (!keep(x, y, o)) continue; + const key = ((px[o]! >> 4) << 8) | ((px[o + 1]! >> 4) << 4) | (px[o + 2]! >> 4); + let bin = bins.get(key); + if (!bin) bins.set(key, (bin = [0, 0, 0, 0])); + bin[0]!++; + for (let c = 0; c < 3; c++) bin[c + 1]! += px[o + c]!; + } + const out: { seed: number[]; n: number; sum: number[] }[] = []; + for (const [n, ...sum] of [...bins.values()].sort((a, b) => b[0]! - a[0]!)) { + const rgb = sum.map((s) => s / n!); + const home = out.find((c) => dist2(c.seed, rgb) <= radius * radius); + if (!home) out.push({ seed: rgb, n: n!, sum }); + else { + home.n += n!; + sum.forEach((s, c) => (home.sum[c]! += s)); + } + } + return out.map((c) => ({ rgb: c.sum.map((s) => s / c.n), n: c.n })).sort((a, b) => b.n - a.n); +} + +/** --color auto: the commonest cluster among pixels that match no colour holding 2% of the ROI's border ring. */ +function autoColor(f: Frame, tol: number): Cluster | null { + const { w, h } = f; + const ring = (x: number, y: number) => + x < BORDER || y < BORDER || x >= w - BORDER || y >= h - BORDER; + const border = clusters(f, [0, 0, w - 1, h - 1], ring, tol); + const total = border.reduce((s, c) => s + c.n, 0); + const ground = border.filter((c) => c.n >= 0.02 * total).map((c) => c.rgb); + const unlike = (x: number, y: number, o: number) => + !ring(x, y) && + ground.every((c) => dist2(c, [f.px[o]!, f.px[o + 1]!, f.px[o + 2]!]) > tol * tol); + return clusters(f, [0, 0, w - 1, h - 1], unlike, tol)[0] ?? null; +} + +/** The pixel test for --color. */ +function matcher(spec: string, tol: number): Match { + if (spec === "white") return (r, g, b) => Math.min(r, g, b) >= 255 - tol; + if (spec === "dark") return (r, g, b) => Math.max(r, g, b) <= tol; + // what a solo render on #00ff00 leaves once the green is keyed out + if (spec === "alpha") return (r, g, b) => r + (255 - g) + b >= 90; + const m = /^#?([0-9a-f]{6})$/i.exec(spec); + if (!m) + throw new Fix( + `--color must be #rrggbb, white, dark, auto or alpha, got "${spec}"`, + "--color #fce7bc", + ); + const c = [0, 2, 4].map((i) => parseInt(m[1]!.slice(i, i + 2), 16)); + return (r, g, b) => dist2(c, [r, g, b]) <= tol * tol; +} + +/** The 8-connected blobs of the pixels `match` passes: each one's size and box, and every pixel's blob (0: none). */ +// fallow-ignore-next-line complexity +export function blobs({ px, w, h }: Frame, match: Match): { id: Int32Array; list: Blob[] } { + const mask = new Uint8Array(w * h); + for (let i = 0; i < w * h; i++) + mask[i] = match(px[3 * i]!, px[3 * i + 1]!, px[3 * i + 2]!) ? 1 : 0; + const id = new Int32Array(w * h); + const list: Blob[] = [{ n: 0, x0: w, y0: h, x1: -1, y1: -1 }]; + for (let s = 0; s < w * h; s++) { + if (!mask[s] || id[s]) continue; + const blob = { n: 0, x0: w, y0: h, x1: -1, y1: -1 }; + const c = list.push(blob) - 1; + const stack = [s]; + id[s] = c; + for (let p = stack.pop(); p !== undefined; p = stack.pop()) { + const x = p % w; + const y = (p - x) / w; + blob.n++; + [blob.x0, blob.y0] = [Math.min(blob.x0, x), Math.min(blob.y0, y)]; + [blob.x1, blob.y1] = [Math.max(blob.x1, x), Math.max(blob.y1, y)]; + for (let ny = Math.max(0, y - 1); ny <= Math.min(h - 1, y + 1); ny++) + for (let nx = Math.max(0, x - 1); nx <= Math.min(w - 1, x + 1); nx++) { + const q = ny * w + nx; + if (!mask[q] || id[q]) continue; + id[q] = c; + stack.push(q); + } + } + } + return { id, list }; +} + +/** Blobs under BLOB of the graphic's largest are speckle (a highlight on a jacket, a slice of other text). */ +const floorOf = (...lists: Blob[][]) => + Math.max(MIN_BLOB, BLOB * lists.flat().reduce((m, b) => Math.max(m, b.n), 0)); + +/** The box and pixel count of the blobs holding `floor` pixels or more, or null. */ +function graphic(list: Blob[], floor: number) { + const kept = list.filter((b) => b.n >= floor); + if (!kept.length) return null; + const edge = (k: "x0" | "y0" | "x1" | "y1", f: (...v: number[]) => number) => + f(...kept.map((b) => b[k])); + return { + x0: edge("x0", Math.min), + y0: edge("y0", Math.min), + x1: edge("x1", Math.max), + y1: edge("y1", Math.max), + ink: kept.reduce((s, b) => s + b.n, 0), + }; +} +type Graphic = NonNullable>; + +/** One frame's graphic, with its text line bands: runs of rows holding it, a one-row gap bridged, under 3 rows + * dropped. */ +function measure(f: Frame, match: Match) { + const { id, list } = blobs(f, match); + const floor = floorOf(list); + const g = graphic(list, floor); + if (!g) return null; + const lines: number[][] = []; + for (let y = g.y0; y <= g.y1; y++) { + let on = false; + for (let x = g.x0; x <= g.x1 && !on; x++) + on = id[y * f.w + x]! > 0 && list[id[y * f.w + x]!]!.n >= floor; + if (!on) continue; + const last = lines.at(-1); + if (last && y - last[1]! <= 2) last[1] = y; + else lines.push([y, y]); + } + return { ...g, lines: lines.filter(([a, b]) => b! - a! + 1 >= 3) }; +} + +/** Frames of `roi` from `start` for `seconds`, every one, at `width` (0: full size). */ +function read(video: string, start: number, seconds: number, roi: Roi, width: number): Frame[] { + const { frames, w, h } = readFrames(video, { start, seconds, roi, width }); + return frames.map((px) => ({ px, w, h })); +} + +/** One frame with the box drawn (and the union in yellow), cropped round it. */ +function picture( + video: string, + t: number, + b: number[], + [W, H]: number[], + out: string, + union: number[] | null = null, +) { + const outer = union ?? b; + const pad = Math.max(16, Math.round(0.25 * Math.max(outer[2]!, outer[3]!))); + const cx = Math.max(0, outer[0]! - pad); + const cy = Math.max(0, outer[1]! - pad); + const cw = Math.min(W!, outer[0]! + outer[2]! + pad) - cx; + const ch = Math.min(H!, outer[1]! + outer[3]! + pad) - cy; + const draw = ([x, y, w, h]: number[], color: string, thick: number) => + `drawbox=x=${x! - cx}:y=${y! - cy}:w=${w}:h=${h}:color=${color}:t=${thick}`; + const vf = [ + `crop=${cw}:${ch}:${cx}:${cy}`, + ...(union ? [draw(union, "yellow", 1)] : []), + draw(b, "red", 2), + ]; + runFfmpeg(["-y", "-ss", String(t), "-i", video, "-frames:v", "1", "-vf", vf.join(","), out]); + return out; +} + +/** The --color spec as a test, choosing auto's colour on the sample of `fs` where it covers most. */ +function choose(spec: string, tol: number, fs: Frame[]) { + if (spec !== "auto") return { match: matcher(spec, tol), color: spec }; + const picks = [0, 0.25, 0.5, 0.75, 1].map((p) => + autoColor(fs[Math.round(p * (fs.length - 1))]!, tol), + ); + const best = picks.reduce((a, b) => ((b?.n ?? 0) > (a?.n ?? 0) ? b : a), null); + if (!best) + throw new Fix( + "--color auto found nothing unlike the ROI's border", + "pass --color #rrggbb or white", + ); + return { match: matcher(hexOf(best.rgb), tol), color: hexOf(best.rgb) }; +} + +/** The no-pixels error, naming what the ROI does hold. */ +function nothing(f: Frame, color: string, tol: number) { + const top = clusters(f, [0, 0, f.w - 1, f.h - 1], () => true, tol).slice(0, 3); + const names = top + .map((c) => `${hexOf(c.rgb)} ${Math.round((100 * c.n) / (f.w * f.h))}%`) + .join(", "); + const pick = autoColor(f, tol); + return new Fix( + `no pixels of ${color} (within --tol ${tol}) in the ROI; it holds ${names}`, + pick + ? `pass --color ${hexOf(pick.rgb)} (what --color auto picks here) or one of those colours` + : "draw --roi round the graphic with some of its ground, then --color auto", + ); +} + +/** In video pixels. */ +const toVideo = (m: Graphic, [rx, ry]: Roi, kx: number, ky: number) => ({ + x: Math.round(rx + m.x0 * kx), + y: Math.round(ry + m.y0 * ky), + w: Math.round((m.x1 - m.x0 + 1) * kx), + h: Math.round((m.y1 - m.y0 + 1) * ky), +}); + +export interface BoxOptions { + video: string; + info: VideoInfo; + roi: Roi; + color: string; + tol: number; + dir: string; +} + +const frameAtStart = (t: number, fps: number) => Math.ceil(t * fps - 1e-6); + +/** The graphic at one time. */ +export function boxAt(o: BoxOptions, t: number) { + const one = readFrameAt(o.info, t, { roi: o.roi }); + const f: Frame | undefined = one.frames[0] && { px: one.frames[0], w: one.w, h: one.h }; + if (!f) + throw new Fix( + `no frame at ${t}s`, + `pick a time before ${round(o.info.duration - 1 / o.info.fps, 2)}s`, + ); + const { match, color } = choose(o.color, o.tol, [f]); + const m = measure(f, match); + if (!m) throw nothing(f, color, o.tol); + const box = toVideo(m, o.roi, 1, 1); + const inBox = clusters(f, [m.x0, m.y0, m.x1, m.y1], () => true, o.tol); + const area = box.w * box.h; + const lines = m.lines.map(([a, b]) => ({ y: o.roi[1] + a!, h: b! - a! + 1 })); + const heights = lines.map((l) => l.h).sort((a, b) => a - b); + return { + t, + frame: frameAt(t, o.info.fps), + color, + box, + ink_px: m.ink, + colors: inBox.slice(0, 3).map((c) => ({ hex: hexOf(c.rgb), share: round(c.n / area, 2) })), + lines, + cap_height_px: heights[(heights.length - 1) >> 1] ?? null, + image: picture( + o.video, + t, + [box.x, box.y, box.w, box.h], + [o.info.width, o.info.height], + join(o.dir, `box-${t}s.png`), + ), + }; +} + +interface RangeRow extends Row { + frame: number; + box: { x: number; y: number; w: number; h: number } | null; +} + +/** Runs where the box is on screen and none of x, y, w, h moves past `quant`, 0.3 s or longer. */ +function holds(rows: RangeRow[], quant: number, fps: number) { + const still = rows.map((r, i) => { + const prev = rows[i - 1]; + if (!prev || r.x == null || prev.x == null || r.y == null || prev.y == null) return false; + return ( + Math.abs(r.x - prev.x) <= quant && + Math.abs(r.y - prev.y) <= quant && + Math.abs(r.w - prev.w) <= quant && + Math.abs(r.h - prev.h) <= quant + ); + }); + const out: { start_s: number; seconds: number }[] = []; + for (let i = 0, run = 0; i <= still.length; i++) { + if (still[i]) run++; + else { + if (run + 1 >= HOLD_S * fps) + out.push({ start_s: rows[i - run - 1]!.t, seconds: round((run + 1) / fps, 2) }); + run = 0; + } + } + return out; +} + +/** Over a range: when it is seen, its union and median box; with `track`, each move fitted and the still holds. */ +export function boxRange(opts: BoxOptions, start: number, end: number, track: boolean) { + const { info, roi } = opts; + const width = Math.min(RANGE_W, roi[2]); + const fs = read(opts.video, start, end - start, roi, width); + if (!fs.length) + throw new Fix( + `no frames between ${start}s and ${end}s`, + `keep --end at most ${round(info.duration, 2)}`, + ); + const kx = roi[2] / width; + const ky = roi[3] / fs[0]!.h; + const { match, color } = choose(opts.color, opts.tol, fs); + // speckle is judged against the graphic's largest blob in the whole range, so a sliver entering is not swamped + const found = fs.map((f) => blobs(f, match).list); + const floor = floorOf(...found); + const frame0 = frameAtStart(start, info.fps); + const empty = (t: number, frame: number): RangeRow => ({ + t, + frame, + x: null, + y: null, + w: 0, + h: 0, + ink: 0, + box: null, + }); + const rows = found.map((list, i): RangeRow => { + const frame = frame0 + i; + const t = round(frame / info.fps, 3); + const m = graphic(list, floor); + if (!m) return empty(t, frame); + const b = toVideo(m, roi, kx, ky); + return { + t, + frame, + x: round(b.x + b.w / 2, 1), + y: round(b.y + b.h / 2, 1), + w: b.w, + h: b.h, + ink: Math.round(m.ink * kx * ky), + box: b, + }; + }); + const most = Math.max(...rows.map((r) => r.ink)); + const present = rows.map((r) => (r.box && r.ink < PRESENT * most ? empty(r.t, r.frame) : r)); + const seen = present.filter((r) => r.box); + if (!seen.length) throw nothing(fs[fs.length >> 1]!, color, opts.tol); + const med = (v: number[]) => [...v].sort((a, b) => a - b)[(v.length - 1) >> 1]!; + const boxes = seen.flatMap((r) => (r.box ? [r.box] : [])); + const x0 = Math.min(...boxes.map((b) => b.x)); + const y0 = Math.min(...boxes.map((b) => b.y)); + const union = [ + x0, + y0, + Math.max(...boxes.map((b) => b.x + b.w)) - x0, + Math.max(...boxes.map((b) => b.y + b.h)) - y0, + ]; + const last = seen.at(-1)!; + const lastBox = boxes.at(-1)!; + const out = { + start, + end, + fps: round(info.fps, 3), + color, + frames: present.length, + seen_frames: seen.length, + first_seen_s: seen[0]!.t, + first_seen_frame: seen[0]!.frame, + last_seen_s: last.t, + last_seen_frame: last.frame, + box_union: union, + box_median: (["x", "y", "w", "h"] as const).map((k) => med(boxes.map((b) => b[k]))), + ink_px_median: med(seen.map((r) => r.ink)), + image: picture( + opts.video, + last.t, + [lastBox.x, lastBox.y, lastBox.w, lastBox.h], + [info.width, info.height], + join(opts.dir, `box-${start}-${end}s.png`), + union, + ), + }; + if (!track) return out; + // Each move of x, y (the box's centre), w, h and ink, and the still holds. + const quant = Math.max(kx, ky); + const minPx = Math.max(2 * quant, 0.02 * Math.max(roi[2], roi[3])); + const maxInk = Math.max(...present.map((r) => r.ink)); + const moves = (["x", "y", "w", "h", "ink"] as const) + .flatMap((k) => + k === "ink" + ? fitMoves(present, k, info.fps, 0.1 * maxInk, 0.02 * maxInk) + : fitMoves(present, k, info.fps, minPx, quant), + ) + .sort((a, b) => a.start_s - b.start_s); + return { + ...out, + moves: printable(moves).map((m) => ({ ...m, next: undefined })), + holds: holds(present, quant, info.fps), + }; +} diff --git a/packages/cli/src/measure/continuity.test.ts b/packages/cli/src/measure/continuity.test.ts new file mode 100644 index 00000000000..d226b99fe15 --- /dev/null +++ b/packages/cli/src/measure/continuity.test.ts @@ -0,0 +1,50 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; +import { continuity } from "./continuity.js"; + +const ffmpeg = findFFmpeg(); +const dir = mkdtempSync(join(tmpdir(), "hf-continuity-test-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +function clip(name: string, args: string[]): string { + const out = join(dir, name); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", ...args], + ...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-qp", "0", out], + ]); + return out; +} + +describe.skipIf(!ffmpeg || !findFFprobe())("continuity", () => { + it("flags a frame the original does not have as a jump", () => { + const original = clip("orig.mp4", ["-f", "lavfi", "-i", "testsrc2=d=2:r=30:s=160x90"]); + const yours = clip("jump.mp4", [ + ...["-i", original, "-vf"], + "drawbox=x=0:y=0:w=160:h=90:color=red:t=fill:enable='eq(n,30)'", + ]); + const r = continuity(original, yours, { outDir: dir }); + const jumps = r.events.filter((e) => e.kind === "jump"); + expect(jumps).toHaveLength(1); + expect(jumps[0]!.peak_frame).toBeGreaterThanOrEqual(30); + expect(jumps[0]!.peak_frame).toBeLessThanOrEqual(31); + expect(existsSync(jumps[0]!.sheet)).toBe(true); + }); + + it("flags a shape drawn over a still background as structure", () => { + const original = clip("still.mp4", ["-f", "lavfi", "-i", "color=c=0x305070:s=160x90:r=30:d=2"]); + const yours = clip("box.mp4", [ + ...["-i", original, "-vf"], + "drawbox=x=100:y=20:w=24:h=24:color=white:t=fill:enable='between(n,20,29)'", + ]); + const r = continuity(original, yours, { outDir: dir }); + const shapes = r.events.filter((e) => e.kind === "structure"); + expect(shapes.length).toBeGreaterThan(0); + expect(shapes[0]!.peak_frame).toBeGreaterThanOrEqual(20); + expect(shapes[0]!.peak_frame).toBeLessThanOrEqual(29); + expect(shapes[0]!.at![0]).toBeGreaterThan(80); + }); +}); diff --git a/packages/cli/src/measure/continuity.ts b/packages/cli/src/measure/continuity.ts new file mode 100644 index 00000000000..e2457cf271b --- /dev/null +++ b/packages/cli/src/measure/continuity.ts @@ -0,0 +1,329 @@ +// Seams a rebuild has and its original does not, ported from the Studio's hf-media continuity: on the original's +// clock, a jump (a whole-frame change the original lacks), flicker (a change over background the original keeps +// still) and structure (an extra, missing or misplaced shape, from blurred edge maps). Screening candidates: open each +// event's sheet before acting. The per-pixel work is plain TypeScript so it needs no OpenCV download. +import { join } from "node:path"; +import { decode, Fix, frameAt, probeVideo, round, writeImage, type VideoInfo } from "./media.js"; + +const WIDTH = 160; +const NEIGHBOURS = 2; // a spike must beat the original's max within ±this many frames +const STATIC_WINDOW = 12; // still mask: the original's per-pixel change stays low for ±this many frames ... +const STATIC_LEVEL = 6; // ... below this level +const STATIC_MIN_SHARE = 0.05; // fewer still pixels than this share: no flicker reading at that frame +const [JUMP_ABS, JUMP_RATIO] = [3, 2]; +const [FLICKER_ABS, FLICKER_RATIO, FLICKER_LOCAL] = [2, 3, 1.8]; +const STRUCTURE_K = 4; // a structure spike is above the film's median + k × MAD ... +const STRUCTURE_LOCAL = 1.5; // ... above this multiple of its median within ±1 s ... +const STRUCTURE_MIN = 1.5; // ... and above this floor (mean-normalised edge difference) +const STRUCTURE_BOX = 0.15; // local window of the structure difference, share of the width +const EVENT_GAP = 2; + +export type ContinuityKind = "jump" | "flicker" | "structure"; + +/** The weighted mean under `k` (odd length, centred) along rows, then along columns, with clamped edges. */ +function separable(src: Float32Array, w: number, h: number, k: number[]): Float32Array { + const r = (k.length - 1) / 2; + const sum = k.reduce((s, v) => s + v, 0); + const pass = (a: Float32Array, horizontal: boolean) => { + const out = new Float32Array(a.length); + for (let y = 0; y < h; y++) + for (let x = 0; x < w; x++) { + let s = 0; + for (let i = -r; i <= r; i++) { + const xx = horizontal ? Math.min(w - 1, Math.max(0, x + i)) : x; + const yy = horizontal ? y : Math.min(h - 1, Math.max(0, y + i)); + s += a[yy * w + xx]! * k[i + r]!; + } + out[y * w + x] = s / sum; + } + return out; + }; + return pass(pass(src, true), false); +} + +/** Separable Gaussian with clamped edges; cv2.GaussianBlur with sigma and an automatic kernel, near enough. */ +function gaussian(src: Float32Array, w: number, h: number, sigma: number): Float32Array { + const r = Math.ceil(3 * sigma); + const k = Array.from({ length: 2 * r + 1 }, (_, i) => + Math.exp(-((i - r) ** 2) / (2 * sigma * sigma)), + ); + return separable(src, w, h, k); +} + +/** Box mean over a size×size window with clamped edges (cv2.blur). */ +function box(src: Float32Array, w: number, h: number, size: number): Float32Array { + return separable(src, w, h, new Array(2 * Math.floor(size / 2) + 1).fill(1)); +} + +/** Blurred edge strength: Gaussian σ1, 3×3 Sobel magnitude, Gaussian σ2. */ +function edges(g: Uint8Array, w: number, h: number): Float32Array { + const b = gaussian(Float32Array.from(g), w, h, 1); + const at = (x: number, y: number) => + b[Math.min(h - 1, Math.max(0, y)) * w + Math.min(w - 1, Math.max(0, x))]!; + const mag = new Float32Array(w * h); + for (let y = 0; y < h; y++) + for (let x = 0; x < w; x++) { + const gx = + at(x + 1, y - 1) + + 2 * at(x + 1, y) + + at(x + 1, y + 1) - + at(x - 1, y - 1) - + 2 * at(x - 1, y) - + at(x - 1, y + 1); + const gy = + at(x - 1, y + 1) + + 2 * at(x, y + 1) + + at(x + 1, y + 1) - + at(x - 1, y - 1) - + 2 * at(x, y - 1) - + at(x + 1, y - 1); + mag[y * w + x] = Math.hypot(gx, gy); + } + return gaussian(mag, w, h, 2); +} + +const mean = (a: ArrayLike) => { + let s = 0; + for (let i = 0; i < a.length; i++) s += a[i]!; + return s / (a.length || 1); +}; +const median = (v: number[]) => { + const s = [...v].sort((a, b) => a - b); + return s[Math.floor(s.length / 2)] ?? 0; +}; + +interface Row { + frame: number; + original_change: number | null; + change: number | null; + original_static_change: number | null; + static_change: number | null; + structure: number; + at: [number, number]; +} + +export interface ContinuityEvent { + kind: ContinuityKind; + start_s: number; + end_s: number; + peak_s: number; + peak_frame: number; + value: number; + original_value: number | null; + at?: [number, number]; + sheet: string; +} + +/** The original's frames [a, b) and yours on the original's clock, grey at WIDTH. */ +function frames(path: string, a: number, b: number, h: number, fps?: string): Uint8Array[] { + const select = `select=between(n\\,${a}\\,${b - 1})`; + const filters = [ + ...(fps ? [`fps=${fps}`] : []), + select, + `setpts=N/(${fps ?? "1"})/TB`, + `scale=${WIDTH}:${h}`, + ]; + return decode(["-i", path], filters, WIDTH, h, true).frames; +} + +/** One sheet: the original's frames over yours, the peak marked by a white bar under it. */ +function sheet( + orig: Uint8Array[], + yours: Uint8Array[], + w: number, + h: number, + peak: number, + out: string, +) { + const W = orig.length * (w + 4); + const H = 2 * h + 12; + const px = new Uint8Array(W * H).fill(30); + orig.forEach((f, i) => { + for (let y = 0; y < h; y++) { + px.set(f.subarray(y * w, (y + 1) * w), y * W + i * (w + 4)); + px.set(yours[i]!.subarray(y * w, (y + 1) * w), (h + 6 + y) * W + i * (w + 4)); + } + if (i === peak) + for (let y = 2 * h + 8; y < H; y++) + px.fill(255, y * W + i * (w + 4), y * W + i * (w + 4) + w); + }); + return writeImage(out, px, W, H, true); +} + +// fallow-ignore-next-line complexity +export function continuity( + reference: string, + yoursPath: string, + opts: { start?: number; end?: number; outDir: string }, +): { events: ContinuityEvent[]; thresholds: Record; frames: number } { + const ref: VideoInfo = probeVideo(reference); + const yoursInfo = probeVideo(yoursPath); + const h = Math.max(2, Math.round((ref.height * WIDTH) / ref.width / 2) * 2); + const k0 = frameAt(opts.start ?? 0, ref.fps); + const k1 = Math.min(ref.frames, Math.ceil((opts.end ?? ref.duration) * ref.fps - 1e-6)); + const yours = frames(yoursInfo.path, k0, k1, h, ref.fpsText); + if (yours.length < 3) + throw new Fix( + "fewer than three frames of yours fall in the window", + "widen --start/--end, or check yours plays that long", + ); + const low = Math.max(0, k0 - STATIC_WINDOW - 1); + const high = Math.min(ref.frames, k0 + yours.length + STATIC_WINDOW + 1); + const stack = frames(ref.path, low, high, h, ref.fpsText); + const n = WIDTH * h; + const moved = stack.map((f, k) => { + const d = new Uint8Array(n); + if (k) for (let i = 0; i < n; i++) d[i] = Math.abs(f[i]! - stack[k - 1]![i]!); + return d; + }); + const boxSize = Math.max(3, Math.round(STRUCTURE_BOX * WIDTH)); + const scale = ref.width / WIDTH; + const rows: Row[] = []; + for (let m = 0; m < yours.length; m++) { + const j = k0 + m; + const k = j - low; + if (!stack[k]) break; + // Still pixels: the original's change stays under STATIC_LEVEL for ±STATIC_WINDOW frames. + const mask = new Uint8Array(n).fill(1); + for ( + let q = Math.max(0, k - STATIC_WINDOW); + q <= Math.min(stack.length - 1, k + STATIC_WINDOW); + q++ + ) + for (let i = 0; i < n; i++) if (moved[q]![i]! >= STATIC_LEVEL) mask[i] = 0; + const [a, b] = [edges(stack[k]!, WIDTH, h), edges(yours[m]!, WIDTH, h)]; + const norm = (mean(a) + mean(b)) / 2 + 1e-6; + const diff = box( + a.map((v, i) => Math.abs(v - b[i]!) / norm), + WIDTH, + h, + boxSize, + ); + let best = 0; + for (let i = 1; i < n; i++) if (diff[i]! > diff[best]!) best = i; + const row: Row = { + frame: j, + original_change: j > 0 && k > 0 ? mean(moved[k]!) : null, + change: null, + original_static_change: null, + static_change: null, + structure: diff[best]!, + at: [Math.round((best % WIDTH) * scale), Math.round(Math.floor(best / WIDTH) * scale)], + }; + if (m > 0 && k > 0) { + const prev = yours[m - 1]!; + let [all, still, stillOrig, count] = [0, 0, 0, 0]; + for (let i = 0; i < n; i++) { + const d = Math.abs(yours[m]![i]! - prev[i]!); + all += d; + if (mask[i]) { + still += d; + stillOrig += moved[k]![i]!; + count++; + } + } + row.change = all / n; + if (count / n >= STATIC_MIN_SHARE) { + row.static_change = still / count; + row.original_static_change = stillOrig / count; + } + } + rows.push(row); + } + + const around = (key: keyof Row, at: number, reach: number, reduce: (v: number[]) => number) => { + const v = rows + .slice(Math.max(0, at - reach), at + reach + 1) + .map((r) => r[key]) + .filter((x): x is number => typeof x === "number"); + return v.length ? reduce(v) : null; + }; + const structures = rows.map((r) => r.structure); + const med = median(structures); + const mad = Math.max(median(structures.map((s) => Math.abs(s - med))), 0.05 * med); + const structureThreshold = Math.max(STRUCTURE_MIN, med + STRUCTURE_K * mad); + const localReach = Math.max(1, Math.round(ref.fps)); + const flagged: { at: number; kind: ContinuityKind; value: number; base: number | null }[] = []; + // fallow-ignore-next-line complexity + rows.forEach((r, i) => { + if (r.change !== null) { + const base = around("original_change", i, NEIGHBOURS, (v) => Math.max(...v)) ?? 0; + if (r.change > Math.max(base + JUMP_ABS, base * JUMP_RATIO)) + flagged.push({ at: i, kind: "jump", value: r.change, base }); + else if (r.static_change !== null) { + const sb = around("original_static_change", i, NEIGHBOURS, (v) => Math.max(...v)) ?? 0; + const own = around("static_change", i, STATIC_WINDOW, median) ?? 0; + if (r.static_change > Math.max(sb + FLICKER_ABS, sb * FLICKER_RATIO, own * FLICKER_LOCAL)) + flagged.push({ at: i, kind: "flicker", value: r.static_change, base: sb }); + } + } + if ( + r.structure > structureThreshold && + r.structure > STRUCTURE_LOCAL * (around("structure", i, localReach, median) ?? 0) + ) + flagged.push({ at: i, kind: "structure", value: r.structure, base: med }); + }); + + // Flagged frames of one kind within EVENT_GAP of each other are one event, peaking at its largest value. + const groups: { + kind: ContinuityKind; + first: number; + last: number; + peak: number; + value: number; + base: number | null; + }[] = []; + for (const kind of ["jump", "flicker", "structure"] as const) { + let open: (typeof groups)[number] | null = null; + for (const f of flagged.filter((x) => x.kind === kind)) { + if (open && f.at - open.last <= EVENT_GAP) { + open.last = f.at; + if (f.value > open.value) Object.assign(open, { peak: f.at, value: f.value, base: f.base }); + continue; + } + open = { kind, first: f.at, last: f.at, peak: f.at, value: f.value, base: f.base }; + groups.push(open); + } + } + groups.sort((x, y) => x.first - y.first); + const t = (i: number) => round(rows[i]!.frame / ref.fps, 3); + const events = groups.map((g, number) => { + const shown = [g.peak - 2, g.peak - 1, g.peak, g.peak + 1].filter( + (i) => i >= 0 && i < rows.length, + ); + const path = sheet( + shown.map((i) => stack[rows[i]!.frame - low]!), + shown.map((i) => yours[i]!), + WIDTH, + h, + shown.indexOf(g.peak), + join( + opts.outDir, + `event-${String(number + 1).padStart(2, "0")}-${g.kind}-${rows[g.peak]!.frame}.png`, + ), + ); + return { + kind: g.kind, + start_s: t(g.first), + end_s: t(g.last), + peak_s: t(g.peak), + peak_frame: rows[g.peak]!.frame, + value: round(g.value, 2), + original_value: g.base === null ? null : round(g.base, 2), + ...(g.kind === "structure" ? { at: rows[g.peak]!.at } : {}), + sheet: path, + }; + }); + return { + events, + frames: rows.length, + thresholds: { + jump_abs: JUMP_ABS, + jump_ratio: JUMP_RATIO, + flicker_abs: FLICKER_ABS, + flicker_ratio: FLICKER_RATIO, + structure: round(structureThreshold, 3), + structure_median: round(med, 3), + }, + }; +} diff --git a/packages/cli/src/measure/curve.ts b/packages/cli/src/measure/curve.ts index b5448780a45..65fde28fafb 100644 --- a/packages/cli/src/measure/curve.ts +++ b/packages/cli/src/measure/curve.ts @@ -9,7 +9,7 @@ export type Prop = "x" | "y" | "w" | "h" | "ink"; const PROPS: Prop[] = ["x", "y", "w", "h", "ink"]; /** One frame of an element: its box centre and size (video pixels) and ink; x and y are null when it is absent. */ -interface Row { +export interface Row { t: number; x: number | null; y: number | null; @@ -34,7 +34,7 @@ const elastic = (p: number): Ease => [ (u) => (u <= 0 ? 0 : u >= 1 ? 1 : 2 ** (-10 * u) * Math.sin(((u - p / 4) * 2 * Math.PI) / p) + 1), ]; /** GSAP's named eases as functions of normalized time. */ -const EASES: Ease[] = [ +export const EASES: Ease[] = [ ["none", (u) => u], ...[1, 2, 3, 4].flatMap((k) => [pw(k), pwo(k), pwio(k)]), ["sine.out", (u) => Math.sin((u * Math.PI) / 2)], @@ -179,7 +179,7 @@ function fit( /** Every move of one property: stretches of change separated by holds of 0.15 s or more (shorter stills are the * turn at an overshoot's peak). A stretch that returns to where it began is a pulse (peak and length, no ease). */ // fallow-ignore-next-line complexity -function fitMoves( +export function fitMoves( rows: readonly Row[], prop: Prop, fps: number, diff --git a/packages/cli/src/measure/flow.ts b/packages/cli/src/measure/flow.ts new file mode 100644 index 00000000000..bd208ccdc82 --- /dev/null +++ b/packages/cli/src/measure/flow.ts @@ -0,0 +1,285 @@ +// Dense optical flow inside one element's box, ported from hf-media's flow.py. Two uses: comparing a rebuild with +// its original on one clock (motion energy, the shape of the motion over time, path, scale and rotation; fonts and +// content do not move these numbers as they do SSIM), and reversing a move into tweens. hf-media's DIS flow is not in +// opencv.js, so this is Farneback's; thresholds carry over but are calibrated on the reverse bench. +import { EASES } from "./curve.js"; +import { readClip, round, videoClip, type Roi, type VideoInfo } from "./media.js"; +import { greyMat, loadOpenCv } from "./opencv.js"; + +// Below this mean flow (pixels a frame at the analysis size) a frame counts as still. +const STILL = 0.05; +// Flow under this magnitude is noise for the affine fit. +const FIT_MIN = 0.3; +// Flow needs room for its windows: thin boxes are read taller. +const MIN_HEIGHT = 64; + +export interface FlowRow { + mag: number; + share: number; + tx: number; + ty: number; + scale: number; + rot: number; +} + +/** Per pair of frames: mean flow, the share of moving pixels, and the affine of the moving part. */ +// fallow-ignore-next-line complexity +export async function flowSeries( + info: VideoInfo, + start: number, + end: number, + roi: Roi, + maxWidth: number, +): Promise { + const cv = await loadOpenCv((m) => console.error(m)); + let width = Math.min(maxWidth, roi[2]); + if ((width * roi[3]) / roi[2] < MIN_HEIGHT) width = Math.ceil((MIN_HEIGHT * roi[2]) / roi[3]); + const { frames, w, h } = readClip(videoClip(info.path, start, end - start, info), { + roi, + width, + gray: true, + }); + const rows: FlowRow[] = []; + const grid: number[] = []; + for (let y = 0; y < h; y += 4) for (let x = 0; x < w; x += 4) grid.push(x, y); + for (let i = 1; i < frames.length; i++) { + const [a, b, flow] = [ + greyMat(cv, frames[i - 1]!, w, h), + greyMat(cv, frames[i]!, w, h), + new cv.Mat(), + ]; + try { + cv.calcOpticalFlowFarneback(a, b, flow, 0.5, 5, 25, 3, 7, 1.5, 0); + const f = flow.data32F; + let [sum, moving] = [0, 0]; + for (let p = 0; p < w * h; p++) { + const m = Math.hypot(f[2 * p]!, f[2 * p + 1]!); + sum += m; + if (m > FIT_MIN) moving++; + } + const row: FlowRow = { + mag: sum / (w * h), + share: moving / (w * h), + tx: 0, + ty: 0, + scale: 1, + rot: 0, + }; + const from: number[] = []; + const to: number[] = []; + for (let g = 0; g < grid.length; g += 2) { + const p = grid[g + 1]! * w + grid[g]!; + const [dx, dy] = [f[2 * p]!, f[2 * p + 1]!]; + if (Math.hypot(dx, dy) <= FIT_MIN) continue; + from.push(grid[g]!, grid[g + 1]!); + to.push(grid[g]! + dx, grid[g + 1]! + dy); + } + if (from.length >= 24) { + const fit = similarity(from, to); + if (fit) + Object.assign(row, { tx: fit.tx / w, ty: fit.ty / h, scale: fit.scale, rot: fit.rot }); + } + rows.push(row); + } finally { + for (const mat of [a, b, flow]) mat.delete(); + } + } + return rows; +} + +interface Similarity { + a: number; + b: number; + tx: number; + ty: number; +} + +/** Least squares in closed form over the points `keep`. */ +function fitSimilarity(from: number[], to: number[], keep: number[]): Similarity | null { + let [px, py, qx, qy] = [0, 0, 0, 0]; + for (const i of keep) { + px += from[2 * i]!; + py += from[2 * i + 1]!; + qx += to[2 * i]!; + qy += to[2 * i + 1]!; + } + [px, py, qx, qy] = [px / keep.length, py / keep.length, qx / keep.length, qy / keep.length]; + let [sa, sb, ss] = [0, 0, 0]; + for (const i of keep) { + const [ux, uy, vx, vy] = [ + from[2 * i]! - px, + from[2 * i + 1]! - py, + to[2 * i]! - qx, + to[2 * i + 1]! - qy, + ]; + sa += ux * vx + uy * vy; + sb += ux * vy - uy * vx; + ss += ux * ux + uy * uy; + } + if (ss < 1e-9) return null; + const [a, b] = [sa / ss, sb / ss]; + return { a, b, tx: qx - (a * px - b * py), ty: qy - (b * px + a * py) }; +} + +const residual = (f: Similarity, from: number[], to: number[], i: number) => + Math.hypot( + f.a * from[2 * i]! - f.b * from[2 * i + 1]! + f.tx - to[2 * i]!, + f.b * from[2 * i]! + f.a * from[2 * i + 1]! + f.ty - to[2 * i + 1]!, + ); + +/** + * The scale, rotation and translation the most points agree on (`from` and `to` are flat x,y lists): RANSAC over + * two-point hypotheses with a fixed seed, then least squares on the inliers. opencv.js has no + * estimateAffinePartial2D, and a plain fit is pulled off by the smeared flow at a moving element's edges. + */ +export function similarity( + from: number[], + to: number[], +): { tx: number; ty: number; scale: number; rot: number } | null { + const n = from.length / 2; + let seed = 1709; + const pick = () => { + seed = (seed * 1103515245 + 12345) % 2147483648; + return seed % n; + }; + let best: number[] = []; + for (let k = 0; k < 200; k++) { + const [i, j] = [pick(), pick()]; + if (i === j) continue; + const f = fitSimilarity(from, to, [i, j]); + if (!f) continue; + const inliers: number[] = []; + for (let p = 0; p < n; p++) if (residual(f, from, to, p) <= 1) inliers.push(p); + if (inliers.length > best.length) best = inliers; + } + if (best.length < Math.max(6, 0.2 * n)) return null; + const fit = fitSimilarity(from, to, best); + if (!fit) return null; + return { + tx: fit.tx, + ty: fit.ty, + scale: Math.hypot(fit.a, fit.b), + rot: (Math.atan2(fit.b, fit.a) * 180) / Math.PI, + }; +} + +/** The best Pearson r of two curves within ±3 frames, and its lag. */ +function corr(a: number[], b: number[]): [number | null, number] { + let best: [number | null, number] = [null, 0]; + for (let lag = -3; lag <= 3; lag++) { + const x = a.slice(Math.max(0, -lag), a.length - Math.max(0, lag)); + const y = b.slice(Math.max(0, lag), b.length - Math.max(0, -lag)); + const n = Math.min(x.length, y.length); + if (n < 4) continue; + const mx = x.slice(0, n).reduce((s, v) => s + v, 0) / n; + const my = y.slice(0, n).reduce((s, v) => s + v, 0) / n; + let [sxy, sx, sy] = [0, 0, 0]; + for (let i = 0; i < n; i++) { + sxy += (x[i]! - mx) * (y[i]! - my); + sx += (x[i]! - mx) ** 2; + sy += (y[i]! - my) ** 2; + } + if (sx < 1e-18 || sy < 1e-18) continue; + const c = sxy / Math.sqrt(sx * sy); + if (best[0] === null || c > best[0]) best = [c, lag]; + } + return best; +} + +/** Yours against the original, frame for frame on one clock. */ +export function compareFlow(ref: FlowRow[], yours: FlowRow[]) { + const n = Math.min(ref.length, yours.length); + const [a, b] = [ref.slice(0, n), yours.slice(0, n)]; + const energy = (rows: FlowRow[]) => + rows.reduce((s, r) => s + r.mag, 0) / Math.max(1, rows.length); + const [re, ye] = [energy(a), energy(b)]; + const still = re < STILL; + const ratio = still ? (ye < STILL ? 1 : null) : ye / re; + const [c, lag] = corr( + a.map((r) => r.mag), + b.map((r) => r.mag), + ); + const mean = (v: number[]) => v.reduce((s, x) => s + x, 0) / Math.max(1, v.length); + return { + frames: n, + original_still: still, + energy_ratio: ratio === null ? null : round(ratio, 3), + motion_corr: c === null ? null : round(c, 3), + lag_frames: lag, + path_error: round( + mean(a.map((r, i) => Math.abs(r.tx - b[i]!.tx) + Math.abs(r.ty - b[i]!.ty))), + 4, + ), + scale_error: round(mean(a.map((r, i) => Math.abs(r.scale - b[i]!.scale))), 4), + rotation_error: round(mean(a.map((r, i) => Math.abs(r.rot - b[i]!.rot))), 3), + }; +} + +function fitEase(progress: number[]): [string, number] { + const n = progress.length; + let best: [string, number] = ["none", Infinity]; + for (const [name, f] of EASES) { + const err = progress.reduce((s, p, i) => s + (f(i / Math.max(1, n - 1)) - p) ** 2, 0) / n; + if (err < best[1]) best = [name, err]; + } + return [best[0], round(Math.sqrt(best[1]), 4)]; +} + +/** Moves between holds, each as a tween: when, how far (source pixels), scale, rotation, ease and overshoot. */ +// fallow-ignore-next-line complexity +export function flowMoves(rows: FlowRow[], fps: number, [, , w, h]: Roi, start: number) { + const pos: [number, number, number, number][] = [[0, 0, 1, 0]]; + for (const r of rows) { + const [x, y, s, a] = pos.at(-1)!; + pos.push([x + r.tx * w, y + r.ty * h, s * r.scale, a + r.rot]); + } + // A frame belongs to a move when the element itself shifts, scales or turns, however slowly it eases out. + const active = rows.map( + (r) => + Math.hypot(r.tx * w, r.ty * h) >= 0.25 || + Math.abs(r.scale - 1) >= 0.003 || + Math.abs(r.rot) >= 0.15, + ); + const moves: Record[] = []; + for (let i = 0; i < active.length; ) { + if (!active[i]) { + i++; + continue; + } + let j = i; + while (j + 1 < active.length && (active[j + 1] || (j + 2 < active.length && active[j + 2]))) + j++; + const [s0, s1] = [pos[i]!, pos[j + 1]!]; + const [dx, dy] = [s1[0] - s0[0], s1[1] - s0[1]]; + const along = pos + .slice(i, j + 2) + .map((p) => (Math.abs(dx) >= Math.abs(dy) ? p[0] - s0[0] : p[1] - s0[1])); + const total = Math.abs(dx) >= Math.abs(dy) ? dx : dy; + const ds = s1[2] - s0[2]; + const progress = + Math.abs(total) >= 1 + ? along.map((v) => v / total) + : pos.slice(i, j + 2).map((p) => (Math.abs(ds) > 1e-3 ? (p[2] - s0[2]) / ds : 0)); + const head = { + start_s: round(start + i / fps, 3), + x: round(dx, 1), + y: round(dy, 1), + scale: round(s1[2] / s0[2], 3), + }; + if (j === i) moves.push({ kind: "jump", ...head }); + else { + const [ease, err] = fitEase(progress); + moves.push({ + kind: "tween", + ...head, + duration_s: round((j + 1 - i) / fps, 3), + rotation: round(s1[3] - s0[3], 2), + ease, + ease_error: err, + overshoot: round(Math.max(0, Math.max(...progress) - 1), 3), + }); + } + i = j + 1; + } + return moves; +} diff --git a/packages/cli/src/measure/objects.test.ts b/packages/cli/src/measure/objects.test.ts new file mode 100644 index 00000000000..ff524c48a42 --- /dev/null +++ b/packages/cli/src/measure/objects.test.ts @@ -0,0 +1,54 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; +import { probeVideo } from "./media.js"; +import { objects } from "./objects.js"; + +const ffmpeg = findFFmpeg(); +const dir = mkdtempSync(join(tmpdir(), "hf-objects-test-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +/** White 16 px squares centred on `points` over black. */ +function squares(name: string, points: [number, number][]): string { + const out = join(dir, name); + const boxes = points.map( + ([x, y]) => + `drawbox=x=${Math.round(x - 8)}:y=${Math.round(y - 8)}:w=16:h=16:color=white:t=fill`, + ); + execFileSync(ffmpeg!, [ + ...["-nostdin", "-v", "error", "-y", "-f", "lavfi", "-i", "color=black:s=320x180:d=0.5:r=10"], + ...["-vf", boxes.join(","), "-pix_fmt", "yuv444p", "-c:v", "libx264", "-crf", "0", out], + ]); + return out; +} + +const measure = (video: string) => objects(video, probeVideo(video), 0.2, [0, 0, 320, 180]); + +describe.skipIf(!ffmpeg || !findFFprobe())("objects", () => { + it("reads six squares on a circle as an evenly spaced ring", () => { + const ring = Array.from({ length: 6 }, (_, i): [number, number] => [ + 160 + 60 * Math.cos((i * Math.PI) / 3), + 90 + 60 * Math.sin((i * Math.PI) / 3), + ]); + const r = measure(squares("ring.mp4", ring)); + expect(r.count).toBe(6); + expect(r.arrangement).toMatchObject({ + kind: "ring", + count: 6, + evenly_spaced: true, + touching: false, + }); + }); + + it("reads a 3x2 layout as a grid", () => { + const grid: [number, number][] = [60, 160, 260].flatMap((x) => + [50, 130].map((y): [number, number] => [x, y]), + ); + const r = measure(squares("grid.mp4", grid)); + expect(r.count).toBe(6); + expect(r.arrangement).toMatchObject({ kind: "grid", rows: 2, cols: 3 }); + }); +}); diff --git a/packages/cli/src/measure/objects.ts b/packages/cli/src/measure/objects.ts new file mode 100644 index 00000000000..5a6239da682 --- /dev/null +++ b/packages/cli/src/measure/objects.ts @@ -0,0 +1,198 @@ +// Separate elements at one instant, ported from light-reverse's measure.objects: count, sizes, and whether they +// form a ring, grid, row, column or stack. Elements are 8-connected blobs of pixels far from the background (the +// ROI border's median); positions and sizes are shares of the full frame. +import { blobs, type Frame } from "./box.js"; +import { Fix, readFrameAt, round, type Roi, type VideoInfo } from "./media.js"; + +const MAX_W = 960; +const MIN_SHARE = 0.0003; + +interface Found { + cx: number; + cy: number; + w: number; + h: number; + x0: number; + y0: number; + area: number; +} + +/** Where working pixels sit in the frame: offset of the ROI and video pixels per working pixel. */ +interface Geometry { + W: number; + H: number; + x: number; + y: number; + sx: number; + sy: number; +} + +const toShare = (g: Geometry, x: number, y: number) => ({ + x: round((g.x + x * g.sx) / g.W, 3), + y: round((g.y + y * g.sy) / g.H, 3), +}); + +const mean = (v: number[]) => v.reduce((s, x) => s + x, 0) / (v.length || 1); +const std = (v: number[]) => Math.sqrt(mean(v.map((x) => (x - mean(v)) ** 2))); +const median = (v: number[]) => { + const s = [...v].sort((a, b) => a - b); + return s.length % 2 ? s[s.length >> 1]! : (s[s.length / 2 - 1]! + s[s.length / 2]!) / 2; +}; + +/** How many runs `values` form when gaps over `gap` split them. */ +const runs = (values: number[], gap: number) => { + const v = [...values].sort((a, b) => a - b); + return 1 + v.slice(1).filter((x, i) => x - v[i]! > gap).length; +}; + +/** The arrangement of the largest group of like-sized elements; a stray glyph or crumb joins nothing. */ +// fallow-ignore-next-line complexity +function arrangement(all: Found[], g: Geometry) { + let found = all; + if (found.length >= 3) { + const med = median(found.map((o) => o.area)); + const like = found.filter((o) => o.area >= 0.6 * med && o.area <= 1.6 * med); + if (like.length >= 3) found = like; + } + if (found.length < 3) return { kind: "fewer than 3 elements", count: found.length }; + const areas = found.map((o) => o.area); + const similar = std(areas) / mean(areas) < 0.35; + const c = found.map((o) => [o.cx * g.sx, o.cy * g.sy] as const); // video px + const size = median(found.map((o) => Math.max(o.w * g.sx, o.h * g.sy))); + const boxes = found.map((o) => [o.x0, o.y0, o.x0 + o.w, o.y0 + o.h] as const); + const overlaps = boxes + .flatMap((a, i) => + boxes.slice(i + 1).map((b) => a[0] < b[2] && b[0] < a[2] && a[1] < b[3] && b[1] < a[3]), + ) + .filter(Boolean).length; + if (similar) { + const centre = [mean(c.map((p) => p[0])), mean(c.map((p) => p[1]))] as const; + const r = c.map((p) => Math.hypot(p[0] - centre[0], p[1] - centre[1])); + if (mean(r) > size * 0.6 && std(r) / mean(r) < 0.12) { + const ang = c + .map( + (p) => + ((((Math.atan2(p[1] - centre[1], p[0] - centre[0]) * 180) / Math.PI) % 360) + 360) % + 360, + ) + .sort((a, b) => a - b); + const gaps = [...ang.slice(1), ang[0]! + 360].map((a, i) => a - ang[i]!); + const radius = mean(r); + return { + kind: "ring", + count: found.length, + centre: toShare(g, centre[0] / g.sx, centre[1] / g.sy), + ring_diameter_share_of_width: round((2 * radius) / g.W, 4), + element_to_ring_diameter: round(size / (2 * radius), 2), + evenly_spaced: std(gaps) < 12, + touching: 2 * radius * Math.sin(Math.PI / found.length) <= size * 1.1, + }; + } + const rows = runs( + c.map((p) => p[1]), + size * 0.5, + ); + const cols = runs( + c.map((p) => p[0]), + size * 0.5, + ); + if (rows * cols >= found.length && found.length >= 0.7 * rows * cols && (rows > 1 || cols > 1)) + return { + kind: rows > 1 && cols > 1 ? "grid" : rows === 1 ? "row" : "column", + rows, + cols, + element_share_of_width: round(size / g.W, 3), + }; + } + if (overlaps >= found.length - 1) { + const order = c.map((_, i) => i).sort((i, j) => c[i]![0] + c[i]![1] - (c[j]![0] + c[j]![1])); + const steps = order + .slice(1) + .map((i, k) => [c[i]![0] - c[order[k]!]![0], c[i]![1] - c[order[k]!]![1]]); + return { + kind: "stack", + count: found.length, + overlapping_pairs: overlaps, + mean_offset_share: { + x: round(mean(steps.map((s) => s[0]!)) / g.W, 3), + y: round(mean(steps.map((s) => s[1]!)) / g.H, 3), + }, + }; + } + return { + kind: "scattered", + count: found.length, + overlapping_pairs: overlaps, + similar_sizes: similar, + }; +} + +const hex = (c: number[]) => + `#${c.map((v) => Math.round(v).toString(16).padStart(2, "0")).join("")}`; + +/** The elements standing out from the background at `t` inside `roi`. */ +// fallow-ignore-next-line complexity +export function objects(video: string, info: VideoInfo, t: number, roi: Roi, tol = 40) { + const width = roi[2] > MAX_W ? MAX_W : 0; + const { frames, w, h } = readFrameAt(info, t, { roi, width }); + const px = frames[0]; + if (!px) + throw new Fix( + `no frame at ${t}s`, + `pick a time before ${round(info.duration - 1 / info.fps, 2)}s`, + ); + const f: Frame = { px, w, h }; + const g: Geometry = { + W: info.width, + H: info.height, + x: roi[0], + y: roi[1], + sx: roi[2] / w, + sy: roi[3] / h, + }; + // the background is the median of the ROI's outermost rows and columns + const border: number[][] = [[], [], []]; + for (let y = 0; y < h; y++) + for (let x = 0; x < w; x++) + if (x === 0 || y === 0 || x === w - 1 || y === h - 1) + for (let ch = 0; ch < 3; ch++) border[ch]!.push(px[(y * w + x) * 3 + ch]!); + const bg = border.map(median); + const { id, list } = blobs( + f, + (r, gr, b) => Math.max(Math.abs(r - bg[0]!), Math.abs(gr - bg[1]!), Math.abs(b - bg[2]!)) > tol, + ); + const sums = list.map(() => [0, 0]); + for (let i = 0; i < id.length; i++) + if (id[i]) { + sums[id[i]!]![0]! += i % w; + sums[id[i]!]![1]! += Math.floor(i / w); + } + const found = list + .flatMap((b, i): Found[] => + i === 0 || b.n < MIN_SHARE * w * h + ? [] + : [ + { + cx: sums[i]![0]! / b.n, + cy: sums[i]![1]! / b.n, + w: b.x1 - b.x0 + 1, + h: b.y1 - b.y0 + 1, + x0: b.x0, + y0: b.y0, + area: b.n, + }, + ], + ) + .sort((a, b) => b.area - a.area); + return { + t, + background: hex(bg), + count: found.length, + elements: found.slice(0, 24).map((o) => ({ + ...toShare(g, o.cx, o.cy), + w: round((o.w * g.sx) / g.W, 3), + h: round((o.h * g.sy) / g.H, 3), + })), + arrangement: arrangement(found, g), + }; +} diff --git a/packages/cli/src/measure/opencv.test.ts b/packages/cli/src/measure/opencv.test.ts new file mode 100644 index 00000000000..bb82644b55e --- /dev/null +++ b/packages/cli/src/measure/opencv.test.ts @@ -0,0 +1,79 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { ffmpeg, loadOpenCvFirst, noTools } from "./diff.test-helpers.js"; +import { flowMoves, flowSeries, similarity } from "./flow.js"; +import { probeVideo } from "./media.js"; +import { trackTemplate } from "./track.js"; + +const dir = mkdtempSync(join(tmpdir(), "hf-opencv-test-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +describe("similarity", () => { + it("recovers a shift, scale and turn despite outliers", () => { + const [from, to]: [number[], number[]] = [[], []]; + const [s, a, tx, ty] = [1.1, (5 * Math.PI) / 180, 7, -3]; + for (let i = 0; i < 60; i++) { + const [x, y] = [(i * 37) % 100, (i * 53) % 80]; + from.push(x, y); + const outlier = i % 5 === 0; + to.push( + outlier ? x + 40 : s * (Math.cos(a) * x - Math.sin(a) * y) + tx, + outlier ? y - 25 : s * (Math.sin(a) * x + Math.cos(a) * y) + ty, + ); + } + const fit = similarity(from, to)!; + expect(fit.scale).toBeCloseTo(1.1, 3); + expect(fit.rot).toBeCloseTo(5, 2); + expect(fit.tx).toBeCloseTo(7, 2); + }); +}); + +describe.skipIf(noTools)("opencv tools", () => { + loadOpenCvFirst(); + // A textured 48 px square slides 400 px right from 0.3 s to 1.3 s. + const video = join(dir, "slide.mp4"); + if (!noTools) { + execFileSync(ffmpeg!, [ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=s=48x48", + "-frames:v", + "1", + join(dir, "tex.png"), + ]); + execFileSync(ffmpeg!, [ + ...["-v", "error", "-y", "-f", "lavfi", "-i", "color=c=0x202830:s=640x360:r=30:d=2"], + ...["-loop", "1", "-i", join(dir, "tex.png"), "-filter_complex"], + "[0][1]overlay=x='40+400*min(max((t-0.3)/1.0,0),1)':y=150:shortest=1", + ...["-pix_fmt", "yuv420p", "-c:v", "libx264", "-crf", "10", video], + ]); + } + + it("follows a template and fits its move", async () => { + const r = await trackTemplate({ + info: probeVideo(video), + at: 0, + box: [40, 150, 48, 48], + to: 2, + }); + expect(r.lost).toBeNull(); + const move = r.moves.find((m) => m.prop === "x"); + expect(move).toMatchObject({ from: 64, to: 464, start_s: 0.3, duration_s: 1, ease: "none" }); + }); + + it("reverses the slide from optical flow", async () => { + const info = probeVideo(video); + const rows = await flowSeries(info, 0, 2, [0, 100, 640, 160], 320); + const [move] = flowMoves(rows, info.fps, [0, 100, 640, 160], 0); + expect(move).toMatchObject({ kind: "tween", start_s: 0.3, duration_s: 1 }); + expect(Number(move!.x)).toBeGreaterThan(360); + expect(Number(move!.x)).toBeLessThan(440); + }); +}); diff --git a/packages/cli/src/measure/rhythm.test.ts b/packages/cli/src/measure/rhythm.test.ts new file mode 100644 index 00000000000..e83851bc9e6 --- /dev/null +++ b/packages/cli/src/measure/rhythm.test.ts @@ -0,0 +1,32 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js"; +import { rhythm } from "./rhythm.js"; + +const ffmpeg = findFFmpeg(); +const dir = mkdtempSync(join(tmpdir(), "hf-rhythm-test-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); +const run = (args: string[]) => execFileSync(ffmpeg!, ["-nostdin", "-v", "error", "-y", ...args]); + +describe.skipIf(!ffmpeg || !findFFprobe())("rhythm", () => { + it("reads a held picture as a static camera and a centred zoom as a push in", () => { + const still = join(dir, "still.png"); + run(["-f", "lavfi", "-i", "testsrc=s=320x180:d=1", "-frames:v", "1", still]); + const held = join(dir, "held.mp4"); + run(["-loop", "1", "-i", still, "-t", "2", "-r", "30", "-pix_fmt", "yuv420p", held]); + const zoom = join(dir, "zoom.mp4"); + run([ + ...["-i", still, "-vf"], + "zoompan=z='1+0.003*on':x='iw/2-iw/zoom/2':y='ih/2-ih/zoom/2':d=60:s=320x180:fps=30", + ...["-pix_fmt", "yuv420p", zoom], + ]); + expect(rhythm(held).shots.map((s) => s.camera)).toEqual(["static"]); + const pushed = rhythm(zoom).shots; + expect(pushed).toHaveLength(1); + expect(pushed[0]!.camera).toBe("push in"); + expect(pushed[0]!.zoom).toBeGreaterThan(1.03); + }); +}); diff --git a/packages/cli/src/measure/rhythm.ts b/packages/cli/src/measure/rhythm.ts new file mode 100644 index 00000000000..f3e6756e973 --- /dev/null +++ b/packages/cli/src/measure/rhythm.ts @@ -0,0 +1,219 @@ +// How a film moves over time, ported from the Studio's motion-rhythm.mjs: the share of frames where the picture +// changes, the still holds, the rate the animation updates at (on ones, twos, threes) and the camera's zoom and pan +// inside each shot. Zoom and pan are the dominant whole-frame transform every 0.5 s, so a large moving subject can +// read as camera; confirm a move on frames. +import { findCuts } from "./cuts.js"; +import { meanAbsDiff, probeVideo, readFrames, round, type Roi } from "./media.js"; + +const CUT = 25; +const W = 160; + +function bilinear(img: Uint8Array, w: number, h: number, x: number, y: number) { + const [x0, y0] = [Math.floor(x), Math.floor(y)]; + if (x0 < 0 || y0 < 0 || x0 + 1 >= w || y0 + 1 >= h) return -1; + const [fx, fy, i] = [x - x0, y - y0, y0 * w + x0]; + return ( + (img[i]! * (1 - fx) + img[i + 1]! * fx) * (1 - fy) + + (img[i + w]! * (1 - fx) + img[i + w + 1]! * fx) * fy + ); +} + +interface CameraStep { + s: number; + dx: number; + dy: number; + residual: number; + still: number; +} + +/** Best s, dx, dy with B(x) ≈ A(c + (x − c)/s + d): s > 1 is a push in. */ +function cameraStep(a: Uint8Array, b: Uint8Array, w: number, h: number): CameraStep { + const [cx, cy, m] = [w / 2, h / 2, Math.round(w * 0.1)]; + const cost = (s: number, dx: number, dy: number) => { + let [sum, n] = [0, 0]; + for (let y = m; y < h - m; y += 2) + for (let x = m; x < w - m; x += 2) { + const v = bilinear(a, w, h, cx + (x - cx) / s + dx, cy + (y - cy) / s + dy); + if (v >= 0) { + sum += Math.abs(v - b[y * w + x]!); + n++; + } + } + return n ? sum / n : 1e9; + }; + let best = [cost(1, 0, 0), 1, 0, 0]; + const search = (ss: number[], dxs: number[], dys: number[]) => { + for (const s of ss) + for (const dx of dxs) + for (const dy of dys) { + const c = cost(s, dx, dy); + if (c < best[0]!) best = [c, s, dx, dy]; + } + }; + const range = (c: number, step: number, n: number) => + Array.from({ length: 2 * n + 1 }, (_, i) => c + (i - n) * step); + // coarse first, wide enough for a handheld swing (±10% zoom, ±10% of the width), then fine around the best + search(range(1, 0.02, 5), range(0, 4, 4), range(0, 4, 4)); + search(range(best[1]!, 0.005, 2), range(best[2]!, 1, 2), range(best[3]!, 1, 2)); + search(range(best[1]!, 0.0025, 2), range(best[2]!, 0.5, 2), range(best[3]!, 0.5, 2)); + return { s: best[1]!, dx: best[2]!, dy: best[3]!, residual: best[0]!, still: cost(1, 0, 0) }; +} + +/** True when the whole frame moved: the best zoom and pan leave well under the change a still camera leaves. A + * presenter swaying in front of a locked camera changes a small area, and moving it barely helps. */ +const cameraMoved = (step: CameraStep) => step.residual < 0.7 * step.still; + +interface Shot { + start: number; + end?: number; + zoom: number; + pan_x: number; + pan_y: number; + path: number; + residuals: number[]; +} + +export interface Rhythm { + start: number; + seconds: number; + fps: number; + moving_share: number; + longest_hold_s: number; + median_hold_s: number; + holds_over_1s_per_min: number; + update_fps: number; + still_threshold: number; + cuts_per_min: number; + camera_moves_per_min: number; + shots: { + start: number; + end: number; + camera: string; + zoom: number; + pan_x: number; + pan_y: number; + path: number; + }[]; +} + +/** `still` is the reference's still threshold, so yours is measured at its level. */ +// fallow-ignore-next-line complexity +export function rhythm(path: string, start = 0, seconds = 0, roi?: Roi, still?: number): Rhythm { + const p = probeVideo(path); + const secs = seconds || p.duration - start; + const { frames: fs, h } = readFrames(path, { start, seconds: secs, roi, width: W, gray: true }); + const d = fs.slice(1).map((f, i) => meanAbsDiff(f, fs[i]!)); + // mean |Δluma| on a 160 px frame below which nothing visibly changed; grain and compression raise the floor + // (capped, so a window that never stops moving is not read as noise) + const STILL = + still ?? + Math.min( + 1.5, + Math.max(0.5, 2 * ([...d].sort((x, y) => x - y)[Math.floor(d.length * 0.1)] ?? 0)), + ); + // Shots split where measure cuts and diff find cuts; CUT stays the ceiling of what counts as motion. + const cuts = findCuts(fs).map((c) => c.frame); + // a frame is moving when it or a neighbour changes, so animation on twos or threes still counts as motion + const moving = d.map((_, i) => + [d[i - 1], d[i], d[i + 1]].some((v) => v !== undefined && v > STILL && v <= CUT), + ); + const holds: number[] = []; + for (let i = 0, run = 0; i <= moving.length; i++) { + if (i < moving.length && !moving[i] && d[i]! <= CUT) run++; + else { + if (run) holds.push(run / p.fps); + run = 0; + } + } + // inside stretches that are moving, a repeated frame means the animation updates on twos or threes + let [inMotion, repeats] = [0, 0]; + for (let i = 2; i < d.length - 2; i++) { + const near = Math.max(d[i - 2]!, d[i - 1]!, d[i + 1]!, d[i + 2]!); + if (near > STILL && near <= CUT) { + inMotion++; + if (d[i]! < 0.2 * near) repeats++; + } + } + const step = Math.max(1, Math.round(p.fps / 2)); + const shots: Shot[] = []; + const fresh = (t: number): Shot => ({ + start: t, + zoom: 1, + pan_x: 0, + pan_y: 0, + path: 0, + residuals: [], + }); + let shot = fresh(0); + for (let i = 0; i + step < fs.length; i += step) { + const cut = cuts.find((c) => c > i && c <= i + step); + if (cut) { + shots.push({ ...shot, end: cut / p.fps }); + shot = fresh(cut / p.fps); + continue; + } + const c = cameraStep(fs[i]!, fs[i + step]!, W, h); + shot.zoom *= c.s; + shot.pan_x += c.dx / W; + shot.pan_y += c.dy / h; + if (cameraMoved(c)) shot.path += Math.hypot(c.dx / W, c.dy / h) + Math.abs(Math.log(c.s)); + shot.residuals.push(c.residual); + } + shots.push({ ...shot, end: fs.length / p.fps }); + // what is left after the best whole-frame transform; far above the film's usual level, the frame changed in a way + // no zoom or pan explains (a swing past the search, a rotation, a large subject crossing the frame) + const all = shots.flatMap((s) => s.residuals).sort((x, y) => x - y); + const UNEXPLAINED = Math.max(12, 3 * (all[Math.floor(all.length / 2)] ?? 0)); + const kind = (s: Shot) => { + if ((s.end ?? 0) - s.start < 0.6) return "short"; + const z = s.zoom - 1; + if (Math.abs(z) > 0.03) return z > 0 ? "push in" : "pull out"; + if (Math.hypot(s.pan_x, s.pan_y) > 0.03) return "pan/drift"; + if (s.path > 0.08) return "handheld"; + if (s.residuals.some((r) => r > UNEXPLAINED)) return "moving, unexplained (confirm on frames)"; + return "static"; + }; + const sorted = [...holds].sort((x, y) => x - y); + return { + start, + seconds: round(secs, 2), + fps: round(p.fps, 2), + moving_share: round(moving.filter(Boolean).length / Math.max(1, d.length), 2), + longest_hold_s: round(sorted.at(-1) ?? 0, 2), + median_hold_s: round(sorted[Math.floor(sorted.length / 2)] ?? 0, 2), + holds_over_1s_per_min: round((holds.filter((x) => x >= 1).length / secs) * 60, 1), + update_fps: round(p.fps * (1 - repeats / Math.max(1, inMotion)), 1), + still_threshold: round(STILL, 2), + cuts_per_min: round((cuts.length / secs) * 60, 1), + camera_moves_per_min: round( + (shots.filter((s) => /push|pull|pan|handheld/.test(kind(s))).length / secs) * 60, + 1, + ), + shots: shots.map((s) => ({ + start: round(start + s.start, 2), + end: round(start + (s.end ?? 0), 2), + camera: kind(s), + zoom: round(s.zoom, 3), + pan_x: round(s.pan_x, 3), + pan_y: round(s.pan_y, 3), + path: round(s.path, 3), + })), + }; +} + +const RATIO_KEYS = [ + "moving_share", + "longest_hold_s", + "median_hold_s", + "holds_over_1s_per_min", + "update_fps", + "cuts_per_min", + "camera_moves_per_min", +] as const; + +/** Yours over the reference's, key by key; null where the reference's is 0. */ +export function compareRhythm(ref: Rhythm, yours: Rhythm) { + return Object.fromEntries( + RATIO_KEYS.map((k) => [k, ref[k] ? round(yours[k] / ref[k], 2) : null]), + ); +} diff --git a/packages/cli/src/measure/track.ts b/packages/cli/src/measure/track.ts new file mode 100644 index 00000000000..e85b5c966a4 --- /dev/null +++ b/packages/cli/src/measure/track.ts @@ -0,0 +1,123 @@ +// One textured thing followed frame by frame (a cursor, a hand, a product), ported from hf-media's track: the box at +// one moment is the template, each next frame is searched near the last match by normalized cross-correlation, and +// the track ends at a cut, a weak match or an ambiguous one rather than guessing. Moves come out as eases. +import { findCuts } from "./cuts.js"; +import { fitMoves, printable, type Row } from "./curve.js"; +import { readClip, round, videoClip, type Roi, type VideoInfo } from "./media.js"; +import { greyMat, loadOpenCv } from "./opencv.js"; + +const WORK_W = 960; + +export interface TrackOptions { + info: VideoInfo; + /** The moment the template is cut from, and the box it is cut from (source pixels). */ + at: number; + box: Roi; + to: number; + /** How far it may move between two frames, in source pixels. */ + radius?: number; + minScore?: number; + /** How far the best match must beat the runner-up elsewhere. */ + minMargin?: number; +} + +export interface Tracked { + rows: Row[]; + scores: number[]; + lost: { t: number; reason: "cut" | "low_score" | "ambiguous"; score?: number } | null; + moves: ReturnType; + /** Working size the search ran at, for drawing. */ + scale: number; +} + +// fallow-ignore-next-line complexity +export async function trackTemplate(o: TrackOptions): Promise { + const cv = await loadOpenCv((m) => console.error(m)); + const k = Math.min(1, WORK_W / o.info.width); + const { frames, w, h } = readClip(videoClip(o.info.path, o.at, o.to - o.at, o.info), { + width: Math.round(o.info.width * k), + gray: true, + }); + const cuts = new Set(findCuts(frames).map((c) => c.frame)); + const [bx, by, bw, bh] = o.box.map((v) => Math.round(v * k)); + const tw = Math.max(4, Math.min(bw!, w - bx!)); + const th = Math.max(4, Math.min(bh!, h - by!)); + const templ = new Uint8Array(tw * th); + for (let y = 0; y < th; y++) + templ.set(frames[0]!.subarray((by! + y) * w + bx!, (by! + y) * w + bx! + tw), y * tw); + const tMat = greyMat(cv, templ, tw, th); + const radius = Math.max(2, Math.round((o.radius ?? 48) * k)); + const fps = o.info.fps; + const centre = (x: number, y: number): Row => ({ + t: 0, + x: (x + tw / 2) / k, + y: (y + th / 2) / k, + w: tw / k, + h: th / k, + ink: 1, + }); + const rows: Row[] = [{ ...centre(bx!, by!), t: round(o.at, 3) }]; + const scores = [1]; + let [px, py] = [bx!, by!]; + let lost: Tracked["lost"] = null; + try { + for (let i = 1; i < frames.length; i++) { + const t = round(o.at + i / fps, 3); + if (cuts.has(i)) { + lost = { t, reason: "cut" }; + break; + } + const [x0, y0] = [Math.max(0, px - radius), Math.max(0, py - radius)]; + const [x1, y1] = [Math.min(w, px + tw + radius), Math.min(h, py + th + radius)]; + const sw = x1 - x0; + const sh = y1 - y0; + const area = new Uint8Array(sw * sh); + for (let y = 0; y < sh; y++) + area.set(frames[i]!.subarray((y0 + y) * w + x0, (y0 + y) * w + x1), y * sw); + const aMat = greyMat(cv, area, sw, sh); + const result = new cv.Mat(); + try { + cv.matchTemplate(aMat, tMat, result, cv.TM_CCOEFF_NORMED); + const { maxVal, maxLoc } = cv.minMaxLoc(result); + // The runner-up away from the best match's own correlation lobe: a second cursor, a repeated icon. + let second = -Infinity; + const [rw, dx, dy] = [result.cols, Math.max(1, tw >> 1), Math.max(1, th >> 1)]; + result.data32F.forEach((v, j) => { + const [cx, cy] = [j % rw, Math.floor(j / rw)]; + if ((Math.abs(cx - maxLoc.x) > dx || Math.abs(cy - maxLoc.y) > dy) && v > second) + second = v; + }); + const margin = Number.isFinite(second) ? maxVal - second : 1; + if (maxVal < (o.minScore ?? 0.6) || margin < (o.minMargin ?? 0.05)) { + lost = { + t, + reason: maxVal < (o.minScore ?? 0.6) ? "low_score" : "ambiguous", + score: round(maxVal, 3), + }; + break; + } + [px, py] = [x0 + maxLoc.x, y0 + maxLoc.y]; + rows.push({ ...centre(px, py), t }); + scores.push(maxVal); + } finally { + aMat.delete(); + result.delete(); + } + } + } finally { + tMat.delete(); + } + const quant = 1 / k; + const moves = (["x", "y"] as const) + .flatMap((p) => + fitMoves( + rows, + p, + fps, + Math.max(3 * quant, 0.01 * Math.max(o.info.width, o.info.height)), + quant, + ), + ) + .sort((a, b) => a.start_s - b.start_s); + return { rows, scores, lost, moves: printable(moves), scale: k }; +}