feat(cli): measure trail, box, rhythm, objects, track, flow and continuity

docs/packages/cli.mdx

@@ -1649,12 +1649,20 @@ npx hyperframes measure face ref.mp4 --start 1 --end 2.5
 | `cuts`       | Hard cuts from the frame-to-frame difference, and the shots between them                |
 | `events`     | Cuts, flashes, appears, disappears and holds; `--map` adds the sound's hits             |
 | `frames`     | Frames as pictures: chosen moments, an overview sheet, a range, or strips around events |
+| `trail`      | A window's motion as one picture: onion skin, the moving part's path, a heat map        |
 | `extract`    | A still or a clip cut from the video, cropped to a region, as project media             |
+| `box`        | One graphic found by its colour: box, ink, colours, text lines; `--track` adds moves    |
 | `curve`      | Each move of one element: start, duration, overshoot and the closest GSAP ease          |
 | `energy`     | How much two clips move, and whether at the same moments                                |
+| `ink`        | The bounds, size and lines of the text or marks in a region of one frame                |
 | `look`       | Brightness, contrast, saturation and warmth, alone or against another video             |
+| `rhythm`     | Moving share, holds, update rate (ones, twos), cuts, and each shot's zoom or pan        |
+| `objects`    | Separate elements at one time: count, sizes, and a ring, grid, row, column or stack     |
+| `track`      | One textured thing (a cursor, a hand) followed from its box until a cut or lost match   |
+| `flow`       | One element's moves from optical flow as tweens, or against your render's               |
 | `face`       | The face's size and position over a window                                              |
 | `sync`       | The lag between mouth motion and the voice                                              |
+| `continuity` | Seams your render has and the original lacks: jumps, flicker, extra or missing shapes   |
 
 ### `diff`
 

packages/cli/src/commands/measure.ts

@@ -11,20 +11,31 @@ export const examples: Example[] = [
     "hyperframes measure events ref.mp4 --map",
   ],
   ["The frames around every event, as strips", "hyperframes measure frames ref.mp4 --events"],
+  [
+    "One element's box, colours, text lines and moves",
+    "hyperframes measure box ref.mp4 --start 1 --end 2.5 --roi 200,300,600,200 --track",
+  ],
 ];
 
 const subCommands = {
   probe: () => import("./measure/probe.js").then((m) => m.default),
   cuts: () => import("./measure/cuts.js").then((m) => m.default),
   events: () => import("./measure/events.js").then((m) => m.default),
   frames: () => import("./measure/frames.js").then((m) => m.default),
+  trail: () => import("./measure/trail.js").then((m) => m.default),
   extract: () => import("./measure/extract.js").then((m) => m.default),
+  box: () => import("./measure/box.js").then((m) => m.default),
   curve: () => import("./measure/curve.js").then((m) => m.default),
   energy: () => import("./measure/energy.js").then((m) => m.default),
   ink: () => import("./measure/ink.js").then((m) => m.default),
   look: () => import("./measure/look.js").then((m) => m.default),
+  rhythm: () => import("./measure/rhythm.js").then((m) => m.default),
+  objects: () => import("./measure/objects.js").then((m) => m.default),
+  track: () => import("./measure/track.js").then((m) => m.default),
+  flow: () => import("./measure/flow.js").then((m) => m.default),
   face: () => import("./measure/face.js").then((m) => m.default),
   sync: () => import("./measure/sync.js").then((m) => m.default),
+  continuity: () => import("./measure/continuity.js").then((m) => m.default),
 };
 
 export const MEASURE_OPS = Object.keys(subCommands);
@@ -33,7 +44,7 @@ export default defineCommand({
   meta: {
     name: "measure",
     description:
-      "Measure a video file as JSON: clock, cuts, events, frames, crops, eases, motion energy, ink, look, faces, lip sync",
+      "Measure a video file as JSON: clock, cuts, events, frames, motion trails, crops, element boxes, eases, motion energy, ink, look, rhythm, objects, tracking, optical flow, faces, lip sync, continuity",
   },
   subCommands,
 });

packages/cli/src/commands/measure/box.ts

@@ -0,0 +1,75 @@
+import { defineCommand } from "../../utils/defineCommand.js";
+import type { Example } from "../_examples.js";
+import {
+  num,
+  jsonArg,
+  outArg,
+  roiArg,
+  runMeasure,
+  videoAndRoi,
+  videoArg,
+  window,
+} from "../../measure/command.js";
+import { boxAt, boxRange } from "../../measure/box.js";
+import { Fix, outDir } from "../../measure/media.js";
+
+export const examples: Example[] = [
+  [
+    "One graphic's box, colours, text lines and cap height at one time",
+    "hyperframes measure box ref.mp4 --at 1 --roi 240,280,600,540",
+  ],
+  [
+    "When it is on screen over a range, and each move it makes with its ease",
+    "hyperframes measure box ref.mp4 --start 3.5 --end 4.5 --roi 240,280,600,540 --track",
+  ],
+];
+
+export default defineCommand({
+  meta: {
+    name: "box",
+    description:
+      "Where one graphic sits, found by its colour: box, ink, colours, text lines, cap height; over a range its moves",
+  },
+  args: {
+    video: videoArg,
+    at: { type: "string", description: "One time in seconds" },
+    start: { type: "string", alias: ["from"], description: "Range start in seconds" },
+    end: { type: "string", alias: ["to"], description: "Range end in seconds" },
+    roi: {
+      ...roiArg,
+      description: "Region round the graphic with some of its ground (default: whole frame)",
+    },
+    color: {
+      type: "string",
+      description:
+        "Its pixels: auto (the commonest colour unlike the ROI border), white, dark, alpha or #rrggbb (default auto)",
+      default: "auto",
+    },
+    tol: { type: "string", description: "RGB distance that still matches the colour (40)" },
+    track: {
+      type: "boolean",
+      description: "Over a range: each move of its centre, size and ink, with the ease",
+      default: false,
+    },
+    out: outArg,
+    json: jsonArg,
+  },
+  run: ({ args }) =>
+    runMeasure(() => {
+      const { info, roi } = videoAndRoi(args.video, args.roi);
+      const opts = {
+        video: info.path,
+        info,
+        roi,
+        color: args.color ?? "auto",
+        tol: num(args.tol, "--tol", 40),
+        dir: outDir(args.out, "measure/box"),
+      };
+      if (args.at !== undefined)
+        return { video: info.path, roi, ...boxAt(opts, num(args.at, "--at", 0)) };
+      if (args.start === undefined || args.end === undefined)
+        throw new Fix("give --at t, or --start and --end", "--start 3.5 --end 4.5");
+      const [start, end] = window(info, args.start, args.end);
+      return { video: info.path, roi, ...boxRange(opts, start, end, args.track === true) };
+    }),
+});

packages/cli/src/commands/measure/continuity.ts

@@ -0,0 +1,53 @@
+import { resolve } from "node:path";
+import { defineCommand } from "../../utils/defineCommand.js";
+import type { Example } from "../_examples.js";
+import { jsonArg, num, outArg, runMeasure } from "../../measure/command.js";
+import { continuity } from "../../measure/continuity.js";
+import { outDir } from "../../measure/media.js";
+
+export const examples: Example[] = [
+  [
+    "Seams a chained shot has that the original does not",
+    "hyperframes measure continuity ref.mp4 renders/final.mp4 --start 4 --end 12",
+  ],
+];
+
+export default defineCommand({
+  meta: {
+    name: "continuity",
+    description:
+      "Seams yours has and the original does not: jumps, background flicker, and extra or missing shapes, on the original's clock",
+  },
+  args: {
+    reference: { type: "positional", description: "The original", required: true },
+    yours: { type: "positional", description: "The rebuild's render", required: true },
+    start: { type: "string", alias: ["from"], description: "First second (default 0)" },
+    end: { type: "string", alias: ["to"], description: "Last second (default: the end)" },
+    out: outArg,
+    json: jsonArg,
+  },
+  async run({ args }) {
+    await runMeasure(() => {
+      const r = continuity(resolve(String(args.reference)), resolve(String(args.yours)), {
+        start: num(args.start, "--start"),
+        end: num(args.end, "--end"),
+        outDir: outDir(args.out, "measure/continuity"),
+      });
+      const count = (kind: string) => r.events.filter((e) => e.kind === kind).length;
+      return {
+        reference: resolve(String(args.reference)),
+        yours: resolve(String(args.yours)),
+        frames: r.frames,
+        counts: { jump: count("jump"), flicker: count("flicker"), structure: count("structure") },
+        events: r.events,
+        thresholds: r.thresholds,
+        reading:
+          "jump: a whole-frame change the original lacks; flicker: a change over background the original keeps still; structure: an extra, missing or misplaced shape at `at` (source pixels); open each sheet (original over yours, peak marked) before acting",
+        limitations: [
+          "frames pair by time on the original's clock, so a retimed or offset rebuild reads as structure everywhere",
+          "events are candidates to look at, not verdicts",
+        ],
+      };
+    });
+  },
+});

packages/cli/src/commands/measure/flow.ts

@@ -0,0 +1,91 @@
+import { defineCommand } from "../../utils/defineCommand.js";
+import type { Example } from "../_examples.js";
+import {
+  jsonArg,
+  num,
+  roiArg,
+  runMeasure,
+  videoAndRoi,
+  videoArg,
+  window,
+} from "../../measure/command.js";
+import { compareFlow, flowMoves, flowSeries } from "../../measure/flow.js";
+import { parseRoi, probeVideo, type Roi } from "../../measure/media.js";
+
+export const examples: Example[] = [
+  [
+    "A card's moves from optical flow, as tweens",
+    "hyperframes measure flow ref.mp4 --start 1 --end 2.4 --roi 300,200,640,400",
+  ],
+  [
+    "Does the rebuild's element move like the original's",
+    "hyperframes measure flow ref.mp4 --start 1 --end 2.4 --roi 300,200,640,400 --compare renders/v1.mp4",
+  ],
+];
+
+export default defineCommand({
+  meta: {
+    name: "flow",
+    description:
+      "One element's motion from optical flow inside its box: its moves as tweens, or compared with another video's (energy, timing, path, scale, rotation)",
+  },
+  args: {
+    video: videoArg,
+    start: { type: "string", alias: ["from"], description: "First second", required: true },
+    end: { type: "string", alias: ["to"], description: "Last second", required: true },
+    roi: roiArg,
+    compare: {
+      type: "string",
+      description: "Another video over the same window, on the same clock",
+    },
+    "compare-start": {
+      type: "string",
+      description: "Second the window starts in it (default: --start)",
+    },
+    "compare-roi": { type: "string", description: "Its box (default: --roi scaled by the widths)" },
+    json: jsonArg,
+  },
+  async run({ args }) {
+    await runMeasure(async () => {
+      const { info, roi } = videoAndRoi(args.video, args.roi);
+      const [a, b] = window(info, args.start, args.end);
+      if (args.compare === undefined) {
+        const rows = await flowSeries(info, a, b, roi, 320);
+        return {
+          video: info.path,
+          start: a,
+          end: b,
+          roi,
+          moves: flowMoves(rows, info.fps, roi, a),
+        };
+      }
+      const other = probeVideo(String(args.compare));
+      // Each axis scales on its own: a generated shot is rarely the original's exact aspect (768x1344 for 9:16).
+      const [kx, ky] = [other.width / info.width, other.height / info.height];
+      const scaled: Roi = [
+        Math.round(roi[0] * kx),
+        Math.round(roi[1] * ky),
+        Math.round(roi[2] * kx),
+        Math.round(roi[3] * ky),
+      ];
+      const otherRoi =
+        args["compare-roi"] !== undefined
+          ? parseRoi(args["compare-roi"], other.width, other.height)
+          : scaled;
+      const oa = num(args["compare-start"], "--compare-start", a);
+      const [ref, yours] = await Promise.all([
+        flowSeries(info, a, b, roi, 160),
+        flowSeries(other, oa, oa + (b - a), otherRoi, 160),
+      ]);
+      return {
+        video: info.path,
+        compare: other.path,
+        start: a,
+        end: b,
+        ...compareFlow(ref, yours),
+        reading:
+          "energy_ratio near 1 moves as much; motion_corr near 1 at lag_frames 0 moves at the same moments; path, scale and rotation errors are per frame",
+      };
+    });
+  },
+});

packages/cli/src/commands/measure/objects.ts

@@ -0,0 +1,38 @@
+import { defineCommand } from "../../utils/defineCommand.js";
+import type { Example } from "../_examples.js";
+import { jsonArg, num, roiArg, runMeasure, videoAndRoi, videoArg } from "../../measure/command.js";
+import { objects } from "../../measure/objects.js";
+
+export const examples: Example[] = [
+  [
+    "How many elements are on screen and how they are arranged (ring, grid, row, stack)",
+    "hyperframes measure objects ref.mp4 --at 2.4 --roi 300,200,1320,680",
+  ],
+];
+
+export default defineCommand({
+  meta: {
+    name: "objects",
+    description:
+      "Separate elements at one time: count, sizes and arrangement (ring, grid, row, column, stack) as shares of the frame",
+  },
+  args: {
+    video: videoArg,
+    at: { type: "string", description: "Time in seconds", required: true },
+    roi: roiArg,
+    tol: {
+      type: "string",
+      description: "Distance from the background that counts as an element (40)",
+    },
+    json: jsonArg,
+  },
+  run: ({ args }) =>
+    runMeasure(() => {
+      const { info, roi } = videoAndRoi(args.video, args.roi);
+      return {
+        video: info.path,
+        roi,
+        ...objects(info.path, info, num(args.at, "--at", 0), roi, num(args.tol, "--tol", 40)),
+      };
+    }),
+});

packages/cli/src/commands/measure/rhythm.ts

@@ -0,0 +1,69 @@
+import { defineCommand } from "../../utils/defineCommand.js";
+import type { Example } from "../_examples.js";
+import {
+  compareWindowArgs,
+  jsonArg,
+  num,
+  roiArg,
+  runMeasure,
+  videoAndRoi,
+  videoArg,
+  window,
+} from "../../measure/command.js";
+import { parseRoi, probeVideo } from "../../measure/media.js";
+import { compareRhythm, rhythm } from "../../measure/rhythm.js";
+
+export const examples: Example[] = [
+  [
+    "How the film moves: holds, update rate, cuts and camera per shot",
+    "hyperframes measure rhythm ref.mp4",
+  ],
+  [
+    "The same numbers for your render, as ratios to the original's",
+    "hyperframes measure rhythm ref.mp4 --start 4 --end 9 --compare renders/final.mp4",
+  ],
+];
+
+export default defineCommand({
+  meta: {
+    name: "rhythm",
+    description:
+      "How a film moves: moving share, holds, update rate (ones, twos), cuts, and each shot's zoom, pan or handheld",
+  },
+  args: {
+    video: videoArg,
+    start: { type: "string", alias: ["from"], description: "Window start in seconds (default 0)" },
+    end: { type: "string", alias: ["to"], description: "Window end in seconds (default: the end)" },
+    roi: roiArg,
+    // fallow-ignore-next-line code-duplication
+    compare: {
+      type: "string",
+      description: "A second video (your render) measured at the original's still level",
+    },
+    ...compareWindowArgs,
+    json: jsonArg,
+  },
+  run: ({ args }) =>
+    runMeasure(() => {
+      const { info, roi } = videoAndRoi(args.video, args.roi);
+      const [start, end] = window(info, args.start, args.end);
+      const region = args.roi === undefined ? undefined : roi;
+      const ref = rhythm(info.path, start, end - start, region);
+      if (!args.compare) return { video: info.path, ...ref };
+      const other = probeVideo(args.compare);
+      const yoursRoi = args["compare-roi"]
+        ? parseRoi(args["compare-roi"], other.width, other.height)
+        : region;
+      const from = num(args["compare-start"], "--compare-start", start);
+      const yours = rhythm(other.path, from, end - start, yoursRoi, ref.still_threshold);
+      const { shots: refShots, ...refSummary } = ref;
+      const { shots: yoursShots, ...yoursSummary } = yours;
+      return {
+        reference: refSummary,
+        yours: yoursSummary,
+        ratio: compareRhythm(ref, yours),
+        reference_shots: refShots,
+        yours_shots: yoursShots,
+      };
+    }),
+});