From 14bcfc5922a1d0ea34b8686f944aa5b8aba99027 Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:13:55 +0100 Subject: [PATCH 1/9] feat(scrawl): ink is recognised, numbered and grouped on one keyframe Native captures called every stroke group a plain "mark", so an agent could not tell a circle from an underline, and marks on one screen each took a keyframe. Port Scrawl's rules as pure TypeScript: shapes with from/to and the circled box, marks grouped on one keyframe while nothing happened to the screen, numbered in drawing order in the text and scrawl.json but not on the image. The keyframe draws all its marks' ink, which has faded from the video by then. Whole sentences still own keyframes. Co-Authored-By: Claude Opus 5.5 --- FORK.md | 8 + apps/web/src/scrawl/markGroups.test.ts | 91 ++++++ apps/web/src/scrawl/markGroups.ts | 90 ++++++ apps/web/src/scrawl/markShapes.test.ts | 324 ++++++++++++++++++++ apps/web/src/scrawl/markShapes.ts | 399 +++++++++++++++++++++++++ apps/web/src/scrawl/narration.test.ts | 42 +++ apps/web/src/scrawl/narration.ts | 125 +++++++- apps/web/src/scrawl/scrawlBundle.ts | 90 +++--- 8 files changed, 1116 insertions(+), 53 deletions(-) create mode 100644 apps/web/src/scrawl/markGroups.test.ts create mode 100644 apps/web/src/scrawl/markGroups.ts create mode 100644 apps/web/src/scrawl/markShapes.test.ts create mode 100644 apps/web/src/scrawl/markShapes.ts diff --git a/FORK.md b/FORK.md index cc27d2eb494b2..6210a7c0ba768 100644 --- a/FORK.md +++ b/FORK.md @@ -165,6 +165,14 @@ the contact sheet land in the composer on screen. Esc in T3 Code discards a capt - Speech and pointing do not run in step, so sentences are never cut at a click: whole sentences own a keyframe, click bursts fold into one (`clicked ×3`), and clicks the screen did not answer read as `pointed`. On 32 real captures this took cuts through sentences from 59 of 127 lines to 1. +- Ink is recognised with Scrawl's rules (`circled`, `underlined`, `drew a line`, `drew an arrow`, + else `marked`; `scrawl.json` has `shape`, `from`, `to`). Marks drawn on one screen share a keyframe + that draws all their ink, until a click or drag, a scene change, 20 s or 12 strokes. Several on a + keyframe are numbered in drawing order in the text and `scrawl.json` (`✎① 0:06 circled “…”`), never + on the image. No mark is dropped; over the keyframe budget, neighbouring groups merge. On Scrawl's + 43 captures all 124 ink marks get the shape and box of Scrawl's current `sketch.rs`. Scrawl compares + the screen around the ink, but here the video shows the ink, so scene changes decide "same screen". + The laser and pins are not here: Scrawl's Windows build records no laser and Shift+move is the ink. - Uses the dictation model above, with word times mapped back across the pauses it cuts out. - The recording pill in the corner shows in the video and keyframes; `setContentProtection` does not keep it out on Windows 10 22H2. diff --git a/apps/web/src/scrawl/markGroups.test.ts b/apps/web/src/scrawl/markGroups.test.ts new file mode 100644 index 0000000000000..783f77a2b12f0 --- /dev/null +++ b/apps/web/src/scrawl/markGroups.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { groupMarks, placeMarks } from "./markGroups.ts"; +import { buildNarration } from "./narration.ts"; + +const mark = (id: string, t0: number, strokes = 1) => ({ + id, + t0, + t1: t0 + 500, + strokes: Array.from({ length: strokes }, (_, index) => index), +}); +const idsOf = (groups: ReadonlyArray>) => + groups.map((group) => group.map(({ id }) => id)); + +describe("groupMarks", () => { + const free = { actionTimes: [], sceneChanges: [], budget: 12 }; + + it("shares a keyframe while nothing happened to the screen between the marks", () => { + const marks = [mark("m1", 1_000), mark("m2", 5_000), mark("m3", 9_000)]; + expect(idsOf(groupMarks(marks, free))).toEqual([["m1", "m2", "m3"]]); + }); + + it.each([ + ["a click", { actionTimes: [3_000] }], + ["a scene change", { sceneChanges: [3_000] }], + ])("closes the group at %s", (_name, cause) => { + const marks = [mark("m1", 1_000), mark("m2", 5_000)]; + expect(idsOf(groupMarks(marks, { ...free, ...cause }))).toEqual([["m1"], ["m2"]]); + }); + + it("ignores a click made while the ink was being drawn", () => { + const marks = [mark("m1", 1_000), mark("m2", 5_000)]; + expect(idsOf(groupMarks(marks, { ...free, actionTimes: [1_200, 5_200] }))).toEqual([ + ["m1", "m2"], + ]); + }); + + it("closes the group past 20 s and past 12 strokes", () => { + const late = [mark("m1", 0), mark("m2", 10_000), mark("m3", 20_000)]; + expect(idsOf(groupMarks(late, free))).toEqual([["m1", "m2"], ["m3"]]); + const inky = [mark("m1", 0, 6), mark("m2", 2_000, 6), mark("m3", 4_000, 1)]; + expect(idsOf(groupMarks(inky, free))).toEqual([["m1", "m2"], ["m3"]]); + }); + + it("merges over budget, same-screen least ink first, else the most alike, dropping none", () => { + // Kept apart by clicks, all on one screen: the lightest neighbours go together. + const clicked = [mark("m1", 0, 5), mark("m2", 2_000), mark("m3", 4_000), mark("m4", 6_000, 5)]; + expect( + idsOf( + groupMarks(clicked, { + actionTimes: [1_000, 3_000, 5_000], + sceneChanges: [], + budget: 3, + }), + ), + ).toEqual([["m1"], ["m2", "m3"], ["m4"]]); + // A busier gap stays a boundary. + const screens = [mark("m1", 0), mark("m2", 2_000), mark("m3", 4_000)]; + expect( + idsOf( + groupMarks(screens, { + actionTimes: [], + sceneChanges: [1_000, 1_500, 3_000], + budget: 2, + }), + ), + ).toEqual([["m1"], ["m2", "m3"]]); + expect( + idsOf(groupMarks(screens, { actionTimes: [], sceneChanges: [1_000, 3_000], budget: 1 })), + ).toEqual([["m1", "m2", "m3"]]); + }); +}); + +describe("placeMarks", () => { + it("gives every mark its keyframe, and a number only among several", () => { + const marks = [mark("m1", 1_000), mark("m2", 3_000), mark("m3", 30_000)]; + const lines = buildNarration({ + words: [], + events: [], + marks, + durationMs: 40_000, + markGroups: [["m1", "m2"], ["m3"]], + }); + expect(placeMarks(marks, lines)).toMatchObject([ + { id: "m1", frame: "frames/frame-01.png", number: 1 }, + { id: "m2", frame: "frames/frame-01.png", number: 2 }, + { id: "m3", frame: "frames/frame-02.png" }, + ]); + expect(placeMarks(marks, lines)[2]).not.toHaveProperty("number"); + }); +}); diff --git a/apps/web/src/scrawl/markGroups.ts b/apps/web/src/scrawl/markGroups.ts new file mode 100644 index 0000000000000..f9a30cf9c6456 --- /dev/null +++ b/apps/web/src/scrawl/markGroups.ts @@ -0,0 +1,90 @@ +// Which marks share one keyframe: those drawn on the same screen, so one image answers for all their +// ink. Scrawl compares the luma around the ink, in frames that do not show it. Here the video shows +// the ink, so the same question goes to the scene changes that already have the ink filtered out. + +import type { NarrationLine } from "./narration"; + +interface GroupableMark { + readonly id: string; + readonly t0: number; + readonly t1: number; + readonly strokes: ReadonlyArray; +} + +// Past these one frame stops being legible and its words drift from its ink. +const GROUP_MS = 20_000; +const GROUP_STROKES = 12; + +export function groupMarks( + marks: ReadonlyArray, + input: { + /** Clicks and drags: what the hand did to the screen between two marks. */ + readonly actionTimes: ReadonlyArray; + readonly sceneChanges: ReadonlyArray; + readonly budget: number; + }, +): Array> { + const changesBetween = (from: Mark, to: Mark) => + input.sceneChanges.filter((t) => t > from.t1 && t <= to.t1).length; + const acted = (from: Mark, to: Mark) => input.actionTimes.some((t) => t > from.t1 && t < to.t0); + const inkOf = (group: ReadonlyArray) => + group.reduce((sum, mark) => sum + mark.strokes.length, 0); + + const groups: Array> = []; + for (const mark of marks.toSorted((a, b) => a.t1 - b.t1)) { + const open = groups.at(-1); + const last = open?.at(-1); + if ( + open !== undefined && + last !== undefined && + mark.t1 - open[0]!.t0 <= GROUP_MS && + inkOf(open) + mark.strokes.length <= GROUP_STROKES && + !acted(last, mark) && + changesBetween(last, mark) === 0 + ) { + open.push(mark); + } else { + groups.push([mark]); + } + } + + // Over budget: merge neighbours, same-screen pairs with the least ink first, else the most alike. + while (groups.length > input.budget && groups.length > 1) { + let best = 1; + let bestChanges = Infinity; + let bestInk = Infinity; + for (let index = 1; index < groups.length; index++) { + const earlier = groups[index - 1]!; + const later = groups[index]!; + const changes = changesBetween(earlier.at(-1)!, later[0]!); + const ink = inkOf(earlier) + inkOf(later); + if (changes < bestChanges || (changes === bestChanges && ink < bestInk)) { + best = index; + bestChanges = changes; + bestInk = ink; + } + } + const [later] = groups.splice(best, 1); + groups[best - 1]!.push(...later!); + } + return groups; +} + +/** What `scrawl.json` says about a mark once the keyframes are known. */ +export function placeMarks( + marks: ReadonlyArray, + lines: ReadonlyArray>, +): Array { + const places = new Map(); + lines.forEach((line, index) => { + const frame = `frames/frame-${String(index + 1).padStart(2, "0")}.png`; + for (const action of line.actions) { + if (action.kind !== "mark") continue; + places.set(action.id, { + frame, + ...(action.number === undefined ? {} : { number: action.number }), + }); + } + }); + return marks.map((mark) => ({ ...mark, ...places.get(mark.id) })); +} diff --git a/apps/web/src/scrawl/markShapes.test.ts b/apps/web/src/scrawl/markShapes.test.ts new file mode 100644 index 0000000000000..265f62b171aee --- /dev/null +++ b/apps/web/src/scrawl/markShapes.test.ts @@ -0,0 +1,324 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { marksOf, type InkMark, type InkStroke, type Point } from "./markShapes.ts"; + +const stroke = (points: ReadonlyArray, t0 = 0, t1 = t0 + 100): InkStroke => ({ + t0, + t1, + points: points.flatMap(([x, y]) => [x, y]), +}); +const shapeOf = (points: ReadonlyArray, width = 1000, height = 1000) => + marksOf([stroke(points)], width, height)[0]!.shape; + +/** Corners joined by points every 4 px on the 1000 px test frame, as a mouse samples them. */ +const polyline = (corners: ReadonlyArray): Array => { + const points: Array = []; + for (let index = 1; index < corners.length; index++) { + const a = corners[index - 1]!; + const b = corners[index]!; + const count = Math.max(1, Math.floor(Math.hypot(b[0] - a[0], b[1] - a[1]) / 0.004)); + for (let k = 0; k < count; k++) { + points.push([a[0] + ((b[0] - a[0]) * k) / count, a[1] + ((b[1] - a[1]) * k) / count]); + } + } + points.push(corners.at(-1)!); + return points; +}; + +const ellipse = (c: Point, r: Point, from: number, to: number): Array => + Array.from({ length: 61 }, (_, k) => { + const a = from + ((to - from) * k) / 60; + return [c[0] + r[0] * Math.cos(a), c[1] + r[1] * Math.sin(a)]; + }); + +const turnsOf = (c: Point, r: Point, turns: number): Array => { + const count = Math.floor(120 * turns); + return Array.from({ length: count + 1 }, (_, k) => { + const a = (2 * Math.PI * turns * k) / count; + return [c[0] + r[0] * Math.cos(a), c[1] + r[1] * Math.sin(a)]; + }); +}; + +const scribble = (count: number): Array => + Array.from({ length: count }, (_, i) => [ + 0.2 + 0.1 * Math.sin(0.9 * i), + 0.2 + 0.022 * Math.sin(0.37 * i), + ]); + +// Real strokes kept in Scrawl's own tests (flat x, y, dt in 0..10000): a flat loop round a table +// cell, an arrow drawn in one go, and two underlines whose end hooks back like a lone barb. +const REAL_LOOP = [ + 3617, 3170, 0, 3611, 3184, 48, 3595, 3194, 76, 3582, 3200, 96, 3560, 3210, 106, 3529, 3217, 126, + 3495, 3225, 146, 3449, 3229, 156, 3399, 3229, 175, 3363, 3229, 195, 3321, 3229, 215, 3276, 3229, + 225, 3249, 3229, 245, 3214, 3225, 263, 3192, 3218, 281, 3153, 3207, 290, 3136, 3199, 310, 3093, + 3172, 330, 3069, 3152, 350, 3051, 3132, 370, 3040, 3117, 388, 3033, 3103, 397, 3026, 3084, 417, + 3024, 3051, 437, 3024, 3030, 447, 3024, 3009, 457, 3024, 2971, 467, 3024, 2928, 487, 3026, 2890, + 505, 3041, 2841, 515, 3073, 2779, 544, 3139, 2696, 562, 3197, 2640, 581, 3228, 2616, 591, 3330, + 2563, 611, 3398, 2544, 631, 3477, 2533, 641, 3550, 2529, 661, 3623, 2529, 681, 3702, 2539, 691, + 3780, 2579, 710, 3852, 2634, 730, 3913, 2676, 747, 3945, 2698, 757, 3983, 2729, 776, 4011, 2746, + 785, 4035, 2767, 804, 4055, 2790, 823, 4078, 2823, 834, 4088, 2848, 853, 4099, 2885, 873, 4103, + 2918, 883, 4103, 2990, 920, 4081, 3027, 938, 4045, 3067, 948, 4002, 3105, 968, 3921, 3159, 987, + 3900, 3171, 997, 3849, 3192, 1017, 3803, 3205, 1027, 3762, 3217, 1052, 3720, 3225, 1062, 3678, + 3225, 1080, 3636, 3225, 1090, 3594, 3225, 1110, 3566, 3225, 1128, 3535, 3225, 1148, 3516, 3222, + 1158, 3502, 3218, 1178, +]; +const REAL_ARROW = [ + 4057, 2996, 0, 4047, 2996, 68, 4075, 2996, 199, 4130, 3004, 209, 4236, 3044, 229, 4312, 3073, 248, + 4406, 3106, 256, 4490, 3121, 276, 4567, 3131, 296, 4606, 3131, 316, 4633, 3131, 326, 4653, 3131, + 365, 4621, 3111, 431, 4570, 3090, 451, 4507, 3062, 469, 4404, 3018, 488, 4373, 3007, 506, 4345, + 3000, 516, 4331, 3000, 534, 4322, 2997, 573, 4337, 2995, 624, 4376, 2995, 644, 4436, 2999, 664, + 4509, 3023, 674, 4549, 3039, 684, 4607, 3061, 694, 4638, 3076, 713, 4652, 3083, 723, 4663, 3086, + 733, 4674, 3092, 743, 4662, 3107, 830, 4648, 3115, 840, 4609, 3147, 850, 4530, 3217, 880, 4512, + 3233, 889, 4494, 3249, 899, 4483, 3259, 907, 4467, 3275, 917, 4458, 3283, 934, 4481, 3284, 1308, +]; +const REAL_HOOKED_UNDERLINES = [ + [ + 3589, 6331, 0, 3612, 6331, 30, 3665, 6347, 49, 3718, 6356, 68, 3787, 6365, 78, 3817, 6365, 86, + 3856, 6365, 95, 3871, 6365, 105, 3963, 6365, 115, 4001, 6365, 134, 4062, 6365, 143, 4139, 6365, + 153, 4169, 6365, 172, 4214, 6365, 182, 4244, 6377, 199, 4258, 6377, 218, 4272, 6381, 227, 4302, + 6381, 247, 4355, 6390, 266, 4392, 6405, 276, 4407, 6405, 286, 4422, 6411, 295, 4429, 6415, 305, + 4439, 6422, 314, 4445, 6431, 334, 4450, 6442, 353, 4453, 6451, 371, 4455, 6465, 401, 4450, 6476, + 411, 4442, 6491, 430, 4436, 6500, 449, 4426, 6510, 458, 4415, 6519, 477, 4408, 6528, 487, 4394, + 6544, 497, 4372, 6560, 506, 4357, 6567, 516, 4342, 6573, 524, 4305, 6573, 538, 4274, 6573, 548, + 4267, 6573, 558, 4244, 6565, 567, 4223, 6559, 576, + ], + [ + 7252, 494, 0, 7273, 484, 18, 7288, 477, 28, 7319, 468, 37, 7341, 468, 46, 7356, 461, 56, 7371, + 461, 64, 7409, 461, 81, 7477, 461, 91, 7508, 461, 99, 7530, 469, 109, 7545, 476, 119, 7555, 476, + 129, 7559, 485, 165, 7547, 504, 185, 7536, 524, 195, 7528, 532, 224, 7514, 541, 234, 7460, 567, + 254, 7407, 601, 264, + ], +]; + +const fromTriples = (triples: ReadonlyArray): Array => { + const points: Array = []; + for (let index = 0; index + 2 < triples.length; index += 3) { + points.push([triples[index]! / 10_000, triples[index + 1]! / 10_000]); + } + return points; +}; + +const expectBox = (mark: InkMark, want: ReadonlyArray) => { + mark.boxNorm.forEach((value, index) => expect(value).toBeCloseTo(want[index]!, 2)); +}; + +describe("a single stroke's shape", () => { + it("is measured in pixels", () => { + expect( + shapeOf([ + [0.5, 0.5], + [0.505, 0.505], + ]), + ).toBe("tap"); + expect( + shapeOf([ + [0.1, 0.5], + [0.8, 0.5], + ]), + ).toBe("underline"); + expect( + shapeOf([ + [0.2, 0.2], + [0.5, 0.6], + ]), + ).toBe("line"); + // Twice as tall as wide in fractions, round on a frame half as high. + expect(shapeOf(ellipse([0.5, 0.5], [0.1, 0.2], 0, 2 * Math.PI), 1000, 500)).toBe("circle"); + }); + + it.each([ + ["round", ellipse([0.3, 0.3], [0.1, 0.1], 0, 2 * Math.PI)], + ["flat", ellipse([0.3, 0.3], [0.1, 0.05], 0, 2 * Math.PI)], + ["open by 30°", ellipse([0.3, 0.3], [0.115, 0.05], Math.PI / 2, Math.PI * 2.5 - 0.55)], + ["open along the other axis", ellipse([0.3, 0.3], [0.05, 0.1], 0.2, 2 * Math.PI - 0.2)], + ["twice round", turnsOf([0.3, 0.3], [0.1, 0.08], 2)], + ["five times round", turnsOf([0.3, 0.3], [0.1, 0.1], 5)], + ["flat, three times round", turnsOf([0.3, 0.3], [0.12, 0.03], 3)], + ])("calls a %s loop a circle", (_name, points) => { + expect(shapeOf(points)).toBe("circle"); + }); + + it.each]>([ + [ + "check mark", + polyline([ + [0.1, 0.2], + [0.13, 0.24], + [0.22, 0.12], + ]), + ], + [ + "V", + polyline([ + [0.1, 0.1], + [0.15, 0.25], + [0.2, 0.1], + ]), + ], + [ + "zigzag", + polyline([ + [0.1, 0.2], + [0.13, 0.16], + [0.16, 0.2], + [0.19, 0.16], + [0.22, 0.2], + ]), + ], + [ + "back and forth", + polyline([ + [0.1, 0.2], + [0.3, 0.21], + [0.11, 0.225], + [0.3, 0.24], + [0.11, 0.255], + ]), + ], + ["scribble", scribble(60)], + ["long scribble", scribble(600)], + [ + "hooked underline", + polyline([ + [0.1, 0.3], + [0.4, 0.3], + [0.405, 0.27], + ]), + ], + [ + "one-sided barb", + polyline([ + [0.1, 0.3], + [0.3, 0.25], + [0.26, 0.24], + ]), + ], + ["C", ellipse([0.3, 0.3], [0.1, 0.1], Math.PI / 4, 1.75 * Math.PI)], + ["flat C", ellipse([0.3, 0.3], [0.1, 0.025], 0.8, 2 * Math.PI - 0.8)], + ["flat loop four times round", turnsOf([0.3, 0.3], [0.12, 0.03], 4)], + ])("does not call a %s an arrow or a circle", (_name, points) => { + expect(["arrow", "circle"]).not.toContain(shapeOf(points)); + }); + + it("does not call a real underline with a hooked end an arrow", () => { + for (const pts of REAL_HOOKED_UNDERLINES) { + expect(shapeOf(fromTriples(pts), 2560, 1440)).not.toBe("arrow"); + } + }); + + it("takes what a tailed loop circled from the loop alone", () => { + const flat = [0.2, 0.25, 0.4, 0.35]; + const tailed = [ + ...ellipse([0.3, 0.3], [0.1, 0.05], 0, 2 * Math.PI), + ...polyline([ + [0.4, 0.3], + [0.52, 0.38], + ]), + ]; + const overshoot = ellipse([0.3, 0.3], [0.1, 0.05], 0, 2.6 * Math.PI); + const leadIn = [ + ...polyline([ + [0.18, 0.2], + [0.3, 0.25], + ]), + ...ellipse([0.3, 0.3], [0.1, 0.05], -Math.PI / 2, 1.6 * Math.PI), + ]; + for (const points of [tailed, overshoot, leadIn]) { + const mark = marksOf([stroke(points)], 1000, 1000)[0]!; + expect(mark.shape).toBe("circle"); + expectBox(mark, flat); + } + }); + + it("recognises the real loop and the real one-stroke arrow, tip included", () => { + const [loop, arrow] = marksOf( + [ + stroke(fromTriples(REAL_LOOP), 6_208, 7_386), + stroke(fromTriples(REAL_ARROW), 10_283, 11_591), + ], + 1728, + 1117, + ); + expect(loop).toMatchObject({ + shape: "circle", + description: "circled region x 0.30–0.41, y 0.25–0.32", + }); + expect(arrow).toMatchObject({ + shape: "arrow", + from: [0.4057, 0.2996], + to: [0.4653, 0.3131], + description: "drew an arrow from x 0.41, y 0.30 pointing to x 0.47, y 0.31", + }); + }); + + it("is not hidden by sampling a long stroke, nor slowed by one", () => { + const zigzag: Array = []; + for (let index = 0; index < 400; index++) { + const a = (2 * Math.PI * index) / 400; + const p: Point = [0.3 + 0.12 * Math.cos(a), 0.3 + 0.03 * Math.sin(a)]; + zigzag.push(p, [p[0] + 0.004, p[1] + 0.008], [p[0] - 0.004, p[1] - 0.008]); + } + zigzag.push(zigzag[0]!); + expect(shapeOf(zigzag)).toBe("mark"); + expect(shapeOf(turnsOf([0.3, 0.3], [0.12, 0.03], 100))).toBe("mark"); + + const long: Array = Array.from({ length: 3_000 }, (_, k) => { + const a = (2 * Math.PI * k) / 2_500; + return [0.3 + 0.1 * Math.cos(a), 0.3 + 0.05 * Math.sin(a)]; + }); + const mark = marksOf([stroke(long)], 10_000, 10_000)[0]!; + expect(mark.shape).toBe("circle"); + expectBox(mark, [0.2, 0.25, 0.4, 0.35]); + }); +}); + +describe("marksOf", () => { + it("points a two-stroke arrow toward its head in either drawing direction", () => { + const head: ReadonlyArray = [ + [0.42, 0.5], + [0.5, 0.5], + [0.5, 0.58], + ]; + const shafts: ReadonlyArray> = [ + [ + [0.2, 0.8], + [0.5, 0.5], + ], + [ + [0.5, 0.5], + [0.2, 0.8], + ], + ]; + for (const shaft of shafts) { + const marks = marksOf([stroke(shaft, 0, 100), stroke(head, 200, 300)], 1000, 1000); + expect(marks).toHaveLength(1); + expect(marks[0]).toMatchObject({ shape: "arrow", from: [0.2, 0.8], to: [0.5, 0.5] }); + } + }); + + it("starts a new mark after a pause and after six strokes, keeping every stroke", () => { + const dot: ReadonlyArray = [ + [0.5, 0.5], + [0.6, 0.5], + ]; + const close = Array.from({ length: 8 }, (_, index) => stroke(dot, index * 200, index * 200)); + const marks = marksOf([...close, stroke(dot, 5_000, 5_100)], 1000, 1000); + expect(marks.map((mark) => mark.strokes)).toEqual([[0, 1, 2, 3, 4, 5], [6, 7], [8]]); + expect(marks.map((mark) => mark.id)).toEqual(["m1", "m2", "m3"]); + expect(marks[0]).toMatchObject({ shape: "underline", t0: 0, t1: 1_000 }); + }); + + it("keeps a group of differing strokes a plain mark, with its stroke count", () => { + const line: ReadonlyArray = [ + [0.2, 0.2], + [0.5, 0.6], + ]; + const marks = marksOf( + [stroke(line), stroke(ellipse([0.8, 0.8], [0.05, 0.05], 0, 2 * Math.PI), 200, 300)], + 1000, + 1000, + ); + expect(marks[0]!.shape).toBe("mark"); + expect(marks[0]!.description).toContain("(2 strokes)"); + }); +}); diff --git a/apps/web/src/scrawl/markShapes.ts b/apps/web/src/scrawl/markShapes.ts new file mode 100644 index 0000000000000..070de4b20fcf4 --- /dev/null +++ b/apps/web/src/scrawl/markShapes.ts @@ -0,0 +1,399 @@ +// What a hand drew: strokes close in time form a mark, and a mark is recognised by its shape. Scrawl's +// rules (src/sketch.rs), measured in pixels because a circle on a wide screen is flat in fractions. + +export type Point = readonly [number, number]; +type Box = readonly [number, number, number, number]; + +export interface InkStroke { + readonly t0: number; + readonly t1: number; + /** x, y, x, y, ... as fractions of the frame. */ + readonly points: ReadonlyArray; +} + +/** Scrawl's serialized names: `scrawl.json` has to stay readable by its CLI. */ +export type MarkShape = "circle" | "underline" | "line" | "arrow" | "tap" | "mark"; + +export interface InkMark { + readonly id: string; + readonly strokes: ReadonlyArray; + readonly t0: number; + readonly t1: number; + readonly boxNorm: Box; + readonly shape: MarkShape; + readonly from?: Point; + readonly to?: Point; + readonly description: string; +} + +const MARK_GAP_MS = 1_200; +const MAX_MARK_STROKES = 6; +const MAX_LOOP_POINTS = 512; + +const pointsOf = (stroke: InkStroke): Array => { + const points: Array = []; + for (let index = 0; index + 1 < stroke.points.length; index += 2) { + points.push([stroke.points[index]!, stroke.points[index + 1]!]); + } + return points; +}; + +const boundsOf = (points: ReadonlyArray): Box => { + if (points.length === 0) return [0, 0, 0, 0]; + let [x0, y0] = points[0]!; + let [x1, y1] = points[0]!; + for (const [x, y] of points) { + x0 = Math.min(x0, x); + y0 = Math.min(y0, y); + x1 = Math.max(x1, x); + y1 = Math.max(y1, y); + } + return [x0, y0, x1, y1]; +}; + +const distance = (a: Point, b: Point) => Math.hypot(a[0] - b[0], a[1] - b[1]); + +/** Ramanujan's approximation of the perimeter of the ellipse filling the box. */ +const perimeterOf = ([x0, y0, x1, y1]: Box) => { + const w = x1 - x0; + const h = y1 - y0; + return (Math.PI / 2) * (3 * (w + h) - Math.sqrt((3 * w + h) * (w + 3 * h))); +}; + +/** + * Whether a path goes round something. Flatter loops, to 5:1 (a wide cell or a line of text gets a + * flat loop), must also close within their short side, so a flat C doesn't count, and go round at + * most about three times, so a scribble over text doesn't, though it closes and fills its box too. + */ +const encloses = (gap: number, maxGap: number, length: number, box: Box, area: number) => { + const w = box[2] - box[0]; + const h = box[3] - box[1]; + const big = Math.max(w, h); + const small = Math.min(w, h); + return ( + gap < maxGap * big && + length > 2 * big && + small >= 18 && + area > 0.32 * w * h && + (small > 0.42 * big || + (small > 0.2 * big && gap < 0.6 * small && length <= 3.5 * perimeterOf(box))) + ); +}; + +/** + * The loop inside a stroke that circles something once before or after a lead-in, tail or overshoot, + * at least 0.6 of the stroke, whose ends meet closest. Every pair of ends is quadratic, so long + * strokes try every nth point, but judge each stretch by the whole path so sampling can't hide turns. + */ +const loopIn = (points: ReadonlyArray): readonly [number, number] | null => { + const count = points.length; + if (count < 3) return null; + const run = [0]; + const twiceArea = [0]; + for (let index = 1; index < count; index++) { + const a = points[index - 1]!; + const b = points[index]!; + run.push(run[index - 1]! + distance(a, b)); + twiceArea.push(twiceArea[index - 1]! + a[0] * b[1] - b[0] * a[1]); + } + const total = run[count - 1]!; + const step = Math.ceil(count / MAX_LOOP_POINTS); + const ends: Array = []; + for (let index = 0; index < count; index += step) ends.push(index); + if (ends.at(-1) !== count - 1) ends.push(count - 1); + let best: { closeness: number; from: number; to: number } | null = null; + for (let k = 0; k < ends.length; k++) { + const from = ends[k]!; + const a = points[from]!; + let [x0, y0] = a; + let [x1, y1] = a; + for (const to of ends.slice(k + 1)) { + const z = points[to]!; + x0 = Math.min(x0, z[0]); + y0 = Math.min(y0, z[1]); + x1 = Math.max(x1, z[0]); + y1 = Math.max(y1, z[1]); + const length = run[to]! - run[from]!; + if (length < 0.6 * total) continue; + const gap = distance(a, z); + const closeness = gap / Math.max(x1 - x0, y1 - y0); + if (best !== null && closeness >= best.closeness) continue; + const box: Box = [x0, y0, x1, y1]; + const twice = twiceArea[to]! - twiceArea[from]! + z[0] * a[1] - a[0] * z[1]; + if ( + length <= 1.5 * perimeterOf(box) && + encloses(gap, 0.15, length, box, 0.5 * Math.abs(twice)) + ) { + best = { closeness, from, to }; + } + } + } + return best === null ? null : [best.from, best.to]; +}; + +const offLine = (p: Point, a: Point, b: Point) => + Math.abs((b[0] - a[0]) * (a[1] - p[1]) - (a[0] - p[0]) * (b[1] - a[1])) / distance(a, b); + +const cosine = (degrees: number) => Math.cos((degrees * Math.PI) / 180); + +/** + * The tip of an arrow drawn in one stroke: a straight shaft, where the stroke turns back into one or + * two straight barbs, and a head reaching both sides of the shaft. An underline whose end hooks back + * to one side, ticks, V's and zigzags fail. + */ +const arrowTip = (points: ReadonlyArray, big: number): number | null => { + const start = points[0]!; + let tip = 0; + let turned = false; + for (let index = 0; index < points.length; index++) { + const d = distance(start, points[index]!); + const far = distance(start, points[tip]!); + if (d >= far) { + tip = index; + } else if (far >= 0.5 * big && d < 0.85 * far) { + turned = true; + break; + } + } + const t = points[tip]!; + const shaft = distance(start, t); + if (!turned || points.slice(0, tip).some((p) => offLine(p, start, t) > 0.1 * shaft)) return null; + + // The direction the shaft arrives in, over its last 30%. + let arc = 0; + for (let index = 0; index < tip; index++) arc += distance(points[index]!, points[index + 1]!); + let travelled = 0; + let back = 0; + for (let index = 0; index < tip; index++) { + travelled += distance(points[index]!, points[index + 1]!); + if (travelled >= 0.7 * arc) { + back = index; + break; + } + } + const along = distance(points[back]!, t); + if (along < 1) return null; + const dir: Point = [(t[0] - points[back]![0]) / along, (t[1] - points[back]![1]) / along]; + + // Barbs: runs away from the tip by at least `slack`, each from the nearest point before it. + const slack = 0.15 * shaft; + const barbs: Array = []; + let out = tip; + let near = tip; + let rising = true; + for (let index = tip + 1; index < points.length; index++) { + const p = points[index]!; + if ((p[0] - t[0]) * dir[0] + (p[1] - t[1]) * dir[1] > slack) return null; + const d = distance(t, p); + if (rising) { + const outAt = distance(t, points[out]!); + if (d >= outAt) { + out = index; + } else if (d < outAt - slack) { + barbs.push([near, out]); + near = index; + rising = false; + } + } else { + const nearAt = distance(t, points[near]!); + if (d <= nearAt) { + near = index; + } else if (d > nearAt + slack) { + out = index; + rising = true; + } + } + } + if (rising && distance(t, points[out]!) >= slack) barbs.push([near, out]); + + const measured: Array<{ readonly length: number; readonly cos: number }> = []; + for (const [from, end] of barbs) { + const a = points[from]!; + const z = points[end]!; + const span = distance(a, z); + if (points.slice(from, end + 1).some((p) => offLine(p, a, z) > 0.15 * span)) return null; + const e: Point = [z[0] - t[0], z[1] - t[1]]; + const length = Math.hypot(e[0], e[1]); + const cos = -(e[0] * dir[0] + e[1] * dir[1]) / length; + if (length > 0.9 * shaft || cos < cosine(75)) return null; + measured.push({ length, cos }); + } + const [one, two] = measured; + const head = + one !== undefined && measured.length === 1 + ? one.length <= 0.6 * shaft && one.cos <= cosine(15) + : one !== undefined && two !== undefined && measured.length === 2 + ? Math.max(one.cos, two.cos) <= cosine(5) + : false; + + // The head spreads to both sides of the shaft; an underline that hooks back doesn't. + let left = 0; + let right = 0; + for (const p of points.slice(tip)) { + const side = dir[0] * (p[1] - t[1]) - dir[1] * (p[0] - t[0]); + left = Math.max(left, side); + right = Math.max(right, -side); + } + return head && Math.min(left, right) > 0.3 * Math.max(left, right) ? tip : null; +}; + +interface Ink { + readonly shape: MarkShape; + /** What the stroke covers: a circle's loop without its lead-in or tail, else the whole stroke. */ + readonly region: Box; + /** A one-stroke arrow's shaft start and tip. */ + readonly ends?: readonly [Point, Point]; +} + +const inkOf = (norm: ReadonlyArray, width: number, height: number): Ink => { + const region = boundsOf(norm); + const points = norm.map(([x, y]): Point => [x * width, y * height]); + const first = points[0]; + if (first === undefined) return { shape: "tap", region }; + const last = points.at(-1)!; + const box = boundsOf(points); + const w = box[2] - box[0]; + const h = box[3] - box[1]; + let length = 0; + let shoelace = last[0] * first[1] - first[0] * last[1]; + for (let index = 1; index < points.length; index++) { + const a = points[index - 1]!; + const b = points[index]!; + length += distance(a, b); + shoelace += a[0] * b[1] - b[0] * a[1]; + } + const chord = distance(first, last); + const loopBox = (): Box | null => { + const loop = loopIn(points); + return loop === null ? null : boundsOf(norm.slice(loop[0], loop[1] + 1)); + }; + if (Math.hypot(w, h) < 14) return { shape: "tap", region }; + if (encloses(chord, 0.35, length, box, 0.5 * Math.abs(shoelace))) { + // A stroke that goes round more than once keeps its whole box. + const loop = length <= 1.5 * perimeterOf(box) ? loopBox() : null; + return { shape: "circle", region: loop ?? region }; + } + if (chord > 0.88 * length) return { shape: w > 2.5 * h ? "underline" : "line", region }; + const loop = loopBox(); + if (loop !== null) return { shape: "circle", region: loop }; + const tip = arrowTip(points, Math.max(w, h)); + if (tip !== null) return { shape: "arrow", region, ends: [norm[0]!, norm[tip]!] }; + return { shape: "mark", region }; +}; + +const at = ([x, y]: Point) => `x ${x.toFixed(2)}, y ${y.toFixed(2)}`; + +const describeMark = ( + mark: Pick, + strokeCount: number, +): string => { + const [x0, y0, x1, y1] = mark.boxNorm; + const many = strokeCount > 1 ? ` (${strokeCount} strokes)` : ""; + const region = `x ${x0.toFixed(2)}–${x1.toFixed(2)}, y ${y0.toFixed(2)}–${y1.toFixed(2)}`; + switch (mark.shape) { + case "circle": + return `circled region ${region}${many}`; + case "underline": + return `underlined x ${x0.toFixed(2)}–${x1.toFixed(2)} at y ${((y0 + y1) / 2).toFixed(2)}${many}`; + case "line": + return `drew a line from ${at(mark.from ?? [x0, y0])} to ${at(mark.to ?? [x1, y1])}`; + case "arrow": + return `drew an arrow from ${at(mark.from ?? [x0, y0])} pointing to ${at(mark.to ?? [x1, y1])}`; + case "tap": + return `tapped ${at([(x0 + x1) / 2, (y0 + y1) / 2])}${many}`; + case "mark": + return `marked region ${region}${many}`; + } +}; + +/** The shape one group of strokes makes together: `box` is what a circle circled. */ +const recognise = ( + group: ReadonlyArray>, + width: number, + height: number, +): Pick & { readonly box?: Box } => { + const sized = group + .map((points) => { + const box = boundsOf(points); + return { + points, + box, + ink: inkOf(points, width, height), + size: (box[2] - box[0]) * width + (box[3] - box[1]) * height, + }; + }) + .toSorted((a, b) => b.size - a.size); + const main = sized[0]!; + const rest = sized.slice(1); + const shape = main.ink.shape; + const a = main.points[0] ?? ([0, 0] as const); + const z = main.points.at(-1) ?? a; + + if ((shape === "line" || shape === "underline") && (rest.length === 1 || rest.length === 2)) { + const pixels = ([x, y]: Point): Point => [x * width, y * height]; + const shaft = distance(pixels(a), pixels(z)); + const headAt = (end: Point) => + rest.every( + ({ box }) => + distance(pixels([(box[0] + box[2]) / 2, (box[1] + box[3]) / 2]), pixels(end)) < + 0.35 * shaft, + ); + if (headAt(z)) return { shape: "arrow", from: a, to: z }; + if (headAt(a)) return { shape: "arrow", from: z, to: a }; + } + if (shape === "circle") { + const allCircles = rest.every(({ ink }) => ink.shape === "circle"); + const box = allCircles + ? boundsOf( + sized.flatMap(({ ink: { region } }): Array => [ + [region[0], region[1]], + [region[2], region[3]], + ]), + ) + : main.ink.region; + return { shape, box }; + } + if (!rest.every(({ ink }) => ink.shape === shape)) return { shape: "mark" }; + if (shape === "line") return rest.length === 0 ? { shape, from: a, to: z } : { shape: "mark" }; + if (shape === "arrow") { + return rest.length === 0 && main.ink.ends !== undefined + ? { shape, from: main.ink.ends[0], to: main.ink.ends[1] } + : { shape: "mark" }; + } + return { shape }; +}; + +/** Strokes are in drawing order, `width` and `height` the frame's size in pixels. */ +export function marksOf( + strokes: ReadonlyArray, + width: number, + height: number, +): Array { + const groups: Array> = []; + strokes.forEach((stroke, index) => { + const group = groups.at(-1); + if ( + group !== undefined && + group.length < MAX_MARK_STROKES && + stroke.t0 - strokes[group.at(-1)!]!.t1 <= MARK_GAP_MS + ) { + group.push(index); + } else { + groups.push([index]); + } + }); + return groups.map((group, index) => { + const points = group.map((strokeIndex) => pointsOf(strokes[strokeIndex]!)); + const { box, ...recognised } = recognise(points, width, height); + const boxNorm = box ?? boundsOf(points.flat()); + return { + id: `m${index + 1}`, + strokes: group, + t0: strokes[group[0]!]!.t0, + t1: Math.max(...group.map((strokeIndex) => strokes[strokeIndex]!.t1)), + boxNorm, + ...recognised, + description: describeMark({ ...recognised, boxNorm }, group.length), + }; + }); +} diff --git a/apps/web/src/scrawl/narration.test.ts b/apps/web/src/scrawl/narration.test.ts index 2c82b0b07cf7b..7990a98c86462 100644 --- a/apps/web/src/scrawl/narration.test.ts +++ b/apps/web/src/scrawl/narration.test.ts @@ -129,6 +129,48 @@ describe("buildNarration", () => { expect(formatNarrationLine(lines.at(-1)!, 1)).toBe("- #2 (0:20) clicked"); }); + it("draws a group of marks on one keyframe, numbered, without cutting a sentence", () => { + const words = [ + ...say("Erst etwas Einleitung.", 0), + ...say("Dieses Feld hier soll weg.", 2_000), + ...say("Und dann noch dieser.", 5_000), + ...say("Das hier auch noch.", 7_000), + ]; + const lines = buildNarration({ + words, + events: [{ kind: "click", t: 12_000, x: 0.5, y: 0.5 }], + marks: [ + { id: "m1", t0: 2_200, t1: 2_800, shape: "circle" }, + { id: "m2", t0: 3_000, t1: 3_200, shape: "underline", strokes: [1, 2] }, + { id: "m3", t0: 7_200, t1: 7_800, shape: "arrow" }, + ], + durationMs: 14_000, + sceneChanges: [4_000], + markGroups: [["m1", "m2", "m3"]], + }); + expect(lines).toHaveLength(2); + expect(lines[0]!.frameT).toBe(7_800); + expect(lines[0]!.text).toBe(words.map((word) => word.text).join(" ")); + expect(formatNarrationLine(lines[0]!, 0)).toBe( + [ + "- #1 (0:07) “Erst etwas Einleitung.”", + " ✎① 0:02 circled", + " ✎② 0:03 underlined (2 strokes) “Dieses Feld hier soll weg. Und dann noch dieser.”", + " ✎③ 0:07 drew an arrow “Das hier auch noch.”", + ].join("\n"), + ); + }); + + it("names a lone mark by its shape and leaves it unnumbered", () => { + const lines = buildNarration({ + words: say("Das hier.", 0), + events: [], + marks: [{ id: "m1", t0: 100, t1: 300, shape: "circle" }], + durationMs: 2_000, + }); + expect(formatNarrationLine(lines[0]!, 0)).toBe("- #1 (0:00) [circled] “Das hier.”"); + }); + it("keeps every mark over budget, and speech ahead of the first keyframe with it", () => { const marks = Array.from({ length: 9 }, (_, index) => ({ id: `m${index}`, diff --git a/apps/web/src/scrawl/narration.ts b/apps/web/src/scrawl/narration.ts index a9b67540af783..27137072fe679 100644 --- a/apps/web/src/scrawl/narration.ts +++ b/apps/web/src/scrawl/narration.ts @@ -27,6 +27,17 @@ export interface CaptureMark { readonly t0: number; readonly t1: number; readonly shape?: string; + readonly strokes?: ReadonlyArray; +} + +export interface MarkAction { + readonly kind: "mark"; + readonly t0: number; + readonly t1: number; + readonly id: string; + readonly text: string; + /** Its place in drawing order, when the keyframe draws several marks. */ + readonly number?: number; } export type NarrationAction = @@ -41,7 +52,7 @@ export type NarrationAction = } | { readonly kind: "drag"; readonly t0: number; readonly t1: number; readonly direction: string } | { readonly kind: "scroll"; readonly t0: number; readonly t1: number } - | { readonly kind: "mark"; readonly t0: number; readonly t1: number; readonly id: string } + | MarkAction | { readonly kind: "scene"; readonly t0: number; readonly t1: number }; export interface NarrationLine { @@ -51,6 +62,11 @@ export interface NarrationLine { readonly frameT: number; readonly text: string; readonly actions: ReadonlyArray; + /** Whole sentences, with the marks drawn over each. */ + readonly sentences: ReadonlyArray<{ + readonly text: string; + readonly marks: ReadonlyArray; + }>; } interface Sentence { @@ -73,7 +89,7 @@ const SHORT_DRAG = 0.15; const ATTACH_MS = 3_000; const SETTLE_MS = 300; // Scrawl's keyframe budget: each image costs a reading model about 1.5k tokens. -const budgetFor = (durationMs: number) => +export const budgetFor = (durationMs: number) => Math.min(12, 6 + Math.max(0, Math.ceil((durationMs - 40_000) / 20_000))); const COVERAGE_MS = 35_000; const COVERAGE_WORDS = 100; @@ -122,6 +138,20 @@ export function splitSentences(words: ReadonlyArray): Array return sentences; } +const MARK_VERBS: Record = { + circle: "circled", + underline: "underlined", + line: "drew a line", + arrow: "drew an arrow", + tap: "tapped", +}; + +const markText = ({ shape, strokes }: CaptureMark) => { + const verb = (shape === undefined ? undefined : MARK_VERBS[shape]) ?? "marked"; + const count = strokes?.length ?? 1; + return count > 1 ? `${verb} (${count} strokes)` : verb; +}; + const directionOf = (dx: number, dy: number) => Math.abs(dx) >= Math.abs(dy) ? (dx < 0 ? "left" : "right") : dy < 0 ? "up" : "down"; @@ -134,6 +164,7 @@ export function foldActions( t0: mark.t0, t1: mark.t1, id: mark.id, + text: markText(mark), })); const bursts: Array<{ t0: number; @@ -198,13 +229,36 @@ const attach = (sentences: Array, actions: ReadonlyArray sentence.actions.reduce((best, action) => Math.max(best, SCORES[action.kind]), 0); +/** Sentences inside a group of marks: its keyframe starts earlier and nothing cuts into it. */ +const insideMarkGroups = ( + sentences: ReadonlyArray, + markGroups: ReadonlyArray>, +) => { + const inside = new Set(); + for (const ids of markGroups) { + const holding = sentences.flatMap((sentence, index) => + sentence.actions.some((action) => action.kind === "mark" && ids.includes(action.id)) + ? [index] + : [], + ); + for (let index = (holding[0] ?? 0) + 1; index <= (holding.at(-1) ?? -1); index++) { + inside.add(index); + } + } + return inside; +}; + /** The sentences that own a keyframe: every mark, then the strongest actions, then long stretches. */ -const chooseFrameOwners = (sentences: ReadonlyArray, durationMs: number) => { +const chooseFrameOwners = ( + sentences: ReadonlyArray, + durationMs: number, + inside: ReadonlySet, +) => { const budget = budgetFor(durationMs); const owners = new Set(); const ranked = sentences .map((sentence, index) => ({ index, score: scoreOf(sentence) })) - .filter((entry) => entry.score > 0) + .filter((entry) => entry.score > 0 && !inside.has(entry.index)) .sort((a, b) => b.score - a.score || a.index - b.index); for (const { index, score } of ranked) { if (score >= SCORES.mark || owners.size < budget) owners.add(index); @@ -221,7 +275,7 @@ const chooseFrameOwners = (sentences: ReadonlyArray, durationMs: numbe } stretchWords += sentence.words.length; const long = sentence.t1 - stretchStart > COVERAGE_MS || stretchWords > COVERAGE_WORDS; - if (long && owners.size < budget) { + if (long && owners.size < budget && !inside.has(index)) { owners.add(index); stretchStart = sentence.t0; stretchWords = sentence.words.length; @@ -267,6 +321,8 @@ export function buildNarration(input: { readonly durationMs: number; /** When the screen changed; without it clicks are taken at face value. */ readonly sceneChanges?: ReadonlyArray; + /** Mark ids that share one keyframe; without it each sentence with a mark has its own. */ + readonly markGroups?: ReadonlyArray>; }): Array { const sentences = splitSentences(input.words); const folded = foldActions(input.events, input.marks); @@ -277,9 +333,13 @@ export function buildNarration(input: { // Nothing said, done or changed: the picture is still what the capture shows. if (sentences.length === 0) { const frameT = input.durationMs / 2; - return [{ t0: 0, t1: input.durationMs, frameT, text: "", actions: [] }]; + return [{ t0: 0, t1: input.durationMs, frameT, text: "", actions: [], sentences: [] }]; } - const owners = chooseFrameOwners(sentences, input.durationMs); + const owners = chooseFrameOwners( + sentences, + input.durationMs, + insideMarkGroups(sentences, input.markGroups ?? []), + ); const groups: Array> = []; let ownerSeen = false; sentences.forEach((sentence, index) => { @@ -290,7 +350,15 @@ export function buildNarration(input: { ownerSeen ||= owner; }); return groups.map((group) => { - const actions = group.flatMap((sentence) => sentence.actions); + const drawn = group + .flatMap((sentence) => sentence.actions) + .filter((action) => action.kind === "mark") + .toSorted((a, b) => a.t0 - b.t0); + const numbered = (action: NarrationAction): NarrationAction => + action.kind === "mark" && drawn.length > 1 + ? { ...action, number: drawn.findIndex((mark) => mark.id === action.id) + 1 } + : action; + const actions = group.flatMap((sentence) => sentence.actions).map(numbered); const t0 = group[0]!.t0; return { t0, @@ -298,23 +366,27 @@ export function buildNarration(input: { frameT: frameTimeOf(t0, actions, input.durationMs), text: group.flatMap((sentence) => sentence.words).join(" "), actions, + sentences: group.map((sentence) => ({ + text: sentence.words.join(" "), + marks: sentence.actions.map(numbered).filter((action) => action.kind === "mark"), + })), }; }); } const formatClock = (ms: number) => { - const seconds = Math.round(ms / 1000); + const seconds = Math.floor(ms / 1000); return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; }; const describeActions = (actions: ReadonlyArray) => { - let marks = 0; + const marks: Array = []; let clicks = 0; let pointed = 0; let scrolled = false; const drags = new Set(); for (const action of actions) { - if (action.kind === "mark") marks++; + if (action.kind === "mark") marks.push(action.text); else if (action.kind === "clicks") { if (action.changed === false) pointed += action.count; else clicks += action.count; @@ -323,7 +395,7 @@ const describeActions = (actions: ReadonlyArray) => { } const counted = (word: string, count: number) => (count > 1 ? `${word} ×${count}` : word); return [ - marks > 0 ? counted("marked", marks) : null, + ...marks, clicks > 0 ? counted("clicked", clicks) : null, pointed > 0 ? counted("pointed", pointed) : null, ...[...drags].map((direction) => `dragged ${direction}`), @@ -333,9 +405,38 @@ const describeActions = (actions: ReadonlyArray) => { .join(", "); }; +const CIRCLED = "①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳"; + +const markLabel = (mark: MarkAction) => { + const number = mark.number === undefined ? "" : (CIRCLED[mark.number - 1] ?? `(${mark.number})`); + return `✎${number} ${formatClock(mark.t0)} ${mark.text}`; +}; + +/** + * Several marks on a keyframe: each starts a line with the whole sentences said from it on, as + * Scrawl's agent text does, so "this one" reads next to the number. The image only shows ink. + */ +const formatNumberedMarks = (head: string, line: NarrationLine) => { + const others = describeActions(line.actions.filter((action) => action.kind !== "mark")); + const lead: Array = []; + const marked: Array<{ readonly label: string; readonly said: Array }> = []; + for (const sentence of line.sentences) { + for (const mark of sentence.marks) marked.push({ label: markLabel(mark), said: [] }); + if (sentence.text !== "") (marked.at(-1)?.said ?? lead).push(sentence.text); + } + const quoted = (said: ReadonlyArray) => (said.length > 0 ? ` “${said.join(" ")}”` : ""); + return [ + `${head}${others === "" ? "" : ` [${others}]`}${quoted(lead)}`, + ...marked.map(({ label, said }) => ` ${label}${quoted(said)}`), + ].join("\n"); +}; + /** `- #2 (0:09) [clicked ×3] “…”`: the shape Scrawl pastes, which agents and skills already read. */ export function formatNarrationLine(line: NarrationLine, index: number): string { const head = `- #${index + 1} (${formatClock(line.frameT)})`; + if (line.actions.filter((action) => action.kind === "mark").length > 1) { + return formatNumberedMarks(head, line); + } const actions = describeActions(line.actions); if (line.text === "") { const shown = line.actions.length > 0 ? "screen changed" : "nothing said or done"; diff --git a/apps/web/src/scrawl/scrawlBundle.ts b/apps/web/src/scrawl/scrawlBundle.ts index f90df615ecd55..363267b48834d 100644 --- a/apps/web/src/scrawl/scrawlBundle.ts +++ b/apps/web/src/scrawl/scrawlBundle.ts @@ -9,10 +9,12 @@ import { type ScrawlStroke, } from "@t3tools/contracts"; +import { groupMarks, placeMarks } from "./markGroups"; +import { marksOf } from "./markShapes"; import { + budgetFor, buildNarration, formatNarrationLine, - type CaptureMark, type NarrationLine, type TimedWord, } from "./narration"; @@ -20,7 +22,6 @@ import { withoutInkChanges } from "./sceneChanges"; const SHEET_LONG_EDGE = 1600; const SHEET_GUTTER = 6; -const STROKE_GROUP_MS = 1_500; const CROP_PADDING = 0.08; // Cropping to most of the screen only loses context. const MAX_CROP_AREA = 0.7; @@ -43,13 +44,6 @@ export interface CaptureRecording { readonly wav: Uint8Array; } -interface ManifestMark extends CaptureMark { - readonly strokes: ReadonlyArray; - readonly boxNorm: readonly [number, number, number, number]; - readonly shape: string; - readonly description: string; -} - // Scrawl stores flat [x, y, dt] triples with positions in 0..=10000. const STROKE_SCALE = 10_000; const manifestStroke = (stroke: ScrawlStroke, width: number, height: number) => { @@ -80,33 +74,6 @@ export interface BuiltBundle { readonly lines: ReadonlyArray; } -const strokesToMarks = (strokes: ReadonlyArray): Array => { - const groups: Array<{ t0: number; t1: number; strokes: Array }> = []; - strokes.forEach((stroke, index) => { - const open = groups.at(-1); - if (open !== undefined && stroke.t0 - open.t1 <= STROKE_GROUP_MS) { - open.t1 = stroke.t1; - open.strokes.push(index); - } else { - groups.push({ t0: stroke.t0, t1: stroke.t1, strokes: [index] }); - } - }); - return groups.map((group, index) => { - const xs: Array = []; - const ys: Array = []; - for (const strokeIndex of group.strokes) { - strokes[strokeIndex]!.points.forEach((value, at) => (at % 2 === 0 ? xs : ys).push(value)); - } - return { - id: `m${index + 1}`, - ...group, - boxNorm: [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)], - shape: "mark", - description: `marked (${group.strokes.length} ${group.strokes.length === 1 ? "stroke" : "strokes"})`, - }; - }); -}; - /** Everything the hand touched, padded, or null when that is most of the screen anyway. */ function activityBox(recording: Pick): { readonly x: number; @@ -206,6 +173,33 @@ const drawClickRings = ( } }; +/** A group's earlier ink has faded from the video by the time its keyframe is taken. */ +const drawMarkInk = ( + context: CanvasRenderingContext2D, + line: NarrationLine, + marks: ReadonlyArray<{ readonly id: string; readonly strokes: ReadonlyArray }>, + strokes: ReadonlyArray, + width: number, + height: number, +) => { + context.strokeStyle = "#ff2d2d"; + context.lineWidth = Math.max(3, width / 640); + context.lineCap = "round"; + context.lineJoin = "round"; + for (const action of line.actions) { + if (action.kind !== "mark") continue; + const mark = marks.find((candidate) => candidate.id === action.id); + for (const strokeIndex of mark?.strokes ?? []) { + const { points } = strokes[strokeIndex]!; + context.beginPath(); + for (let index = 0; index + 1 < points.length; index += 2) { + context.lineTo(points[index]! * width, points[index + 1]! * height); + } + context.stroke(); + } + } +}; + const drawChip = (context: CanvasRenderingContext2D, label: string, height: number) => { // Blind reading tests in Scrawl: the number needs its own white square to be read as a number. const size = Math.max(24, height / 18); @@ -287,13 +281,21 @@ export async function buildBundle( t0: shift(stroke.t0), t1: shift(stroke.t1), })); - const marks = strokesToMarks(strokes); + const marks = marksOf(strokes, recording.session.width, recording.session.height); + const sceneChanges = withoutInkChanges(recording.sceneChanges, strokes, SCRAWL_INK_HOLD_MS); const lines = buildNarration({ words: recording.words, events, marks, durationMs: recording.durationMs, - sceneChanges: withoutInkChanges(recording.sceneChanges, strokes, SCRAWL_INK_HOLD_MS), + sceneChanges, + markGroups: groupMarks(marks, { + actionTimes: events.flatMap((event) => + event.kind === "click" || event.kind === "drag" ? [event.t] : [], + ), + sceneChanges, + budget: budgetFor(recording.durationMs), + }).map((group) => group.map((mark) => mark.id)), }); const box = activityBox({ input: { events, strokes } }); @@ -311,6 +313,7 @@ export async function buildBundle( const context = full.getContext("2d")!; context.drawImage(video, 0, 0); drawClickRings(context, line, full.width, full.height); + drawMarkInk(context, line, marks, strokes, full.width, full.height); const name = `frame-${String(index + 1).padStart(2, "0")}.png`; if (box === null) { await write(`frames/${name}`, await bytesOf(await toBlob(full))); @@ -367,17 +370,22 @@ export async function buildBundle( words: recording.words, }, strokes: strokes.map((stroke) => manifestStroke(stroke, width, height)), - marks, + marks: placeMarks(marks, lines), events: events.map((event) => (event.kind === "click" ? { ...event, button: "left" } : event)), keyframes: lines.map((line, index) => { - const main = line.actions.find((action) => action.kind === "mark") ?? line.actions.at(-1); + const drawn = line.actions.filter((action) => action.kind === "mark"); + const main = drawn[0] ?? line.actions.at(-1); return { index: index + 1, t: Math.round(line.frameT), reason: main === undefined ? "speech" : KEYFRAME_REASONS[main.kind], file: `frames/frame-${String(index + 1).padStart(2, "0")}.png`, - event: main?.kind ?? "speech", + event: + drawn.length > 0 + ? drawn.map(({ id }) => marks.find((mark) => mark.id === id)?.description).join("; ") + : (main?.kind ?? "speech"), said: line.text, + ...(drawn[0] === undefined ? {} : { markId: drawn[0].id }), }; }), contactSheet: "contact-sheet.png", From 1c5747c3c7dd4cf74193e5d5b2572adcca31c82c Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:20:51 +0100 Subject: [PATCH 2/9] feat(scrawl): rebindable shortcut, past captures, crash recovery, video stays on disk The shortcut was hardcoded, a capture held its video in memory, a crash lost the result, and nobody could see or keep what retention deletes. Settings now rebinds the shortcut and says when another program holds it, the video and audio only live on disk, the next launch rebuilds interrupted captures, and a history dialog finds, pins and bins captures with the date retention acts on. Co-Authored-By: Claude Opus 5.5 --- FORK.md | 15 +- apps/desktop/src/ipc/DesktopIpcHandlers.ts | 20 + apps/desktop/src/ipc/channels.ts | 10 + .../desktop/src/ipc/methods/clientSettings.ts | 2 + apps/desktop/src/ipc/methods/scrawl.ts | 97 ++++ apps/desktop/src/ipc/methods/snapShot.ts | 3 + apps/desktop/src/preload.ts | 12 + apps/desktop/src/scrawl/DesktopScrawl.ts | 416 ++++++++++++++---- apps/desktop/src/scrawl/captureStore.test.ts | 192 ++++++++ apps/desktop/src/scrawl/captureStore.ts | 286 ++++++++++++ apps/web/src/components/AppSidebarLayout.tsx | 2 + apps/web/src/components/CommandPalette.tsx | 3 + apps/web/src/components/chat/ChatComposer.tsx | 3 +- .../components/chat/ComposerScrawlButton.tsx | 22 +- .../components/scrawl/ScrawlHistoryDialog.tsx | 354 +++++++++++++++ .../src/components/scrawl/ScrawlRuntime.tsx | 62 +++ .../components/scrawl/scrawlPaletteItems.tsx | 59 +++ .../components/scrawl/useScrawlCaptures.ts | 42 ++ .../settings/KeybindingsSettings.logic.ts | 1 + .../settings/ScrawlSettingsSection.tsx | 273 ++++++++++++ .../components/settings/SnapShotSettings.tsx | 2 + .../src/components/settings/settingsSearch.ts | 18 + .../settings/useSnapShotShortcutRecorder.tsx | 6 +- apps/web/src/scrawl/sceneChanges.ts | 11 + apps/web/src/scrawl/scrawlBundle.ts | 4 +- apps/web/src/scrawl/scrawlCapture.ts | 160 ++++--- apps/web/src/scrawl/scrawlFiles.test.ts | 28 ++ apps/web/src/scrawl/scrawlFiles.ts | 28 ++ apps/web/src/scrawl/scrawlHistory.ts | 120 +++++ apps/web/src/scrawl/scrawlRecovery.test.ts | 35 ++ apps/web/src/scrawl/scrawlRecovery.ts | 239 ++++++++++ docs/user/screen-capture.md | 35 ++ packages/contracts/src/ipc.ts | 25 +- packages/contracts/src/keybindings.ts | 1 + packages/contracts/src/scrawl.ts | 59 +++ packages/contracts/src/settings.ts | 19 + packages/shared/src/keybindings.ts | 1 + 37 files changed, 2501 insertions(+), 164 deletions(-) create mode 100644 apps/desktop/src/scrawl/captureStore.test.ts create mode 100644 apps/desktop/src/scrawl/captureStore.ts create mode 100644 apps/web/src/components/scrawl/ScrawlHistoryDialog.tsx create mode 100644 apps/web/src/components/scrawl/ScrawlRuntime.tsx create mode 100644 apps/web/src/components/scrawl/scrawlPaletteItems.tsx create mode 100644 apps/web/src/components/scrawl/useScrawlCaptures.ts create mode 100644 apps/web/src/components/settings/ScrawlSettingsSection.tsx create mode 100644 apps/web/src/scrawl/scrawlFiles.test.ts create mode 100644 apps/web/src/scrawl/scrawlFiles.ts create mode 100644 apps/web/src/scrawl/scrawlHistory.ts create mode 100644 apps/web/src/scrawl/scrawlRecovery.test.ts create mode 100644 apps/web/src/scrawl/scrawlRecovery.ts create mode 100644 docs/user/screen-capture.md diff --git a/FORK.md b/FORK.md index cc27d2eb494b2..33112eff9ce73 100644 --- a/FORK.md +++ b/FORK.md @@ -160,8 +160,9 @@ the contact sheet land in the composer on screen. Esc in T3 Code discards a capt - Desktop app only: a browser cannot see the whole screen, and mobile has no such thing. Clicks and ink need Windows; elsewhere a capture has voice and video. -- While the Scrawl app runs it holds Alt+Shift+S. T3 Code logs that and takes the shortcut within - 15 seconds of Scrawl closing. The composer button works either way. +- While the Scrawl app runs it holds Alt+Shift+S. Settings > SnapShots > Screen capture says so, + and T3 Code takes the shortcut within 15 seconds of Scrawl closing. The composer button and the + command palette work either way. Rebind it in the same place. - Speech and pointing do not run in step, so sentences are never cut at a click: whole sentences own a keyframe, click bursts fold into one (`clicked ×3`), and clicks the screen did not answer read as `pointed`. On 32 real captures this took cuts through sentences from 59 of 127 lines to 1. @@ -169,5 +170,13 @@ the contact sheet land in the composer on screen. Esc in T3 Code discards a capt - The recording pill in the corner shows in the video and keyframes; `setContentProtection` does not keep it out on Windows 10 22H2. - Bundles are in `/scrawl/captures` in Scrawl's format, so the Scrawl skill and the - `scrawl` CLI (`inspect`, `zoom`, `frames`) read them. Video and audio are deleted after 7 days. + `scrawl` CLI (`inspect`, `zoom`, `frames`) read them. +- Video and audio go to disk as they are recorded and stay out of memory until the build, which reads + them back. A crash or reload during a capture loses at most the last seconds: the next launch + rebuilds it once from `unfinished.json`, `timing.json`, `audio.pcm` and the video. A capture that + fails is kept and offers a manual rebuild. Scene changes come from sampling the video again. +- Past captures (Alt+Shift+H, palette, Settings) find, insert, pin, or bin a capture and show the + date retention acts on. Same presets and `.pinned` marker as Scrawl: by default video goes to the + Recycle Bin after 7 days and the capture after 30. The list and the hourly pass share one plan + (`apps/desktop/src/scrawl/captureStore.ts`), so the dates shown are what happens. - Code: `apps/desktop/src/scrawl/`, `apps/web/src/scrawl/`, `packages/contracts/src/scrawl.ts`. diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index f378757607428..58785999d87c7 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -12,9 +12,19 @@ import { import { dictationAppend, dictationCancel, dictationFinish } from "./methods/dictation.ts"; import { scrawlAppendVideo, + scrawlBeginRebuild, scrawlCancel, + scrawlCheckShortcut, + scrawlGetShortcutState, + scrawlListCaptures, + scrawlListUnfinished, + scrawlMarkBuilt, + scrawlReadFile, + scrawlReveal, + scrawlSetPinned, scrawlStart, scrawlStop, + scrawlTrash, scrawlWriteFile, } from "./methods/scrawl.ts"; import { @@ -112,6 +122,16 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(scrawlCancel); yield* ipc.handle(scrawlAppendVideo); yield* ipc.handle(scrawlWriteFile); + yield* ipc.handle(scrawlReadFile); + yield* ipc.handle(scrawlListUnfinished); + yield* ipc.handle(scrawlBeginRebuild); + yield* ipc.handle(scrawlMarkBuilt); + yield* ipc.handle(scrawlListCaptures); + yield* ipc.handle(scrawlSetPinned); + yield* ipc.handle(scrawlTrash); + yield* ipc.handle(scrawlReveal); + yield* ipc.handle(scrawlGetShortcutState); + yield* ipc.handle(scrawlCheckShortcut); yield* ipc.handle(dictationCancel); yield* ipc.handle(getLocalEnvironmentBearerToken); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 58d11aa9e8e3e..f2466a33a1bae 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -44,6 +44,16 @@ export const SCRAWL_STOP_CHANNEL = "desktop:scrawl-stop"; export const SCRAWL_CANCEL_CHANNEL = "desktop:scrawl-cancel"; export const SCRAWL_APPEND_VIDEO_CHANNEL = "desktop:scrawl-append-video"; export const SCRAWL_WRITE_FILE_CHANNEL = "desktop:scrawl-write-file"; +export const SCRAWL_READ_FILE_CHANNEL = "desktop:scrawl-read-file"; +export const SCRAWL_LIST_UNFINISHED_CHANNEL = "desktop:scrawl-list-unfinished"; +export const SCRAWL_BEGIN_REBUILD_CHANNEL = "desktop:scrawl-begin-rebuild"; +export const SCRAWL_MARK_BUILT_CHANNEL = "desktop:scrawl-mark-built"; +export const SCRAWL_LIST_CAPTURES_CHANNEL = "desktop:scrawl-list-captures"; +export const SCRAWL_SET_PINNED_CHANNEL = "desktop:scrawl-set-pinned"; +export const SCRAWL_TRASH_CHANNEL = "desktop:scrawl-trash"; +export const SCRAWL_REVEAL_CHANNEL = "desktop:scrawl-reveal"; +export const SCRAWL_GET_SHORTCUT_STATE_CHANNEL = "desktop:scrawl-get-shortcut-state"; +export const SCRAWL_CHECK_SHORTCUT_CHANNEL = "desktop:scrawl-check-shortcut"; export const GET_LOCAL_ENVIRONMENT_BEARER_TOKEN_CHANNEL = "desktop:get-local-environment-bearer-token"; export const GET_CLIENT_SETTINGS_CHANNEL = "desktop:get-client-settings"; diff --git a/apps/desktop/src/ipc/methods/clientSettings.ts b/apps/desktop/src/ipc/methods/clientSettings.ts index 7a647a58dc431..f7d89d10ee397 100644 --- a/apps/desktop/src/ipc/methods/clientSettings.ts +++ b/apps/desktop/src/ipc/methods/clientSettings.ts @@ -4,6 +4,7 @@ import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as DesktopClientSettings from "../../settings/DesktopClientSettings.ts"; +import * as DesktopScrawl from "../../scrawl/DesktopScrawl.ts"; import * as DesktopSnapShot from "../../snapShot/DesktopSnapShot.ts"; import * as IpcChannels from "../channels.ts"; import * as DesktopIpc from "../DesktopIpc.ts"; @@ -27,5 +28,6 @@ export const setClientSettings = DesktopIpc.makeIpcMethod({ const snapShot = yield* DesktopSnapShot.DesktopSnapShot; yield* clientSettings.set(settings); yield* snapShot.configure(settings); + yield* (yield* DesktopScrawl.DesktopScrawl).configure(settings); }), }); diff --git a/apps/desktop/src/ipc/methods/scrawl.ts b/apps/desktop/src/ipc/methods/scrawl.ts index b82618fa4642d..5b3a32a9f6232 100644 --- a/apps/desktop/src/ipc/methods/scrawl.ts +++ b/apps/desktop/src/ipc/methods/scrawl.ts @@ -1,9 +1,16 @@ import { ScrawlAppendVideoInput, ScrawlCaptureRef, + ScrawlCaptureSummary, + ScrawlReadFileInput, + ScrawlSetPinnedInput, + ScrawlShortcutAvailability, + ScrawlShortcutState, ScrawlStartResult, ScrawlStopResult, + ScrawlUnfinished, ScrawlWriteFileInput, + SnapShotKeyChord, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; @@ -56,3 +63,93 @@ export const scrawlWriteFile = makeIpcMethod({ yield* (yield* DesktopScrawl.DesktopScrawl).writeFile(input); }), }); + +export const scrawlReadFile = makeIpcMethod({ + channel: IpcChannels.SCRAWL_READ_FILE_CHANNEL, + payload: ScrawlReadFileInput, + result: Schema.Uint8Array, + handler: Effect.fn("desktop.ipc.scrawl.readFile")(function* (input) { + return yield* (yield* DesktopScrawl.DesktopScrawl).readFile(input); + }), +}); + +export const scrawlListUnfinished = makeIpcMethod({ + channel: IpcChannels.SCRAWL_LIST_UNFINISHED_CHANNEL, + payload: Schema.Void, + result: Schema.Array(ScrawlUnfinished), + handler: Effect.fn("desktop.ipc.scrawl.listUnfinished")(function* () { + return yield* (yield* DesktopScrawl.DesktopScrawl).listUnfinished(); + }), +}); + +export const scrawlBeginRebuild = makeIpcMethod({ + channel: IpcChannels.SCRAWL_BEGIN_REBUILD_CHANNEL, + payload: ScrawlCaptureRef, + result: ScrawlUnfinished, + handler: Effect.fn("desktop.ipc.scrawl.beginRebuild")(function* (input) { + return yield* (yield* DesktopScrawl.DesktopScrawl).beginRebuild(input); + }), +}); + +export const scrawlMarkBuilt = makeIpcMethod({ + channel: IpcChannels.SCRAWL_MARK_BUILT_CHANNEL, + payload: ScrawlCaptureRef, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.scrawl.markBuilt")(function* (input) { + yield* (yield* DesktopScrawl.DesktopScrawl).markBuilt(input); + }), +}); + +export const scrawlListCaptures = makeIpcMethod({ + channel: IpcChannels.SCRAWL_LIST_CAPTURES_CHANNEL, + payload: Schema.Void, + result: Schema.Array(ScrawlCaptureSummary), + handler: Effect.fn("desktop.ipc.scrawl.listCaptures")(function* () { + return yield* (yield* DesktopScrawl.DesktopScrawl).listCaptures(); + }), +}); + +export const scrawlSetPinned = makeIpcMethod({ + channel: IpcChannels.SCRAWL_SET_PINNED_CHANNEL, + payload: ScrawlSetPinnedInput, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.scrawl.setPinned")(function* (input) { + yield* (yield* DesktopScrawl.DesktopScrawl).setPinned(input); + }), +}); + +export const scrawlTrash = makeIpcMethod({ + channel: IpcChannels.SCRAWL_TRASH_CHANNEL, + payload: ScrawlCaptureRef, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.scrawl.trash")(function* (input) { + yield* (yield* DesktopScrawl.DesktopScrawl).trash(input); + }), +}); + +export const scrawlReveal = makeIpcMethod({ + channel: IpcChannels.SCRAWL_REVEAL_CHANNEL, + payload: ScrawlCaptureRef, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.scrawl.reveal")(function* (input) { + yield* (yield* DesktopScrawl.DesktopScrawl).reveal(input); + }), +}); + +export const scrawlGetShortcutState = makeIpcMethod({ + channel: IpcChannels.SCRAWL_GET_SHORTCUT_STATE_CHANNEL, + payload: Schema.Void, + result: ScrawlShortcutState, + handler: Effect.fn("desktop.ipc.scrawl.getShortcutState")(function* () { + return yield* (yield* DesktopScrawl.DesktopScrawl).shortcutState; + }), +}); + +export const scrawlCheckShortcut = makeIpcMethod({ + channel: IpcChannels.SCRAWL_CHECK_SHORTCUT_CHANNEL, + payload: SnapShotKeyChord, + result: ScrawlShortcutAvailability, + handler: Effect.fn("desktop.ipc.scrawl.checkShortcut")(function* (shortcut) { + return yield* (yield* DesktopScrawl.DesktopScrawl).checkShortcut(shortcut); + }), +}); diff --git a/apps/desktop/src/ipc/methods/snapShot.ts b/apps/desktop/src/ipc/methods/snapShot.ts index dff64081b70c6..2a6187a76545b 100644 --- a/apps/desktop/src/ipc/methods/snapShot.ts +++ b/apps/desktop/src/ipc/methods/snapShot.ts @@ -18,6 +18,7 @@ import type * as Electron from "electron"; import * as ElectronWindow from "../../electron/ElectronWindow.ts"; import * as ElectronDialog from "../../electron/ElectronDialog.ts"; +import * as DesktopScrawl from "../../scrawl/DesktopScrawl.ts"; import * as DesktopSnapShot from "../../snapShot/DesktopSnapShot.ts"; import * as IpcChannels from "../channels.ts"; import * as DesktopIpc from "../DesktopIpc.ts"; @@ -158,6 +159,8 @@ export const setSnapShotShortcutSuppressed = DesktopIpc.makeIpcMethod({ handler: Effect.fn("desktop.ipc.snapShot.setShortcutSuppressed")(function* (suppressed, event) { yield* ensureTrustedSnapShotSender(event); yield* (yield* DesktopSnapShot.DesktopSnapShot).setShortcutSuppressed(suppressed); + // One recorder records both shortcuts, and a held global shortcut never reaches it. + yield* (yield* DesktopScrawl.DesktopScrawl).setShortcutSuppressed(suppressed); }), }); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 34a785cd56d19..27526fa2ddcda 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -127,6 +127,18 @@ contextBridge.exposeInMainWorld("desktopBridge", { scrawlCancel: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_CANCEL_CHANNEL, input), scrawlAppendVideo: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_APPEND_VIDEO_CHANNEL, input), scrawlWriteFile: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_WRITE_FILE_CHANNEL, input), + scrawlReadFile: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_READ_FILE_CHANNEL, input), + scrawlListUnfinished: () => ipcRenderer.invoke(IpcChannels.SCRAWL_LIST_UNFINISHED_CHANNEL), + scrawlBeginRebuild: (input) => + ipcRenderer.invoke(IpcChannels.SCRAWL_BEGIN_REBUILD_CHANNEL, input), + scrawlMarkBuilt: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_MARK_BUILT_CHANNEL, input), + scrawlListCaptures: () => ipcRenderer.invoke(IpcChannels.SCRAWL_LIST_CAPTURES_CHANNEL), + scrawlSetPinned: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_SET_PINNED_CHANNEL, input), + scrawlTrash: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_TRASH_CHANNEL, input), + scrawlReveal: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_REVEAL_CHANNEL, input), + scrawlGetShortcutState: () => ipcRenderer.invoke(IpcChannels.SCRAWL_GET_SHORTCUT_STATE_CHANNEL), + scrawlCheckShortcut: (shortcut) => + ipcRenderer.invoke(IpcChannels.SCRAWL_CHECK_SHORTCUT_CHANNEL, shortcut), getClientSettings: () => ipcRenderer.invoke(IpcChannels.GET_CLIENT_SETTINGS_CHANNEL), setClientSettings: (settings) => ipcRenderer.invoke(IpcChannels.SET_CLIENT_SETTINGS_CHANNEL, settings), diff --git a/apps/desktop/src/scrawl/DesktopScrawl.ts b/apps/desktop/src/scrawl/DesktopScrawl.ts index f63fe2e36f6c7..69d0b5cb2c7bc 100644 --- a/apps/desktop/src/scrawl/DesktopScrawl.ts +++ b/apps/desktop/src/scrawl/DesktopScrawl.ts @@ -4,20 +4,32 @@ // @effect-diagnostics globalDateInEffect:off -- Same clock as above. // @effect-diagnostics nodeBuiltinImport:off -- Bundles are plain files written outside any Effect service. // The desktop half of a Scrawl capture: the global shortcut, the mouse and Shift watcher, the ink -// and the recording pill, and the bundle on disk. The renderer records and builds the bundle. +// and the recording pill, and the bundles on disk. The renderer records and builds the bundle. import { + DEFAULT_SCRAWL_SHORTCUT, SCRAWL_INK_HOLD_MS, + SCRAWL_SHORTCUT_STATE_ACTION, SCRAWL_TOGGLE_ACTION, + type ClientSettings, type ScrawlAppendVideoInput, type ScrawlCaptureRef, + type ScrawlCaptureSummary, + type ScrawlReadFileInput, + type ScrawlRetention, + type ScrawlSetPinnedInput, + type ScrawlShortcutAvailability, + type ScrawlShortcutState, type ScrawlStartResult, type ScrawlStopResult, + type ScrawlUnfinished, type ScrawlWriteFileInput, + type SnapShotKeyChord, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as Electron from "electron"; import * as NodeFSP from "node:fs/promises"; @@ -25,11 +37,24 @@ import * as NodePath from "node:path"; import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import { loadWindowsForegroundApi } from "../electron/WindowsForeground.ts"; +import * as DesktopClientSettings from "../settings/DesktopClientSettings.ts"; import { activeWindow } from "../snapShot/ActiveWindow.ts"; +import { snapShotShortcutSystemConflict, toElectronAccelerator } from "../snapShot/snapShot.ts"; import * as DesktopWindow from "../window/DesktopWindow.ts"; +import { + PINNED_FILE, + REBUILT_ONCE_FILE, + planRebuilds, + planRetention, + readCaptureFolder, + readCaptureFolders, + removeJournal, + summarize, + unfinishedOf, + writeJournal, +} from "./captureStore.ts"; import { ScrawlInputTracker } from "./inputTracker.ts"; -const DEFAULT_SHORTCUT = "Alt+Shift+S"; // Another program (the Scrawl app uses the same keys) may hold the shortcut and let go of it later. const SHORTCUT_RETRY_MS = 15_000; // Short presses must not fall between two looks. Polling needs no hook, no message pump and no @@ -37,9 +62,10 @@ const SHORTCUT_RETRY_MS = 15_000; const INPUT_POLL_MS = 10; const VK_LBUTTON = 0x01; const VK_SHIFT = 0x10; -// Like Scrawl: frames, text and contact sheet stay, the heavy media goes. -const MEDIA_KEPT_MS = 7 * 24 * 60 * 60 * 1000; -const MEDIA_FILES = ["recording.mp4", "recording.webm", "audio.wav"]; +// The input a crash of the whole app can lose. A reload loses none: this process keeps it. +const JOURNAL_INTERVAL_MS = 5_000; +const RETENTION_INTERVAL_MS = 60 * 60 * 1000; +const MAX_READ_BYTES = 32 * 1024 * 1024; const PILL_WIDTH = 96; const PILL_HEIGHT = 30; @@ -62,6 +88,24 @@ export class DesktopScrawl extends Context.Service< input: ScrawlAppendVideoInput, ) => Effect.Effect; readonly writeFile: (input: ScrawlWriteFileInput) => Effect.Effect; + readonly readFile: ( + input: ScrawlReadFileInput, + ) => Effect.Effect; + readonly listUnfinished: () => Effect.Effect>; + readonly beginRebuild: ( + input: ScrawlCaptureRef, + ) => Effect.Effect; + readonly markBuilt: (input: ScrawlCaptureRef) => Effect.Effect; + readonly listCaptures: () => Effect.Effect>; + readonly setPinned: (input: ScrawlSetPinnedInput) => Effect.Effect; + readonly trash: (input: ScrawlCaptureRef) => Effect.Effect; + readonly reveal: (input: ScrawlCaptureRef) => Effect.Effect; + readonly configure: (settings: ClientSettings) => Effect.Effect; + readonly setShortcutSuppressed: (suppressed: boolean) => Effect.Effect; + readonly shortcutState: Effect.Effect; + readonly checkShortcut: ( + shortcut: SnapShotKeyChord, + ) => Effect.Effect; } >()("@t3tools/desktop/scrawl/DesktopScrawl") {} @@ -78,19 +122,6 @@ const PILL_PAGE = `0:00 `; -const dropOldMedia = async (capturesDir: string, now: number) => { - const bundles = await NodeFSP.readdir(capturesDir).catch(() => [] as Array); - for (const bundle of bundles) { - if (!bundle.endsWith(".scrawl")) continue; - const folder = NodePath.join(capturesDir, bundle); - const stat = await NodeFSP.stat(folder).catch(() => null); - if (stat === null || now - stat.mtimeMs < MEDIA_KEPT_MS) continue; - await Promise.all( - MEDIA_FILES.map((file) => NodeFSP.rm(NodePath.join(folder, file), { force: true })), - ).catch(() => undefined); - } -}; - const dataUrl = (html: string) => `data:text/html;charset=utf-8,${encodeURIComponent(html)}`; const overlayOptions = (bounds: Electron.Rectangle): Electron.BrowserWindowConstructorOptions => ({ @@ -107,6 +138,18 @@ const overlayOptions = (bounds: Electron.Rectangle): Electron.BrowserWindowConst webPreferences: { sandbox: true, contextIsolation: true, backgroundThrottling: false }, }); +const probeShortcut = (candidate: string): ScrawlShortcutAvailability => { + try { + if (!Electron.globalShortcut.register(candidate, () => undefined)) { + return { available: false, message: "Another program is using this shortcut" }; + } + Electron.globalShortcut.unregister(candidate); + return { available: true, message: null }; + } catch { + return { available: false, message: "The system could not register this shortcut" }; + } +}; + const toError = (cause: unknown) => new DesktopScrawlError({ reason: cause instanceof Error ? cause.message : String(cause) }); @@ -119,6 +162,9 @@ interface RunningCapture { readonly ink: Electron.BrowserWindow; readonly pill: Electron.BrowserWindow; readonly inputTimer: ReturnType; + readonly journalTimer: ReturnType; + readonly session: ScrawlStartResult; + journalWrites: Promise; inkTimer: ReturnType | null; readonly clockTimer: ReturnType; } @@ -126,12 +172,24 @@ interface RunningCapture { const make = Effect.gen(function* () { const environment = yield* DesktopEnvironment.DesktopEnvironment; const desktopWindow = yield* DesktopWindow.DesktopWindow; + const clientSettings = yield* DesktopClientSettings.DesktopClientSettings; const services = yield* Effect.context(); const capturesDir = NodePath.join(environment.baseDir, "scrawl", "captures"); + const initialSettings = yield* clientSettings.get.pipe( + Effect.map(Option.getOrNull), + Effect.orElseSucceed(() => null), + ); let running: RunningCapture | null = null; - // Finished captures keep writing their frames and summary after the watcher stopped. - const bundles = new Map(); let nextIndex = 0; + let retention: ScrawlRetention = initialSettings?.scrawlRetention ?? "week"; + let accelerator = toElectronAccelerator( + initialSettings?.scrawlShortcut ?? DEFAULT_SCRAWL_SHORTCUT, + ); + let shortcutHeld = false; + let shortcutSuppressed = false; + let shortcutRetry: ReturnType | null = null; + // Ids are checked by the contract, and finished captures keep writing frames and a summary. + const bundleOf = (id: string) => NodePath.join(capturesDir, `${id}.scrawl`); const ink = (capture: RunningCapture, script: string) => { if (!capture.ink.isDestroyed()) void capture.ink.webContents.executeJavaScript(script, true); @@ -179,7 +237,6 @@ const make = Effect.gen(function* () { // The renderer owns the capture and only starts when it has none, so one left here lost its // renderer to a reload. Without this its ink and pill would stay on screen for good. if (running !== null) tearDown(running); - void dropOldMedia(capturesDir, Date.now()); const cursor = Electron.screen.getCursorScreenPoint(); const display = Electron.screen.getDisplayNearestPoint(cursor); const sources = yield* Effect.tryPromise({ @@ -234,8 +291,24 @@ const make = Effect.gen(function* () { void pill.loadURL(dataUrl(PILL_PAGE)); pill.showInactive(); + const scale = display.scaleFactor; + const session: ScrawlStartResult = { + id, + sourceId: source.id, + width: Math.round(display.bounds.width * scale), + height: Math.round(display.bounds.height * scale), + startedAtMs, + app: front?.owner.name.replace(/\.exe$/i, "") ?? "", + windowTitle: front?.title ?? "", + bundlePath, + }; const capture: RunningCapture = { id, + session, + journalWrites: Promise.resolve(), + journalTimer: setInterval(() => { + if (running !== null) void saveJournal(running); + }, JOURNAL_INTERVAL_MS), bundlePath, startedAtMs, tracker, @@ -253,22 +326,26 @@ const make = Effect.gen(function* () { }, 1000), }; running = capture; - bundles.set(id, bundlePath); - const scale = display.scaleFactor; - return { - id, - sourceId: source.id, - width: Math.round(display.bounds.width * scale), - height: Math.round(display.bounds.height * scale), - startedAtMs, - app: front?.owner.name.replace(/\.exe$/i, "") ?? "", - windowTitle: front?.title ?? "", - bundlePath, - } satisfies ScrawlStartResult; + saveJournal(capture); + return session; }); + const saveJournal = (capture: RunningCapture) => { + const { events, strokes } = capture.tracker.result(); + const journal = { + session: capture.session, + input: { events: [...events], strokes: [...strokes] }, + }; + capture.journalWrites = capture.journalWrites + .then(() => writeJournal(capture.bundlePath, journal)) + // Only a crash reads it, and the next write brings it up to date. + .catch(() => undefined); + return capture.journalWrites; + }; + const tearDown = (capture: RunningCapture) => { clearInterval(capture.inputTimer); + clearInterval(capture.journalTimer); if (capture.inkTimer !== null) clearTimeout(capture.inkTimer); clearInterval(capture.clockTimer); if (!capture.ink.isDestroyed()) capture.ink.destroy(); @@ -283,40 +360,31 @@ const make = Effect.gen(function* () { const capture = running; capture.tracker.shiftUp(); tearDown(capture); + yield* Effect.promise(() => saveJournal(capture)); const { events, strokes } = capture.tracker.result(); return { events: [...events], strokes: [...strokes] }; }); const cancel = Effect.fn("DesktopScrawl.cancel")(function* (input: ScrawlCaptureRef) { - if (running?.id === input.id) tearDown(running); - const bundlePath = bundles.get(input.id); - bundles.delete(input.id); - if (bundlePath !== undefined) { - yield* Effect.promise(() => - NodeFSP.rm(bundlePath, { recursive: true, force: true }).catch(() => undefined), - ); - } + const capture = running?.id === input.id ? running : null; + if (capture !== null) tearDown(capture); + yield* Effect.promise(async () => { + await capture?.journalWrites; + await NodeFSP.rm(bundleOf(input.id), { recursive: true, force: true }).catch(() => undefined); + }); }); const appendVideo = Effect.fn("DesktopScrawl.appendVideo")(function* ( input: ScrawlAppendVideoInput, ) { - const bundlePath = bundles.get(input.id); - if (bundlePath === undefined) { - return yield* new DesktopScrawlError({ reason: "Unknown capture." }); - } yield* Effect.tryPromise({ - try: () => NodeFSP.appendFile(NodePath.join(bundlePath, input.name), input.bytes), + try: () => NodeFSP.appendFile(NodePath.join(bundleOf(input.id), input.name), input.bytes), catch: toError, }); }); const writeFile = Effect.fn("DesktopScrawl.writeFile")(function* (input: ScrawlWriteFileInput) { - const bundlePath = bundles.get(input.id); - if (bundlePath === undefined) { - return yield* new DesktopScrawlError({ reason: "Unknown capture." }); - } - const target = NodePath.join(bundlePath, input.path); + const target = NodePath.join(bundleOf(input.id), input.path); yield* Effect.tryPromise({ try: async () => { await NodeFSP.mkdir(NodePath.dirname(target), { recursive: true }); @@ -326,39 +394,231 @@ const make = Effect.gen(function* () { }); }); - yield* Effect.acquireRelease( - Effect.promise(() => { - const run = Effect.runForkWith(services); - const register = () => - Electron.globalShortcut.register(DEFAULT_SHORTCUT, () => { - run( - desktopWindow - .dispatchMenuAction(SCRAWL_TOGGLE_ACTION, { reveal: false }) - .pipe(Effect.ignore), - ); + const readFile = Effect.fn("DesktopScrawl.readFile")(function* (input: ScrawlReadFileInput) { + return yield* Effect.tryPromise({ + try: async () => { + const file = await NodeFSP.open(NodePath.join(bundleOf(input.id), input.path), "r"); + try { + const bytes = new Uint8Array(Math.min(input.length, MAX_READ_BYTES)); + const { bytesRead } = await file.read(bytes, 0, bytes.length, input.offset); + return bytes.subarray(0, bytesRead); + } finally { + await file.close(); + } + }, + catch: toError, + }); + }); + + const listUnfinished = Effect.fn("DesktopScrawl.listUnfinished")(function* () { + // Only a renderer without a capture of its own asks, so one still here lost its renderer. + if (running !== null) { + const orphan = running; + orphan.tracker.shiftUp(); + tearDown(orphan); + yield* Effect.promise(() => saveJournal(orphan)); + } + return yield* Effect.promise(async () => { + const { owed, empty } = planRebuilds(await readCaptureFolders(capturesDir), null); + await Promise.all( + empty.map((capture) => + NodeFSP.rm(capture.folder, { recursive: true, force: true }).catch(() => undefined), + ), + ); + return owed.map(unfinishedOf); + }); + }); + + const beginRebuild = Effect.fn("DesktopScrawl.beginRebuild")(function* (input: ScrawlCaptureRef) { + return yield* Effect.tryPromise({ + try: async () => { + const capture = await readCaptureFolder(bundleOf(input.id)); + if (capture === null) throw new Error("This capture no longer exists."); + await NodeFSP.writeFile(NodePath.join(capture.folder, REBUILT_ONCE_FILE), ""); + return unfinishedOf(capture); + }, + catch: toError, + }); + }); + + const markBuilt = Effect.fn("DesktopScrawl.markBuilt")(function* (input: ScrawlCaptureRef) { + yield* Effect.promise(() => removeJournal(bundleOf(input.id)).catch(() => undefined)); + }); + + const listCaptures = Effect.fn("DesktopScrawl.listCaptures")(function* () { + return yield* Effect.promise(async () => { + const captures = await readCaptureFolders(capturesDir); + return captures + .map((capture) => summarize(capture, retention, running?.id ?? null)) + .toSorted((left, right) => right.createdAtMs - left.createdAtMs); + }); + }); + + const setPinned = Effect.fn("DesktopScrawl.setPinned")(function* (input: ScrawlSetPinnedInput) { + const marker = NodePath.join(bundleOf(input.id), PINNED_FILE); + yield* Effect.tryPromise({ + try: () => + input.pinned ? NodeFSP.writeFile(marker, "") : NodeFSP.rm(marker, { force: true }), + catch: toError, + }); + }); + + const trash = Effect.fn("DesktopScrawl.trash")(function* (input: ScrawlCaptureRef) { + if (running?.id === input.id) { + return yield* new DesktopScrawlError({ reason: "This capture is still recording." }); + } + yield* Effect.tryPromise({ + try: () => Electron.shell.trashItem(bundleOf(input.id)), + catch: toError, + }); + }); + + const reveal = Effect.fn("DesktopScrawl.reveal")(function* (input: ScrawlCaptureRef) { + yield* Effect.tryPromise({ + try: async () => { + const failure = await Electron.shell.openPath(bundleOf(input.id)); + if (failure !== "") throw new Error(failure); + }, + catch: toError, + }); + }); + + const run = Effect.runForkWith(services); + + /** Moves to the bin, like Scrawl: the list named the date, and a wrong one can still be undone. */ + const runRetention = async () => { + if (running !== null) return; + const actions = planRetention( + await readCaptureFolders(capturesDir), + retention, + null, + Date.now(), + ); + for (const action of actions) { + const paths = action.kind === "bundle" ? [action.folder] : action.paths; + for (const path of paths) { + await Electron.shell.trashItem(path).catch((cause: unknown) => { + run(Effect.logWarning(`scrawl: could not move ${path} to the bin: ${String(cause)}`)); }); - const whenReady = async () => { - await Electron.app.whenReady(); - if (register()) return null; - run(Effect.logWarning(`scrawl: ${DEFAULT_SHORTCUT} is taken by another program`)); - const retry = setInterval(() => { - if (register()) clearInterval(retry); - }, SHORTCUT_RETRY_MS); - retry.unref?.(); - return retry; - }; - return whenReady(); + } + } + }; + + const publishShortcutState = () => { + run( + desktopWindow + .dispatchMenuAction(SCRAWL_SHORTCUT_STATE_ACTION, { reveal: false }) + .pipe(Effect.ignore), + ); + }; + + const takeShortcut = () => { + try { + shortcutHeld = Electron.globalShortcut.register(accelerator, () => { + run( + desktopWindow + .dispatchMenuAction(SCRAWL_TOGGLE_ACTION, { reveal: false }) + .pipe(Effect.ignore), + ); + }); + } catch { + shortcutHeld = false; + } + return shortcutHeld; + }; + + const releaseShortcut = () => { + if (shortcutRetry !== null) clearInterval(shortcutRetry); + shortcutRetry = null; + if (shortcutHeld) Electron.globalShortcut.unregister(accelerator); + shortcutHeld = false; + }; + + const applyShortcut = () => { + if (shortcutSuppressed || takeShortcut()) return; + run(Effect.logWarning(`scrawl: ${accelerator} is taken by another program`)); + const retry = setInterval(() => { + if (!takeShortcut()) return; + clearInterval(retry); + if (shortcutRetry === retry) shortcutRetry = null; + publishShortcutState(); + }, SHORTCUT_RETRY_MS); + retry.unref?.(); + shortcutRetry = retry; + }; + + const configure = (settings: ClientSettings) => + Effect.sync(() => { + retention = settings.scrawlRetention; + const next = toElectronAccelerator(settings.scrawlShortcut); + if (next === accelerator) return; + releaseShortcut(); + accelerator = next; + applyShortcut(); + publishShortcutState(); + }); + + const setShortcutSuppressed = (suppressed: boolean) => + Effect.sync(() => { + if (suppressed === shortcutSuppressed) return; + shortcutSuppressed = suppressed; + releaseShortcut(); + applyShortcut(); + publishShortcutState(); + }); + + const shortcutState = Effect.sync((): ScrawlShortcutState => ({ + accelerator, + // Letting go while the user records a new shortcut is not another program holding it. + registered: shortcutHeld || shortcutSuppressed, + })); + + const checkShortcut = (shortcut: SnapShotKeyChord) => + Effect.sync((): ScrawlShortcutAvailability => { + const conflict = snapShotShortcutSystemConflict(shortcut); + if (conflict !== null) return { available: false, message: conflict }; + const candidate = toElectronAccelerator(shortcut); + // Electron refuses a second registration of keys this app already holds. + if (candidate === accelerator && shortcutHeld) return { available: true, message: null }; + return probeShortcut(candidate); + }); + + yield* Effect.acquireRelease( + Effect.promise(async () => { + await Electron.app.whenReady(); + applyShortcut(); + void runRetention(); + const retentionTimer = setInterval(() => void runRetention(), RETENTION_INTERVAL_MS); + retentionTimer.unref?.(); + return retentionTimer; }), - (retry) => - Effect.promise(async () => { - const timer = await retry; - if (timer !== null) clearInterval(timer); - Electron.globalShortcut.unregister(DEFAULT_SHORTCUT); + (retentionTimer) => + Effect.sync(() => { + clearInterval(retentionTimer); + releaseShortcut(); if (running !== null) tearDown(running); }), ); - return DesktopScrawl.of({ start, stop, cancel, appendVideo, writeFile }); + return DesktopScrawl.of({ + start, + stop, + cancel, + appendVideo, + writeFile, + readFile, + listUnfinished, + beginRebuild, + markBuilt, + listCaptures, + setPinned, + trash, + reveal, + configure, + setShortcutSuppressed, + shortcutState, + checkShortcut, + }); }); export const layer = Layer.effect(DesktopScrawl, make); diff --git a/apps/desktop/src/scrawl/captureStore.test.ts b/apps/desktop/src/scrawl/captureStore.test.ts new file mode 100644 index 0000000000000..d387b59b77a67 --- /dev/null +++ b/apps/desktop/src/scrawl/captureStore.test.ts @@ -0,0 +1,192 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Real folders on disk are what these tests are about. +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, describe, expect, it } from "vite-plus/test"; + +import { + PINNED_FILE, + REBUILT_ONCE_FILE, + UNFINISHED_FILE, + idTimeMs, + planRebuilds, + planRetention, + readCaptureFolder, + readCaptureFolders, + removeJournal, + retentionDates, + type CaptureFolder, +} from "./captureStore.ts"; + +const DAY_MS = 24 * 60 * 60 * 1000; +const CREATED_AT = Date.UTC(2026, 0, 1); + +const capture = (overrides: Partial = {}): CaptureFolder => ({ + id: "scrawl-a-0", + folder: "/captures/scrawl-a-0.scrawl", + built: true, + videoBytes: 100, + tried: false, + pinned: false, + journaled: false, + journal: null, + createdAtMs: CREATED_AT, + changedAtMs: CREATED_AT, + durationMs: 1_000, + app: "", + windowTitle: "", + sizeBytes: 200, + mediaBytes: 100, + mediaItems: ["recording.mp4", "audio.wav"], + ...overrides, +}); + +const temporaryFolders: Array = []; +const makeCapturesDir = async () => { + const dir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "scrawl-store-")); + temporaryFolders.push(dir); + return dir; +}; +const makeBundle = async (dir: string, id: string, files: Record) => { + const folder = NodePath.join(dir, `${id}.scrawl`); + await NodeFSP.mkdir(folder); + for (const [name, text] of Object.entries(files)) { + await NodeFSP.writeFile(NodePath.join(folder, name), text); + } + return folder; +}; + +afterEach(async () => { + await Promise.all( + temporaryFolders.splice(0).map((dir) => NodeFSP.rm(dir, { recursive: true, force: true })), + ); +}); + +describe("planRebuilds", () => { + it("rebuilds a capture without a manifest once, and never the one recording", async () => { + const dir = await makeCapturesDir(); + const journal = JSON.stringify({ + session: { + id: "scrawl-crashed-0", + sourceId: "screen:0", + width: 100, + height: 100, + startedAtMs: CREATED_AT, + app: "Figma", + windowTitle: "Board", + bundlePath: "x", + }, + input: { events: [{ kind: "click", t: 5, x: 0.1, y: 0.2 }], strokes: [] }, + }); + await makeBundle(dir, "scrawl-crashed-0", { + [UNFINISHED_FILE]: journal, + "recording.mp4": "video", + }); + await makeBundle(dir, "scrawl-old-0", { "recording.webm": "video" }); + await makeBundle(dir, "scrawl-live-0", { + [UNFINISHED_FILE]: journal, + "recording.mp4": "video", + }); + await makeBundle(dir, "scrawl-built-0", { + "scrawl.json": "{}", + [UNFINISHED_FILE]: journal, + "recording.mp4": "video", + }); + await makeBundle(dir, "scrawl-tried-0", { + [REBUILT_ONCE_FILE]: "", + [UNFINISHED_FILE]: journal, + "recording.mp4": "video", + }); + await makeBundle(dir, "scrawl-empty-0", { [UNFINISHED_FILE]: journal }); + await makeBundle(dir, "not-a-capture", { "recording.mp4": "video" }); + + const { owed, empty } = planRebuilds(await readCaptureFolders(dir), "scrawl-live-0"); + + expect(owed.map((entry) => entry.id).toSorted()).toEqual(["scrawl-crashed-0", "scrawl-old-0"]); + expect(owed.find((entry) => entry.id === "scrawl-crashed-0")?.journal?.input.events).toEqual([ + { kind: "click", t: 5, x: 0.1, y: 0.2 }, + ]); + expect(empty.map((entry) => entry.id)).toEqual(["scrawl-empty-0"]); + }); + + it("treats half a journal as no journal, so the video is still rebuilt", async () => { + const dir = await makeCapturesDir(); + const folder = await makeBundle(dir, "scrawl-torn-0", { + [UNFINISHED_FILE]: '{"session":{"id"', + "recording.mp4": "video", + }); + const read = await readCaptureFolder(folder); + expect(read?.journal).toBeNull(); + expect(planRebuilds(read === null ? [] : [read], null).owed).toHaveLength(1); + }); + + it("a finished build clears the journal and stays built", async () => { + const dir = await makeCapturesDir(); + const folder = await makeBundle(dir, "scrawl-done-0", { + "scrawl.json": "{}", + [UNFINISHED_FILE]: "{}", + [REBUILT_ONCE_FILE]: "", + "timing.json": "{}", + "audio.pcm": "pcm", + "recording.mp4": "video", + }); + await removeJournal(folder); + expect((await NodeFSP.readdir(folder)).toSorted()).toEqual(["recording.mp4", "scrawl.json"]); + }); +}); + +describe("planRetention", () => { + const media = { kind: "media", paths: ["recording.mp4", "audio.wav"] }; + const bundle = { kind: "bundle", folder: capture().folder }; + const withJoinedPaths = (actions: ReturnType) => + actions.map((action) => + action.kind === "media" + ? { kind: "media", paths: action.paths.map((path) => NodePath.basename(path)) } + : action, + ); + + it.each([ + ["young", capture(), "week", 6, []], + ["media due", capture(), "week", 7, [media]], + ["bundle due", capture(), "week", 30, [bundle]], + ["longer preset", capture(), "month", 29, []], + ["longer preset, media due", capture(), "month", 30, [media]], + ["kept forever", capture(), "forever", 3_650, []], + ["pinned", capture({ pinned: true }), "week", 365, []], + ["owes a build", capture({ built: false }), "week", 365, []], + ["media already gone", capture({ mediaItems: [] }), "week", 7, []], + ["changed just now", capture({ changedAtMs: CREATED_AT + 7 * DAY_MS }), "week", 7, []], + ["rebuild failed", capture({ built: false, tried: true }), "week", 7, [media]], + ] as const)("%s", (_name, entry, retention, ageDays, expected) => { + const actions = planRetention([entry], retention, null, CREATED_AT + ageDays * DAY_MS); + expect(withJoinedPaths(actions)).toEqual(expected); + }); + + it("leaves the capture being recorded alone", () => { + const entry = capture(); + expect(planRetention([entry], "week", entry.id, CREATED_AT + 365 * DAY_MS)).toEqual([]); + }); + + it("removes on exactly the dates the list shows", () => { + const entry = capture(); + const { mediaAtMs, bundleAtMs } = retentionDates(entry, "week", null); + expect(planRetention([entry], "week", null, mediaAtMs! - 1)).toEqual([]); + expect(planRetention([entry], "week", null, mediaAtMs!)[0]?.kind).toBe("media"); + expect(planRetention([entry], "week", null, bundleAtMs! - 1)[0]?.kind).toBe("media"); + expect(planRetention([entry], "week", null, bundleAtMs!)[0]?.kind).toBe("bundle"); + }); +}); + +describe("readCaptureFolder", () => { + it("dates a capture by its id, not by the rebuild that wrote its manifest", async () => { + const dir = await makeCapturesDir(); + const id = `scrawl-${CREATED_AT.toString(36)}-0`; + const folder = await makeBundle(dir, id, { + "scrawl.json": JSON.stringify({ createdAtMs: CREATED_AT + 9 * DAY_MS, durationMs: 42 }), + [PINNED_FILE]: "", + }); + const read = await readCaptureFolder(folder); + expect(read).toMatchObject({ createdAtMs: CREATED_AT, durationMs: 42, pinned: true }); + expect(idTimeMs("scrawl-1730000000000-0")).toBeNull(); + }); +}); diff --git a/apps/desktop/src/scrawl/captureStore.ts b/apps/desktop/src/scrawl/captureStore.ts new file mode 100644 index 0000000000000..216ccdafaec00 --- /dev/null +++ b/apps/desktop/src/scrawl/captureStore.ts @@ -0,0 +1,286 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Bundles are plain folders read outside any Effect service. +// What is in the captures folder: which capture still owes a build, and what retention will remove +// and when. The list a user reads and the pass that deletes run on the same dates, so what the list +// promises is what happens. + +import { + ScrawlUnfinished, + type ScrawlCaptureStatus, + type ScrawlCaptureSummary, + type ScrawlRetention, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +/** Written at the start of a capture and removed once `scrawl.json` is there. */ +export const UNFINISHED_FILE = "unfinished.json"; +/** One rebuild try, so a rebuild that crashes the app cannot repeat at every launch. */ +export const REBUILT_ONCE_FILE = ".recovering"; +/** The same marker the Scrawl app uses to keep a bundle forever. */ +export const PINNED_FILE = ".pinned"; +const MANIFEST_FILE = "scrawl.json"; +const VIDEO_FILES = ["recording.mp4", "recording.webm"]; +// Only a build needs these: what the renderer journals next to the video while it records. +const JOURNAL_FILES = [UNFINISHED_FILE, REBUILT_ONCE_FILE, "timing.json", "audio.pcm"]; +// Frames, text and contact sheet stay; these are nearly all of a bundle's size. +const MEDIA_ITEMS = [ + ...VIDEO_FILES, + "audio.wav", + "audio.pcm", + "frames-full", + "contact-sheet-full.png", + "zoom", +]; +const DAY_MS = 24 * 60 * 60 * 1000; +const CAPTURE_FOLDER = /^(scrawl-[a-z0-9-]{1,64})\.scrawl$/; +// Ids carry their start in base 36. A Scrawl app bundle copied here counts in base 10 instead. +const MIN_ID_TIME_MS = Date.UTC(2020, 0, 1); +const MAX_ID_TIME_MS = Date.UTC(2100, 0, 1); + +const RETENTION_DAYS: Record< + ScrawlRetention, + { readonly media: number | null; readonly bundle: number | null } +> = { + week: { media: 7, bundle: 30 }, + month: { media: 30, bundle: 90 }, + forever: { media: null, bundle: null }, +}; + +export interface CaptureFolder { + readonly id: string; + readonly folder: string; + readonly built: boolean; + readonly videoBytes: number; + readonly tried: boolean; + readonly pinned: boolean; + readonly journaled: boolean; + readonly journal: ScrawlUnfinished | null; + readonly createdAtMs: number; + /** When an entry of the folder was last added, removed or renamed. */ + readonly changedAtMs: number; + readonly durationMs: number; + readonly app: string; + readonly windowTitle: string; + readonly sizeBytes: number; + readonly mediaBytes: number; + readonly mediaItems: ReadonlyArray; +} + +const sizeOf = async (path: string): Promise => { + const stat = await NodeFSP.lstat(path).catch(() => null); + if (stat === null) return 0; + if (!stat.isDirectory()) return stat.size; + const entries = await NodeFSP.readdir(path).catch(() => [] as Array); + const sizes = await Promise.all(entries.map((entry) => sizeOf(NodePath.join(path, entry)))); + return sizes.reduce((sum, size) => sum + size, 0); +}; + +const exists = (path: string) => + NodeFSP.access(path).then( + () => true, + () => false, + ); + +const readJson = async (path: string): Promise => { + const text = await NodeFSP.readFile(path, "utf8").catch(() => null); + if (text === null) return null; + try { + return JSON.parse(text); + } catch { + return null; + } +}; + +const decodeJournal = Schema.decodeUnknownSync(ScrawlUnfinished); +const readJournal = async (folder: string): Promise => { + const value = await readJson(NodePath.join(folder, UNFINISHED_FILE)); + if (value === null) return null; + try { + return decodeJournal(value); + } catch { + return null; + } +}; + +export const idTimeMs = (id: string): number | null => { + const time = Number.parseInt(id.split("-")[1] ?? "", 36); + return time >= MIN_ID_TIME_MS && time < MAX_ID_TIME_MS ? time : null; +}; + +const asString = (value: unknown) => (typeof value === "string" ? value : ""); +const asNumber = (value: unknown) => (typeof value === "number" ? value : null); +const asRecord = (value: unknown): Record => + typeof value === "object" && value !== null ? (value as Record) : {}; + +export const readCaptureFolder = async (folder: string): Promise => { + const id = CAPTURE_FOLDER.exec(NodePath.basename(folder))?.[1]; + if (id === undefined) return null; + const stat = await NodeFSP.stat(folder).catch(() => null); + if (stat === null || !stat.isDirectory()) return null; + const manifest = await readJson(NodePath.join(folder, MANIFEST_FILE)); + const facts = asRecord(manifest); + const target = asRecord(facts.target); + const journal = await readJournal(folder); + const mediaSizes = await Promise.all( + MEDIA_ITEMS.map(async (item) => { + const path = NodePath.join(folder, item); + return (await exists(path)) ? { item, bytes: await sizeOf(path) } : null; + }), + ); + const media = mediaSizes.filter((entry) => entry !== null); + return { + id, + folder, + built: manifest !== null, + videoBytes: media.reduce( + (sum, { item, bytes }) => (VIDEO_FILES.includes(item) ? sum + bytes : sum), + 0, + ), + tried: await exists(NodePath.join(folder, REBUILT_ONCE_FILE)), + pinned: await exists(NodePath.join(folder, PINNED_FILE)), + journaled: await exists(NodePath.join(folder, UNFINISHED_FILE)), + journal, + // The id before the manifest: a rebuild writes the time of the rebuild there. + createdAtMs: idTimeMs(id) ?? asNumber(facts.createdAtMs) ?? stat.birthtimeMs, + changedAtMs: Math.max(stat.mtimeMs, stat.ctimeMs), + durationMs: asNumber(facts.durationMs) ?? 0, + app: asString(target.app) || (journal?.session.app ?? ""), + windowTitle: asString(target.windowTitle) || (journal?.session.windowTitle ?? ""), + sizeBytes: await sizeOf(folder), + mediaBytes: media.reduce((sum, { bytes }) => sum + bytes, 0), + mediaItems: media.map(({ item }) => item), + }; +}; + +export const readCaptureFolders = async (capturesDir: string) => { + const entries = await NodeFSP.readdir(capturesDir).catch(() => [] as Array); + const folders = await Promise.all( + entries.map((entry) => readCaptureFolder(NodePath.join(capturesDir, entry))), + ); + return folders.filter((folder) => folder !== null); +}; + +const captureStatus = (capture: CaptureFolder, recordingId: string | null): ScrawlCaptureStatus => { + if (capture.id === recordingId) return "recording"; + if (capture.built) return "built"; + return capture.tried || capture.videoBytes === 0 ? "failed" : "unfinished"; +}; + +export const planRebuilds = ( + captures: ReadonlyArray, + recordingId: string | null, +): { + readonly owed: ReadonlyArray; + /** Journal but not one byte of video: the app went down in a capture's first seconds. */ + readonly empty: ReadonlyArray; +} => ({ + owed: captures.filter((capture) => captureStatus(capture, recordingId) === "unfinished"), + empty: captures.filter( + (capture) => + capture.id !== recordingId && + !capture.built && + capture.journaled && + capture.videoBytes === 0 && + !capture.pinned, + ), +}); + +/** What a build needs; a capture from before the journal has its video and nothing else. */ +export const unfinishedOf = (capture: CaptureFolder): ScrawlUnfinished => + capture.journal ?? { + session: { + id: capture.id, + sourceId: "", + width: 0, + height: 0, + startedAtMs: capture.createdAtMs, + app: "", + windowTitle: "", + bundlePath: capture.folder, + }, + input: { events: [], strokes: [] }, + }; + +export const retentionDates = ( + capture: CaptureFolder, + retention: ScrawlRetention, + recordingId: string | null, +): { readonly mediaAtMs: number | null; readonly bundleAtMs: number | null } => { + const status = captureStatus(capture, recordingId); + if (capture.pinned || status === "recording" || status === "unfinished") { + return { mediaAtMs: null, bundleAtMs: null }; + } + const due = (days: number | null) => + // A bundle changed within the last day (copied here, unpinned just now) waits a day. + days === null + ? null + : Math.max(capture.createdAtMs + days * DAY_MS, capture.changedAtMs + DAY_MS); + const { media, bundle } = RETENTION_DAYS[retention]; + return { + mediaAtMs: capture.mediaItems.length === 0 ? null : due(media), + bundleAtMs: due(bundle), + }; +}; + +export type RetentionAction = + | { readonly kind: "bundle"; readonly folder: string } + | { readonly kind: "media"; readonly paths: ReadonlyArray }; + +export const planRetention = ( + captures: ReadonlyArray, + retention: ScrawlRetention, + recordingId: string | null, + now: number, +): ReadonlyArray => + captures.flatMap((capture): ReadonlyArray => { + const { mediaAtMs, bundleAtMs } = retentionDates(capture, retention, recordingId); + if (bundleAtMs !== null && now >= bundleAtMs) { + return [{ kind: "bundle", folder: capture.folder }]; + } + if (mediaAtMs !== null && now >= mediaAtMs) { + return [ + { + kind: "media", + paths: capture.mediaItems.map((item) => NodePath.join(capture.folder, item)), + }, + ]; + } + return []; + }); + +export const summarize = ( + capture: CaptureFolder, + retention: ScrawlRetention, + recordingId: string | null, +): ScrawlCaptureSummary => { + const { mediaAtMs, bundleAtMs } = retentionDates(capture, retention, recordingId); + return { + id: capture.id, + bundlePath: capture.folder, + status: captureStatus(capture, recordingId), + createdAtMs: capture.createdAtMs, + durationMs: capture.durationMs, + app: capture.app, + windowTitle: capture.windowTitle, + sizeBytes: capture.sizeBytes, + mediaBytes: capture.mediaBytes, + pinned: capture.pinned, + mediaRemovedAtMs: mediaAtMs, + bundleRemovedAtMs: bundleAtMs, + }; +}; + +/** Never half a journal: a crash during this write keeps the previous one. */ +export const writeJournal = async (folder: string, journal: ScrawlUnfinished) => { + const target = NodePath.join(folder, UNFINISHED_FILE); + const temporary = `${target}.tmp`; + await NodeFSP.writeFile(temporary, JSON.stringify(journal)); + await NodeFSP.rename(temporary, target); +}; + +export const removeJournal = async (folder: string) => { + await Promise.all( + JOURNAL_FILES.map((file) => NodeFSP.rm(NodePath.join(folder, file), { force: true })), + ); +}; diff --git a/apps/web/src/components/AppSidebarLayout.tsx b/apps/web/src/components/AppSidebarLayout.tsx index 91df5616f7f50..f188a648fa0bb 100644 --- a/apps/web/src/components/AppSidebarLayout.tsx +++ b/apps/web/src/components/AppSidebarLayout.tsx @@ -12,6 +12,7 @@ import { useLocation, useRouter, type AnyRouter } from "@tanstack/react-router"; import { moveWorkspaceFocus, workspaceAreaProps } from "../lib/workspaceFocus"; import { isCommandPaletteOpen } from "../commandPaletteBus"; import { isElectron } from "../env"; +import { ScrawlRuntime } from "./scrawl/ScrawlRuntime"; import { releaseThreadUndoShortcut, threadUndoClaimsShortcut, @@ -476,6 +477,7 @@ export function AppSidebarLayout({ children }: { children: ReactNode }) { {isElectron ? : null} + {isElectron ? : null} ); diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index deb9092726641..e29c54ceb6d0c 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -158,6 +158,7 @@ import { resolveProjectPickerTarget, resolveWslProjectSelection, } from "../wslPaths"; +import { scrawlPaletteItems } from "./scrawl/scrawlPaletteItems"; import { resolveCommandPaletteProject, ADDON_ICON_CLASS, @@ -1525,6 +1526,8 @@ function OpenCommandPaletteDialog(props: { }); } + actionItems.push(...scrawlPaletteItems()); + actionItems.push({ kind: "action", value: "action:settings", diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index d40066b10aa89..701ad9e152d40 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -233,6 +233,7 @@ import { useDictation } from "../../dictation/useDictation"; import { ComposerDictationButton } from "./ComposerDictationButton"; import { ComposerScrawlButton } from "./ComposerScrawlButton"; import { cancelScrawl, type ScrawlDelivery } from "../../scrawl/scrawlCapture"; +import { isScrawlHistoryOpen } from "../../scrawl/scrawlHistory"; import { DictationHistoryDialog } from "./DictationHistoryDialog"; import { toastManager } from "../ui/toast"; import { PlayIcon } from "lucide-react"; @@ -2502,7 +2503,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const handler = (event: globalThis.KeyboardEvent) => { if (event.key === "Escape") { // The open history takes Esc for itself, so closing it cannot drop a running recording. - if (dictationHistoryOpen) return; + if (dictationHistoryOpen || isScrawlHistoryOpen()) return; if (cancelScrawl()) { event.preventDefault(); event.stopPropagation(); diff --git a/apps/web/src/components/chat/ComposerScrawlButton.tsx b/apps/web/src/components/chat/ComposerScrawlButton.tsx index 61f528088dd73..854092b05e5cd 100644 --- a/apps/web/src/components/chat/ComposerScrawlButton.tsx +++ b/apps/web/src/components/chat/ComposerScrawlButton.tsx @@ -1,7 +1,8 @@ -import { SCRAWL_TOGGLE_ACTION } from "@t3tools/contracts"; import { LoaderCircleIcon, ScreenShareIcon } from "lucide-react"; import { useEffect, useSyncExternalStore } from "react"; +import { useClientSettings } from "../../hooks/useSettings"; +import { formatSnapShotShortcutLabel } from "../../lib/snapShotShortcut"; import { getScrawlState, isScrawlAvailable, @@ -19,25 +20,16 @@ const NOTICE_TONES = { error: "text-destructive", } as const; -const SHORTCUT = "Alt+Shift+S"; - /** Records the screen with voice, clicks and ink. Only the desktop app can see the whole screen. */ export function ComposerScrawlButton(props: { readonly delivery: ScrawlDelivery }) { const { phase, notice } = useSyncExternalStore(subscribeScrawl, getScrawlState); const { delivery } = props; + const shortcut = useClientSettings((settings) => settings.scrawlShortcut); + const shortcutLabel = formatSnapShotShortcutLabel(shortcut); // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- offers this composer to a capture running outside React useEffect(() => registerScrawlDelivery(delivery), [delivery]); - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to the global shortcut, which arrives as a desktop menu action - useEffect( - () => - window.desktopBridge?.onMenuAction((action) => { - if (action === SCRAWL_TOGGLE_ACTION) toggleScrawl(); - }), - [], - ); - if (!isScrawlAvailable()) return null; const recording = phase === "recording"; const busy = phase === "starting" || phase === "building"; @@ -45,7 +37,7 @@ export function ComposerScrawlButton(props: { readonly delivery: ScrawlDelivery phase === "building" ? "Building the capture" : recording - ? `Recording this screen · ${SHORTCUT} finishes · Esc discards` + ? `Recording this screen · ${shortcutLabel} finishes · Esc discards` : null; return ( <> @@ -80,7 +72,9 @@ export function ComposerScrawlButton(props: { readonly delivery: ScrawlDelivery {busy ? : } - {recording ? `Finish capture (${SHORTCUT})` : `Capture the screen (${SHORTCUT})`} + {recording + ? `Finish capture (${shortcutLabel})` + : `Capture the screen (${shortcutLabel})`} diff --git a/apps/web/src/components/scrawl/ScrawlHistoryDialog.tsx b/apps/web/src/components/scrawl/ScrawlHistoryDialog.tsx new file mode 100644 index 0000000000000..e075513d4e149 --- /dev/null +++ b/apps/web/src/components/scrawl/ScrawlHistoryDialog.tsx @@ -0,0 +1,354 @@ +import type { ScrawlCaptureSummary } from "@t3tools/contracts"; +import { useNavigate } from "@tanstack/react-router"; +import { + CheckIcon, + CopyIcon, + CornerDownLeftIcon, + FolderOpenIcon, + PinIcon, + PinOffIcon, + RotateCcwIcon, + Trash2Icon, +} from "lucide-react"; +import { useEffect, useRef, useState, useSyncExternalStore } from "react"; + +import { useCopyToClipboard } from "../../hooks/useCopyToClipboard"; +import { useClientSettings } from "../../hooks/useSettings"; +import { hasScrawlComposer, insertPastCapture } from "../../scrawl/scrawlCapture"; +import { readCaptureFile } from "../../scrawl/scrawlFiles"; +import { + captureFate, + captureListChanged, + captureTitle, + formatBytes, + formatDuration, + getScrawlHistoryState, + setScrawlHistoryOpen, + subscribeScrawlHistory, +} from "../../scrawl/scrawlHistory"; +import { rebuildCapture } from "../../scrawl/scrawlRecovery"; +import { cn } from "~/lib/utils"; +import { Button } from "../ui/button"; +import { Dialog, DialogFooter, DialogHeader, DialogPopup, DialogTitle } from "../ui/dialog"; +import { ScrollArea } from "../ui/scroll-area"; +import { useScrawlCaptures } from "./useScrawlCaptures"; + +const RETENTION_SUMMARY = { + week: "Video stays for 7 days, the rest for 30. Pinned captures stay", + month: "Video stays for 30 days, the rest for 90. Pinned captures stay", + forever: "Captures stay until you delete them", +} as const; + +const dayLabel = (timestamp: number, now: Date) => { + const date = new Date(timestamp); + const yesterday = new Date(now); + yesterday.setDate(now.getDate() - 1); + if (date.toDateString() === now.toDateString()) return "Today"; + if (date.toDateString() === yesterday.toDateString()) return "Yesterday"; + return date.toLocaleDateString(undefined, { + weekday: "long", + day: "numeric", + month: "long", + ...(date.getFullYear() === now.getFullYear() ? {} : { year: "numeric" }), + }); +}; + +/** Reads the sheet only once its row scrolls into view; a long history must open at once. */ +function useContactSheetUrl(id: string, enabled: boolean) { + const [url, setUrl] = useState(null); + const [visible, setVisible] = useState(false); + const targetRef = useRef(null); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- watches the row's visibility with an IntersectionObserver + useEffect(() => { + const target = targetRef.current; + if (target === null || !enabled) return; + const observer = new IntersectionObserver((entries) => { + if (entries.some((entry) => entry.isIntersecting)) setVisible(true); + }); + observer.observe(target); + return () => observer.disconnect(); + }, [enabled]); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- reads the image from disk over the bridge and owns its object URL + useEffect(() => { + const read = window.desktopBridge?.scrawlReadFile; + if (!visible || read === undefined) return; + let cancelled = false; + let created: string | null = null; + void readCaptureFile(read, id, "contact-sheet.png", "image/png").then( + (blob) => { + if (cancelled) return; + created = URL.createObjectURL(blob); + setUrl(created); + }, + // Without a sheet the row just has no picture. + () => undefined, + ); + return () => { + cancelled = true; + if (created !== null) URL.revokeObjectURL(created); + }; + }, [id, visible]); + + return { url, targetRef }; +} + +function CaptureRow(props: { + readonly capture: ScrawlCaptureSummary; + readonly heading: string | null; + readonly building: boolean; + readonly now: number; + readonly composerOnScreen: boolean; + readonly onChanged: () => void; + readonly onError: (message: string) => void; + readonly onInsert: (capture: ScrawlCaptureSummary) => void; +}) { + const { capture, building, onChanged, onError } = props; + const built = capture.status === "built"; + const { url, targetRef } = useContactSheetUrl(capture.id, built); + const [expanded, setExpanded] = useState(false); + const { copyToClipboard, isCopied } = useCopyToClipboard(); + const bridge = window.desktopBridge; + const fate = captureFate(capture, props.now); + const title = captureTitle(capture); + + const act = (action: Promise | undefined) => { + void action?.then(onChanged, (error: unknown) => + onError(error instanceof Error ? error.message : String(error)), + ); + }; + + const copyText = () => { + const read = bridge?.scrawlReadFile; + if (read === undefined) return; + void readCaptureFile(read, capture.id, "summary.md") + .then((blob) => blob.text()) + .then( + (text) => copyToClipboard(text.trimEnd(), undefined), + () => onError("This capture has no text to copy"), + ); + }; + + return ( +
  • + {props.heading !== null ? ( +

    + {props.heading} +

    + ) : null} +
    + +
    + {built ? ( + <> + + + + ) : ( + + )} + + + +
    +
    + {expanded && url !== null ? ( + {`Contact + ) : null} +
    • + ); +} + +export function ScrawlHistoryDialog() { + const retention = useClientSettings((settings) => settings.scrawlRetention); + const navigate = useNavigate(); + const { building } = useSyncExternalStore(subscribeScrawlHistory, getScrawlHistoryState); + const { captures, failed } = useScrawlCaptures(); + const [actionError, setActionError] = useState(null); + const error = failed ? "Past captures could not be read. Reopen this to try again" : actionError; + const [nowDate] = useState(() => new Date()); + const pendingInsertRef = useRef(null); + + const close = () => setScrawlHistoryOpen(false); + const composerOnScreen = hasScrawlComposer(); + + return ( + { + if (!open) close(); + }} + onOpenChangeComplete={(open) => { + const capture = pendingInsertRef.current; + if (open || capture === null) return; + pendingInsertRef.current = null; + void insertPastCapture(capture.id, capture.bundlePath); + }} + > + + + Past screen captures + +
    + + {captures !== null && captures.length === 0 ? ( +

    No captures yet

    + ) : null} +
      + {(captures ?? []).map((capture, index, all) => { + const day = dayLabel(capture.createdAtMs, nowDate); + const previous = all[index - 1]; + const continues = + previous !== undefined && dayLabel(previous.createdAtMs, nowDate) === day; + return ( + { + setActionError(null); + captureListChanged(); + }} + onError={setActionError} + onInsert={(target) => { + pendingInsertRef.current = target; + close(); + }} + /> + ); + })} +
    +     +
    + +

    + {error ?? RETENTION_SUMMARY[retention]} +

    + +     +     +     + ); +} diff --git a/apps/web/src/components/scrawl/ScrawlRuntime.tsx b/apps/web/src/components/scrawl/ScrawlRuntime.tsx new file mode 100644 index 0000000000000..c0413432a6093 --- /dev/null +++ b/apps/web/src/components/scrawl/ScrawlRuntime.tsx @@ -0,0 +1,62 @@ +import { useAtomValue } from "@effect/atom-react"; +import { SCRAWL_TOGGLE_ACTION } from "@t3tools/contracts"; +import { useEffect, useSyncExternalStore } from "react"; + +import { resolveShortcutCommand } from "../../keybindings"; +import { getTerminalFocusOwner } from "../../lib/terminalFocus"; +import { isScrawlAvailable, toggleScrawl } from "../../scrawl/scrawlCapture"; +import { + getScrawlHistoryState, + isScrawlHistoryOpen, + setScrawlHistoryOpen, + subscribeScrawlHistory, +} from "../../scrawl/scrawlHistory"; +import { rebuildUnfinishedCaptures } from "../../scrawl/scrawlRecovery"; +import { primaryServerKeybindingsAtom } from "../../state/server"; +import { ScrawlHistoryDialog } from "./ScrawlHistoryDialog"; + +/** + * Lives as long as the app shell, because captures do: the shortcut works on pages without a + * composer, and captures the app went down on are rebuilt at launch whatever page opens first. + */ +export function ScrawlRuntime() { + const keybindings = useAtomValue(primaryServerKeybindingsAtom); + const { open } = useSyncExternalStore(subscribeScrawlHistory, getScrawlHistoryState); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to the global shortcut, which arrives as a desktop menu action + useEffect( + () => + window.desktopBridge?.onMenuAction((action) => { + if (action === SCRAWL_TOGGLE_ACTION) toggleScrawl(); + }), + [], + ); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- starts the one rebuild pass of this page load, which runs outside React + useEffect(() => { + if (!isScrawlAvailable()) return; + void rebuildUnfinishedCaptures().catch((error: unknown) => { + console.warn("scrawl: unfinished captures could not be listed", error); + }); + }, []); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- window keydown listener for the past captures keybinding + useEffect(() => { + if (!isScrawlAvailable()) return; + const handler = (event: KeyboardEvent) => { + if (event.target instanceof HTMLElement && event.target.closest("[data-keybinding-capture]")) + return; + const command = resolveShortcutCommand(event, keybindings, { + context: { terminalFocus: getTerminalFocusOwner() !== null }, + }); + if (command !== "scrawl.history") return; + event.preventDefault(); + event.stopPropagation(); + if (!event.repeat) setScrawlHistoryOpen(!isScrawlHistoryOpen()); + }; + window.addEventListener("keydown", handler, true); + return () => window.removeEventListener("keydown", handler, true); + }, [keybindings]); + + return open ? : null; +} diff --git a/apps/web/src/components/scrawl/scrawlPaletteItems.tsx b/apps/web/src/components/scrawl/scrawlPaletteItems.tsx new file mode 100644 index 0000000000000..07136a77a050c --- /dev/null +++ b/apps/web/src/components/scrawl/scrawlPaletteItems.tsx @@ -0,0 +1,59 @@ +import { CircleStopIcon, HistoryIcon, ScreenShareIcon, Trash2Icon } from "lucide-react"; + +import { + cancelScrawl, + getScrawlState, + isScrawlAvailable, + toggleScrawl, +} from "../../scrawl/scrawlCapture"; +import { setScrawlHistoryOpen } from "../../scrawl/scrawlHistory"; +import { ITEM_ICON_CLASS, type CommandPaletteActionItem } from "../CommandPalette.logic"; + +export function scrawlPaletteItems(): ReadonlyArray { + if (!isScrawlAvailable()) return []; + const { phase } = getScrawlState(); + const past: CommandPaletteActionItem = { + kind: "action", + value: "action:scrawl-history", + searchTerms: ["past screen captures", "scrawl history", "recordings", "reopen capture"], + title: "Past screen captures", + icon: , + shortcutCommand: "scrawl.history", + run: async () => setScrawlHistoryOpen(true), + }; + if (phase === "recording") { + return [ + { + kind: "action", + value: "action:scrawl-finish", + searchTerms: ["finish screen capture", "stop recording", "scrawl"], + title: "Finish screen capture", + icon: , + run: async () => toggleScrawl(), + }, + { + kind: "action", + value: "action:scrawl-discard", + searchTerms: ["discard screen capture", "cancel recording", "scrawl"], + title: "Discard screen capture", + icon: , + run: async () => { + cancelScrawl(); + }, + }, + past, + ]; + } + if (phase !== "idle") return [past]; + return [ + { + kind: "action", + value: "action:scrawl-start", + searchTerms: ["capture the screen", "screen recording", "record screen", "scrawl"], + title: "Capture the screen", + icon: , + run: async () => toggleScrawl(), + }, + past, + ]; +} diff --git a/apps/web/src/components/scrawl/useScrawlCaptures.ts b/apps/web/src/components/scrawl/useScrawlCaptures.ts new file mode 100644 index 0000000000000..5cd3803b2a4a3 --- /dev/null +++ b/apps/web/src/components/scrawl/useScrawlCaptures.ts @@ -0,0 +1,42 @@ +import type { ScrawlCaptureSummary } from "@t3tools/contracts"; +import { useEffect, useState } from "react"; + +import { getScrawlHistoryState, subscribeScrawlHistory } from "../../scrawl/scrawlHistory"; + +/** The captures folder as the desktop app reads it, again whenever something changed it. */ +export function useScrawlCaptures() { + const [captures, setCaptures] = useState | null>(null); + const [failed, setFailed] = useState(false); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- reads the folder over the desktop bridge, and follows changes made outside this component + useEffect(() => { + let request = 0; + const load = () => { + const current = ++request; + void window.desktopBridge?.scrawlListCaptures?.().then( + (list) => { + if (current !== request) return; + setCaptures(list.filter((capture) => capture.status !== "recording")); + setFailed(false); + }, + () => { + if (current === request) setFailed(true); + }, + ); + }; + load(); + let revision = getScrawlHistoryState().revision; + const unsubscribe = subscribeScrawlHistory(() => { + const next = getScrawlHistoryState().revision; + if (next === revision) return; + revision = next; + load(); + }); + return () => { + request = -1; + unsubscribe(); + }; + }, []); + + return { captures, failed }; +} diff --git a/apps/web/src/components/settings/KeybindingsSettings.logic.ts b/apps/web/src/components/settings/KeybindingsSettings.logic.ts index 231f6789940ba..5886cacda0e89 100644 --- a/apps/web/src/components/settings/KeybindingsSettings.logic.ts +++ b/apps/web/src/components/settings/KeybindingsSettings.logic.ts @@ -376,6 +376,7 @@ export function commandLabel(command: KeybindingCommand): string { if (command === "composer.dictate") return "Composer: Dictate"; if (command === "composer.dictateStructured") return "Composer: Dictate as Structured Prompt"; if (command === "composer.dictationHistory") return "Composer: Dictation History"; + if (command === "scrawl.history") return "Past Screen Captures"; if (command === "thread.steerQueuedMessage") return "Queue: Send First Queued Message"; if (command === "thread.editQueuedMessage") return "Queue: Edit Last Queued Message"; if (command === "thread.copyReference") return "Pull Request: Copy Link or Thread"; diff --git a/apps/web/src/components/settings/ScrawlSettingsSection.tsx b/apps/web/src/components/settings/ScrawlSettingsSection.tsx new file mode 100644 index 0000000000000..a83ee5655851b --- /dev/null +++ b/apps/web/src/components/settings/ScrawlSettingsSection.tsx @@ -0,0 +1,273 @@ +import { useAtomValue } from "@effect/atom-react"; +import { + DEFAULT_SCRAWL_SHORTCUT, + SCRAWL_SHORTCUT_STATE_ACTION, + isModifierPairShortcut, + type ScrawlCaptureSummary, + type ScrawlRetention, + type ScrawlShortcutAvailability, + type ScrawlShortcutState, + type SnapShotKeyChord, +} from "@t3tools/contracts"; +import { useEffect, useRef, useState } from "react"; + +import { updateClientSettings, useClientSettings } from "../../hooks/useSettings"; +import { + formatSnapShotShortcutLabel, + sameSnapShotShortcut, + snapShotKeybindingConflict, +} from "../../lib/snapShotShortcut"; +import { isScrawlAvailable } from "../../scrawl/scrawlCapture"; +import { formatBytes, setScrawlHistoryOpen } from "../../scrawl/scrawlHistory"; +import { primaryServerKeybindingsAtom } from "../../state/server"; +import { useScrawlCaptures } from "../scrawl/useScrawlCaptures"; +import { Button } from "../ui/button"; +import { Menu, MenuPopup, MenuRadioGroup, MenuRadioItem, MenuTrigger } from "../ui/menu"; +import { SelectButton } from "../ui/select"; +import { commandLabel } from "./KeybindingsSettings.logic"; +import { SettingsRow, SettingsSection, SettingsUnavailableGroup } from "./settingsLayout"; +import { searchableSetting } from "./settingsSearch"; +import { useSnapShotShortcutRecorder } from "./useSnapShotShortcutRecorder"; + +const RETENTION_LABELS: Record = { + week: "Video 7 days, capture 30", + month: "Video 30 days, capture 90", + forever: "Forever", +}; + +type ShortcutCheck = + | { readonly status: "idle" } + | { readonly status: "checking"; readonly shortcut: SnapShotKeyChord } + | { + readonly status: "checked"; + readonly shortcut: SnapShotKeyChord; + readonly availability: ScrawlShortcutAvailability; + }; + +/** Follows the desktop app's word on the shortcut: another program can let go of it at any time. */ +function useShortcutState() { + const [state, setState] = useState(null); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to the desktop app, which announces when it takes or loses the shortcut + useEffect(() => { + const bridge = window.desktopBridge; + if (bridge?.scrawlGetShortcutState === undefined) return; + const refresh = () => { + void bridge.scrawlGetShortcutState?.().then(setState, () => undefined); + }; + refresh(); + return bridge.onMenuAction((action) => { + if (action === SCRAWL_SHORTCUT_STATE_ACTION) refresh(); + }); + }, []); + return state; +} + +const overviewText = (captures: ReadonlyArray) => { + if (captures.length === 0) return "Find a capture again, pin it, or see when it goes"; + const bytes = captures.reduce((sum, capture) => sum + capture.sizeBytes, 0); + const owed = captures.filter((capture) => capture.status === "failed").length; + const count = captures.length === 1 ? "1 capture" : `${captures.length} captures`; + return [`${count} using ${formatBytes(bytes)}`, owed > 0 ? `${owed} could not be rebuilt` : null] + .filter(Boolean) + .join(", "); +}; + +export function ScrawlSettingsSection() { + const saved = useClientSettings((settings) => settings.scrawlShortcut); + const retention = useClientSettings((settings) => settings.scrawlRetention); + const snapShotShortcut = useClientSettings((settings) => settings.snapShotShortcut); + const keybindings = useAtomValue(primaryServerKeybindingsAtom); + const shortcutState = useShortcutState(); + const { captures } = useScrawlCaptures(); + const available = isScrawlAvailable(); + + const [candidate, setCandidate] = useState(saved); + const [latestCheck, setCheck] = useState({ status: "idle" }); + const [saving, setSaving] = useState(false); + const checkIdRef = useRef(0); + + const [renderedSaved, setRenderedSaved] = useState(saved); + if (!sameSnapShotShortcut(renderedSaved, saved)) { + setRenderedSaved(saved); + setCandidate(saved); + setCheck({ status: "idle" }); + } + + const changed = !sameSnapShotShortcut(candidate, saved); + const check = + latestCheck.status !== "idle" && sameSnapShotShortcut(latestCheck.shortcut, candidate) + ? latestCheck + : ({ status: "idle" } as const); + const keybindingConflict = changed ? snapShotKeybindingConflict(candidate, keybindings) : null; + const usedBySnapShots = changed && sameSnapShotShortcut(candidate, snapShotShortcut); + const canSave = + changed && + keybindingConflict === null && + !usedBySnapShots && + check.status === "checked" && + check.availability.available; + + const checkShortcut = async (shortcut: SnapShotKeyChord) => { + const checkId = ++checkIdRef.current; + setCandidate(shortcut); + setCheck({ status: "checking", shortcut }); + const availability = await window.desktopBridge?.scrawlCheckShortcut?.(shortcut).catch(() => ({ + available: false, + message: "Could not check this shortcut", + })); + if (availability === undefined || checkId !== checkIdRef.current) return; + setCheck({ status: "checked", shortcut, availability }); + }; + + const reset = () => { + checkIdRef.current++; + setCandidate(saved); + setCheck({ status: "idle" }); + }; + + const { recording, stopRecording, input } = useSnapShotShortcutRecorder({ + shortcut: candidate, + disabled: saving || !available, + allowModifierPairs: false, + subject: "screen capture", + onStart: () => { + checkIdRef.current++; + setCheck({ status: "idle" }); + }, + onRecord: (shortcut) => { + if (!isModifierPairShortcut(shortcut)) void checkShortcut(shortcut); + }, + onError: (message) => + setCheck({ + status: "checked", + shortcut: candidate, + availability: { available: false, message }, + }), + }); + + const save = async (shortcut: SnapShotKeyChord) => { + setSaving(true); + try { + await updateClientSettings({ scrawlShortcut: shortcut }); + } catch { + setCheck({ + status: "checked", + shortcut, + availability: { available: false, message: "The shortcut could not be saved" }, + }); + } finally { + setSaving(false); + } + }; + + const savedLabel = formatSnapShotShortcutLabel(saved); + const status = recording + ? "Press your shortcut. Esc cancels" + : keybindingConflict !== null + ? `T3 Code already uses this for "${commandLabel(keybindingConflict)}"` + : usedBySnapShots + ? "SnapShots already uses this shortcut" + : check.status === "checking" + ? "Checking shortcut…" + : check.status === "checked" + ? check.availability.available + ? "Ready to save" + : (check.availability.message ?? "This shortcut is not available") + : shortcutState?.registered === false + ? `Another program is holding ${savedLabel}. T3 Code takes it as soon as it is free, and the composer button works until then` + : undefined; + + return ( + + + + {input} + {changed ? ( + <> + + + + ) : !sameSnapShotShortcut(saved, DEFAULT_SCRAWL_SHORTCUT) ? ( + + ) : null} + + } + /> + + } + className="w-auto min-w-0" + disabled={!available} + > + {RETENTION_LABELS[retention]} + + + + void updateClientSettings({ scrawlRetention: value as ScrawlRetention }) + } + > + {(Object.keys(RETENTION_LABELS) as Array).map((value) => ( + + {RETENTION_LABELS[value]} + + ))} + + + + } + /> + setScrawlHistoryOpen(true)} + > + Show captures + + } + /> + + + ); +} diff --git a/apps/web/src/components/settings/SnapShotSettings.tsx b/apps/web/src/components/settings/SnapShotSettings.tsx index c69f1e7869149..622e10aeebe15 100644 --- a/apps/web/src/components/settings/SnapShotSettings.tsx +++ b/apps/web/src/components/settings/SnapShotSettings.tsx @@ -47,6 +47,7 @@ import { SettingsRow, SettingsSection, } from "./settingsLayout"; +import { ScrawlSettingsSection } from "./ScrawlSettingsSection"; import { searchableSetting } from "./settingsSearch"; import { Button } from "../ui/button"; import { Menu, MenuItem, MenuPopup, MenuRadioGroup, MenuRadioItem, MenuTrigger } from "../ui/menu"; @@ -560,6 +561,7 @@ export function SnapShotSettings() { ) : null} + {wizard && state ? ( void; onStart?: () => void; onError: (message: string) => void; @@ -127,8 +129,8 @@ export function useSnapShotShortcutRecorder({ disabled={disabled} aria-label={ displayShortcut - ? `Record snapshot shortcut, currently ${formatSnapShotShortcutLabel(displayShortcut)}` - : "Change snapshot shortcut" + ? `Record ${subject} shortcut, currently ${formatSnapShotShortcutLabel(displayShortcut)}` + : `Change ${subject} shortcut` } aria-pressed={recording} data-keybinding-capture="" diff --git a/apps/web/src/scrawl/sceneChanges.ts b/apps/web/src/scrawl/sceneChanges.ts index 7f5035feb5258..15f097b578d23 100644 --- a/apps/web/src/scrawl/sceneChanges.ts +++ b/apps/web/src/scrawl/sceneChanges.ts @@ -3,6 +3,17 @@ // the fourth-worst block, and real transitions across at least four blocks still register. export const THUMB_WIDTH = 64; +/** Perceived brightness of an RGBA thumbnail, one value per pixel. */ +export const lumaOf = (rgba: Uint8ClampedArray) => { + const luma = new Float32Array(rgba.length / 4); + for (let index = 0; index < luma.length; index++) { + const offset = index * 4; + luma[index] = + (0.2126 * rgba[offset]! + 0.7152 * rgba[offset + 1]! + 0.0722 * rgba[offset + 2]!) / 255; + } + return luma; +}; + export const THUMB_HEIGHT = 36; const COLUMNS = 8; const ROWS = 6; diff --git a/apps/web/src/scrawl/scrawlBundle.ts b/apps/web/src/scrawl/scrawlBundle.ts index f90df615ecd55..edc4c1814d983 100644 --- a/apps/web/src/scrawl/scrawlBundle.ts +++ b/apps/web/src/scrawl/scrawlBundle.ts @@ -146,7 +146,7 @@ function activityBox(recording: Pick): { return width * height > MAX_CROP_AREA ? null : { x, y, width, height }; } -const seekTo = (video: HTMLVideoElement, seconds: number) => +export const seekTo = (video: HTMLVideoElement, seconds: number) => new Promise((resolve, reject) => { const onSeeked = () => { video.removeEventListener("seeked", onSeeked); @@ -163,7 +163,7 @@ const seekTo = (video: HTMLVideoElement, seconds: number) => video.currentTime = seconds; }); -const loadVideo = async (blob: Blob) => { +export const loadVideo = async (blob: Blob) => { const video = document.createElement("video"); video.muted = true; video.preload = "auto"; diff --git a/apps/web/src/scrawl/scrawlCapture.ts b/apps/web/src/scrawl/scrawlCapture.ts index f431c3b7cb04d..3d8ce00c5090b 100644 --- a/apps/web/src/scrawl/scrawlCapture.ts +++ b/apps/web/src/scrawl/scrawlCapture.ts @@ -6,13 +6,29 @@ import type { DesktopBridge, ScrawlStartResult } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; import { startMicrophoneCapture, type MicrophoneCapture } from "../dictation/microphoneCapture"; +import { toastManager } from "../components/ui/toast"; import { DEFAULT_DICTATION_LANGUAGE } from "../dictation/dictationLanguages"; import { DICTATION_LANGUAGE_KEY } from "../dictation/useDictation"; import { getLocalStorageItem } from "../hooks/useLocalStorage"; import { randomUUID } from "../lib/utils"; import { wordsOrWhole } from "./narration"; import { buildBundle } from "./scrawlBundle"; -import { findSceneChanges, THUMB_HEIGHT, THUMB_WIDTH, type LumaSample } from "./sceneChanges"; +import { readCaptureFile, readOptionalCaptureFile } from "./scrawlFiles"; +import { + claimBuild, + encodeWav, + releaseBuild, + SCRAWL_AUDIO_FILE, + SCRAWL_TIMING_FILE, + type ScrawlTiming, +} from "./scrawlHistory"; +import { + findSceneChanges, + lumaOf, + THUMB_HEIGHT, + THUMB_WIDTH, + type LumaSample, +} from "./sceneChanges"; const SEND_INTERVAL_MS = 500; const SAMPLE_INTERVAL_MS = 750; @@ -45,9 +61,7 @@ interface Running { readonly mic: MicrophoneCapture; readonly recorder: MediaRecorder; readonly videoName: string; - readonly chunks: Array; readonly samples: Array; - readonly pcm: Array; readonly micStartedAt: number; readonly language: string; readonly pcmTimer: ReturnType; @@ -70,9 +84,11 @@ const setState = (next: ScrawlState) => { for (const listener of listeners) listener(); }; -const notify = (notice: NonNullable) => { +const notify = (notice: NonNullable, phase: ScrawlPhase = "idle") => { if (noticeTimer !== null) clearTimeout(noticeTimer); - setState({ phase: "idle", notice }); + setState({ phase, notice }); + // Notices show above the composer; Settings has none, and the word must still reach the user. + if (delivery === null) toastManager.add({ type: notice.tone, title: notice.text }); noticeTimer = setTimeout(() => { noticeTimer = null; if (state.notice === notice) setState({ ...state, notice: null }); @@ -100,6 +116,8 @@ const scrawlBridge = (): DesktopBridge | null => export const isScrawlAvailable = () => scrawlBridge() !== null; +export const hasScrawlComposer = () => delivery !== null; + const openScreen = (sourceId: string) => navigator.mediaDevices.getUserMedia({ audio: false, @@ -135,14 +153,7 @@ const startSampling = (track: MediaStreamTrack, startedAt: number, samples: Arra context.drawImage(frame, 0, 0, THUMB_WIDTH, THUMB_HEIGHT); frame.close(); const { data } = context.getImageData(0, 0, THUMB_WIDTH, THUMB_HEIGHT); - const luma = new Float32Array(THUMB_WIDTH * THUMB_HEIGHT); - for (let index = 0; index < luma.length; index++) { - const offset = index * 4; - luma[index] = - (0.2126 * data[offset]! + 0.7152 * data[offset + 1]! + 0.0722 * data[offset + 2]!) / - 255; - } - samples.push({ t, luma }); + samples.push({ t, luma: lumaOf(data) }); }) // A frame missed while the track starts or ends only thins the samples. .catch(() => undefined) @@ -159,38 +170,20 @@ const decodeBase64 = (value: string) => { return bytes; }; -/** 16 kHz mono 16-bit, as the dictation pipeline records it. */ -const encodeWav = (pieces: ReadonlyArray) => { - const length = pieces.reduce((sum, piece) => sum + piece.length, 0); - const wav = new Uint8Array(44 + length); - const view = new DataView(wav.buffer); - const ascii = (offset: number, text: string) => { - for (let index = 0; index < text.length; index++) { - view.setUint8(offset + index, text.charCodeAt(index)); - } - }; - ascii(0, "RIFF"); - view.setUint32(4, 36 + length, true); - ascii(8, "WAVEfmt "); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); - view.setUint16(22, 1, true); - view.setUint32(24, 16_000, true); - view.setUint32(28, 32_000, true); - view.setUint16(32, 2, true); - view.setUint16(34, 16, true); - ascii(36, "data"); - view.setUint32(40, length, true); - let offset = 44; - for (const piece of pieces) { - wav.set(piece, offset); - offset += piece.length; - } - return wav; +/** Everything recorded goes to disk as it comes: a crash keeps it, and memory stays flat. */ +const appendToDisk = ( + bridge: DesktopBridge, + capture: Running, + name: string, + bytes: () => Promise | Uint8Array, +) => { + capture.writes = capture.writes.then(async () => { + await bridge.scrawlAppendVideo?.({ id: capture.session.id, name, bytes: await bytes() }); + }); }; const sendPcm = (bridge: DesktopBridge, capture: Running, pcm: string) => { - capture.pcm.push(pcm); + appendToDisk(bridge, capture, SCRAWL_AUDIO_FILE, () => decodeBase64(pcm)); capture.sending = capture.sending.then(async () => { // The recording goes on without words when the speech model is missing; the end says so. await bridge @@ -222,9 +215,7 @@ const start = async (bridge: DesktopBridge) => { mic, recorder, videoName: format.name, - chunks: [], samples, - pcm: [], micStartedAt, language: getLocalStorageItem(DICTATION_LANGUAGE_KEY, Schema.String) ?? DEFAULT_DICTATION_LANGUAGE, @@ -238,18 +229,29 @@ const start = async (bridge: DesktopBridge) => { }; recorder.onstart = () => { capture.videoStartedAt = Date.now(); + const timing: ScrawlTiming = { + micStartedAtMs: capture.micStartedAt, + videoStartedAtMs: capture.videoStartedAt, + language: capture.language, + sessionId: capture.sessionId, + }; + void bridge + .scrawlWriteFile?.({ + id: capture.session.id, + path: SCRAWL_TIMING_FILE, + bytes: new TextEncoder().encode(JSON.stringify(timing)), + }) + // A rebuild without it estimates the offset from the lengths of video and audio. + .catch(() => undefined); }; recorder.ondataavailable = (event) => { if (event.data.size === 0) return; - capture.chunks.push(event.data); - // Written as it comes, so a crash keeps the video recorded so far. - capture.writes = capture.writes.then(async () => { - await bridge.scrawlAppendVideo?.({ - id: capture.session.id, - name: capture.videoName, - bytes: new Uint8Array(await event.data.arrayBuffer()), - }); - }); + appendToDisk( + bridge, + capture, + capture.videoName, + async () => new Uint8Array(await event.data.arrayBuffer()), + ); }; recorder.start(VIDEO_TIMESLICE_MS); running = capture; @@ -282,19 +284,27 @@ const stopRecorder = (recorder: MediaRecorder) => recorder.stop(); }); -const deliver = async (text: string, contactSheet: Blob, bundlePath: string) => { +const deliver = async ( + text: string, + contactSheet: Blob, + bundlePath: string, + phase: ScrawlPhase = "idle", +) => { if (delivery === null || !delivery.insert(`${text}\n`)) { // The clipboard refuses a window that is not in front, and this one usually is not. const copied = await navigator.clipboard?.writeText(text).then( () => true, () => false, ); - notify({ - text: copied - ? "No composer can take the capture · its text is on the clipboard" - : `No composer can take the capture · it is saved in ${bundlePath}`, - tone: "warning", - }); + notify( + { + text: copied + ? "No composer can take the capture · its text is on the clipboard" + : `No composer can take the capture · it is saved in ${bundlePath}`, + tone: "warning", + }, + phase, + ); return; } const attached = await delivery.attachImage( @@ -304,12 +314,29 @@ const deliver = async (text: string, contactSheet: Blob, bundlePath: string) => attached ? { text: "Capture added to the composer", tone: "success" } : { text: "Capture text added · the contact sheet could not be attached", tone: "warning" }, + phase, ); }; +/** Puts a past capture into the composer on screen, as if it had just finished. */ +export const insertPastCapture = async (id: string, bundlePath: string) => { + const read = scrawlBridge()?.scrawlReadFile; + if (read === undefined) return; + // Never ends a capture in progress: this is not the end of the one being recorded. + const phase = state.phase; + try { + const text = await (await readCaptureFile(read, id, "summary.md")).text(); + const sheet = await readCaptureFile(read, id, "contact-sheet.png", "image/png"); + await deliver(text.trimEnd(), sheet, bundlePath, phase); + } catch (error) { + notify({ text: `The capture could not be read: ${describe(error)}`, tone: "error" }, phase); + } +}; + const finish = async (bridge: DesktopBridge, capture: Running) => { running = null; setState({ phase: "building", notice: null }); + claimBuild(capture.session.id); try { const input = await bridge.scrawlStop!({ id: capture.session.id }); const durationMs = Date.now() - capture.micStartedAt; @@ -318,6 +345,9 @@ const finish = async (bridge: DesktopBridge, capture: Running) => { release(capture); await capture.sending; await capture.writes; + const read = bridge.scrawlReadFile!; + const { id } = capture.session; + const pcm = await readOptionalCaptureFile(read, id, SCRAWL_AUDIO_FILE); const transcript = await bridge.dictationFinish?.({ sessionId: capture.sessionId, words: true, @@ -335,16 +365,18 @@ const finish = async (bridge: DesktopBridge, capture: Running) => { words, transcriptText, model: "parakeet", - video: new Blob(capture.chunks, { type: capture.recorder.mimeType }), + // Only now, for as long as the build runs; nothing of it stayed around during the capture. + video: await readCaptureFile(read, id, capture.videoName, capture.recorder.mimeType), videoName: capture.videoName, videoOffsetMs: capture.videoStartedAt - capture.micStartedAt, inputOffsetMs: capture.micStartedAt - capture.session.startedAtMs, durationMs, sceneChanges: findSceneChanges(capture.samples), - wav: encodeWav(capture.pcm.map(decodeBase64)), + wav: encodeWav(new Uint8Array((await pcm?.arrayBuffer()) ?? new ArrayBuffer(0))), }, (path, bytes) => bridge.scrawlWriteFile!({ id: capture.session.id, path, bytes }), ); + await bridge.scrawlMarkBuilt?.({ id: capture.session.id }).catch(() => undefined); await deliver(built.text, built.contactSheet, capture.session.bundlePath); if (transcript !== undefined && !transcript.ok) { notify({ text: `Capture has no transcript: ${transcript.reason}`, tone: "warning" }); @@ -352,9 +384,11 @@ const finish = async (bridge: DesktopBridge, capture: Running) => { } catch (error) { release(capture); notify({ - text: `The capture could not be built: ${describe(error)}. Its video is in ${capture.session.bundlePath}`, + text: `The capture could not be built: ${describe(error)}. Rebuild it from Past captures`, tone: "error", }); + } finally { + releaseBuild(capture.session.id); } }; diff --git a/apps/web/src/scrawl/scrawlFiles.test.ts b/apps/web/src/scrawl/scrawlFiles.test.ts new file mode 100644 index 0000000000000..26547d834f50c --- /dev/null +++ b/apps/web/src/scrawl/scrawlFiles.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { readCaptureFile, type ScrawlFileReader } from "./scrawlFiles"; + +const readerOver = (file: Uint8Array): ScrawlFileReader => { + return async ({ offset, length }) => file.subarray(offset, offset + length); +}; + +describe("readCaptureFile", () => { + it.each([ + ["empty", 0], + ["within one piece", 1_000], + ["exactly one piece", 8 * 1024 * 1024], + ["several pieces", 8 * 1024 * 1024 * 2 + 17], + ])("returns the whole file in order: %s", async (_name, size) => { + const file = new Uint8Array(size); + for (let index = 0; index < size; index += 1) file[index] = index % 251; + const blob = await readCaptureFile( + readerOver(file), + "scrawl-a-0", + "recording.mp4", + "video/mp4", + ); + expect(blob.type).toBe("video/mp4"); + const read = Buffer.from(await blob.arrayBuffer()); + expect(read.equals(file)).toBe(true); + }); +}); diff --git a/apps/web/src/scrawl/scrawlFiles.ts b/apps/web/src/scrawl/scrawlFiles.ts new file mode 100644 index 0000000000000..03664951c95f8 --- /dev/null +++ b/apps/web/src/scrawl/scrawlFiles.ts @@ -0,0 +1,28 @@ +// A capture's files stay on disk. A Blob made of Blobs is handed to the browser's blob store piece +// by piece, which pages to disk when large, so this keeps nothing in this process's heap. + +import type { DesktopBridge } from "@t3tools/contracts"; + +const PIECE_BYTES = 8 * 1024 * 1024; + +export type ScrawlFileReader = NonNullable; + +export const readCaptureFile = async ( + read: ScrawlFileReader, + id: string, + path: string, + type = "", +): Promise => { + const pieces: Array = []; + let offset = 0; + for (;;) { + const bytes = await read({ id, path, offset, length: PIECE_BYTES }); + if (bytes.length > 0) pieces.push(new Blob([bytes as Uint8Array])); + offset += bytes.length; + if (bytes.length < PIECE_BYTES) return new Blob(pieces, { type }); + } +}; + +/** A file the capture may not have got to write, such as audio from a muted recording. */ +export const readOptionalCaptureFile = (read: ScrawlFileReader, id: string, path: string) => + readCaptureFile(read, id, path).catch(() => null); diff --git a/apps/web/src/scrawl/scrawlHistory.ts b/apps/web/src/scrawl/scrawlHistory.ts new file mode 100644 index 0000000000000..56a636e44f987 --- /dev/null +++ b/apps/web/src/scrawl/scrawlHistory.ts @@ -0,0 +1,120 @@ +// What the history dialog, Settings and the rebuild share: whether the dialog is open, which captures +// are being built right now, and the journal files a capture keeps next to its video. + +import type { ScrawlCaptureSummary } from "@t3tools/contracts"; + +export const SCRAWL_AUDIO_FILE = "audio.pcm"; +export const SCRAWL_TIMING_FILE = "timing.json"; + +export interface ScrawlTiming { + readonly micStartedAtMs: number; + readonly videoStartedAtMs: number; + readonly language: string; + readonly sessionId: string; +} + +interface HistoryState { + readonly open: boolean; + /** Bumped when the list on disk changed behind the dialog's back. */ + readonly revision: number; + readonly building: ReadonlySet; +} + +let state: HistoryState = { open: false, revision: 0, building: new Set() }; +const listeners = new Set<() => void>(); + +const setState = (next: HistoryState) => { + state = next; + for (const listener of listeners) listener(); +}; + +export const subscribeScrawlHistory = (listener: () => void) => { + listeners.add(listener); + return () => listeners.delete(listener); +}; +export const getScrawlHistoryState = () => state; +export const isScrawlHistoryOpen = () => state.open; +export const setScrawlHistoryOpen = (open: boolean) => setState({ ...state, open }); + +export const captureListChanged = () => setState({ ...state, revision: state.revision + 1 }); + +export const claimBuild = (id: string) => + setState({ ...state, building: new Set(state.building).add(id) }); +export const releaseBuild = (id: string) => { + const building = new Set(state.building); + building.delete(id); + setState({ ...state, building, revision: state.revision + 1 }); +}; + +/** 16 kHz mono 16-bit, as the dictation pipeline records it. */ +export const encodeWav = (pcm: Uint8Array) => { + const wav = new Uint8Array(44 + pcm.length); + const view = new DataView(wav.buffer); + const ascii = (offset: number, text: string) => { + for (let index = 0; index < text.length; index++) { + view.setUint8(offset + index, text.charCodeAt(index)); + } + }; + ascii(0, "RIFF"); + view.setUint32(4, 36 + pcm.length, true); + ascii(8, "WAVEfmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, 16_000, true); + view.setUint32(28, 32_000, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + ascii(36, "data"); + view.setUint32(40, pcm.length, true); + wav.set(pcm, 44); + return wav; +}; + +const formatDay = (ms: number, now: number) => + new Date(ms).toLocaleDateString(undefined, { + day: "numeric", + month: "short", + ...(new Date(ms).getFullYear() === new Date(now).getFullYear() ? {} : { year: "numeric" }), + }); + +const SOON_MS = 2 * 24 * 60 * 60 * 1000; + +/** What retention will do to a capture, in the words its row shows. */ +export const captureFate = ( + capture: Pick< + ScrawlCaptureSummary, + "pinned" | "status" | "mediaBytes" | "mediaRemovedAtMs" | "bundleRemovedAtMs" + >, + now: number, +): { readonly text: string; readonly soon: boolean } => { + if (capture.pinned) return { text: "Kept until you delete it", soon: false }; + if (capture.status === "unfinished") return { text: "Kept until it is rebuilt", soon: false }; + const { mediaRemovedAtMs, bundleRemovedAtMs } = capture; + if (mediaRemovedAtMs !== null && mediaRemovedAtMs !== bundleRemovedAtMs) { + return { + text: `Video deleted ${formatDay(Math.max(mediaRemovedAtMs, now), now)}`, + soon: mediaRemovedAtMs - now < SOON_MS, + }; + } + if (bundleRemovedAtMs !== null) { + return { + text: `Deleted ${formatDay(Math.max(bundleRemovedAtMs, now), now)}`, + soon: bundleRemovedAtMs - now < SOON_MS, + }; + } + return { text: capture.mediaBytes === 0 ? "Video removed" : "Kept", soon: false }; +}; + +export const captureTitle = (capture: Pick) => + [capture.app, capture.windowTitle].filter(Boolean).join(": ") || "Screen"; + +export const formatBytes = (bytes: number) => + bytes >= 1_000_000_000 + ? `${(bytes / 1_000_000_000).toFixed(1)} GB` + : `${Math.max(1, Math.round(bytes / 1_000_000))} MB`; + +export const formatDuration = (ms: number) => { + const seconds = Math.round(ms / 1000); + return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; +}; diff --git a/apps/web/src/scrawl/scrawlRecovery.test.ts b/apps/web/src/scrawl/scrawlRecovery.test.ts new file mode 100644 index 0000000000000..cf5254301719f --- /dev/null +++ b/apps/web/src/scrawl/scrawlRecovery.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { rebuildClock } from "./scrawlRecovery"; + +describe("rebuildClock", () => { + const timing = { + micStartedAtMs: 1_200, + videoStartedAtMs: 1_500, + language: "en", + sessionId: "session", + }; + + it("uses the journaled start of each clock", () => { + expect(rebuildClock({ startedAtMs: 1_000, timing, pcmBytes: 32_000, videoMs: 900 })).toEqual({ + videoOffsetMs: 300, + inputOffsetMs: 200, + durationMs: 1_200, + }); + }); + + it("estimates the offset without a journal, and never by more than a second", () => { + expect( + rebuildClock({ startedAtMs: 1_000, timing: null, pcmBytes: 99_200, videoMs: 3_000 }), + ).toEqual({ videoOffsetMs: 100, inputOffsetMs: 0, durationMs: 3_100 }); + expect( + rebuildClock({ startedAtMs: 1_000, timing: null, pcmBytes: 320_000, videoMs: 3_000 }), + ).toMatchObject({ videoOffsetMs: 1_000, durationMs: 10_000 }); + }); + + it("takes its length from the video when the app died before any audio reached the disk", () => { + expect(rebuildClock({ startedAtMs: 1_000, timing, pcmBytes: 0, videoMs: 4_000 })).toMatchObject( + { durationMs: 4_300 }, + ); + }); +}); diff --git a/apps/web/src/scrawl/scrawlRecovery.ts b/apps/web/src/scrawl/scrawlRecovery.ts new file mode 100644 index 0000000000000..dc4ae7ae88fec --- /dev/null +++ b/apps/web/src/scrawl/scrawlRecovery.ts @@ -0,0 +1,239 @@ +// Builds a capture the app went down on: the video and audio are on disk, and the desktop app kept +// the clicks and ink. Scene changes were never journaled, so they come out of the video again. + +import type { DesktopBridge, ScrawlUnfinished } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; + +import { toastManager } from "../components/ui/toast"; +import { DEFAULT_DICTATION_LANGUAGE } from "../dictation/dictationLanguages"; +import { randomUUID } from "../lib/utils"; +import { wordsOrWhole, type TimedWord } from "./narration"; +import { buildBundle, loadVideo, seekTo } from "./scrawlBundle"; +import { readCaptureFile, readOptionalCaptureFile } from "./scrawlFiles"; +import { + claimBuild, + encodeWav, + releaseBuild, + SCRAWL_AUDIO_FILE, + SCRAWL_TIMING_FILE, + setScrawlHistoryOpen, + type ScrawlTiming, +} from "./scrawlHistory"; +import { + findSceneChanges, + lumaOf, + THUMB_HEIGHT, + THUMB_WIDTH, + type LumaSample, +} from "./sceneChanges"; + +const PCM_BYTES_PER_MS = 32; +// About half a minute of speech per append, well under what a dictation chunk may carry. +const PCM_PIECE_BYTES = 960_000; +const SAMPLE_INTERVAL_MS = 750; +const MAX_SAMPLES = 600; +// The video starts this much after the audio at most (measured in Scrawl: 50 to 500 ms). +const MAX_ESTIMATED_VIDEO_OFFSET_MS = 1_000; + +const decodeTiming = Schema.decodeUnknownSync( + Schema.fromJsonString( + Schema.Struct({ + micStartedAtMs: Schema.Number, + videoStartedAtMs: Schema.Number, + language: Schema.String, + sessionId: Schema.String, + }), + ), +); + +const parseTiming = (text: string | null): ScrawlTiming | null => { + if (text === null) return null; + try { + return decodeTiming(text); + } catch { + return null; + } +}; + +/** All times count from the start of the audio, the clock a live capture would have used. */ +export const rebuildClock = (input: { + readonly startedAtMs: number; + readonly timing: ScrawlTiming | null; + readonly pcmBytes: number; + readonly videoMs: number; +}) => { + const { startedAtMs, timing, pcmBytes, videoMs } = input; + const audioMs = pcmBytes / PCM_BYTES_PER_MS; + const videoOffsetMs = + timing !== null + ? timing.videoStartedAtMs - timing.micStartedAtMs + : Math.min(MAX_ESTIMATED_VIDEO_OFFSET_MS, Math.max(0, audioMs - videoMs)); + return { + videoOffsetMs, + inputOffsetMs: timing === null ? 0 : timing.micStartedAtMs - startedAtMs, + durationMs: Math.max(audioMs, videoOffsetMs + videoMs), + }; +}; + +const toBase64 = (bytes: Uint8Array) => { + let binary = ""; + for (let index = 0; index < bytes.length; index += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(index, index + 0x8000)); + } + return btoa(binary); +}; + +const transcribe = async (bridge: DesktopBridge, pcm: Uint8Array, language: string) => { + const sessionId = randomUUID(); + for (let offset = 0; offset < pcm.length; offset += PCM_PIECE_BYTES) { + await bridge.dictationAppend?.({ + sessionId, + language, + pcm: toBase64(pcm.subarray(offset, offset + PCM_PIECE_BYTES)), + }); + } + return bridge.dictationFinish?.({ sessionId, words: true }); +}; + +const sampleScenes = async (video: HTMLVideoElement, videoOffsetMs: number, videoMs: number) => { + const canvas = document.createElement("canvas"); + canvas.width = THUMB_WIDTH; + canvas.height = THUMB_HEIGHT; + const context = canvas.getContext("2d", { willReadFrequently: true })!; + const step = Math.max(SAMPLE_INTERVAL_MS, videoMs / MAX_SAMPLES); + const samples: Array = []; + for (let at = 0; at < videoMs; at += step) { + await seekTo(video, at / 1000); + context.drawImage(video, 0, 0, THUMB_WIDTH, THUMB_HEIGHT); + const { data } = context.getImageData(0, 0, THUMB_WIDTH, THUMB_HEIGHT); + samples.push({ t: at + videoOffsetMs, luma: lumaOf(data) }); + } + return samples; +}; + +const readVideo = async (read: NonNullable, id: string) => { + const mp4 = await readOptionalCaptureFile(read, id, "recording.mp4"); + if (mp4 !== null && mp4.size > 0) { + return { blob: new Blob([mp4], { type: "video/mp4" }), name: "recording.mp4" }; + } + return { + blob: await readCaptureFile(read, id, "recording.webm", "video/webm"), + name: "recording.webm", + }; +}; + +const rebuild = async (bridge: DesktopBridge, unfinished: ScrawlUnfinished) => { + const read = bridge.scrawlReadFile!; + const { session, input } = unfinished; + const { id } = session; + const timingFile = await readOptionalCaptureFile(read, id, SCRAWL_TIMING_FILE); + const timing = parseTiming((await timingFile?.text()) ?? null); + // The session the lost renderer was feeding has no one left to finish it. + if (timing !== null) { + await bridge.dictationCancel?.({ sessionId: timing.sessionId }).catch(() => undefined); + } + const pcmFile = await readOptionalCaptureFile(read, id, SCRAWL_AUDIO_FILE); + const pcm = new Uint8Array((await pcmFile?.arrayBuffer()) ?? new ArrayBuffer(0)); + const video = await readVideo(read, id); + + const loaded = await loadVideo(video.blob); + let videoMs: number; + let sceneSamples: Array; + let width: number; + let height: number; + let clock: ReturnType; + try { + videoMs = Math.round(loaded.video.duration * 1000); + clock = rebuildClock({ + startedAtMs: session.startedAtMs, + timing, + pcmBytes: pcm.length, + videoMs, + }); + sceneSamples = await sampleScenes(loaded.video, clock.videoOffsetMs, videoMs); + width = session.width || loaded.video.videoWidth; + height = session.height || loaded.video.videoHeight; + } finally { + loaded.dispose(); + } + + const transcript = + pcm.length === 0 + ? undefined + : await transcribe(bridge, pcm, timing?.language ?? DEFAULT_DICTATION_LANGUAGE).catch( + () => undefined, + ); + const transcriptText = transcript?.ok ? transcript.value.text : ""; + const words: ReadonlyArray = wordsOrWhole( + transcript?.ok ? (transcript.value.words ?? []) : [], + transcriptText, + clock.durationMs, + ); + await buildBundle( + { + session: { ...session, width, height }, + input, + words, + transcriptText, + model: "parakeet", + video: video.blob, + videoName: video.name, + videoOffsetMs: clock.videoOffsetMs, + inputOffsetMs: clock.inputOffsetMs, + durationMs: clock.durationMs, + sceneChanges: findSceneChanges(sceneSamples), + wav: encodeWav(pcm), + }, + (path, bytes) => bridge.scrawlWriteFile!({ id, path, bytes }), + ); + await bridge.scrawlMarkBuilt?.({ id }).catch(() => undefined); +}; + +/** One try. Throws when the capture cannot be built, which leaves it marked as failed. */ +export const rebuildCapture = async (id: string) => { + const bridge = window.desktopBridge; + if (bridge?.scrawlBeginRebuild === undefined) return; + claimBuild(id); + try { + await rebuild(bridge, await bridge.scrawlBeginRebuild({ id })); + } finally { + releaseBuild(id); + } +}; + +let launchRebuildStarted = false; + +/** Once per page load, and the renderer has no capture of its own at that point. */ +export const rebuildUnfinishedCaptures = async () => { + if (launchRebuildStarted) return; + launchRebuildStarted = true; + const unfinished = (await window.desktopBridge?.scrawlListUnfinished?.()) ?? []; + let rebuilt = 0; + let failed = 0; + for (const { session } of unfinished) { + try { + await rebuildCapture(session.id); + rebuilt += 1; + } catch (error) { + console.warn("scrawl: capture could not be rebuilt", session.id, error); + failed += 1; + } + } + if (rebuilt + failed === 0) return; + toastManager.add({ + type: failed === 0 ? "success" : "warning", + title: + failed === 0 + ? rebuilt === 1 + ? "An interrupted capture was recovered" + : `${rebuilt} interrupted captures were recovered` + : failed === 1 + ? "An interrupted capture could not be recovered" + : `${failed} interrupted captures could not be recovered`, + timeout: 0, + actionProps: { + children: "Show captures", + onClick: () => setScrawlHistoryOpen(true), + }, + }); +}; diff --git a/docs/user/screen-capture.md b/docs/user/screen-capture.md new file mode 100644 index 0000000000000..2c5d7decf5d55 --- /dev/null +++ b/docs/user/screen-capture.md @@ -0,0 +1,35 @@ +# Screen capture + +A screen capture records the monitor under your mouse with your voice, your clicks, and red ink, then +puts the result into your draft: what you said next to the moments on screen, and a contact sheet of +those moments. It is available in the desktop app. Clicks and ink need Windows. + +## Taking a capture + +Press **Alt+Shift+S** in any app, select the screen button in the composer, or choose **Capture the +screen** in the command palette. Talk and click as you normally would, and hold **Shift** while +moving the mouse to draw. Press the shortcut again to finish, or Esc in T3 Code to discard. + +## Changing the shortcut + +Open **Settings** > **SnapShots** > **Screen capture**, select the shortcut, press the new keys, then +**Save**. If another program is holding the shortcut, that row says so. T3 Code takes it as soon as +it is free, and the composer button and command palette work until then. + +## Finding a past capture + +Press **Alt+Shift+H**, choose **Past screen captures** in the command palette, or select **Show +captures** in Settings. From there you can put a capture back into your draft, copy its text, or open +its folder. Each row shows what happens to it next. + +## What gets deleted + +By default a capture's video and audio go to the Recycle Bin after 7 days. Its text and images stay +until the whole capture goes after 30 days. Change this under **Keep screen captures**, or pin a +capture in **Past screen captures** to keep it. + +## After a crash + +If the app closes or reloads during a capture, the next launch builds what was recorded and tells you +when it is done. The last few seconds may be missing. A capture that cannot be built stays in the list +with a rebuild button. diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index f286eb1e9f7e6..4fccc156a4b2a 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -22,12 +22,23 @@ import type { import type { ScrawlAppendVideoInput, ScrawlCaptureRef, + ScrawlCaptureSummary, + ScrawlReadFileInput, + ScrawlSetPinnedInput, + ScrawlShortcutAvailability, + ScrawlShortcutState, ScrawlStartResult, ScrawlStopResult, + ScrawlUnfinished, ScrawlWriteFileInput, } from "./scrawl.ts"; import { ExecutionEnvironmentDescriptor } from "./environment.ts"; -import { type ClientSettings, type QuitConfirmationMode, SnapShotShortcut } from "./settings.ts"; +import { + type ClientSettings, + type QuitConfirmationMode, + type SnapShotKeyChord, + SnapShotShortcut, +} from "./settings.ts"; import type { EditorId } from "./editor.ts"; import type { PreviewForwardedShortcut } from "./keybindings.ts"; @@ -1064,6 +1075,18 @@ export interface DesktopBridge { scrawlCancel?: (input: ScrawlCaptureRef) => Promise; scrawlAppendVideo?: (input: ScrawlAppendVideoInput) => Promise; scrawlWriteFile?: (input: ScrawlWriteFileInput) => Promise; + scrawlReadFile?: (input: ScrawlReadFileInput) => Promise; + /** Frees what a lost renderer left running and lists the captures to rebuild on their own. */ + scrawlListUnfinished?: () => Promise>; + /** Marks one try, so a rebuild that crashes the app is not repeated at every launch. */ + scrawlBeginRebuild?: (input: ScrawlCaptureRef) => Promise; + scrawlMarkBuilt?: (input: ScrawlCaptureRef) => Promise; + scrawlListCaptures?: () => Promise>; + scrawlSetPinned?: (input: ScrawlSetPinnedInput) => Promise; + scrawlTrash?: (input: ScrawlCaptureRef) => Promise; + scrawlReveal?: (input: ScrawlCaptureRef) => Promise; + scrawlGetShortcutState?: () => Promise; + scrawlCheckShortcut?: (shortcut: SnapShotKeyChord) => Promise; getLocalEnvironmentBearerToken: () => Promise; getClientSettings: () => Promise; setClientSettings: (settings: ClientSettings) => Promise; diff --git a/packages/contracts/src/keybindings.ts b/packages/contracts/src/keybindings.ts index 97fe73037062d..e08cd6133dd02 100644 --- a/packages/contracts/src/keybindings.ts +++ b/packages/contracts/src/keybindings.ts @@ -102,6 +102,7 @@ export const STATIC_KEYBINDING_COMMANDS = [ "composer.dictate", "composer.dictateStructured", "composer.dictationHistory", + "scrawl.history", "chat.new", "chat.newLocal", "chat.newWithoutProject", diff --git a/packages/contracts/src/scrawl.ts b/packages/contracts/src/scrawl.ts index 665833f19d28b..9fa6e254656a6 100644 --- a/packages/contracts/src/scrawl.ts +++ b/packages/contracts/src/scrawl.ts @@ -72,8 +72,67 @@ export const ScrawlWriteFileInput = Schema.Struct({ }); export type ScrawlWriteFileInput = typeof ScrawlWriteFileInput.Type; +export const ScrawlReadFileInput = Schema.Struct({ + id: ScrawlCaptureId, + path: ScrawlWriteFileInput.fields.path, + offset: Schema.Number, + length: Schema.Number, +}); +export type ScrawlReadFileInput = typeof ScrawlReadFileInput.Type; + +export const ScrawlSetPinnedInput = Schema.Struct({ + id: ScrawlCaptureId, + pinned: Schema.Boolean, +}); +export type ScrawlSetPinnedInput = typeof ScrawlSetPinnedInput.Type; + +/** A capture whose bundle was never built, and what its build still needs. */ +export const ScrawlUnfinished = Schema.Struct({ + session: ScrawlStartResult, + input: ScrawlStopResult, +}); +export type ScrawlUnfinished = typeof ScrawlUnfinished.Type; + +/** `failed` was tried once and is only rebuilt when asked, so a crash cannot repeat each launch. */ +export const ScrawlCaptureStatus = Schema.Literals(["built", "recording", "unfinished", "failed"]); +export type ScrawlCaptureStatus = typeof ScrawlCaptureStatus.Type; + +export const ScrawlCaptureSummary = Schema.Struct({ + id: ScrawlCaptureId, + bundlePath: Schema.String, + status: ScrawlCaptureStatus, + createdAtMs: Schema.Number, + durationMs: Schema.Number, + app: Schema.String, + windowTitle: Schema.String, + sizeBytes: Schema.Number, + mediaBytes: Schema.Number, + pinned: Schema.Boolean, + /** When retention removes the video and audio; null when it never will or they are gone. */ + mediaRemovedAtMs: Schema.NullOr(Schema.Number), + /** When retention moves the whole capture to the bin; null when it never will. */ + bundleRemovedAtMs: Schema.NullOr(Schema.Number), +}); +export type ScrawlCaptureSummary = typeof ScrawlCaptureSummary.Type; + +export const ScrawlShortcutState = Schema.Struct({ + accelerator: Schema.String, + /** False while another program holds the keys; T3 Code keeps trying to take them. */ + registered: Schema.Boolean, +}); +export type ScrawlShortcutState = typeof ScrawlShortcutState.Type; + +export const ScrawlShortcutAvailability = Schema.Struct({ + available: Schema.Boolean, + message: Schema.NullOr(Schema.String), +}); +export type ScrawlShortcutAvailability = typeof ScrawlShortcutAvailability.Type; + /** How long ink stays after Shift is let go. Its fading is not the screen answering a click. */ export const SCRAWL_INK_HOLD_MS = 5_000; /** The menu action the global shortcut sends to the renderer. */ export const SCRAWL_TOGGLE_ACTION = "scrawl-toggle"; + +/** Tells the renderer the shortcut was taken or lost, so Settings never shows a stale answer. */ +export const SCRAWL_SHORTCUT_STATE_ACTION = "scrawl-shortcut-state"; diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index fd453814b14eb..ecbb16c1230f9 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -251,6 +251,17 @@ export function snapShotModifierPairLabel(modifier: SnapShotModifier, apple: boo const label = (apple ? APPLE_MODIFIER_LABELS : OTHER_MODIFIER_LABELS)[modifier]; return `${label} + ${label}`; } +export const ScrawlRetention = Schema.Literals(["week", "month", "forever"]); +export type ScrawlRetention = typeof ScrawlRetention.Type; +export const DEFAULT_SCRAWL_SHORTCUT: SnapShotKeyChord = { + key: "s", + metaKey: false, + ctrlKey: false, + shiftKey: true, + altKey: true, + modKey: false, +}; + const DEFAULT_SNAP_SHOT_SHORTCUT: SnapShotShortcut = { kind: "both-shift-keys", }; @@ -518,6 +529,12 @@ export const ClientSettingsSchema = Schema.Struct({ ), snapShotFlash: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), snapShotAnimations: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), + scrawlShortcut: SnapShotKeyChord.pipe( + Schema.withDecodingDefault(Effect.succeed(DEFAULT_SCRAWL_SHORTCUT)), + ), + scrawlRetention: ScrawlRetention.pipe( + Schema.withDecodingDefault(Effect.succeed("week")), + ), wordWrap: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), // Opt-in TeX typesetting in rendered Markdown. Off keeps math parsing and // KaTeX out of the render path entirely. @@ -1805,6 +1822,8 @@ export const ClientSettingsPatch = Schema.Struct({ snapShotSound: Schema.optionalKey(SnapShotSound), snapShotFlash: Schema.optionalKey(Schema.Boolean), snapShotAnimations: Schema.optionalKey(Schema.Boolean), + scrawlShortcut: Schema.optionalKey(SnapShotKeyChord), + scrawlRetention: Schema.optionalKey(ScrawlRetention), wordWrap: Schema.optionalKey(Schema.Boolean), mathRenderingEnabled: Schema.optionalKey(Schema.Boolean), }); diff --git a/packages/shared/src/keybindings.ts b/packages/shared/src/keybindings.ts index 298829813c5f5..e15058c80c2ea 100644 --- a/packages/shared/src/keybindings.ts +++ b/packages/shared/src/keybindings.ts @@ -211,6 +211,7 @@ export const DEFAULT_KEYBINDINGS: ReadonlyArray = [ { key: "f9", command: "composer.dictate", when: "!terminalFocus" }, { key: "f8", command: "composer.dictateStructured", when: "!terminalFocus" }, { key: "shift+f9", command: "composer.dictationHistory", when: "!terminalFocus" }, + { key: "alt+shift+h", command: "scrawl.history", when: "!terminalFocus" }, { key: "mod+shift+l", command: "composer.previousWorktree", when: "!terminalFocus" }, { key: "mod+shift+k", command: "pullRequest.copyNumber", when: "!terminalFocus" }, { key: "mod+shift+arrowup", command: "modelPicker.previousProvider", when: "modelPickerOpen" }, From d0d0ce85ea4d6fde256b872e672d5aca41186dae Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:21:27 +0100 Subject: [PATCH 3/9] feat(scrawl): crop each keyframe to its moment, pack the sheet in justified rows Native captures used one crop box for the whole capture and a plain grid. Each keyframe now gets its own crop from Scrawl's rules: input its line names, what a click changed, optional window and cursor logs, edges moved off text. The sheet uses Scrawl's row planner and one chip size, the click mark is a thin blue ring, and ordinary actions fill only the first 6 keyframes. On the 43 real Scrawl captures the port reproduces Scrawl's crops (median overlap 1.00 with its logs, 0.91 without) and its sheet sizes exactly. Co-Authored-By: Claude Opus 5.5 --- FORK.md | 7 + apps/web/src/scrawl/autocrop-corpus.json | 1 + apps/web/src/scrawl/autocrop.test.ts | 541 +++++++++++++++ apps/web/src/scrawl/autocrop.ts | 691 +++++++++++++++++++ apps/web/src/scrawl/autocropContent.ts | 351 ++++++++++ apps/web/src/scrawl/contactSheet.test.ts | 112 +++ apps/web/src/scrawl/contactSheet.ts | 176 +++++ apps/web/src/scrawl/frameAnnotations.test.ts | 96 +++ apps/web/src/scrawl/frameAnnotations.ts | 132 ++++ apps/web/src/scrawl/narration.test.ts | 42 ++ apps/web/src/scrawl/narration.ts | 52 +- apps/web/src/scrawl/scrawlBundle.ts | 353 ++++++---- vite.config.ts | 2 + 13 files changed, 2387 insertions(+), 169 deletions(-) create mode 100644 apps/web/src/scrawl/autocrop-corpus.json create mode 100644 apps/web/src/scrawl/autocrop.test.ts create mode 100644 apps/web/src/scrawl/autocrop.ts create mode 100644 apps/web/src/scrawl/autocropContent.ts create mode 100644 apps/web/src/scrawl/contactSheet.test.ts create mode 100644 apps/web/src/scrawl/contactSheet.ts create mode 100644 apps/web/src/scrawl/frameAnnotations.test.ts create mode 100644 apps/web/src/scrawl/frameAnnotations.ts diff --git a/FORK.md b/FORK.md index cc27d2eb494b2..e5a17003c1c0f 100644 --- a/FORK.md +++ b/FORK.md @@ -165,6 +165,13 @@ the contact sheet land in the composer on screen. Esc in T3 Code discards a capt - Speech and pointing do not run in step, so sentences are never cut at a click: whole sentences own a keyframe, click bursts fold into one (`clicked ×3`), and clicks the screen did not answer read as `pointed`. On 32 real captures this took cuts through sentences from 59 of 127 lines to 1. +- Each keyframe is cropped on its own to what happened around it, with Scrawl's rules and numbers: + the input its line names, what a click changed, and edges moved off text + (`frames-full/` keeps the whole screen). The contact sheet packs crops of any shape into justified + rows, one chip size throughout. Budget: 6 keyframes up to 40 s, one more per 20 s, at most 12; + ordinary actions fill only the first 6. A window or cursor log improves the crops when + `CaptureRecording` has one. On the 43 real Scrawl captures, given those logs, the crops reproduce + Scrawl's own (median overlap 1.00); without them 0.91. - Uses the dictation model above, with word times mapped back across the pauses it cuts out. - The recording pill in the corner shows in the video and keyframes; `setContentProtection` does not keep it out on Windows 10 22H2. diff --git a/apps/web/src/scrawl/autocrop-corpus.json b/apps/web/src/scrawl/autocrop-corpus.json new file mode 100644 index 0000000000000..77951dd433e2e --- /dev/null +++ b/apps/web/src/scrawl/autocrop-corpus.json @@ -0,0 +1 @@ +[{"name":"arrow-across","truth":{"scenario":"arrow-across","description":"Ink in the browser, then an arrow from the browser to the terminal","screen":[1728.0,1117.0],"whole":[22.0,62.0,1686.0,886.0],"spans":[{"t0":0,"t1":5200,"subject":"card","ideal":[800.0,120.0,900.0,820.0],"targets":[[1049.0,335.0,292.0,20.0]]},{"t0":5200,"t1":13000,"subject":"link","ideal":[8.0,174.0,1392.0,380.0],"targets":[[1030.0,320.0,330.0,194.0],[85.0,363.0,252.0,16.0]]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":2640,"must":[[0.5965,0.2836,0.7904,0.3318]]},{"index":2,"t":6600,"must":[[0.2033,0.2972,0.6916,0.373],[0.2033,0.3179,0.2176,0.3399]]}],"input":{"screen":[1728.0,1117.0],"duration":13000,"keyframes":[2640,6600],"items":[{"kind":"stroke","t0":1800,"t1":2600,"pts":[[0.6291,0.2911,1800],[0.6399,0.2889,1818],[0.6497,0.2859,1836],[0.6642,0.2852,1853],[0.6784,0.2849,1871],[0.6925,0.2836,1889],[0.7071,0.2839,1907],[0.7213,0.2848,1924],[0.7334,0.287,1942],[0.7422,0.29,1960],[0.7576,0.2907,1978],[0.7627,0.2943,1996],[0.7754,0.2963,2013],[0.7825,0.2995,2031],[0.785,0.303,2049],[0.7901,0.3065,2067],[0.7904,0.3101,2084],[0.7857,0.3136,2102],[0.7834,0.3171,2120],[0.7807,0.3208,2138],[0.7656,0.3226,2156],[0.76,0.3259,2173],[0.7472,0.3275,2191],[0.7396,0.3311,2209],[0.7249,0.3316,2227],[0.7108,0.3318,2244],[0.6972,0.3316,2262],[0.6841,0.3313,2280],[0.6704,0.3315,2298],[0.6554,0.3316,2316],[0.6461,0.3285,2333],[0.6309,0.3278,2351],[0.6199,0.3254,2369],[0.615,0.3218,2387],[0.6066,0.319,2404],[0.6012,0.3156,2422],[0.5965,0.3121,2440],[0.5968,0.3085,2458],[0.6026,0.305,2476],[0.6006,0.3012,2493],[0.6112,0.2984,2511],[0.6132,0.2946,2529],[0.6288,0.2932,2547],[0.6363,0.2902,2564],[0.6453,0.2872,2582],[0.6584,0.2856,2600]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":5400,"t1":6200,"pts":[[0.6916,0.373,5400],[0.672,0.3648,5432],[0.6525,0.3571,5464],[0.633,0.3483,5496],[0.6134,0.3402,5528],[0.5939,0.3336,5560],[0.5744,0.3258,5592],[0.5548,0.3197,5624],[0.5353,0.3146,5656],[0.5158,0.31,5688],[0.4963,0.3062,5720],[0.4767,0.3032,5752],[0.4572,0.2992,5784],[0.4377,0.2984,5816],[0.4181,0.2983,5848],[0.3986,0.2972,5880],[0.3791,0.2988,5912],[0.3596,0.2992,5944],[0.34,0.3029,5976],[0.3205,0.3052,6008],[0.301,0.3085,6040],[0.2814,0.3127,6072],[0.2619,0.3174,6104],[0.2424,0.322,6136],[0.2229,0.3265,6168],[0.2033,0.3332,6200]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":6450,"t1":6650,"pts":[[0.2151,0.3179,6450],[0.2033,0.3324,6550],[0.2176,0.3399,6650]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":1848,"pts":[[0.6291,0.2911,0]],"dwell":1.848,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.6599,0.2854,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":1980,"pts":[[0.713,0.2843,1914]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1980,"t1":2046,"pts":[[0.7582,0.2911,1980]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.7846,0.3024,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.7844,0.3155,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.7567,0.3263,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.7112,0.3318,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2376,"pts":[[0.6601,0.3316,2310]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2376,"t1":2442,"pts":[[0.6179,0.324,2376]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2442,"t1":2508,"pts":[[0.5965,0.3117,2442]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2508,"t1":2574,"pts":[[0.6093,0.2989,2508]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2574,"t1":2640,"pts":[[0.6412,0.2886,2574]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2640,"t1":3894,"pts":[[0.6773,0.3103,2640]],"dwell":1.254,"keyframe":null},{"kind":"cursor","t0":3894,"t1":4554,"pts":[[0.677,0.3126,3894]],"dwell":0.66,"keyframe":null},{"kind":"cursor","t0":4554,"t1":4620,"pts":[[0.6725,0.3123,4554]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4620,"t1":4686,"pts":[[0.6686,0.313,4620]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4686,"t1":4752,"pts":[[0.6675,0.3139,4686]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4752,"t1":4818,"pts":[[0.6707,0.3133,4752]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4818,"t1":5148,"pts":[[0.6726,0.3119,4818]],"dwell":0.33,"keyframe":null},{"kind":"cursor","t0":5148,"t1":5214,"pts":[[0.6717,0.3143,5148]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5214,"t1":5280,"pts":[[0.6747,0.3301,5214]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5280,"t1":5346,"pts":[[0.6822,0.355,5280]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5346,"t1":5412,"pts":[[0.689,0.3714,5346]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5412,"t1":5478,"pts":[[0.6842,0.3699,5412]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5478,"t1":5544,"pts":[[0.6439,0.3533,5478]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5544,"t1":5610,"pts":[[0.6037,0.3369,5544]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5610,"t1":5676,"pts":[[0.5634,0.3224,5610]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5676,"t1":5742,"pts":[[0.5231,0.3117,5676]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5742,"t1":5808,"pts":[[0.4828,0.3041,5742]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5808,"t1":5874,"pts":[[0.4426,0.2986,5808]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5874,"t1":5940,"pts":[[0.4023,0.2974,5874]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5940,"t1":6006,"pts":[[0.362,0.2991,5940]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6006,"t1":6072,"pts":[[0.3217,0.305,6006]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6072,"t1":6138,"pts":[[0.2814,0.3127,6072]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6138,"t1":6204,"pts":[[0.2412,0.3223,6138]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6204,"t1":6270,"pts":[[0.2035,0.333,6204]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6270,"t1":6336,"pts":[[0.2066,0.3289,6270]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6336,"t1":6402,"pts":[[0.2097,0.3249,6336]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6402,"t1":6534,"pts":[[0.2129,0.3209,6402]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":6534,"t1":6600,"pts":[[0.2052,0.3301,6534]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6600,"t1":6666,"pts":[[0.2105,0.3362,6600]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6666,"t1":6732,"pts":[[0.2165,0.3354,6666]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6732,"t1":6798,"pts":[[0.1527,0.324,6732]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6798,"t1":8118,"pts":[[0.1066,0.3353,6798]],"dwell":1.32,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8844,"pts":[[0.1064,0.3377,8118]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":8844,"t1":8910,"pts":[[0.1084,0.338,8844]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8910,"t1":8976,"pts":[[0.1105,0.3357,8910]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8976,"t1":9042,"pts":[[0.1114,0.3343,8976]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9042,"t1":10428,"pts":[[0.1415,0.3365,9042]],"dwell":1.386,"keyframe":null},{"kind":"cursor","t0":10428,"t1":10494,"pts":[[0.1396,0.3335,10428]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10494,"t1":10560,"pts":[[0.1372,0.3312,10494]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10560,"t1":11286,"pts":[[0.1361,0.3307,10560]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":11286,"t1":11352,"pts":[[0.1387,0.33,11286]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11352,"t1":12738,"pts":[[0.1397,0.3292,11352]],"dwell":1.386,"keyframe":null},{"kind":"cursor","t0":12738,"t1":12804,"pts":[[0.1402,0.3318,12738]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":12804,"t1":13000,"pts":[[0.141,0.3348,12804]],"dwell":0.196,"keyframe":null},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.0156,0.0732,2640],[0.375,0.0732,2640],[0.0156,0.4634,2640],[0.375,0.4634,2640],[0.1953,0.2683,2640]],"dwell":0.3745,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.4688,0.0976,2640],[0.9844,0.0976,2640],[0.4688,0.1707,2640],[0.9844,0.1707,2640],[0.7266,0.1341,2640]],"dwell":0.1942,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.0313,0.4878,2640],[0.25,0.4878,2640],[0.0313,0.6098,2640],[0.25,0.6098,2640],[0.1406,0.5488,2640]],"dwell":0.1633,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.7969,0.7317,2640],[0.9375,0.7317,2640],[0.7969,0.8049,2640],[0.9375,0.8049,2640],[0.8672,0.7683,2640]],"dwell":0.1014,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.4688,0.2683,2640],[0.5156,0.2683,2640],[0.4688,0.4146,2640],[0.5156,0.4146,2640],[0.4922,0.3415,2640]],"dwell":0.0828,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.0313,0.6829,2640],[0.1563,0.6829,2640],[0.0313,0.7317,2640],[0.1563,0.7317,2640],[0.0938,0.7073,2640]],"dwell":0.0781,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.6094,0.4878,2640],[0.6875,0.4878,2640],[0.6094,0.561,2640],[0.6875,0.561,2640],[0.6484,0.5244,2640]],"dwell":0.0756,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.9219,0.1463,2640],[0.9844,0.1463,2640],[0.9219,0.2195,2640],[0.9844,0.2195,2640],[0.9531,0.1829,2640]],"dwell":0.0676,"keyframe":0}],"snapshots":[[0,[{"id":4101,"app":"Safari","rect":[0.463,0.1074,0.9838,0.8415],"layer":0,"regular":true,"frontmost":true},{"id":4100,"app":"Ghostty","rect":[0.0174,0.0627,0.4919,0.7431],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false},{"id":38,"app":"Dock","rect":[0.0,0.9338,1.0,1.0],"layer":20,"regular":false,"frontmost":false}]]]}},{"name":"fullscreen-app","truth":{"scenario":"fullscreen-app","description":"Zed in macOS full screen; ink on a line and its diagnostic","screen":[1728.0,1117.0],"whole":[194.0,74.0,813.0,1043.0],"spans":[{"t0":0,"t1":8400,"subject":"line","ideal":[228.0,54.0,799.0,254.0],"targets":[[288.0,232.0,374.0,16.0],[686.0,233.0,281.0,15.0]]},{"t0":8400,"t1":12000,"subject":"status","ideal":[94.0,957.0,489.0,160.0],"targets":[[234.0,1097.0,209.0,14.0]]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":4400,"must":[[0.1632,0.227,0.3865,0.2294],[0.3835,0.1928,0.5749,0.2373]]},{"index":2,"t":9020,"must":[[0.1319,0.9979,0.2596,0.9991]]}],"input":{"screen":[1728.0,1117.0],"duration":12000,"keyframes":[4400,9020],"items":[{"kind":"stroke","t0":1900,"t1":2500,"pts":[[0.1632,0.227,1900],[0.1763,0.2281,1935],[0.1895,0.2281,1971],[0.2026,0.2285,2006],[0.2157,0.227,2041],[0.2289,0.2288,2076],[0.242,0.2279,2112],[0.2551,0.2288,2147],[0.2683,0.2289,2182],[0.2814,0.2291,2218],[0.2946,0.2276,2253],[0.3077,0.2293,2288],[0.3208,0.2294,2324],[0.334,0.2282,2359],[0.3471,0.2283,2394],[0.3602,0.2277,2429],[0.3734,0.2284,2465],[0.3865,0.2279,2500]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":3600,"t1":4500,"pts":[[0.4108,0.2004,3600],[0.424,0.1991,3620],[0.4332,0.1966,3640],[0.4463,0.1956,3660],[0.4584,0.1939,3680],[0.4728,0.1949,3700],[0.4866,0.1928,3720],[0.4993,0.1949,3740],[0.5133,0.1951,3760],[0.5276,0.1958,3780],[0.5383,0.1981,3800],[0.5468,0.2008,3820],[0.5514,0.2039,3840],[0.5573,0.2066,3860],[0.5729,0.2086,3880],[0.5708,0.2121,3900],[0.5749,0.2153,3920],[0.566,0.2184,3940],[0.5733,0.222,3960],[0.5627,0.2245,3980],[0.5554,0.2272,4000],[0.5468,0.2295,4020],[0.5365,0.2314,4040],[0.5264,0.2333,4060],[0.5172,0.2359,4080],[0.5026,0.2355,4100],[0.49,0.2362,4120],[0.4769,0.2373,4140],[0.4637,0.2367,4160],[0.4497,0.2366,4180],[0.4399,0.234,4200],[0.4238,0.2341,4220],[0.4193,0.2304,4240],[0.407,0.2288,4260],[0.4021,0.2258,4280],[0.397,0.223,4300],[0.39,0.2202,4320],[0.3856,0.217,4340],[0.3835,0.2137,4360],[0.3935,0.2107,4380],[0.3914,0.2072,4400],[0.3969,0.2041,4420],[0.4056,0.2014,4440],[0.4219,0.2006,4460],[0.4303,0.1981,4480],[0.4433,0.1972,4500]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":8600,"t1":9100,"pts":[[0.1319,0.9979,8600],[0.1395,0.9991,8629],[0.147,0.9991,8659],[0.1545,0.9985,8688],[0.162,0.9991,8718],[0.1695,0.9991,8747],[0.177,0.9991,8776],[0.1845,0.9991,8806],[0.192,0.9991,8835],[0.1995,0.9991,8865],[0.207,0.9991,8894],[0.2145,0.9991,8924],[0.222,0.9991,8953],[0.2296,0.9991,8982],[0.2371,0.9991,9012],[0.2446,0.9991,9041],[0.2521,0.9991,9071],[0.2596,0.9991,9100]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":990,"pts":[[0.2424,0.1137,0]],"dwell":0.99,"keyframe":null},{"kind":"cursor","t0":990,"t1":1056,"pts":[[0.2445,0.1142,990]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1056,"t1":1122,"pts":[[0.2493,0.1128,1056]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1122,"t1":1320,"pts":[[0.2529,0.1111,1122]],"dwell":0.198,"keyframe":null},{"kind":"cursor","t0":1320,"t1":1386,"pts":[[0.2524,0.1093,1320]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1386,"t1":1452,"pts":[[0.2499,0.1087,1386]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1452,"t1":1584,"pts":[[0.248,0.1086,1452]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":1584,"t1":1650,"pts":[[0.2454,0.1067,1584]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1650,"t1":1716,"pts":[[0.2311,0.1175,1650]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1716,"t1":1782,"pts":[[0.2038,0.1514,1716]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1782,"t1":1848,"pts":[[0.1763,0.1922,1782]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.1627,0.2192,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":1980,"pts":[[0.1684,0.2275,1914]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1980,"t1":2046,"pts":[[0.193,0.2282,1980]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.2175,0.2272,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.2421,0.2279,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.2667,0.2289,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.2912,0.228,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2376,"pts":[[0.3158,0.2294,2310]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2376,"t1":2442,"pts":[[0.3404,0.2282,2376]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2442,"t1":2508,"pts":[[0.3649,0.2279,2442]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2508,"t1":2574,"pts":[[0.3867,0.2277,2508]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2574,"t1":2640,"pts":[[0.3881,0.2261,2574]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2640,"t1":2706,"pts":[[0.3896,0.2244,2640]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2706,"t1":2772,"pts":[[0.391,0.2228,2706]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2772,"t1":2838,"pts":[[0.3925,0.2211,2772]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2838,"t1":2904,"pts":[[0.394,0.2195,2838]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2904,"t1":2970,"pts":[[0.3954,0.2178,2904]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2970,"t1":3036,"pts":[[0.3969,0.2161,2970]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3036,"t1":3102,"pts":[[0.3983,0.2145,3036]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3102,"t1":3168,"pts":[[0.3998,0.2128,3102]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3168,"t1":3234,"pts":[[0.4012,0.2112,3168]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3234,"t1":3300,"pts":[[0.4027,0.2095,3234]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3300,"t1":3366,"pts":[[0.4041,0.2079,3300]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3366,"t1":3432,"pts":[[0.4056,0.2062,3366]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3432,"t1":3498,"pts":[[0.4071,0.2046,3432]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3498,"t1":3564,"pts":[[0.4085,0.2029,3498]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3564,"t1":3630,"pts":[[0.41,0.2013,3564]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3630,"t1":3696,"pts":[[0.4286,0.1979,3630]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3696,"t1":3762,"pts":[[0.4699,0.1947,3696]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3762,"t1":3828,"pts":[[0.5148,0.1952,3762]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3828,"t1":3894,"pts":[[0.5486,0.202,3828]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3894,"t1":3960,"pts":[[0.5714,0.2111,3894]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3960,"t1":4026,"pts":[[0.5733,0.222,3960]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4026,"t1":4092,"pts":[[0.5437,0.2301,4026]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4092,"t1":4158,"pts":[[0.5084,0.2357,4092]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4158,"t1":4224,"pts":[[0.465,0.2367,4158]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4224,"t1":4290,"pts":[[0.4229,0.2334,4224]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4290,"t1":4356,"pts":[[0.3995,0.2244,4290]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4356,"t1":4422,"pts":[[0.3839,0.2143,4356]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4422,"t1":4488,"pts":[[0.3978,0.2038,4422]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4488,"t1":4554,"pts":[[0.4355,0.1977,4488]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4554,"t1":5082,"pts":[[0.5012,0.2126,4554]],"dwell":0.528,"keyframe":null},{"kind":"cursor","t0":5082,"t1":5148,"pts":[[0.5042,0.2162,5082]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5148,"t1":5214,"pts":[[0.5074,0.2186,5148]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5214,"t1":5808,"pts":[[0.5088,0.2188,5214]],"dwell":0.594,"keyframe":null},{"kind":"cursor","t0":5808,"t1":5874,"pts":[[0.5081,0.2166,5808]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5874,"t1":5940,"pts":[[0.5062,0.2131,5874]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5940,"t1":6732,"pts":[[0.5046,0.2111,5940]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":6732,"t1":6798,"pts":[[0.508,0.2123,6732]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6798,"t1":7524,"pts":[[0.5098,0.2122,6798]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":7524,"t1":7590,"pts":[[0.5076,0.2126,7524]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7590,"t1":7656,"pts":[[0.5037,0.2156,7590]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7656,"t1":8052,"pts":[[0.5014,0.2182,7656]],"dwell":0.396,"keyframe":null},{"kind":"cursor","t0":8052,"t1":8118,"pts":[[0.5031,0.218,8052]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8250,"pts":[[0.5049,0.2164,8118]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":8250,"t1":8316,"pts":[[0.484,0.2102,8250]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8316,"t1":8382,"pts":[[0.4175,0.2962,8316]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8382,"t1":8448,"pts":[[0.31,0.4951,8382]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.2002,0.733,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8580,"pts":[[0.1328,0.9141,8514]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8580,"t1":8646,"pts":[[0.126,0.9897,8580]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.1437,0.9991,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":8778,"pts":[[0.1605,0.999,8712]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8778,"t1":8844,"pts":[[0.1774,0.9991,8778]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8844,"t1":8910,"pts":[[0.1942,0.9991,8844]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8910,"t1":8976,"pts":[[0.2111,0.9991,8910]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8976,"t1":9042,"pts":[[0.2279,0.9991,8976]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9042,"t1":9108,"pts":[[0.2448,0.9991,9042]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9108,"t1":10164,"pts":[[0.2092,0.9889,9108]],"dwell":1.056,"keyframe":null},{"kind":"cursor","t0":10164,"t1":10230,"pts":[[0.2071,0.9901,10164]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10230,"t1":10296,"pts":[[0.2047,0.9926,10230]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10296,"t1":11022,"pts":[[0.2038,0.9941,10296]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":11022,"t1":11748,"pts":[[0.2032,0.9921,11022]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":11748,"t1":11946,"pts":[[0.202,0.9946,11748]],"dwell":0.198,"keyframe":null},{"kind":"cursor","t0":11946,"t1":12000,"pts":[[0.2046,0.9918,11946]],"dwell":0.054,"keyframe":null},{"kind":"change","t0":4400,"t1":4400,"pts":[[0.1406,0.0244,4400],[0.3906,0.0244,4400],[0.1406,0.2683,4400],[0.3906,0.2683,4400],[0.2656,0.1463,4400]],"dwell":0.2469,"keyframe":0},{"kind":"change","t0":4400,"t1":4400,"pts":[[0.1719,0.3415,4400],[0.3594,0.3415,4400],[0.1719,0.3902,4400],[0.3594,0.3902,4400],[0.2656,0.3659,4400]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":4400,"t1":4400,"pts":[[0.0156,0.0732,4400],[0.0781,0.0732,4400],[0.0156,0.1463,4400],[0.0781,0.1463,4400],[0.0469,0.1098,4400]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":4400,"t1":4400,"pts":[[0.4063,0.1951,4400],[0.5625,0.1951,4400],[0.4063,0.2195,4400],[0.5625,0.2195,4400],[0.4844,0.2073,4400]],"dwell":0.0617,"keyframe":0},{"kind":"change","t0":4400,"t1":4400,"pts":[[0.375,0.3415,4400],[0.4531,0.3415,4400],[0.375,0.3902,4400],[0.4531,0.3902,4400],[0.4141,0.3659,4400]],"dwell":0.0617,"keyframe":0},{"kind":"change","t0":4400,"t1":4400,"pts":[[0.0,0.1707,4400],[0.0469,0.1707,4400],[0.0,0.2439,4400],[0.0469,0.2439,4400],[0.0234,0.2073,4400]],"dwell":0.0586,"keyframe":0},{"kind":"change","t0":4400,"t1":4400,"pts":[[0.1406,0.9756,4400],[0.25,0.9756,4400],[0.1406,1.0,4400],[0.25,1.0,4400],[0.1953,0.9878,4400]],"dwell":0.0516,"keyframe":0},{"kind":"change","t0":4400,"t1":4400,"pts":[[0.4531,0.0,4400],[0.5469,0.0,4400],[0.4531,0.0244,4400],[0.5469,0.0244,4400],[0.5,0.0122,4400]],"dwell":0.0478,"keyframe":0}],"snapshots":[[0,[{"id":4100,"app":"Zed","rect":[0.0,0.0,1.0,1.0],"layer":0,"regular":true,"frontmost":true}]]]}},{"name":"maximized-terminal","truth":{"scenario":"maximized-terminal","description":"One maximized OpenCode terminal; ink on a failing test, then the prompt","screen":[1728.0,1117.0],"whole":[0.0,147.0,1028.0,952.0],"spans":[{"t0":0,"t1":8600,"subject":"test","ideal":[0.0,141.0,1034.0,312.0],"targets":[[39.0,305.0,228.0,16.0]]},{"t0":8600,"t1":14000,"subject":"prompt","ideal":[0.0,418.0,1038.0,691.0],"targets":[[33.0,1053.0,187.0,16.0]]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":3080,"must":[[0.0134,0.2592,0.1676,0.3019]]},{"index":2,"t":5720,"must":[[0.0286,0.3098,0.1813,0.3138]]},{"index":3,"t":9680,"must":[[0.0156,0.9605,0.1308,0.9639]]}],"input":{"screen":[1728.0,1117.0],"duration":14000,"keyframes":[3080,5720,9680],"items":[{"kind":"stroke","t0":2200,"t1":3150,"pts":[[0.0502,0.2634,2200],[0.0582,0.261,2221],[0.069,0.2599,2242],[0.0808,0.2606,2263],[0.0918,0.2592,2284],[0.1024,0.2608,2306],[0.115,0.2598,2327],[0.1238,0.2622,2348],[0.1363,0.2626,2369],[0.144,0.2652,2390],[0.1493,0.268,2411],[0.1559,0.2705,2432],[0.1636,0.273,2453],[0.1617,0.2764,2474],[0.1676,0.2793,2496],[0.1629,0.2824,2517],[0.1676,0.2857,2538],[0.1606,0.2884,2559],[0.1561,0.2913,2580],[0.1509,0.2941,2601],[0.1429,0.2963,2622],[0.1314,0.2973,2643],[0.1248,0.3003,2664],[0.1141,0.3016,2686],[0.1025,0.3019,2707],[0.091,0.3006,2728],[0.0798,0.3014,2749],[0.0678,0.3018,2770],[0.0569,0.3006,2791],[0.05,0.2976,2812],[0.0394,0.2964,2833],[0.0293,0.2947,2854],[0.0283,0.2911,2876],[0.0243,0.2883,2897],[0.0204,0.2856,2918],[0.0145,0.2828,2939],[0.0189,0.2797,2960],[0.0134,0.2764,2981],[0.0193,0.2736,3002],[0.0214,0.2704,3023],[0.0293,0.268,3044],[0.0342,0.2649,3066],[0.0479,0.2644,3087],[0.0543,0.2614,3108],[0.0667,0.2614,3129],[0.0775,0.261,3150]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":5200,"t1":5700,"pts":[[0.0286,0.3109,5200],[0.0376,0.3098,5229],[0.0466,0.3112,5259],[0.0555,0.3104,5288],[0.0645,0.3124,5318],[0.0735,0.3106,5347],[0.0825,0.3116,5376],[0.0915,0.3118,5406],[0.1005,0.3114,5435],[0.1095,0.3115,5465],[0.1184,0.3131,5494],[0.1274,0.3117,5524],[0.1364,0.3115,5553],[0.1454,0.3134,5582],[0.1544,0.3138,5612],[0.1634,0.3124,5641],[0.1724,0.3125,5671],[0.1813,0.3131,5700]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":9000,"t1":9600,"pts":[[0.0156,0.9614,9000],[0.0224,0.9605,9035],[0.0292,0.9622,9071],[0.0359,0.9611,9106],[0.0427,0.9624,9141],[0.0495,0.9614,9176],[0.0563,0.9626,9212],[0.063,0.9621,9247],[0.0698,0.9637,9282],[0.0766,0.9624,9318],[0.0834,0.9633,9353],[0.0901,0.9639,9388],[0.0969,0.962,9424],[0.1037,0.9634,9459],[0.1104,0.9625,9494],[0.1172,0.9638,9529],[0.124,0.9638,9565],[0.1308,0.9638,9600]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":924,"pts":[[0.2378,0.2485,0]],"dwell":0.924,"keyframe":null},{"kind":"cursor","t0":924,"t1":990,"pts":[[0.2367,0.2463,924]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":990,"t1":1386,"pts":[[0.2358,0.2451,990]],"dwell":0.396,"keyframe":null},{"kind":"cursor","t0":1386,"t1":1848,"pts":[[0.2346,0.2459,1386]],"dwell":0.462,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.2307,0.2382,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":1980,"pts":[[0.2057,0.2312,1914]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1980,"t1":2046,"pts":[[0.1569,0.2311,1980]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.1021,0.2374,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.0631,0.2478,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.0504,0.2595,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.07,0.26,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2376,"pts":[[0.105,0.2606,2310]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2376,"t1":2442,"pts":[[0.1389,0.2635,2376]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2442,"t1":2508,"pts":[[0.1595,0.2717,2442]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2508,"t1":2574,"pts":[[0.1648,0.2811,2508]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2574,"t1":2640,"pts":[[0.1574,0.2905,2574]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2640,"t1":2706,"pts":[[0.1332,0.2971,2640]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2706,"t1":2772,"pts":[[0.1028,0.3019,2706]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2772,"t1":2838,"pts":[[0.0668,0.3017,2772]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2838,"t1":2904,"pts":[[0.0372,0.296,2838]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2904,"t1":2970,"pts":[[0.023,0.2874,2904]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2970,"t1":3036,"pts":[[0.0163,0.2781,2970]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3036,"t1":3102,"pts":[[0.0261,0.2689,3036]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3102,"t1":3168,"pts":[[0.0525,0.2623,3102]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3168,"t1":3234,"pts":[[0.0771,0.2615,3168]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3234,"t1":3300,"pts":[[0.0755,0.2631,3234]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3300,"t1":3366,"pts":[[0.0739,0.2647,3300]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3366,"t1":3432,"pts":[[0.0724,0.2663,3366]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3432,"t1":3498,"pts":[[0.0708,0.2679,3432]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3498,"t1":3564,"pts":[[0.0692,0.2695,3498]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3564,"t1":3630,"pts":[[0.0676,0.2711,3564]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3630,"t1":3696,"pts":[[0.0661,0.2727,3630]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3696,"t1":3762,"pts":[[0.0645,0.2743,3696]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3762,"t1":3828,"pts":[[0.0629,0.2759,3762]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3828,"t1":3894,"pts":[[0.0613,0.2775,3828]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3894,"t1":3960,"pts":[[0.0598,0.2791,3894]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3960,"t1":4026,"pts":[[0.0582,0.2807,3960]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4026,"t1":4092,"pts":[[0.0566,0.2823,4026]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4092,"t1":4158,"pts":[[0.055,0.2839,4092]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4158,"t1":4224,"pts":[[0.0535,0.2855,4158]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4224,"t1":4290,"pts":[[0.0519,0.2871,4224]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4290,"t1":4356,"pts":[[0.0503,0.2887,4290]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4356,"t1":4422,"pts":[[0.0487,0.2903,4356]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4422,"t1":4488,"pts":[[0.0472,0.292,4422]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4488,"t1":4554,"pts":[[0.0456,0.2936,4488]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4554,"t1":4620,"pts":[[0.044,0.2952,4554]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4620,"t1":4686,"pts":[[0.0424,0.2968,4620]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4686,"t1":4752,"pts":[[0.0409,0.2984,4686]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4752,"t1":4818,"pts":[[0.0393,0.3,4752]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4818,"t1":4884,"pts":[[0.0377,0.3016,4818]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4884,"t1":4950,"pts":[[0.0361,0.3032,4884]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4950,"t1":5016,"pts":[[0.0346,0.3048,4950]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5016,"t1":5082,"pts":[[0.033,0.3064,5016]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5082,"t1":5148,"pts":[[0.0314,0.308,5082]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5148,"t1":5214,"pts":[[0.0298,0.3096,5148]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5214,"t1":5280,"pts":[[0.0329,0.3104,5214]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5280,"t1":5346,"pts":[[0.053,0.3106,5280]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5346,"t1":5412,"pts":[[0.0732,0.3107,5346]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5412,"t1":5478,"pts":[[0.0934,0.3117,5412]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5478,"t1":5544,"pts":[[0.1135,0.3122,5478]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5544,"t1":5610,"pts":[[0.1337,0.3116,5544]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5610,"t1":5676,"pts":[[0.1538,0.3138,5610]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5676,"t1":5742,"pts":[[0.174,0.3126,5676]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5742,"t1":5808,"pts":[[0.1803,0.3101,5742]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5808,"t1":5874,"pts":[[0.1681,0.3053,5808]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5874,"t1":5940,"pts":[[0.1466,0.3021,5874]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5940,"t1":6006,"pts":[[0.1305,0.3026,5940]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6006,"t1":7392,"pts":[[0.1272,0.3061,6006]],"dwell":1.386,"keyframe":null},{"kind":"cursor","t0":7392,"t1":7458,"pts":[[0.1307,0.3049,7392]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7458,"t1":8250,"pts":[[0.1333,0.3029,7458]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":8250,"t1":8316,"pts":[[0.1323,0.3013,8250]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8316,"t1":8448,"pts":[[0.1304,0.3014,8316]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.1206,0.3013,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8580,"pts":[[0.1047,0.3273,8514]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8580,"t1":8646,"pts":[[0.0834,0.398,8580]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.0587,0.5092,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":8778,"pts":[[0.0343,0.6422,8712]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8778,"t1":8844,"pts":[[0.0148,0.7723,8778]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8844,"t1":8910,"pts":[[0.0038,0.8757,8844]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8910,"t1":8976,"pts":[[0.0032,0.9374,8910]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8976,"t1":9042,"pts":[[0.0114,0.9592,8976]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9042,"t1":9108,"pts":[[0.0237,0.9608,9042]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9108,"t1":9174,"pts":[[0.0363,0.9612,9108]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9174,"t1":9240,"pts":[[0.049,0.9614,9174]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9240,"t1":9306,"pts":[[0.0617,0.9622,9240]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9306,"t1":9372,"pts":[[0.0743,0.9628,9306]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9372,"t1":9438,"pts":[[0.087,0.9636,9372]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9438,"t1":9504,"pts":[[0.0997,0.9626,9438]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9504,"t1":9570,"pts":[[0.1123,0.9629,9504]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9570,"t1":9636,"pts":[[0.125,0.9638,9570]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9636,"t1":9768,"pts":[[0.1308,0.9638,9636]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":9768,"t1":9834,"pts":[[0.1271,0.9589,9768]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9834,"t1":9900,"pts":[[0.1111,0.9535,9834]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9900,"t1":9966,"pts":[[0.0913,0.9503,9900]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9966,"t1":10032,"pts":[[0.081,0.9516,9966]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10032,"t1":10956,"pts":[[0.0802,0.9536,10032]],"dwell":0.924,"keyframe":null},{"kind":"cursor","t0":10956,"t1":11022,"pts":[[0.0832,0.9553,10956]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11022,"t1":11088,"pts":[[0.0875,0.9561,11022]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11088,"t1":13200,"pts":[[0.09,0.9559,11088]],"dwell":2.112,"keyframe":null},{"kind":"cursor","t0":13200,"t1":13266,"pts":[[0.0885,0.9524,13200]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":13266,"t1":13332,"pts":[[0.0869,0.9486,13266]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":13332,"t1":13860,"pts":[[0.0858,0.9469,13332]],"dwell":0.528,"keyframe":null},{"kind":"cursor","t0":13860,"t1":13992,"pts":[[0.085,0.9484,13860]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":13992,"t1":14000,"pts":[[0.0847,0.9504,13992]],"dwell":0.008,"keyframe":null},{"kind":"change","t0":3080,"t1":3080,"pts":[[0.0156,0.3902,3080],[0.25,0.3902,3080],[0.0156,0.561,3080],[0.25,0.561,3080],[0.1328,0.4756,3080]],"dwell":0.2,"keyframe":0},{"kind":"change","t0":3080,"t1":3080,"pts":[[0.0156,0.1707,3080],[0.1875,0.1707,3080],[0.0156,0.3659,3080],[0.1875,0.3659,3080],[0.1016,0.2683,3080]],"dwell":0.1831,"keyframe":0},{"kind":"change","t0":3080,"t1":3080,"pts":[[0.0,0.0732,3080],[0.3438,0.0732,3080],[0.0,0.1463,3080],[0.3438,0.1463,3080],[0.1719,0.1098,3080]],"dwell":0.1586,"keyframe":0},{"kind":"change","t0":3080,"t1":3080,"pts":[[0.0156,0.9268,3080],[0.1406,0.9268,3080],[0.0156,1.0,3080],[0.1406,1.0,3080],[0.0781,0.9634,3080]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":3080,"t1":3080,"pts":[[0.2344,0.3902,3080],[0.375,0.3902,3080],[0.2344,0.4146,3080],[0.375,0.4146,3080],[0.3047,0.4024,3080]],"dwell":0.0586,"keyframe":0},{"kind":"change","t0":3080,"t1":3080,"pts":[[0.1094,0.0,3080],[0.1875,0.0,3080],[0.1094,0.0244,3080],[0.1875,0.0244,3080],[0.1484,0.0122,3080]],"dwell":0.0437,"keyframe":0},{"kind":"change","t0":3080,"t1":3080,"pts":[[0.0938,0.1707,3080],[0.1563,0.1707,3080],[0.0938,0.1951,3080],[0.1563,0.1951,3080],[0.125,0.1829,3080]],"dwell":0.039,"keyframe":0},{"kind":"change","t0":3080,"t1":3080,"pts":[[0.1094,0.3415,3080],[0.1719,0.3415,3080],[0.1094,0.3659,3080],[0.1719,0.3659,3080],[0.1406,0.3537,3080]],"dwell":0.039,"keyframe":0}],"snapshots":[[0,[{"id":4100,"app":"Ghostty","rect":[0.0,0.0295,1.0,1.0],"layer":0,"regular":true,"frontmost":true},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]]]}},{"name":"menu-bar","truth":{"scenario":"menu-bar","description":"Opening the View menu from the menu bar and switching Finder to a list","screen":[1728.0,1117.0],"whole":[152.0,0.0,916.0,738.0],"spans":[{"t0":0,"t1":2400,"subject":"finder","ideal":[160.0,130.0,900.0,600.0],"targets":[]},{"t0":2400,"t1":4700,"subject":"menu","ideal":[152.0,0.0,916.0,738.0],"targets":[]},{"t0":4700,"t1":12000,"subject":"finder","ideal":[160.0,130.0,900.0,600.0],"targets":[]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":2640,"must":[[0.1081,0.0091,0.1127,0.0163]]},{"index":2,"t":4620,"must":[[0.148,0.0642,0.1526,0.0714]]},{"index":3,"t":8580,"must":[[0.1964,0.2694,0.4201,0.3117]]}],"input":{"screen":[1728.0,1117.0],"duration":12000,"keyframes":[2640,4620,8580],"items":[{"kind":"stroke","t0":7800,"t1":8600,"pts":[[0.2163,0.2772,7800],[0.2355,0.2764,7818],[0.2445,0.2738,7836],[0.2578,0.272,7853],[0.275,0.272,7871],[0.2893,0.2704,7889],[0.3055,0.2694,7907],[0.3209,0.2714,7924],[0.3389,0.2701,7942],[0.3514,0.2726,7960],[0.3696,0.2728,7978],[0.384,0.2745,7996],[0.3937,0.2772,8013],[0.4034,0.2797,8031],[0.4136,0.2823,8049],[0.415,0.2855,8067],[0.4201,0.2885,8084],[0.4109,0.2916,8102],[0.411,0.2945,8120],[0.4072,0.2973,8138],[0.4092,0.3008,8156],[0.3922,0.3023,8173],[0.3838,0.3048,8191],[0.3755,0.3076,8209],[0.3574,0.3078,8227],[0.3461,0.3103,8244],[0.3312,0.3117,8262],[0.3144,0.3114,8280],[0.2984,0.3111,8298],[0.2818,0.3113,8316],[0.2663,0.3103,8333],[0.2552,0.3077,8351],[0.2385,0.307,8369],[0.2316,0.3041,8387],[0.2202,0.302,8404],[0.2083,0.2998,8422],[0.203,0.2969,8440],[0.1964,0.294,8458],[0.2049,0.2909,8476],[0.2049,0.288,8493],[0.1999,0.2847,8511],[0.206,0.2817,8529],[0.2183,0.2795,8547],[0.2288,0.2772,8564],[0.2432,0.2757,8582],[0.2548,0.2738,8600]],"dwell":0.0,"keyframe":null},{"kind":"click","t0":2400,"t1":2400,"pts":[[0.1104,0.0127,2400]],"dwell":0.0,"keyframe":null},{"kind":"click","t0":4400,"t1":4400,"pts":[[0.1503,0.0678,4400]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":660,"pts":[[0.4357,0.2282,0]],"dwell":0.66,"keyframe":null},{"kind":"cursor","t0":660,"t1":726,"pts":[[0.438,0.2267,660]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":726,"t1":792,"pts":[[0.4414,0.2225,726]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":792,"t1":1254,"pts":[[0.4433,0.2193,792]],"dwell":0.462,"keyframe":null},{"kind":"cursor","t0":1254,"t1":1320,"pts":[[0.4421,0.2201,1254]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1320,"t1":1782,"pts":[[0.441,0.2222,1320]],"dwell":0.462,"keyframe":null},{"kind":"cursor","t0":1782,"t1":1848,"pts":[[0.4421,0.213,1782]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.4288,0.1878,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":1980,"pts":[[0.3873,0.1488,1914]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1980,"t1":2046,"pts":[[0.3218,0.1003,1980]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.2465,0.0526,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.1789,0.0165,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.1332,0.0003,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.1136,0.0047,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2574,"pts":[[0.1104,0.0127,2310]],"dwell":0.264,"keyframe":null},{"kind":"cursor","t0":2574,"t1":2640,"pts":[[0.115,0.0269,2574]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2640,"t1":2706,"pts":[[0.1422,0.0653,2640]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2706,"t1":3498,"pts":[[0.154,0.0733,2706]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":3498,"t1":3564,"pts":[[0.1522,0.072,3498]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3564,"t1":3630,"pts":[[0.1495,0.0712,3564]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3630,"t1":3762,"pts":[[0.148,0.0713,3630]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":3762,"t1":3828,"pts":[[0.15,0.0714,3762]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3828,"t1":4158,"pts":[[0.1524,0.0705,3828]],"dwell":0.33,"keyframe":null},{"kind":"cursor","t0":4158,"t1":4290,"pts":[[0.1517,0.0682,4158]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":4290,"t1":4620,"pts":[[0.1503,0.0678,4290]],"dwell":0.33,"keyframe":null},{"kind":"cursor","t0":4620,"t1":4686,"pts":[[0.1481,0.0758,4620]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4686,"t1":4752,"pts":[[0.1594,0.1065,4686]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4752,"t1":4818,"pts":[[0.1921,0.1618,4752]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4818,"t1":4884,"pts":[[0.2362,0.2255,4818]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4884,"t1":4950,"pts":[[0.2757,0.2744,4884]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4950,"t1":5016,"pts":[[0.2982,0.2926,4950]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5016,"t1":5874,"pts":[[0.3044,0.2887,5016]],"dwell":0.858,"keyframe":null},{"kind":"cursor","t0":5874,"t1":5940,"pts":[[0.3017,0.2894,5874]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5940,"t1":6006,"pts":[[0.2989,0.2912,5940]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6006,"t1":6864,"pts":[[0.2978,0.2924,6006]],"dwell":0.858,"keyframe":null},{"kind":"cursor","t0":6864,"t1":6930,"pts":[[0.297,0.2905,6864]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6930,"t1":6996,"pts":[[0.2955,0.288,6930]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6996,"t1":7260,"pts":[[0.2945,0.287,6996]],"dwell":0.264,"keyframe":null},{"kind":"cursor","t0":7260,"t1":7326,"pts":[[0.2966,0.2894,7260]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7326,"t1":7524,"pts":[[0.2989,0.2907,7326]],"dwell":0.198,"keyframe":null},{"kind":"cursor","t0":7524,"t1":7590,"pts":[[0.2978,0.2856,7524]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7590,"t1":7656,"pts":[[0.2802,0.2782,7590]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7656,"t1":7722,"pts":[[0.2499,0.2724,7656]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7722,"t1":7788,"pts":[[0.2246,0.2714,7722]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7788,"t1":7854,"pts":[[0.2165,0.2761,7788]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7854,"t1":7920,"pts":[[0.2584,0.272,7854]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7920,"t1":7986,"pts":[[0.3171,0.2709,7920]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7986,"t1":8052,"pts":[[0.3763,0.2736,7986]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8052,"t1":8118,"pts":[[0.4138,0.2829,8052]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8184,"pts":[[0.411,0.2941,8118]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8184,"t1":8250,"pts":[[0.3871,0.3038,8184]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8250,"t1":8316,"pts":[[0.3414,0.3107,8250]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8316,"t1":8382,"pts":[[0.2814,0.3113,8316]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8382,"t1":8448,"pts":[[0.2334,0.3048,8382]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.2,0.2956,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8580,"pts":[[0.2009,0.2842,8514]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8580,"t1":8646,"pts":[[0.2414,0.2759,8580]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.2636,0.2823,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":9966,"pts":[[0.2771,0.2874,8712]],"dwell":1.254,"keyframe":null},{"kind":"cursor","t0":9966,"t1":10032,"pts":[[0.2749,0.289,9966]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10032,"t1":10824,"pts":[[0.2739,0.2903,10032]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":10824,"t1":11880,"pts":[[0.2753,0.2916,10824]],"dwell":1.056,"keyframe":null},{"kind":"cursor","t0":11880,"t1":11946,"pts":[[0.2748,0.2894,11880]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11946,"t1":12000,"pts":[[0.2739,0.2884,11946]],"dwell":0.054,"keyframe":null},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.6563,0.1707,2640],[0.9531,0.1707,2640],[0.6563,0.4878,2640],[0.9531,0.4878,2640],[0.8047,0.3293,2640]],"dwell":0.3068,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.0625,0.0,2640],[0.2031,0.0,2640],[0.0625,0.3659,2640],[0.2031,0.3659,2640],[0.1328,0.1829,2640]],"dwell":0.2268,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.625,0.0732,2640],[0.9688,0.0732,2640],[0.625,0.1463,2640],[0.9688,0.1463,2640],[0.7969,0.1098,2640]],"dwell":0.1586,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.1875,0.6341,2640],[0.6094,0.6341,2640],[0.1875,0.6585,2640],[0.6094,0.6585,2640],[0.3984,0.6463,2640]],"dwell":0.1014,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.3281,0.9268,2640],[0.5469,0.9268,2640],[0.3281,0.9512,2640],[0.5469,0.9512,2640],[0.4375,0.939,2640]],"dwell":0.073,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.5938,0.9268,2640],[0.6875,0.9268,2640],[0.5938,0.9512,2640],[0.6875,0.9512,2640],[0.6406,0.939,2640]],"dwell":0.0478,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.6406,0.8293,2640],[0.7188,0.8293,2640],[0.6406,0.8537,2640],[0.7188,0.8537,2640],[0.6797,0.8415,2640]],"dwell":0.0437,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.3281,0.2439,2640],[0.3906,0.2439,2640],[0.3281,0.2683,2640],[0.3906,0.2683,2640],[0.3594,0.2561,2640]],"dwell":0.039,"keyframe":0},{"kind":"change","t0":4620,"t1":4620,"pts":[[0.0938,0.0,4620],[0.2813,0.0,4620],[0.0938,0.3902,4620],[0.2813,0.3902,4620],[0.1875,0.1951,4620]],"dwell":0.2705,"keyframe":1},{"kind":"change","t0":4620,"t1":4620,"pts":[[0.5,0.1707,4620],[0.5781,0.1707,4620],[0.5,0.3415,4620],[0.5781,0.3415,4620],[0.5391,0.2561,4620]],"dwell":0.1155,"keyframe":1},{"kind":"change","t0":4620,"t1":4620,"pts":[[0.3281,0.1707,4620],[0.3906,0.1707,4620],[0.3281,0.3415,4620],[0.3906,0.3415,4620],[0.3594,0.2561,4620]],"dwell":0.1033,"keyframe":1},{"kind":"change","t0":4620,"t1":4620,"pts":[[0.4375,0.1707,4620],[0.4844,0.1707,4620],[0.4375,0.3415,4620],[0.4844,0.3415,4620],[0.4609,0.2561,4620]],"dwell":0.0895,"keyframe":1},{"kind":"change","t0":4620,"t1":4620,"pts":[[0.4063,0.1951,4620],[0.4219,0.1951,4620],[0.4063,0.2683,4620],[0.4219,0.2683,4620],[0.4141,0.2317,4620]],"dwell":0.0338,"keyframe":1}],"snapshots":[[0,[{"id":4100,"app":"Finder","rect":[0.0926,0.1164,0.6134,0.6535],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Slack","rect":[0.4977,0.0806,0.9722,0.8863],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false},{"id":38,"app":"Dock","rect":[0.0,0.9338,1.0,1.0],"layer":20,"regular":false,"frontmost":false}]],[2460,[{"id":900,"app":"Finder","rect":[0.099,0.0295,0.1916,0.3375],"layer":101,"regular":true,"frontmost":false},{"id":4100,"app":"Finder","rect":[0.0926,0.1164,0.6134,0.6535],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Slack","rect":[0.4977,0.0806,0.9722,0.8863],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false},{"id":38,"app":"Dock","rect":[0.0,0.9338,1.0,1.0],"layer":20,"regular":false,"frontmost":false}]],[3400,[{"id":900,"app":"Finder","rect":[0.099,0.0295,0.1916,0.3375],"layer":101,"regular":true,"frontmost":false},{"id":4100,"app":"Finder","rect":[0.0926,0.1164,0.6134,0.6535],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Slack","rect":[0.4977,0.0806,0.9722,0.8863],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false},{"id":38,"app":"Dock","rect":[0.0,0.9338,1.0,1.0],"layer":20,"regular":false,"frontmost":false}]],[4460,[{"id":4100,"app":"Finder","rect":[0.0926,0.1164,0.6134,0.6535],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Slack","rect":[0.4977,0.0806,0.9722,0.8863],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false},{"id":38,"app":"Dock","rect":[0.0,0.9338,1.0,1.0],"layer":20,"regular":false,"frontmost":false}]]]}},{"name":"overlapping-front","truth":{"scenario":"overlapping-front","description":"Three overlapping windows; all the activity is on the front one","screen":[1728.0,1117.0],"whole":[640.0,260.0,1000.0,780.0],"spans":[{"t0":0,"t1":12000,"subject":"editor","ideal":[640.0,260.0,1000.0,780.0],"targets":[]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":2860,"must":[[0.7577,0.4275,0.9433,0.4693]]},{"index":2,"t":6160,"must":[[0.631,0.4433,0.6356,0.4505]]},{"index":3,"t":8800,"must":[[0.5023,0.93,0.63,0.9327]]}],"input":{"screen":[1728.0,1117.0],"duration":12000,"keyframes":[2860,6160,8800],"items":[{"kind":"stroke","t0":1900,"t1":2800,"pts":[[0.8025,0.4316,1900],[0.8121,0.4292,1920],[0.8251,0.4282,1940],[0.8392,0.4288,1960],[0.8524,0.4275,1980],[0.8651,0.4291,2000],[0.8802,0.4281,2020],[0.8908,0.4304,2040],[0.9059,0.4309,2060],[0.915,0.4333,2080],[0.9214,0.4361,2100],[0.9294,0.4386,2120],[0.9386,0.441,2140],[0.9362,0.4443,2160],[0.9433,0.4472,2180],[0.9377,0.4502,2200],[0.9433,0.4535,2220],[0.9349,0.4561,2240],[0.9295,0.4589,2260],[0.9232,0.4617,2280],[0.9135,0.4639,2300],[0.8997,0.4648,2320],[0.8918,0.4677,2340],[0.8789,0.469,2360],[0.8649,0.4693,2380],[0.8511,0.468,2400],[0.8377,0.4689,2420],[0.8232,0.4692,2440],[0.8102,0.468,2460],[0.8019,0.4651,2480],[0.7891,0.4639,2500],[0.777,0.4622,2520],[0.7757,0.4587,2540],[0.7709,0.456,2560],[0.7661,0.4533,2580],[0.759,0.4506,2600],[0.7643,0.4476,2620],[0.7577,0.4444,2640],[0.7648,0.4416,2660],[0.7673,0.4384,2680],[0.7767,0.4361,2700],[0.7826,0.4331,2720],[0.7991,0.4326,2740],[0.8067,0.4297,2760],[0.8216,0.4297,2780],[0.8346,0.4293,2800]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":8300,"t1":8800,"pts":[[0.5023,0.9309,8300],[0.5098,0.9313,8329],[0.5173,0.93,8359],[0.5248,0.93,8388],[0.5323,0.9306,8418],[0.5399,0.932,8447],[0.5474,0.9322,8476],[0.5549,0.9305,8506],[0.5624,0.9323,8535],[0.5699,0.9325,8565],[0.5774,0.9325,8594],[0.5849,0.9316,8624],[0.5924,0.9315,8653],[0.5999,0.9317,8682],[0.6074,0.9308,8712],[0.6149,0.9325,8741],[0.6225,0.9315,8771],[0.63,0.9327,8800]],"dwell":0.0,"keyframe":null},{"kind":"click","t0":6000,"t1":6000,"pts":[[0.6333,0.4469,6000]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":924,"pts":[[0.5296,0.2738,0]],"dwell":0.924,"keyframe":null},{"kind":"cursor","t0":924,"t1":990,"pts":[[0.5285,0.2715,924]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":990,"t1":1518,"pts":[[0.5276,0.2704,990]],"dwell":0.528,"keyframe":null},{"kind":"cursor","t0":1518,"t1":1650,"pts":[[0.5261,0.2715,1518]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":1650,"t1":1716,"pts":[[0.5316,0.2948,1650]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1716,"t1":1782,"pts":[[0.5997,0.3511,1716]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1782,"t1":1848,"pts":[[0.7101,0.4151,1782]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.7873,0.443,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":1980,"pts":[[0.8092,0.43,1914]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1980,"t1":2046,"pts":[[0.8524,0.4275,1980]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.8953,0.4306,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.9262,0.4376,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.9426,0.4469,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.9338,0.4567,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2376,"pts":[[0.9066,0.4643,2310]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2376,"t1":2442,"pts":[[0.8677,0.4693,2376]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2442,"t1":2508,"pts":[[0.8219,0.4691,2442]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2508,"t1":2574,"pts":[[0.7842,0.4632,2508]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2574,"t1":2640,"pts":[[0.7676,0.4541,2574]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2640,"t1":2706,"pts":[[0.7577,0.4444,2640]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2706,"t1":2772,"pts":[[0.7785,0.4352,2706]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2772,"t1":2838,"pts":[[0.8157,0.4297,2772]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2838,"t1":2904,"pts":[[0.7799,0.4108,2838]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2904,"t1":4224,"pts":[[0.6435,0.4417,2904]],"dwell":1.32,"keyframe":null},{"kind":"cursor","t0":4224,"t1":4290,"pts":[[0.6416,0.4426,4224]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4290,"t1":5478,"pts":[[0.64,0.444,4290]],"dwell":1.188,"keyframe":null},{"kind":"cursor","t0":5478,"t1":5544,"pts":[[0.6411,0.443,5478]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5544,"t1":5610,"pts":[[0.6423,0.4445,5544]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5610,"t1":5742,"pts":[[0.6437,0.4453,5610]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":5742,"t1":5808,"pts":[[0.6387,0.4448,5742]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5808,"t1":5874,"pts":[[0.6345,0.4458,5808]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5874,"t1":7788,"pts":[[0.6333,0.4468,5874]],"dwell":1.914,"keyframe":null},{"kind":"cursor","t0":7788,"t1":7854,"pts":[[0.6315,0.4458,7788]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7854,"t1":7920,"pts":[[0.6186,0.4521,7854]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7920,"t1":7986,"pts":[[0.5971,0.498,7920]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7986,"t1":8052,"pts":[[0.5678,0.5855,7986]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8052,"t1":8118,"pts":[[0.5364,0.6968,8052]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8184,"pts":[[0.5104,0.8046,8118]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8184,"t1":8250,"pts":[[0.4962,0.8839,8184]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8250,"t1":8316,"pts":[[0.4959,0.9225,8250]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8316,"t1":8382,"pts":[[0.5064,0.9311,8316]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8382,"t1":8448,"pts":[[0.5232,0.93,8382]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.5401,0.932,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8580,"pts":[[0.5569,0.931,8514]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8580,"t1":8646,"pts":[[0.5738,0.9325,8580]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.5906,0.9316,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":8778,"pts":[[0.6075,0.9308,8712]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8778,"t1":8844,"pts":[[0.6243,0.9318,8778]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8844,"t1":8910,"pts":[[0.5987,0.9195,8844]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8910,"t1":9768,"pts":[[0.5483,0.9242,8910]],"dwell":0.858,"keyframe":null},{"kind":"cursor","t0":9768,"t1":9834,"pts":[[0.5496,0.9242,9768]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9834,"t1":10494,"pts":[[0.5511,0.9237,9834]],"dwell":0.66,"keyframe":null},{"kind":"cursor","t0":10494,"t1":10560,"pts":[[0.5519,0.9223,10494]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10560,"t1":10626,"pts":[[0.5515,0.9195,10560]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10626,"t1":11880,"pts":[[0.551,0.9168,10626]],"dwell":1.254,"keyframe":null},{"kind":"cursor","t0":11880,"t1":12000,"pts":[[0.5505,0.9194,11880]],"dwell":0.12,"keyframe":null},{"kind":"change","t0":2860,"t1":2860,"pts":[[0.1875,0.1951,2860],[0.4375,0.1951,2860],[0.1875,0.561,2860],[0.4375,0.561,2860],[0.3125,0.378,2860]],"dwell":0.3024,"keyframe":0},{"kind":"change","t0":2860,"t1":2860,"pts":[[0.5,0.2683,2860],[0.7656,0.2683,2860],[0.5,0.5122,2860],[0.7656,0.5122,2860],[0.6328,0.3902,2860]],"dwell":0.2545,"keyframe":0},{"kind":"change","t0":2860,"t1":2860,"pts":[[0.1719,0.0732,2860],[0.5625,0.0732,2860],[0.1719,0.1463,2860],[0.5625,0.1463,2860],[0.3672,0.1098,2860]],"dwell":0.1691,"keyframe":0},{"kind":"change","t0":2860,"t1":2860,"pts":[[0.1875,0.5854,2860],[0.375,0.5854,2860],[0.1875,0.7073,2860],[0.375,0.7073,2860],[0.2813,0.6463,2860]],"dwell":0.1512,"keyframe":0},{"kind":"change","t0":2860,"t1":2860,"pts":[[0.5313,0.561,2860],[0.8281,0.561,2860],[0.5313,0.6098,2860],[0.8281,0.6098,2860],[0.6797,0.5854,2860]],"dwell":0.1203,"keyframe":0},{"kind":"change","t0":2860,"t1":2860,"pts":[[0.0469,0.122,2860],[0.0938,0.122,2860],[0.0469,0.3171,2860],[0.0938,0.3171,2860],[0.0703,0.2195,2860]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":2860,"t1":2860,"pts":[[0.7656,0.439,2860],[0.9375,0.439,2860],[0.7656,0.4634,2860],[0.9375,0.4634,2860],[0.8516,0.4512,2860]],"dwell":0.0647,"keyframe":0},{"kind":"change","t0":2860,"t1":2860,"pts":[[0.1875,0.7805,2860],[0.3125,0.7805,2860],[0.1875,0.8049,2860],[0.3125,0.8049,2860],[0.25,0.7927,2860]],"dwell":0.0552,"keyframe":0}],"snapshots":[[0,[{"id":4100,"app":"Zed","rect":[0.3704,0.2328,0.9491,0.9311],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Ghostty","rect":[0.1736,0.1611,0.7523,0.8415],"layer":0,"regular":true,"frontmost":false},{"id":4102,"app":"Slack","rect":[0.0463,0.0716,0.5671,0.6983],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]]]}},{"name":"scrolling-page","truth":{"scenario":"scrolling-page","description":"Scrolling a long docs page in a big browser window, then inking a paragraph","screen":[1728.0,1117.0],"whole":[384.0,168.0,832.0,452.0],"spans":[{"t0":0,"t1":4900,"subject":"scroll","ideal":[150.0,118.0,1420.0,986.0],"targets":[]},{"t0":4900,"t1":15000,"subject":"para","ideal":[384.0,168.0,832.0,452.0],"targets":[[811.0,332.0,182.0,16.0]]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":1540,"must":[]},{"index":2,"t":2640,"must":[]},{"index":3,"t":5720,"must":[[0.4583,0.2823,0.5907,0.326]]},{"index":4,"t":9240,"must":[[0.5437,0.509,0.5483,0.5162]]},{"index":5,"t":10780,"must":[[0.2419,0.3385,0.6657,0.3416]]}],"input":{"screen":[1728.0,1117.0],"duration":15000,"keyframes":[1540,2640,5720,9240,10780],"items":[{"kind":"stroke","t0":5000,"t1":5800,"pts":[[0.4734,0.2894,5000],[0.4808,0.2873,5018],[0.4898,0.2859,5036],[0.4983,0.2845,5053],[0.5082,0.2844,5071],[0.5175,0.2845,5089],[0.5269,0.2823,5107],[0.5366,0.283,5124],[0.544,0.2857,5142],[0.5547,0.2854,5160],[0.5599,0.2885,5178],[0.5688,0.2897,5196],[0.5774,0.2915,5213],[0.5801,0.2947,5231],[0.5812,0.2978,5249],[0.5842,0.3006,5267],[0.5907,0.3036,5284],[0.5865,0.3067,5302],[0.5867,0.3098,5320],[0.585,0.313,5338],[0.5801,0.3158,5356],[0.5732,0.3181,5373],[0.5636,0.3192,5391],[0.559,0.3222,5409],[0.5499,0.3231,5427],[0.5419,0.3249,5444],[0.5328,0.3259,5462],[0.5231,0.326,5480],[0.5143,0.3239,5498],[0.5059,0.3231,5516],[0.4972,0.3224,5533],[0.489,0.3211,5551],[0.4807,0.3196,5569],[0.4748,0.3172,5587],[0.4694,0.3147,5604],[0.4638,0.3121,5622],[0.4658,0.3088,5640],[0.4583,0.306,5658],[0.4616,0.3029,5676],[0.4604,0.2996,5693],[0.4693,0.2973,5711],[0.4675,0.2937,5729],[0.4772,0.2921,5747],[0.4797,0.2887,5764],[0.4907,0.2882,5782],[0.4961,0.2852,5800]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":10200,"t1":10800,"pts":[[0.2419,0.3387,10200],[0.2668,0.3403,10235],[0.2918,0.3386,10271],[0.3167,0.3385,10306],[0.3416,0.339,10341],[0.3665,0.3401,10376],[0.3915,0.3396,10412],[0.4164,0.3392,10447],[0.4413,0.3398,10482],[0.4663,0.3396,10518],[0.4912,0.3416,10553],[0.5161,0.3414,10588],[0.541,0.3415,10624],[0.566,0.3398,10659],[0.5909,0.3396,10694],[0.6158,0.3402,10729],[0.6408,0.3412,10765],[0.6657,0.3403,10800]],"dwell":0.0,"keyframe":null},{"kind":"scroll","t0":1400,"t1":1400,"pts":[[0.4977,0.547,1400]],"dwell":0.0,"keyframe":null},{"kind":"scroll","t0":2000,"t1":2000,"pts":[[0.4977,0.547,2000]],"dwell":0.0,"keyframe":null},{"kind":"scroll","t0":2600,"t1":2600,"pts":[[0.4977,0.547,2600]],"dwell":0.0,"keyframe":null},{"kind":"scroll","t0":3200,"t1":3200,"pts":[[0.4977,0.547,3200]],"dwell":0.0,"keyframe":null},{"kind":"click","t0":9000,"t1":9000,"pts":[[0.546,0.5126,9000]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":858,"pts":[[0.4799,0.304,0]],"dwell":0.858,"keyframe":null},{"kind":"cursor","t0":858,"t1":924,"pts":[[0.4786,0.3016,858]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":924,"t1":1056,"pts":[[0.4772,0.2997,924]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":1056,"t1":1122,"pts":[[0.4706,0.3108,1056]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1122,"t1":1188,"pts":[[0.4706,0.3725,1122]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1188,"t1":1254,"pts":[[0.4782,0.4655,1188]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1254,"t1":1320,"pts":[[0.4886,0.533,1254]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1320,"t1":3300,"pts":[[0.4977,0.547,1320]],"dwell":1.98,"keyframe":null},{"kind":"cursor","t0":3300,"t1":3366,"pts":[[0.5044,0.5415,3300]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3366,"t1":3432,"pts":[[0.5349,0.3666,3366]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3432,"t1":4620,"pts":[[0.5311,0.3331,3432]],"dwell":1.188,"keyframe":null},{"kind":"cursor","t0":4620,"t1":4686,"pts":[[0.5328,0.3323,4620]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4686,"t1":4818,"pts":[[0.5343,0.3311,4686]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":4818,"t1":4884,"pts":[[0.5304,0.3279,4818]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4884,"t1":4950,"pts":[[0.5122,0.3089,4884]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4950,"t1":5082,"pts":[[0.4808,0.2886,4950]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":5082,"t1":5148,"pts":[[0.5139,0.2844,5082]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5148,"t1":5214,"pts":[[0.5475,0.2856,5148]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5214,"t1":5280,"pts":[[0.5775,0.2916,5214]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5280,"t1":5346,"pts":[[0.5891,0.3028,5280]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5346,"t1":5412,"pts":[[0.5827,0.3143,5346]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5412,"t1":5478,"pts":[[0.5574,0.3224,5412]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5478,"t1":5544,"pts":[[0.5242,0.326,5478]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5544,"t1":5610,"pts":[[0.4923,0.3216,5544]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5610,"t1":5676,"pts":[[0.4677,0.3139,5610]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5676,"t1":5742,"pts":[[0.4616,0.3028,5676]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5742,"t1":5808,"pts":[[0.4746,0.2925,5742]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5808,"t1":6600,"pts":[[0.5152,0.2988,5808]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":6600,"t1":7326,"pts":[[0.5152,0.3008,6600]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":7326,"t1":7392,"pts":[[0.5151,0.3056,7326]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7392,"t1":8052,"pts":[[0.5153,0.3086,7392]],"dwell":0.66,"keyframe":null},{"kind":"cursor","t0":8052,"t1":8118,"pts":[[0.515,0.3061,8052]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8448,"pts":[[0.5143,0.3041,8118]],"dwell":0.33,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.5155,0.3044,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8646,"pts":[[0.5173,0.3043,8514]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.5138,0.3127,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":8778,"pts":[[0.5167,0.3734,8712]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8778,"t1":8844,"pts":[[0.529,0.4619,8778]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8844,"t1":8910,"pts":[[0.5413,0.5103,8844]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8910,"t1":9702,"pts":[[0.546,0.5126,8910]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":9702,"t1":9768,"pts":[[0.5467,0.5093,9702]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9768,"t1":9834,"pts":[[0.5412,0.4897,9768]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9834,"t1":9900,"pts":[[0.5092,0.4581,9834]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9900,"t1":9966,"pts":[[0.4506,0.4168,9900]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9966,"t1":10032,"pts":[[0.3784,0.3744,9966]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10032,"t1":10098,"pts":[[0.3111,0.3416,10032]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10098,"t1":10164,"pts":[[0.2646,0.3266,10098]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10164,"t1":10230,"pts":[[0.2448,0.331,10164]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10230,"t1":10296,"pts":[[0.2631,0.3401,10230]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10296,"t1":10362,"pts":[[0.3097,0.3385,10296]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10362,"t1":10428,"pts":[[0.3563,0.3397,10362]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10428,"t1":10494,"pts":[[0.4029,0.3394,10428]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10494,"t1":10560,"pts":[[0.4496,0.3397,10494]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10560,"t1":10626,"pts":[[0.4962,0.3416,10560]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10626,"t1":10692,"pts":[[0.5428,0.3414,10626]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10692,"t1":10758,"pts":[[0.5894,0.3397,10692]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10758,"t1":10824,"pts":[[0.636,0.341,10758]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10824,"t1":11550,"pts":[[0.468,0.3217,10824]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":11550,"t1":11616,"pts":[[0.4658,0.321,11550]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11616,"t1":12540,"pts":[[0.4638,0.3211,11616]],"dwell":0.924,"keyframe":null},{"kind":"cursor","t0":12540,"t1":12606,"pts":[[0.4665,0.3207,12540]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":12606,"t1":12672,"pts":[[0.4697,0.3186,12606]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":12672,"t1":14124,"pts":[[0.4709,0.3173,12672]],"dwell":1.452,"keyframe":null},{"kind":"cursor","t0":14124,"t1":14190,"pts":[[0.4696,0.319,14124]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":14190,"t1":15000,"pts":[[0.4688,0.3209,14190]],"dwell":0.81,"keyframe":null},{"kind":"change","t0":1540,"t1":1540,"pts":[[0.0781,0.0732,1540],[0.9219,0.0732,1540],[0.0781,1.0,1540],[0.9219,1.0,1540],[0.5,0.5366,1540]],"dwell":0.8843,"keyframe":0},{"kind":"change","t0":1540,"t1":1540,"pts":[[0.9063,0.0488,1540],[0.9219,0.0488,1540],[0.9063,0.878,1540],[0.9219,0.878,1540],[0.9141,0.4634,1540]],"dwell":0.1138,"keyframe":0},{"kind":"change","t0":1540,"t1":1540,"pts":[[0.1094,0.0488,1540],[0.2031,0.0488,1540],[0.1094,0.0976,1540],[0.2031,0.0976,1540],[0.1563,0.0732,1540]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":1540,"t1":1540,"pts":[[0.1094,0.0,1540],[0.2031,0.0,1540],[0.1094,0.0244,1540],[0.2031,0.0244,1540],[0.1563,0.0122,1540]],"dwell":0.0478,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.2344,0.0976,2640],[0.6875,0.0976,2640],[0.2344,1.0,2640],[0.6875,1.0,2640],[0.4609,0.5488,2640]],"dwell":0.6395,"keyframe":1},{"kind":"change","t0":5720,"t1":5720,"pts":[[0.2344,0.0976,5720],[0.6875,0.0976,5720],[0.2344,1.0,5720],[0.6875,1.0,5720],[0.4609,0.5488,5720]],"dwell":0.6395,"keyframe":2}],"snapshots":[[0,[{"id":4100,"app":"Safari","rect":[0.0868,0.0394,0.9086,0.9884],"layer":0,"regular":true,"frontmost":true},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[1440,[{"id":4100,"app":"Safari","rect":[0.0868,0.0394,0.9086,0.9884],"layer":0,"regular":true,"frontmost":true},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[2040,[{"id":4100,"app":"Safari","rect":[0.0868,0.0394,0.9086,0.9884],"layer":0,"regular":true,"frontmost":true},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[2640,[{"id":4100,"app":"Safari","rect":[0.0868,0.0394,0.9086,0.9884],"layer":0,"regular":true,"frontmost":true},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[3240,[{"id":4100,"app":"Safari","rect":[0.0868,0.0394,0.9086,0.9884],"layer":0,"regular":true,"frontmost":true},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]]]}},{"name":"side-by-side","truth":{"scenario":"side-by-side","description":"Editor and browser side by side; all the activity is in the browser","screen":[1728.0,1117.0],"whole":[864.0,33.0,864.0,1084.0],"spans":[{"t0":0,"t1":13000,"subject":"web","ideal":[864.0,33.0,864.0,1084.0],"targets":[]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":2200,"must":[[0.938,0.7716,0.9426,0.7788]]},{"index":2,"t":5060,"must":[[0.8487,0.9225,0.9976,0.9918]]},{"index":3,"t":8580,"must":[[0.6209,0.1875,0.8402,0.398]]}],"input":{"screen":[1728.0,1117.0],"duration":13000,"keyframes":[2200,5060,8580],"items":[{"kind":"stroke","t0":4200,"t1":5100,"pts":[[0.8816,0.9276,4200],[0.8933,0.9269,4220],[0.9033,0.9248,4240],[0.9141,0.9235,4260],[0.9253,0.9225,4280],[0.9358,0.9252,4300],[0.9463,0.9262,4320],[0.9582,0.9265,4340],[0.9639,0.9318,4360],[0.9766,0.9326,4380],[0.9798,0.938,4400],[0.9891,0.9412,4420],[0.9948,0.9456,4440],[0.9917,0.9512,4460],[0.9976,0.9557,4480],[0.9962,0.9607,4500],[0.9904,0.9651,4520],[0.9902,0.9701,4540],[0.9841,0.9741,4560],[0.9789,0.9782,4580],[0.9708,0.9813,4600],[0.9657,0.9862,4620],[0.9563,0.989,4640],[0.9437,0.9877,4660],[0.9349,0.9918,4680],[0.9237,0.9896,4700],[0.9133,0.9897,4720],[0.9026,0.9894,4740],[0.8933,0.987,4760],[0.8819,0.9862,4780],[0.8727,0.9833,4800],[0.8672,0.9788,4820],[0.8595,0.9752,4840],[0.8594,0.9698,4860],[0.8568,0.9653,4880],[0.849,0.9609,4900],[0.8487,0.9559,4920],[0.8498,0.9508,4940],[0.8517,0.9457,4960],[0.8598,0.9418,4980],[0.8627,0.9366,5000],[0.8719,0.9335,5020],[0.8831,0.9319,5040],[0.8918,0.9293,5060],[0.9018,0.928,5080],[0.9115,0.9265,5100]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":7600,"t1":8600,"pts":[[0.6428,0.2314,7600],[0.6575,0.224,7622],[0.6695,0.2151,7644],[0.6816,0.2061,7667],[0.6962,0.2015,7689],[0.7109,0.1976,7711],[0.7258,0.1875,7733],[0.7412,0.197,7756],[0.7573,0.1956,7778],[0.7712,0.2031,7800],[0.7887,0.2041,7822],[0.796,0.2202,7844],[0.807,0.2302,7867],[0.8237,0.2372,7889],[0.8213,0.2556,7911],[0.8285,0.2684,7933],[0.8402,0.2815,7956],[0.8323,0.2971,7978],[0.8304,0.3114,8000],[0.8353,0.3281,8022],[0.82,0.3382,8044],[0.8176,0.3539,8067],[0.8046,0.3626,8089],[0.7937,0.3727,8111],[0.7823,0.3826,8133],[0.7676,0.3868,8156],[0.7533,0.3911,8178],[0.7388,0.398,8200],[0.7234,0.3934,8222],[0.7071,0.3974,8244],[0.6944,0.3863,8267],[0.6814,0.3794,8289],[0.6663,0.375,8311],[0.6563,0.3637,8333],[0.6417,0.3558,8356],[0.6345,0.342,8378],[0.633,0.3263,8400],[0.6254,0.3128,8422],[0.6209,0.2977,8444],[0.6283,0.2828,8467],[0.6269,0.2672,8489],[0.637,0.2546,8511],[0.645,0.2421,8533],[0.6555,0.2316,8556],[0.6598,0.2145,8578],[0.6761,0.2099,8600]],"dwell":0.0,"keyframe":null},{"kind":"click","t0":2000,"t1":2000,"pts":[[0.9403,0.7752,2000]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":1056,"pts":[[0.2572,0.2411,0]],"dwell":1.056,"keyframe":null},{"kind":"cursor","t0":1056,"t1":1122,"pts":[[0.255,0.2412,1056]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1122,"t1":1254,"pts":[[0.2528,0.242,1122]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":1254,"t1":1320,"pts":[[0.2483,0.2662,1254]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1320,"t1":1386,"pts":[[0.2661,0.3097,1320]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1386,"t1":1452,"pts":[[0.3206,0.3764,1386]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1452,"t1":1518,"pts":[[0.4112,0.4637,1452]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1518,"t1":1584,"pts":[[0.5272,0.5615,1518]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1584,"t1":1650,"pts":[[0.6518,0.6563,1584]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1650,"t1":1716,"pts":[[0.7668,0.734,1650]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1716,"t1":1782,"pts":[[0.8564,0.7835,1716]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1782,"t1":1848,"pts":[[0.9119,0.7998,1782]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.9356,0.7873,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":3894,"pts":[[0.9403,0.7752,1914]],"dwell":1.98,"keyframe":null},{"kind":"cursor","t0":3894,"t1":3960,"pts":[[0.9364,0.7748,3894]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3960,"t1":4026,"pts":[[0.9232,0.7966,3960]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4026,"t1":4092,"pts":[[0.9029,0.8446,4026]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4092,"t1":4158,"pts":[[0.8854,0.8948,4092]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4158,"t1":4224,"pts":[[0.8795,0.9227,4158]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4224,"t1":4290,"pts":[[0.8953,0.9265,4224]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4290,"t1":4356,"pts":[[0.9306,0.9239,4290]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4356,"t1":4422,"pts":[[0.9627,0.9307,4356]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4422,"t1":4488,"pts":[[0.9897,0.9416,4422]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4488,"t1":4554,"pts":[[0.997,0.9577,4488]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4554,"t1":4620,"pts":[[0.9859,0.9729,4554]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4620,"t1":4686,"pts":[[0.9657,0.9862,4620]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4686,"t1":4752,"pts":[[0.9315,0.9911,4686]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4752,"t1":4818,"pts":[[0.897,0.988,4752]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4818,"t1":4884,"pts":[[0.8678,0.9793,4818]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4884,"t1":4950,"pts":[[0.8553,0.9644,4884]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4950,"t1":5016,"pts":[[0.8507,0.9483,4950]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5016,"t1":5082,"pts":[[0.8701,0.9341,5016]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5082,"t1":5148,"pts":[[0.9028,0.9278,5082]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5148,"t1":5214,"pts":[[0.9111,0.9291,5148]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5214,"t1":5280,"pts":[[0.9129,0.9425,5214]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5280,"t1":5874,"pts":[[0.9153,0.9511,5280]],"dwell":0.594,"keyframe":null},{"kind":"cursor","t0":5874,"t1":5940,"pts":[[0.9141,0.9526,5874]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5940,"t1":6006,"pts":[[0.912,0.9557,5940]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6006,"t1":6732,"pts":[[0.9109,0.9579,6006]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":6732,"t1":6798,"pts":[[0.9136,0.9546,6732]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6798,"t1":6930,"pts":[[0.9151,0.9513,6798]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":6930,"t1":6996,"pts":[[0.9135,0.9518,6930]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6996,"t1":7062,"pts":[[0.912,0.9543,6996]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7062,"t1":7128,"pts":[[0.9215,0.9369,7062]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7128,"t1":7194,"pts":[[0.9168,0.884,7128]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7194,"t1":7260,"pts":[[0.8912,0.7848,7194]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7260,"t1":7326,"pts":[[0.8473,0.6496,7260]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7326,"t1":7392,"pts":[[0.7927,0.5014,7326]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7392,"t1":7458,"pts":[[0.7379,0.3682,7392]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7458,"t1":7524,"pts":[[0.6923,0.2739,7458]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7524,"t1":7590,"pts":[[0.6618,0.2305,7524]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7590,"t1":7656,"pts":[[0.6448,0.2297,7590]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7656,"t1":7722,"pts":[[0.6758,0.2104,7656]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7722,"t1":7788,"pts":[[0.7182,0.1926,7722]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7788,"t1":7854,"pts":[[0.7637,0.199,7788]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7854,"t1":7920,"pts":[[0.8007,0.2245,7854]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7920,"t1":7986,"pts":[[0.8242,0.2607,7920]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7986,"t1":8052,"pts":[[0.8316,0.3024,7986]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8052,"t1":8118,"pts":[[0.8192,0.3435,8052]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8184,"pts":[[0.7902,0.3758,8118]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8184,"t1":8250,"pts":[[0.7492,0.393,8184]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8250,"t1":8316,"pts":[[0.7039,0.3947,8250]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8316,"t1":8382,"pts":[[0.6641,0.3725,8316]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8382,"t1":8448,"pts":[[0.6342,0.339,8382]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.6221,0.2953,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8580,"pts":[[0.638,0.253,8514]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8580,"t1":8646,"pts":[[0.6615,0.2141,8580]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.6795,0.2237,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":8778,"pts":[[0.7189,0.2735,8712]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8778,"t1":8844,"pts":[[0.7511,0.3007,8778]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8844,"t1":10230,"pts":[[0.7535,0.2984,8844]],"dwell":1.386,"keyframe":null},{"kind":"cursor","t0":10230,"t1":10296,"pts":[[0.7505,0.295,10230]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10296,"t1":12870,"pts":[[0.7481,0.2934,10296]],"dwell":2.574,"keyframe":null},{"kind":"cursor","t0":12870,"t1":12936,"pts":[[0.75,0.295,12870]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":12936,"t1":13000,"pts":[[0.7519,0.2949,12936]],"dwell":0.064,"keyframe":null},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.4063,0.0244,2200],[0.6094,0.0244,2200],[0.4063,1.0,2200],[0.6094,1.0,2200],[0.5078,0.5122,2200]],"dwell":0.4452,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.1406,0.0,2200],[0.3906,0.0,2200],[0.1406,0.2927,2200],[0.3906,0.2927,2200],[0.2656,0.1463,2200]],"dwell":0.2705,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.6406,0.4146,2200],[0.7188,0.4146,2200],[0.6406,0.6098,2200],[0.7188,0.6098,2200],[0.6797,0.5122,2200]],"dwell":0.1235,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.1719,0.3659,2200],[0.3594,0.3659,2200],[0.1719,0.4146,2200],[0.3594,0.4146,2200],[0.2656,0.3902,2200]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.8594,0.9268,2200],[0.9844,0.9268,2200],[0.8594,1.0,2200],[0.9844,1.0,2200],[0.9219,0.9634,2200]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.8438,0.6585,2200],[0.9531,0.6585,2200],[0.8438,0.7317,2200],[0.9531,0.7317,2200],[0.8984,0.6951,2200]],"dwell":0.0895,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.75,0.2439,2200],[0.8125,0.2439,2200],[0.75,0.3415,2200],[0.8125,0.3415,2200],[0.7813,0.2927,2200]],"dwell":0.0781,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.8594,0.7561,2200],[0.9844,0.7561,2200],[0.8594,0.8049,2200],[0.9844,0.8049,2200],[0.9219,0.7805,2200]],"dwell":0.0781,"keyframe":0}],"snapshots":[[0,[{"id":4100,"app":"Zed","rect":[0.0,0.0295,0.5,1.0],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.5,0.0295,1.0,1.0],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[2050,[{"id":4101,"app":"Safari","rect":[0.5,0.0295,1.0,1.0],"layer":0,"regular":true,"frontmost":true},{"id":4100,"app":"Zed","rect":[0.0,0.0295,0.5,1.0],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]]]}},{"name":"stray-cursor","truth":{"scenario":"stray-cursor","description":"Ink on a chart while the cursor strays to far corners and another window","screen":[1728.0,1117.0],"whole":[40.0,60.0,1100.0,820.0],"spans":[{"t0":0,"t1":13000,"subject":"chart","ideal":[40.0,60.0,1100.0,820.0],"targets":[[698.0,260.0,412.0,194.0]]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":2200,"must":[[0.588,0.2596,0.6271,0.3845]]},{"index":2,"t":9020,"must":[[0.4115,0.3945,0.4938,0.3976]]}],"input":{"screen":[1728.0,1117.0],"duration":13000,"keyframes":[2200,9020],"items":[{"kind":"stroke","t0":1400,"t1":2100,"pts":[[0.5899,0.283,1400],[0.5926,0.2784,1416],[0.5943,0.2696,1431],[0.5973,0.2668,1447],[0.6001,0.2633,1462],[0.603,0.2605,1478],[0.6061,0.2602,1493],[0.6092,0.2596,1509],[0.6121,0.2639,1524],[0.6151,0.2656,1540],[0.6178,0.2704,1556],[0.619,0.2804,1571],[0.6221,0.2838,1587],[0.623,0.2927,1602],[0.6244,0.3004,1618],[0.6271,0.3074,1633],[0.6263,0.3172,1649],[0.6266,0.3261,1664],[0.6253,0.3344,1680],[0.6263,0.3444,1696],[0.625,0.3527,1711],[0.6222,0.3575,1727],[0.6208,0.365,1742],[0.6194,0.3732,1758],[0.6161,0.3735,1773],[0.6145,0.3835,1789],[0.6112,0.3802,1804],[0.6085,0.3814,1820],[0.6057,0.3839,1836],[0.6028,0.3845,1851],[0.6006,0.3766,1867],[0.598,0.3741,1882],[0.596,0.3679,1898],[0.5935,0.3636,1913],[0.5911,0.3578,1929],[0.5907,0.3483,1944],[0.5903,0.3395,1960],[0.5884,0.3318,1976],[0.588,0.3226,1991],[0.5891,0.3136,2007],[0.5905,0.3055,2022],[0.592,0.298,2038],[0.5919,0.2875,2053],[0.5937,0.2797,2069],[0.5965,0.2753,2084],[0.5989,0.2694,2100]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":8600,"t1":9100,"pts":[[0.4115,0.3945,8600],[0.4163,0.3952,8629],[0.4211,0.3949,8659],[0.426,0.3961,8688],[0.4308,0.3951,8718],[0.4357,0.3954,8747],[0.4405,0.3952,8776],[0.4454,0.3948,8806],[0.4502,0.3967,8835],[0.4551,0.3967,8865],[0.4599,0.395,8894],[0.4647,0.3976,8924],[0.4696,0.3956,8953],[0.4744,0.3975,8982],[0.4793,0.3972,9012],[0.4841,0.3972,9041],[0.489,0.3957,9071],[0.4938,0.397,9100]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":858,"pts":[[0.506,0.2969,0]],"dwell":0.858,"keyframe":null},{"kind":"cursor","t0":858,"t1":924,"pts":[[0.5037,0.2978,858]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":924,"t1":1122,"pts":[[0.5017,0.2994,924]],"dwell":0.198,"keyframe":null},{"kind":"cursor","t0":1122,"t1":1188,"pts":[[0.5042,0.3051,1122]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1188,"t1":1254,"pts":[[0.5237,0.306,1188]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1254,"t1":1320,"pts":[[0.5558,0.3007,1254]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1320,"t1":1386,"pts":[[0.582,0.2922,1320]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1386,"t1":1452,"pts":[[0.5899,0.2844,1386]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1452,"t1":1518,"pts":[[0.5982,0.2656,1452]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1518,"t1":1584,"pts":[[0.6109,0.2621,1518]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1584,"t1":1650,"pts":[[0.6215,0.2832,1584]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1650,"t1":1716,"pts":[[0.6263,0.3178,1650]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1716,"t1":1782,"pts":[[0.6241,0.3542,1716]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1782,"t1":1848,"pts":[[0.6152,0.3791,1782]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.6034,0.3844,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":1980,"pts":[[0.5934,0.3633,1914]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1980,"t1":2046,"pts":[[0.5883,0.3292,1980]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.5919,0.2924,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.5981,0.2665,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.5727,0.2641,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.5037,0.2948,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2376,"pts":[[0.4322,0.3392,2310]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2376,"t1":2442,"pts":[[0.4046,0.3696,2376]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2442,"t1":2574,"pts":[[0.4051,0.376,2442]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":2574,"t1":2640,"pts":[[0.4004,0.3923,2574]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2640,"t1":2706,"pts":[[0.407,0.4239,2640]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2706,"t1":2772,"pts":[[0.4355,0.4748,2706]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2772,"t1":2838,"pts":[[0.4877,0.5454,2772]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2838,"t1":2904,"pts":[[0.5599,0.6308,2838]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2904,"t1":2970,"pts":[[0.6447,0.7224,2904]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2970,"t1":3036,"pts":[[0.7325,0.8102,2970]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3036,"t1":3102,"pts":[[0.8136,0.8842,3036]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3102,"t1":3168,"pts":[[0.8795,0.9364,3102]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3168,"t1":3234,"pts":[[0.925,0.9626,3168]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3234,"t1":3300,"pts":[[0.9499,0.964,3234]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3300,"t1":4422,"pts":[[0.9606,0.949,3300]],"dwell":1.122,"keyframe":null},{"kind":"cursor","t0":4422,"t1":4488,"pts":[[0.9641,0.9553,4422]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4488,"t1":4554,"pts":[[0.9714,0.9631,4488]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4554,"t1":4752,"pts":[[0.9769,0.9673,4554]],"dwell":0.198,"keyframe":null},{"kind":"cursor","t0":4752,"t1":4818,"pts":[[0.9814,0.9531,4752]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4818,"t1":4884,"pts":[[0.9683,0.9146,4818]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4884,"t1":4950,"pts":[[0.9324,0.8479,4884]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4950,"t1":5016,"pts":[[0.8806,0.7657,4950]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5016,"t1":5082,"pts":[[0.8265,0.689,5016]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5082,"t1":5148,"pts":[[0.7837,0.6376,5082]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5148,"t1":5214,"pts":[[0.76,0.621,5148]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5214,"t1":5544,"pts":[[0.7523,0.6267,5214]],"dwell":0.33,"keyframe":null},{"kind":"cursor","t0":5544,"t1":5610,"pts":[[0.7552,0.6294,5544]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5610,"t1":5676,"pts":[[0.771,0.6181,5610]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5676,"t1":5742,"pts":[[0.8024,0.5769,5676]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5742,"t1":5808,"pts":[[0.8386,0.5207,5742]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5808,"t1":5874,"pts":[[0.8638,0.4736,5808]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5874,"t1":5940,"pts":[[0.8697,0.4511,5874]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5940,"t1":6006,"pts":[[0.8683,0.445,5940]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6006,"t1":6072,"pts":[[0.8639,0.4242,6006]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6072,"t1":6138,"pts":[[0.837,0.401,6072]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6138,"t1":6204,"pts":[[0.7814,0.3756,6138]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6204,"t1":6270,"pts":[[0.7011,0.3503,6204]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6270,"t1":6336,"pts":[[0.6069,0.3282,6270]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6336,"t1":6402,"pts":[[0.5132,0.3124,6336]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6402,"t1":6468,"pts":[[0.434,0.3055,6402]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6468,"t1":6534,"pts":[[0.3796,0.3087,6468]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6534,"t1":6600,"pts":[[0.3532,0.3214,6534]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6600,"t1":6666,"pts":[[0.3472,0.3402,6600]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6666,"t1":6732,"pts":[[0.5512,0.3436,6666]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6732,"t1":7260,"pts":[[0.6069,0.3085,6732]],"dwell":0.528,"keyframe":null},{"kind":"cursor","t0":7260,"t1":7326,"pts":[[0.6082,0.3101,7260]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7326,"t1":8118,"pts":[[0.6098,0.3113,7326]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8250,"pts":[[0.6086,0.3091,8118]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":8250,"t1":8316,"pts":[[0.6032,0.3009,8250]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8316,"t1":8382,"pts":[[0.575,0.3025,8316]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8382,"t1":8448,"pts":[[0.5218,0.3204,8382]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.4631,0.3476,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8580,"pts":[[0.4225,0.3733,8514]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8580,"t1":8646,"pts":[[0.411,0.3906,8580]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.419,0.395,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":8778,"pts":[[0.4299,0.3953,8712]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8778,"t1":8844,"pts":[[0.4408,0.3952,8778]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8844,"t1":8910,"pts":[[0.4516,0.3967,8844]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8910,"t1":8976,"pts":[[0.4625,0.3964,8910]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8976,"t1":9042,"pts":[[0.4734,0.3971,8976]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9042,"t1":9108,"pts":[[0.4843,0.3971,9042]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9108,"t1":9174,"pts":[[0.4897,0.39,9108]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9174,"t1":9240,"pts":[[0.2528,0.5767,9174]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9240,"t1":9636,"pts":[[0.2315,0.6267,9240]],"dwell":0.396,"keyframe":null},{"kind":"cursor","t0":9636,"t1":9702,"pts":[[0.2252,0.6214,9636]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9702,"t1":9768,"pts":[[0.204,0.6352,9702]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9768,"t1":9834,"pts":[[0.1644,0.685,9768]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9834,"t1":9900,"pts":[[0.1127,0.7629,9834]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9900,"t1":9966,"pts":[[0.0614,0.8494,9900]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9966,"t1":10032,"pts":[[0.024,0.9223,9966]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10032,"t1":10098,"pts":[[0.0083,0.9667,10032]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10098,"t1":10758,"pts":[[0.0113,0.9844,10098]],"dwell":0.66,"keyframe":null},{"kind":"cursor","t0":10758,"t1":10824,"pts":[[0.0146,0.9816,10758]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10824,"t1":10890,"pts":[[0.0169,0.9777,10824]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10890,"t1":11022,"pts":[[0.0174,0.9759,10890]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":11022,"t1":11088,"pts":[[0.0205,0.9794,11022]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11088,"t1":11154,"pts":[[0.0361,0.9777,11088]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11154,"t1":11220,"pts":[[0.0669,0.942,11154]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11220,"t1":11286,"pts":[[0.1132,0.8689,11220]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11286,"t1":11352,"pts":[[0.1685,0.7688,11286]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11352,"t1":11418,"pts":[[0.2228,0.6604,11352]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11418,"t1":11484,"pts":[[0.266,0.5638,11418]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11484,"t1":11550,"pts":[[0.2905,0.4949,11484]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11550,"t1":11616,"pts":[[0.295,0.459,11550]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11616,"t1":11682,"pts":[[0.3001,0.4585,11616]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11682,"t1":11748,"pts":[[0.5278,0.3495,11682]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11748,"t1":12738,"pts":[[0.5352,0.328,11748]],"dwell":0.99,"keyframe":null},{"kind":"cursor","t0":12738,"t1":12804,"pts":[[0.5324,0.3232,12738]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":12804,"t1":13000,"pts":[[0.5301,0.3205,12804]],"dwell":0.196,"keyframe":null},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.7188,0.3415,2200],[1.0,0.3415,2200],[0.7188,0.9756,2200],[1.0,0.9756,2200],[0.8594,0.6585,2200]],"dwell":0.4223,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.7344,0.5366,2200],[0.9688,0.5366,2200],[0.7344,0.7561,2200],[0.9688,0.7561,2200],[0.8516,0.6463,2200]],"dwell":0.2268,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.0469,0.0488,2200],[0.6563,0.0488,2200],[0.0469,0.122,2200],[0.6563,0.122,2200],[0.3516,0.0854,2200]],"dwell":0.2112,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.0156,0.7561,2200],[0.6563,0.7561,2200],[0.0156,0.8049,2200],[0.6563,0.8049,2200],[0.3359,0.7805,2200]],"dwell":0.1768,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.4063,0.6829,2200],[0.6094,0.6829,2200],[0.4063,0.7317,2200],[0.6094,0.7317,2200],[0.5078,0.7073,2200]],"dwell":0.0995,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.1563,0.439,2200],[0.25,0.439,2200],[0.1563,0.5366,2200],[0.25,0.5366,2200],[0.2031,0.4878,2200]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.5625,0.9268,2200],[0.6875,0.9268,2200],[0.5625,1.0,2200],[0.6875,1.0,2200],[0.625,0.9634,2200]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":2200,"t1":2200,"pts":[[0.3281,0.2683,2200],[0.375,0.2683,2200],[0.3281,0.3659,2200],[0.375,0.3659,2200],[0.3516,0.3171,2200]],"dwell":0.0676,"keyframe":0}],"snapshots":[[0,[{"id":4100,"app":"Safari","rect":[0.0231,0.0537,0.6597,0.7878],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Slack","rect":[0.5903,0.3402,0.9896,0.9669],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false},{"id":38,"app":"Dock","rect":[0.0,0.9338,1.0,1.0],"layer":20,"regular":false,"frontmost":false}]]]}},{"name":"talk-only","truth":{"scenario":"talk-only","description":"Talking over Slack, then switching to the browser with the keyboard and talking","screen":[1728.0,1117.0],"whole":[192.0,62.0,1276.0,886.0],"spans":[{"t0":0,"t1":6350,"subject":"slack","ideal":[200.0,70.0,1000.0,760.0],"targets":[]},{"t0":6350,"t1":14000,"subject":"web","ideal":[360.0,120.0,1100.0,820.0],"targets":[]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":6600,"must":[]}],"input":{"screen":[1728.0,1117.0],"duration":14000,"keyframes":[6600],"items":[{"kind":"cursor","t0":0,"t1":924,"pts":[[0.4444,0.1747,0]],"dwell":0.924,"keyframe":null},{"kind":"cursor","t0":924,"t1":990,"pts":[[0.4454,0.1723,924]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":990,"t1":2112,"pts":[[0.446,0.1699,990]],"dwell":1.122,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.4441,0.1675,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2970,"pts":[[0.4418,0.1667,2178]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":2970,"t1":4092,"pts":[[0.4406,0.1668,2970]],"dwell":1.122,"keyframe":null},{"kind":"cursor","t0":4092,"t1":4158,"pts":[[0.4412,0.1705,4092]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4158,"t1":4224,"pts":[[0.4433,0.1742,4158]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4224,"t1":5082,"pts":[[0.4445,0.1756,4224]],"dwell":0.858,"keyframe":null},{"kind":"cursor","t0":5082,"t1":5808,"pts":[[0.447,0.1751,5082]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":5808,"t1":5874,"pts":[[0.4444,0.1733,5808]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5874,"t1":6006,"pts":[[0.4429,0.1734,5874]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":6006,"t1":6072,"pts":[[0.4446,0.1726,6006]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6072,"t1":6138,"pts":[[0.4378,0.1981,6072]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6138,"t1":6204,"pts":[[0.4484,0.2842,6138]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6204,"t1":6270,"pts":[[0.4753,0.4151,6204]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6270,"t1":6336,"pts":[[0.5074,0.5393,6270]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6336,"t1":6402,"pts":[[0.5327,0.6084,6336]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6402,"t1":7392,"pts":[[0.547,0.6165,6402]],"dwell":0.99,"keyframe":null},{"kind":"cursor","t0":7392,"t1":8052,"pts":[[0.5464,0.6137,7392]],"dwell":0.66,"keyframe":null},{"kind":"cursor","t0":8052,"t1":8118,"pts":[[0.5489,0.6136,8052]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8184,"pts":[[0.5529,0.6126,8118]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8184,"t1":9504,"pts":[[0.5551,0.6114,8184]],"dwell":1.32,"keyframe":null},{"kind":"cursor","t0":9504,"t1":9570,"pts":[[0.5533,0.6111,9504]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9570,"t1":10626,"pts":[[0.551,0.6121,9570]],"dwell":1.056,"keyframe":null},{"kind":"cursor","t0":10626,"t1":10758,"pts":[[0.5501,0.613,10626]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":10758,"t1":10824,"pts":[[0.55,0.6167,10758]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10824,"t1":12210,"pts":[[0.5503,0.6192,10824]],"dwell":1.386,"keyframe":null},{"kind":"cursor","t0":12210,"t1":12276,"pts":[[0.5488,0.6151,12210]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":12276,"t1":13398,"pts":[[0.5475,0.6131,12276]],"dwell":1.122,"keyframe":null},{"kind":"cursor","t0":13398,"t1":13464,"pts":[[0.5505,0.6134,13398]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":13464,"t1":13926,"pts":[[0.5519,0.613,13464]],"dwell":0.462,"keyframe":null},{"kind":"cursor","t0":13926,"t1":14000,"pts":[[0.5511,0.615,13926]],"dwell":0.074,"keyframe":null},{"kind":"change","t0":6600,"t1":6600,"pts":[[0.2031,0.0732,6600],[0.8438,0.0732,6600],[0.2031,0.8537,6600],[0.8438,0.8537,6600],[0.5234,0.4634,6600]],"dwell":0.7071,"keyframe":0},{"kind":"change","t0":6600,"t1":6600,"pts":[[0.125,0.0976,6600],[0.1719,0.0976,6600],[0.125,0.3171,6600],[0.1719,0.3171,6600],[0.1484,0.2073,6600]],"dwell":0.1014,"keyframe":0},{"kind":"change","t0":6600,"t1":6600,"pts":[[0.3438,0.4878,6600],[0.4375,0.4878,6600],[0.3438,0.561,6600],[0.4375,0.561,6600],[0.3906,0.5244,6600]],"dwell":0.0828,"keyframe":0},{"kind":"change","t0":6600,"t1":6600,"pts":[[0.0469,0.0,6600],[0.2344,0.0,6600],[0.0469,0.0244,6600],[0.2344,0.0244,6600],[0.1406,0.0122,6600]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":6600,"t1":6600,"pts":[[0.3438,0.5854,6600],[0.3906,0.5854,6600],[0.3438,0.6829,6600],[0.3906,0.6829,6600],[0.3672,0.6341,6600]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":6600,"t1":6600,"pts":[[0.4844,0.5854,6600],[0.5156,0.5854,6600],[0.4844,0.6829,6600],[0.5156,0.6829,6600],[0.5,0.6341,6600]],"dwell":0.0552,"keyframe":0},{"kind":"change","t0":6600,"t1":6600,"pts":[[0.7656,0.8049,6600],[0.8281,0.8049,6600],[0.7656,0.8537,6600],[0.8281,0.8537,6600],[0.7969,0.8293,6600]],"dwell":0.0552,"keyframe":0},{"kind":"change","t0":6600,"t1":6600,"pts":[[0.7969,0.1707,6600],[0.8281,0.1707,6600],[0.7969,0.2195,6600],[0.8281,0.2195,6600],[0.8125,0.1951,6600]],"dwell":0.039,"keyframe":0}],"snapshots":[[0,[{"id":4100,"app":"Slack","rect":[0.1157,0.0627,0.6944,0.7431],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.2083,0.1074,0.8449,0.8415],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[6400,[{"id":4101,"app":"Safari","rect":[0.2083,0.1074,0.8449,0.8415],"layer":0,"regular":true,"frontmost":true},{"id":4100,"app":"Slack","rect":[0.1157,0.0627,0.6944,0.7431],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]]]}},{"name":"three-window-hop","truth":{"scenario":"three-window-hop","description":"Thirty seconds hopping between editor, browser and terminal","screen":[1728.0,1117.0],"whole":[0.0,33.0,1728.0,1084.0],"spans":[{"t0":0,"t1":8100,"subject":"editor","ideal":[0.0,33.0,860.0,560.0],"targets":[]},{"t0":8100,"t1":17100,"subject":"web","ideal":[860.0,33.0,868.0,1084.0],"targets":[]},{"t0":17100,"t1":24900,"subject":"term","ideal":[0.0,593.0,860.0,524.0],"targets":[]},{"t0":24900,"t1":30000,"subject":"editor","ideal":[0.0,33.0,860.0,560.0],"targets":[]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":2640,"must":[[0.1561,0.2225,0.4014,0.2663]]},{"index":2,"t":8360,"must":[[0.5524,0.3077,0.557,0.3149]]},{"index":3,"t":11000,"must":[[0.6247,0.1957,0.833,0.3926]]},{"index":4,"t":17380,"must":[[0.2736,0.9439,0.2782,0.9511]]},{"index":5,"t":19580,"must":[[0.0286,0.8115,0.1813,0.8153]]},{"index":6,"t":25300,"must":[[0.2684,0.1353,0.273,0.1425]]}],"input":{"screen":[1728.0,1117.0],"duration":30000,"keyframes":[2640,8360,11000,17380,19580,25300],"items":[{"kind":"stroke","t0":1800,"t1":2700,"pts":[[0.2201,0.227,1800],[0.2337,0.2248,1820],[0.2525,0.2253,1840],[0.2685,0.2229,1860],[0.2872,0.2225,1880],[0.305,0.2236,1900],[0.3206,0.2254,1920],[0.3344,0.2274,1940],[0.3535,0.228,1960],[0.3661,0.2303,1980],[0.371,0.2335,2000],[0.385,0.2356,2020],[0.3965,0.2383,2040],[0.4014,0.2414,2060],[0.3907,0.2447,2080],[0.3959,0.2478,2100],[0.3855,0.2505,2120],[0.3837,0.2536,2140],[0.3773,0.2565,2160],[0.3643,0.2587,2180],[0.3503,0.2605,2200],[0.3412,0.2636,2220],[0.3209,0.2635,2240],[0.3069,0.2658,2260],[0.288,0.2647,2280],[0.2708,0.2663,2300],[0.2544,0.2646,2320],[0.2379,0.2639,2340],[0.2216,0.2629,2360],[0.2091,0.2608,2380],[0.195,0.2592,2400],[0.1816,0.2572,2420],[0.1665,0.2551,2440],[0.1698,0.2515,2460],[0.1606,0.2488,2480],[0.1561,0.2458,2500],[0.157,0.2426,2520],[0.1569,0.2394,2540],[0.1617,0.2363,2560],[0.1731,0.2337,2580],[0.183,0.231,2600],[0.1928,0.2282,2620],[0.2087,0.2265,2640],[0.2294,0.2266,2660],[0.2428,0.2241,2680],[0.2603,0.2231,2700]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":10000,"t1":11000,"pts":[[0.6574,0.2156,10000],[0.6722,0.2096,10022],[0.6886,0.2086,10044],[0.7002,0.1958,10067],[0.7169,0.2021,10089],[0.7315,0.2025,10111],[0.7477,0.1957,10133],[0.7617,0.2029,10156],[0.7746,0.2107,10178],[0.7922,0.212,10200],[0.8044,0.222,10222],[0.8134,0.2346,10244],[0.816,0.2502,10267],[0.82,0.2638,10289],[0.833,0.2753,10311],[0.8294,0.2904,10333],[0.8271,0.3045,10356],[0.8327,0.3201,10378],[0.8218,0.3321,10400],[0.8141,0.3442,10422],[0.8047,0.355,10444],[0.7994,0.37,10467],[0.7824,0.3725,10489],[0.7746,0.3887,10511],[0.7573,0.3871,10533],[0.7425,0.3881,10556],[0.7279,0.389,10578],[0.7127,0.3926,10600],[0.6978,0.3892,10622],[0.6823,0.3865,10644],[0.6689,0.3788,10667],[0.6598,0.3664,10689],[0.6478,0.3573,10711],[0.6359,0.3472,10733],[0.6369,0.3306,10756],[0.6267,0.3188,10778],[0.6285,0.3039,10800],[0.6306,0.29,10822],[0.6247,0.2748,10844],[0.6282,0.2602,10867],[0.6338,0.2461,10889],[0.6473,0.2368,10911],[0.6591,0.2276,10933],[0.6679,0.2154,10956],[0.6785,0.2032,10978],[0.6963,0.205,11000]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":19000,"t1":19500,"pts":[[0.0286,0.812,19000],[0.0376,0.8115,19029],[0.0466,0.8126,19059],[0.0555,0.8135,19088],[0.0645,0.8133,19118],[0.0735,0.8139,19147],[0.0825,0.8132,19176],[0.0915,0.8125,19206],[0.1005,0.8132,19235],[0.1095,0.8136,19265],[0.1184,0.814,19294],[0.1274,0.8138,19324],[0.1364,0.8125,19353],[0.1454,0.8148,19382],[0.1544,0.8146,19412],[0.1634,0.8143,19441],[0.1724,0.8153,19471],[0.1813,0.8153,19500]],"dwell":0.0,"keyframe":null},{"kind":"click","t0":8200,"t1":8200,"pts":[[0.5547,0.3113,8200]],"dwell":0.0,"keyframe":null},{"kind":"click","t0":17200,"t1":17200,"pts":[[0.2759,0.9475,17200]],"dwell":0.0,"keyframe":null},{"kind":"click","t0":25000,"t1":25000,"pts":[[0.2707,0.1389,25000]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":1056,"pts":[[0.1577,0.0765,0]],"dwell":1.056,"keyframe":null},{"kind":"cursor","t0":1056,"t1":1122,"pts":[[0.16,0.0752,1056]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1122,"t1":1518,"pts":[[0.1621,0.0732,1122]],"dwell":0.396,"keyframe":null},{"kind":"cursor","t0":1518,"t1":1584,"pts":[[0.1612,0.0843,1518]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1584,"t1":1650,"pts":[[0.1717,0.1232,1584]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1650,"t1":1716,"pts":[[0.1911,0.1768,1650]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1716,"t1":1782,"pts":[[0.2092,0.2173,1716]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1782,"t1":1848,"pts":[[0.2187,0.2279,1782]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.2589,0.2243,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":1980,"pts":[[0.3159,0.2248,1914]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1980,"t1":2046,"pts":[[0.3661,0.2303,1980]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.398,0.2392,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.3897,0.2494,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.3656,0.2585,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.3181,0.2639,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2376,"pts":[[0.2626,0.2654,2310]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2376,"t1":2442,"pts":[[0.2116,0.2612,2376]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2442,"t1":2508,"pts":[[0.1668,0.2548,2442]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2508,"t1":2574,"pts":[[0.1565,0.2445,2508]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2574,"t1":2640,"pts":[[0.1697,0.2345,2574]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2640,"t1":2706,"pts":[[0.2087,0.2265,2640]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2706,"t1":3564,"pts":[[0.2585,0.2493,2706]],"dwell":0.858,"keyframe":null},{"kind":"cursor","t0":3564,"t1":4290,"pts":[[0.2593,0.2469,3564]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":4290,"t1":4356,"pts":[[0.258,0.2464,4290]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4356,"t1":7656,"pts":[[0.2562,0.2478,4356]],"dwell":3.3,"keyframe":null},{"kind":"cursor","t0":7656,"t1":7722,"pts":[[0.2619,0.2642,7656]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7722,"t1":7788,"pts":[[0.2934,0.2834,7722]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7788,"t1":7854,"pts":[[0.3527,0.3037,7788]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7854,"t1":7920,"pts":[[0.426,0.3208,7854]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7920,"t1":7986,"pts":[[0.4935,0.3301,7920]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7986,"t1":8052,"pts":[[0.5377,0.3288,7986]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8052,"t1":8118,"pts":[[0.5536,0.3177,8052]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8118,"t1":9702,"pts":[[0.5547,0.3113,8118]],"dwell":1.584,"keyframe":null},{"kind":"cursor","t0":9702,"t1":9768,"pts":[[0.5601,0.3144,9702]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9768,"t1":9834,"pts":[[0.5821,0.301,9768]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9834,"t1":9900,"pts":[[0.6167,0.2708,9834]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9900,"t1":9966,"pts":[[0.6467,0.2391,9900]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9966,"t1":10032,"pts":[[0.6581,0.2202,9966]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10032,"t1":10098,"pts":[[0.6794,0.2091,10032]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10098,"t1":10164,"pts":[[0.7229,0.2023,10098]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10164,"t1":10230,"pts":[[0.7666,0.2059,10164]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10230,"t1":10296,"pts":[[0.8076,0.2264,10230]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10296,"t1":10362,"pts":[[0.8241,0.2675,10296]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10362,"t1":10428,"pts":[[0.8287,0.309,10362]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10428,"t1":10494,"pts":[[0.8117,0.347,10428]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10494,"t1":10560,"pts":[[0.7806,0.3762,10494]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10560,"t1":10626,"pts":[[0.7396,0.3883,10560]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10626,"t1":10692,"pts":[[0.6952,0.3887,10626]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10692,"t1":10758,"pts":[[0.6581,0.3651,10692]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10758,"t1":10824,"pts":[[0.6357,0.3293,10758]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10824,"t1":10890,"pts":[[0.6301,0.2888,10824]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10890,"t1":10956,"pts":[[0.6345,0.2456,10890]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10956,"t1":11022,"pts":[[0.6681,0.2152,10956]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11022,"t1":11616,"pts":[[0.7145,0.3218,11022]],"dwell":0.594,"keyframe":null},{"kind":"cursor","t0":11616,"t1":11682,"pts":[[0.7175,0.326,11616]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11682,"t1":13134,"pts":[[0.7198,0.3281,11682]],"dwell":1.452,"keyframe":null},{"kind":"cursor","t0":13134,"t1":13200,"pts":[[0.7214,0.3265,13134]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":13200,"t1":14256,"pts":[[0.722,0.3247,13200]],"dwell":1.056,"keyframe":null},{"kind":"cursor","t0":14256,"t1":16368,"pts":[[0.72,0.3227,14256]],"dwell":2.112,"keyframe":null},{"kind":"cursor","t0":16368,"t1":16434,"pts":[[0.7173,0.3197,16368]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":16434,"t1":16500,"pts":[[0.7033,0.3115,16434]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":16500,"t1":16566,"pts":[[0.6741,0.3275,16500]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":16566,"t1":16632,"pts":[[0.6258,0.3751,16566]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":16632,"t1":16698,"pts":[[0.5611,0.4522,16632]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":16698,"t1":16764,"pts":[[0.487,0.5503,16698]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":16764,"t1":16830,"pts":[[0.4126,0.6567,16764]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":16830,"t1":16896,"pts":[[0.3477,0.7578,16830]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":16896,"t1":16962,"pts":[[0.3003,0.8415,16896]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":16962,"t1":17028,"pts":[[0.2748,0.9,16962]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":17028,"t1":17094,"pts":[[0.2701,0.9332,17028]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":17094,"t1":18546,"pts":[[0.2759,0.9475,17094]],"dwell":1.452,"keyframe":null},{"kind":"cursor","t0":18546,"t1":18612,"pts":[[0.2763,0.9454,18546]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":18612,"t1":18678,"pts":[[0.2709,0.9279,18612]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":18678,"t1":18744,"pts":[[0.2393,0.8992,18678]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":18744,"t1":18810,"pts":[[0.183,0.8626,18744]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":18810,"t1":18876,"pts":[[0.1179,0.8279,18810]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":18876,"t1":18942,"pts":[[0.0641,0.8058,18876]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":18942,"t1":19008,"pts":[[0.0352,0.8025,18942]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":19008,"t1":19074,"pts":[[0.031,0.8119,19008]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":19074,"t1":19140,"pts":[[0.0512,0.8131,19074]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":19140,"t1":19206,"pts":[[0.0714,0.8138,19140]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":19206,"t1":19272,"pts":[[0.0915,0.8125,19206]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":19272,"t1":19338,"pts":[[0.1117,0.8137,19272]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":19338,"t1":19404,"pts":[[0.1318,0.8132,19338]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":19404,"t1":19470,"pts":[[0.152,0.8147,19404]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":19470,"t1":19536,"pts":[[0.1722,0.8153,19470]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":19536,"t1":20328,"pts":[[0.0869,0.7993,19536]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":20328,"t1":20394,"pts":[[0.0842,0.7995,20328]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":20394,"t1":21648,"pts":[[0.0823,0.8003,20394]],"dwell":1.254,"keyframe":null},{"kind":"cursor","t0":21648,"t1":22770,"pts":[[0.0834,0.7981,21648]],"dwell":1.122,"keyframe":null},{"kind":"cursor","t0":22770,"t1":22836,"pts":[[0.0821,0.7974,22770]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":22836,"t1":22902,"pts":[[0.0792,0.7987,22836]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":22902,"t1":22968,"pts":[[0.077,0.8002,22902]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":22968,"t1":23034,"pts":[[0.0777,0.8016,22968]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":23034,"t1":24288,"pts":[[0.0815,0.8023,23034]],"dwell":1.254,"keyframe":null},{"kind":"cursor","t0":24288,"t1":24354,"pts":[[0.0873,0.8053,24288]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":24354,"t1":24420,"pts":[[0.1035,0.7932,24354]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":24420,"t1":24486,"pts":[[0.1291,0.7412,24420]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":24486,"t1":24552,"pts":[[0.1631,0.6477,24486]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":24552,"t1":24618,"pts":[[0.2013,0.5253,24552]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":24618,"t1":24684,"pts":[[0.2373,0.395,24618]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":24684,"t1":24750,"pts":[[0.2649,0.2791,24684]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":24750,"t1":24816,"pts":[[0.2794,0.1959,24750]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":24816,"t1":24882,"pts":[[0.2798,0.1524,24816]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":24882,"t1":25146,"pts":[[0.2707,0.1389,24882]],"dwell":0.264,"keyframe":null},{"kind":"cursor","t0":25146,"t1":25212,"pts":[[0.2699,0.1375,25146]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":25212,"t1":25278,"pts":[[0.2559,0.1381,25212]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":25278,"t1":26532,"pts":[[0.2431,0.1426,25278]],"dwell":1.254,"keyframe":null},{"kind":"cursor","t0":26532,"t1":26598,"pts":[[0.2405,0.1423,26532]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":26598,"t1":26664,"pts":[[0.2375,0.1413,26598]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":26664,"t1":28116,"pts":[[0.2359,0.1412,26664]],"dwell":1.452,"keyframe":null},{"kind":"cursor","t0":28116,"t1":28182,"pts":[[0.2363,0.1431,28116]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":28182,"t1":29304,"pts":[[0.2373,0.145,28182]],"dwell":1.122,"keyframe":null},{"kind":"cursor","t0":29304,"t1":29370,"pts":[[0.2383,0.143,29304]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":29370,"t1":29436,"pts":[[0.238,0.1389,29370]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":29436,"t1":29898,"pts":[[0.2377,0.1368,29436]],"dwell":0.462,"keyframe":null},{"kind":"cursor","t0":29898,"t1":30000,"pts":[[0.2395,0.1399,29898]],"dwell":0.102,"keyframe":null},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.0156,0.6098,2640],[0.3594,0.6098,2640],[0.0156,0.8293,2640],[0.3594,0.8293,2640],[0.1875,0.7195,2640]],"dwell":0.2747,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.1406,0.0,2640],[0.3906,0.0,2640],[0.1406,0.2927,2640],[0.3906,0.2927,2640],[0.2656,0.1463,2640]],"dwell":0.2705,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.4375,0.5122,2640],[0.5,0.5122,2640],[0.4375,1.0,2640],[0.5,1.0,2640],[0.4688,0.7561,2640]],"dwell":0.1746,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.0,0.8537,2640],[0.3906,0.8537,2640],[0.0,0.9024,2640],[0.3906,0.9024,2640],[0.1953,0.878,2640]],"dwell":0.138,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.0156,0.9268,2640],[0.1719,0.9268,2640],[0.0156,1.0,2640],[0.1719,1.0,2640],[0.0938,0.9634,2640]],"dwell":0.1069,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.8438,0.2683,2640],[0.9375,0.2683,2640],[0.8438,0.3659,2640],[0.9375,0.3659,2640],[0.8906,0.3171,2640]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.1719,0.3659,2640],[0.3594,0.3659,2640],[0.1719,0.4146,2640],[0.3594,0.4146,2640],[0.2656,0.3902,2640]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":2640,"t1":2640,"pts":[[0.8438,0.6585,2640],[0.9688,0.6585,2640],[0.8438,0.7317,2640],[0.9688,0.7317,2640],[0.9063,0.6951,2640]],"dwell":0.0956,"keyframe":0},{"kind":"change","t0":8360,"t1":8360,"pts":[[0.4844,0.0488,8360],[0.5313,0.0488,8360],[0.4844,0.5854,8360],[0.5313,0.5854,8360],[0.5078,0.3171,8360]],"dwell":0.1586,"keyframe":1},{"kind":"change","t0":8360,"t1":8360,"pts":[[0.0469,0.0,8360],[0.2656,0.0,8360],[0.0469,0.0244,8360],[0.2656,0.0244,8360],[0.1563,0.0122,8360]],"dwell":0.073,"keyframe":1},{"kind":"change","t0":17380,"t1":17380,"pts":[[0.4531,0.5366,17380],[0.5313,0.5366,17380],[0.4531,1.0,17380],[0.5313,1.0,17380],[0.4922,0.7683,17380]],"dwell":0.1903,"keyframe":3},{"kind":"change","t0":17380,"t1":17380,"pts":[[0.0469,0.0,17380],[0.2344,0.0,17380],[0.0469,0.0244,17380],[0.2344,0.0244,17380],[0.1406,0.0122,17380]],"dwell":0.0676,"keyframe":3},{"kind":"change","t0":17380,"t1":17380,"pts":[[0.2031,0.5366,17380],[0.2969,0.5366,17380],[0.2031,0.561,17380],[0.2969,0.561,17380],[0.25,0.5488,17380]],"dwell":0.0478,"keyframe":3},{"kind":"change","t0":17380,"t1":17380,"pts":[[0.0156,0.5366,17380],[0.0469,0.5366,17380],[0.0156,0.561,17380],[0.0469,0.561,17380],[0.0313,0.5488,17380]],"dwell":0.0276,"keyframe":3},{"kind":"change","t0":25300,"t1":25300,"pts":[[0.4844,0.0488,25300],[0.5313,0.0488,25300],[0.4844,0.561,25300],[0.5313,0.561,25300],[0.5078,0.3049,25300]],"dwell":0.1549,"keyframe":5},{"kind":"change","t0":25300,"t1":25300,"pts":[[0.0469,0.0,25300],[0.25,0.0,25300],[0.0469,0.0244,25300],[0.25,0.0244,25300],[0.1484,0.0122,25300]],"dwell":0.0704,"keyframe":5},{"kind":"change","t0":25300,"t1":25300,"pts":[[0.2031,0.5366,25300],[0.2969,0.5366,25300],[0.2031,0.561,25300],[0.2969,0.561,25300],[0.25,0.5488,25300]],"dwell":0.0478,"keyframe":5},{"kind":"change","t0":25300,"t1":25300,"pts":[[0.0156,0.5366,25300],[0.0469,0.5366,25300],[0.0156,0.561,25300],[0.0469,0.561,25300],[0.0313,0.5488,25300]],"dwell":0.0276,"keyframe":5}],"snapshots":[[0,[{"id":4100,"app":"Zed","rect":[0.0,0.0295,0.4977,0.5309],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.4977,0.0295,1.0,1.0],"layer":0,"regular":true,"frontmost":false},{"id":4102,"app":"Ghostty","rect":[0.0,0.5309,0.4977,1.0],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[8260,[{"id":4101,"app":"Safari","rect":[0.4977,0.0295,1.0,1.0],"layer":0,"regular":true,"frontmost":true},{"id":4100,"app":"Zed","rect":[0.0,0.0295,0.4977,0.5309],"layer":0,"regular":true,"frontmost":false},{"id":4102,"app":"Ghostty","rect":[0.0,0.5309,0.4977,1.0],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[17260,[{"id":4102,"app":"Ghostty","rect":[0.0,0.5309,0.4977,1.0],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.4977,0.0295,1.0,1.0],"layer":0,"regular":true,"frontmost":false},{"id":4100,"app":"Zed","rect":[0.0,0.0295,0.4977,0.5309],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[25060,[{"id":4100,"app":"Zed","rect":[0.0,0.0295,0.4977,0.5309],"layer":0,"regular":true,"frontmost":true},{"id":4102,"app":"Ghostty","rect":[0.0,0.5309,0.4977,1.0],"layer":0,"regular":true,"frontmost":false},{"id":4101,"app":"Safari","rect":[0.4977,0.0295,1.0,1.0],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]]]}},{"name":"tiny-badge","truth":{"scenario":"tiny-badge","description":"A tiny notification badge in a huge browser window","screen":[1728.0,1117.0],"whole":[1236.0,50.0,490.0,349.0],"spans":[{"t0":0,"t1":4900,"subject":"badge","ideal":[1405.0,0.0,323.0,361.0],"targets":[[1619.0,130.0,16.0,16.0]]},{"t0":4900,"t1":12000,"subject":"notifs","ideal":[1256.0,76.0,450.0,303.0],"targets":[[1316.0,170.0,330.0,149.0]]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":2420,"must":[[0.9295,0.1035,0.9579,0.1435]]},{"index":2,"t":5280,"must":[[0.9328,0.1289,0.9374,0.1361]]},{"index":3,"t":8800,"must":[[0.7674,0.2072,0.8756,0.2103]]}],"input":{"screen":[1728.0,1117.0],"duration":12000,"keyframes":[2420,5280,8800],"items":[{"kind":"stroke","t0":1800,"t1":2500,"pts":[[0.9297,0.112,1800],[0.9317,0.1104,1816],[0.9333,0.1085,1831],[0.9352,0.1072,1847],[0.9367,0.1044,1862],[0.939,0.1044,1878],[0.9412,0.1042,1893],[0.9433,0.1043,1909],[0.9457,0.1035,1924],[0.9477,0.105,1940],[0.9498,0.1061,1956],[0.9518,0.1077,1971],[0.9526,0.1109,1987],[0.955,0.112,2002],[0.9551,0.1154,2018],[0.957,0.1175,2033],[0.9568,0.1206,2049],[0.9579,0.1235,2064],[0.957,0.1264,2080],[0.9564,0.1291,2096],[0.9558,0.1318,2111],[0.9558,0.1351,2127],[0.9542,0.137,2142],[0.9529,0.1395,2158],[0.9507,0.1401,2173],[0.949,0.1415,2189],[0.9473,0.1435,2204],[0.9451,0.1423,2220],[0.9432,0.1427,2236],[0.9412,0.1424,2251],[0.9389,0.1431,2267],[0.937,0.142,2282],[0.9356,0.1396,2298],[0.9338,0.1381,2313],[0.9332,0.1351,2329],[0.9311,0.1335,2344],[0.9303,0.1307,2360],[0.931,0.1275,2376],[0.9295,0.1249,2391],[0.93,0.1219,2407],[0.9304,0.1189,2422],[0.9307,0.1159,2438],[0.932,0.1133,2453],[0.9332,0.1107,2469],[0.9351,0.1089,2484],[0.9368,0.107,2500]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":8400,"t1":8900,"pts":[[0.7674,0.2072,8400],[0.7737,0.2087,8429],[0.7801,0.2089,8459],[0.7865,0.2074,8488],[0.7928,0.2086,8518],[0.7992,0.2072,8547],[0.8056,0.2083,8576],[0.8119,0.2075,8606],[0.8183,0.2091,8635],[0.8247,0.2085,8665],[0.831,0.2101,8694],[0.8374,0.208,8724],[0.8438,0.2084,8753],[0.8501,0.209,8782],[0.8565,0.2098,8812],[0.8629,0.2103,8841],[0.8692,0.209,8871],[0.8756,0.2092,8900]],"dwell":0.0,"keyframe":null},{"kind":"click","t0":5000,"t1":5000,"pts":[[0.9351,0.1325,5000]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":1056,"pts":[[0.4365,0.3581,0]],"dwell":1.056,"keyframe":null},{"kind":"cursor","t0":1056,"t1":1122,"pts":[[0.4362,0.356,1056]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1122,"t1":1254,"pts":[[0.4355,0.354,1122]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":1254,"t1":1320,"pts":[[0.4417,0.3689,1254]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1320,"t1":1386,"pts":[[0.4711,0.3754,1320]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1386,"t1":1452,"pts":[[0.5328,0.3614,1386]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1452,"t1":1518,"pts":[[0.6216,0.3276,1452]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1518,"t1":1584,"pts":[[0.7226,0.2801,1518]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1584,"t1":1650,"pts":[[0.8169,0.2279,1584]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1650,"t1":1716,"pts":[[0.8873,0.1802,1650]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1716,"t1":1782,"pts":[[0.9242,0.1433,1716]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1782,"t1":1848,"pts":[[0.9308,0.1178,1782]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.9353,0.1069,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":1980,"pts":[[0.9441,0.104,1914]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1980,"t1":2046,"pts":[[0.9522,0.1095,1980]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.9568,0.1201,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.9558,0.132,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.9502,0.1405,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.9421,0.1425,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2376,"pts":[[0.9342,0.1384,2310]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2376,"t1":2442,"pts":[[0.931,0.1274,2376]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2442,"t1":2508,"pts":[[0.9311,0.1152,2442]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2508,"t1":3960,"pts":[[0.9441,0.1225,2508]],"dwell":1.452,"keyframe":null},{"kind":"cursor","t0":3960,"t1":4026,"pts":[[0.9416,0.1217,3960]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4026,"t1":4488,"pts":[[0.9401,0.1217,4026]],"dwell":0.462,"keyframe":null},{"kind":"cursor","t0":4488,"t1":4752,"pts":[[0.9408,0.1236,4488]],"dwell":0.264,"keyframe":null},{"kind":"cursor","t0":4752,"t1":4818,"pts":[[0.9377,0.1279,4752]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4818,"t1":5148,"pts":[[0.9354,0.1313,4818]],"dwell":0.33,"keyframe":null},{"kind":"cursor","t0":5148,"t1":5214,"pts":[[0.9333,0.1289,5148]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5214,"t1":5280,"pts":[[0.9094,0.1335,5214]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5280,"t1":5346,"pts":[[0.8638,0.1569,5280]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5346,"t1":5412,"pts":[[0.8309,0.1811,5346]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5412,"t1":6468,"pts":[[0.827,0.1918,5412]],"dwell":1.056,"keyframe":null},{"kind":"cursor","t0":6468,"t1":7656,"pts":[[0.8289,0.193,6468]],"dwell":1.188,"keyframe":null},{"kind":"cursor","t0":7656,"t1":7722,"pts":[[0.8276,0.1947,7656]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7722,"t1":7920,"pts":[[0.8264,0.1961,7722]],"dwell":0.198,"keyframe":null},{"kind":"cursor","t0":7920,"t1":8118,"pts":[[0.8278,0.1961,7920]],"dwell":0.198,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8184,"pts":[[0.8278,0.1938,8118]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8184,"t1":8250,"pts":[[0.818,0.1914,8184]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8250,"t1":8316,"pts":[[0.7958,0.1942,8250]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8316,"t1":8382,"pts":[[0.7748,0.2,8316]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8382,"t1":8448,"pts":[[0.7674,0.2058,8382]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.7778,0.2088,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8580,"pts":[[0.792,0.2085,8514]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8580,"t1":8646,"pts":[[0.8063,0.2082,8580]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.8206,0.2088,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":8778,"pts":[[0.8349,0.2088,8712]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8778,"t1":8844,"pts":[[0.8492,0.2089,8778]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8844,"t1":8910,"pts":[[0.8635,0.2102,8844]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8910,"t1":8976,"pts":[[0.8754,0.2084,8910]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8976,"t1":9042,"pts":[[0.8555,0.209,8976]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9042,"t1":10164,"pts":[[0.8535,0.211,9042]],"dwell":1.122,"keyframe":null},{"kind":"cursor","t0":10164,"t1":10230,"pts":[[0.8499,0.21,10164]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10230,"t1":11220,"pts":[[0.8478,0.2102,10230]],"dwell":0.99,"keyframe":null},{"kind":"cursor","t0":11220,"t1":11880,"pts":[[0.8472,0.2085,11220]],"dwell":0.66,"keyframe":null},{"kind":"cursor","t0":11880,"t1":11946,"pts":[[0.8491,0.2102,11880]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":11946,"t1":12000,"pts":[[0.8506,0.2111,11946]],"dwell":0.054,"keyframe":null},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.0156,0.9756,2420],[0.9844,0.9756,2420],[0.0156,1.0,2420],[0.9844,1.0,2420],[0.5,0.9878,2420]],"dwell":0.1537,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.5781,0.2195,2420],[0.6563,0.2195,2420],[0.5781,0.439,2420],[0.6563,0.439,2420],[0.6172,0.3293,2420]],"dwell":0.131,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.7813,0.2683,2420],[0.8281,0.2683,2420],[0.7813,0.439,2420],[0.8281,0.439,2420],[0.8047,0.3537,2420]],"dwell":0.0895,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.3594,0.4878,2420],[0.5156,0.4878,2420],[0.3594,0.5366,2420],[0.5156,0.5366,2420],[0.4375,0.5122,2420]],"dwell":0.0873,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.6875,0.2683,2420],[0.7188,0.2683,2420],[0.6875,0.439,2420],[0.7188,0.439,2420],[0.7031,0.3537,2420]],"dwell":0.073,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.875,0.2683,2420],[0.9063,0.2683,2420],[0.875,0.439,2420],[0.9063,0.439,2420],[0.8906,0.3537,2420]],"dwell":0.073,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.0313,0.1707,2420],[0.0781,0.1707,2420],[0.0313,0.2683,2420],[0.0781,0.2683,2420],[0.0547,0.2195,2420]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.2813,0.2683,2420],[0.3438,0.2683,2420],[0.2813,0.3415,2420],[0.3438,0.3415,2420],[0.3125,0.3049,2420]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":5280,"t1":5280,"pts":[[0.75,0.1463,5280],[0.9688,0.1463,5280],[0.75,0.3171,5280],[0.9688,0.3171,5280],[0.8594,0.2317,5280]],"dwell":0.1933,"keyframe":1}],"snapshots":[[0,[{"id":4100,"app":"Safari","rect":[0.0139,0.043,0.9861,0.983],"layer":0,"regular":true,"frontmost":true},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[5080,[{"id":4100,"app":"Safari","rect":[0.0139,0.043,0.9861,0.983],"layer":0,"regular":true,"frontmost":true},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]]]}},{"name":"two-windows","truth":{"scenario":"two-windows","description":"Comparing two small windows: ink in Notes, ink in Finder, back and forth","screen":[1728.0,1117.0],"whole":[162.0,222.0,1366.0,546.0],"spans":[{"t0":0,"t1":4300,"subject":"notes","ideal":[170.0,230.0,540.0,440.0],"targets":[[214.0,393.0,233.0,16.0]]},{"t0":4300,"t1":7600,"subject":"finder","ideal":[760.0,260.0,760.0,500.0],"targets":[[958.0,445.0,272.0,19.0]]},{"t0":7600,"t1":13000,"subject":"both","ideal":[162.0,222.0,1366.0,546.0],"targets":[[214.0,393.0,233.0,16.0],[958.0,445.0,272.0,19.0]]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":2420,"must":[[0.1162,0.3369,0.2713,0.3803]]},{"index":2,"t":5500,"must":[[0.5404,0.3872,0.7289,0.4266]]},{"index":3,"t":9020,"must":[[0.2666,0.3268,0.6331,0.4066],[0.2666,0.3439,0.281,0.3657]]}],"input":{"screen":[1728.0,1117.0],"duration":13000,"keyframes":[2420,5500,9020],"items":[{"kind":"stroke","t0":1600,"t1":2500,"pts":[[0.1337,0.3471,1600],[0.1399,0.3446,1620],[0.1447,0.3414,1640],[0.1569,0.3409,1660],[0.165,0.3382,1680],[0.1776,0.3392,1700],[0.1882,0.3382,1720],[0.1998,0.3369,1740],[0.2113,0.3377,1760],[0.2224,0.3387,1780],[0.2295,0.3417,1800],[0.2428,0.342,1820],[0.2492,0.3448,1840],[0.2542,0.3476,1860],[0.2573,0.3505,1880],[0.2673,0.3528,1900],[0.2646,0.3561,1920],[0.2713,0.3591,1940],[0.2647,0.3621,1960],[0.2624,0.365,1980],[0.2592,0.3678,2000],[0.2529,0.3703,2020],[0.2465,0.3726,2040],[0.2388,0.3747,2060],[0.2298,0.3763,2080],[0.2204,0.3776,2100],[0.2119,0.3803,2120],[0.2002,0.38,2140],[0.1892,0.3793,2160],[0.179,0.3784,2180],[0.1668,0.3792,2200],[0.1554,0.3784,2220],[0.1499,0.3751,2240],[0.1417,0.3732,2260],[0.132,0.3715,2280],[0.1257,0.3689,2300],[0.1196,0.3662,2320],[0.1194,0.3629,2340],[0.1162,0.3599,2360],[0.1209,0.3568,2380],[0.1223,0.3538,2400],[0.1261,0.3509,2420],[0.1324,0.3485,2440],[0.1347,0.3451,2460],[0.143,0.3429,2480],[0.1517,0.3408,2500]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":4800,"t1":5600,"pts":[[0.5568,0.3953,4800],[0.5706,0.394,4818],[0.5808,0.3923,4836],[0.589,0.3897,4853],[0.6012,0.3883,4871],[0.6148,0.3876,4889],[0.6289,0.3888,4907],[0.6425,0.3872,4924],[0.6549,0.3889,4942],[0.6663,0.3903,4960],[0.6827,0.3897,4978],[0.6908,0.3923,4996],[0.7041,0.3936,5013],[0.7119,0.396,5031],[0.7184,0.3986,5049],[0.7179,0.4017,5067],[0.7289,0.4041,5084],[0.7271,0.4071,5102],[0.724,0.4099,5120],[0.7194,0.4126,5138],[0.7208,0.4158,5156],[0.7121,0.4181,5173],[0.6994,0.4195,5191],[0.6931,0.4222,5209],[0.6828,0.424,5227],[0.6686,0.4243,5244],[0.6563,0.425,5262],[0.6442,0.4264,5280],[0.6311,0.4259,5298],[0.6173,0.4266,5316],[0.606,0.4247,5333],[0.5924,0.4243,5351],[0.5788,0.4235,5369],[0.5704,0.4211,5387],[0.5658,0.4182,5404],[0.5544,0.4164,5422],[0.5467,0.4139,5440],[0.5511,0.4108,5458],[0.5404,0.4082,5476],[0.5492,0.4053,5493],[0.5437,0.4022,5511],[0.5528,0.3998,5529],[0.5558,0.3969,5547],[0.5666,0.395,5564],[0.5727,0.3922,5582],[0.587,0.3913,5600]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":7800,"t1":8500,"pts":[[0.6331,0.4066,7800],[0.6184,0.3984,7828],[0.6038,0.3899,7856],[0.5891,0.3812,7884],[0.5745,0.3734,7912],[0.5598,0.3667,7940],[0.5451,0.3585,7968],[0.5305,0.3531,7996],[0.5158,0.3468,8024],[0.5011,0.3421,8052],[0.4865,0.3372,8080],[0.4718,0.3336,8108],[0.4572,0.3303,8136],[0.4425,0.3284,8164],[0.4278,0.3282,8192],[0.4132,0.3268,8220],[0.3985,0.3276,8248],[0.3839,0.3285,8276],[0.3692,0.3305,8304],[0.3545,0.3333,8332],[0.3399,0.3368,8360],[0.3252,0.3408,8388],[0.3105,0.3449,8416],[0.2959,0.3494,8444],[0.2812,0.3542,8472],[0.2666,0.359,8500]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":8750,"t1":8950,"pts":[[0.278,0.3439,8750],[0.2666,0.359,8850],[0.281,0.3657,8950]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":660,"pts":[[0.1966,0.3561,0]],"dwell":0.66,"keyframe":null},{"kind":"cursor","t0":660,"t1":726,"pts":[[0.197,0.3582,660]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":726,"t1":1056,"pts":[[0.1978,0.3602,726]],"dwell":0.33,"keyframe":null},{"kind":"cursor","t0":1056,"t1":1122,"pts":[[0.1967,0.3586,1056]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1122,"t1":1320,"pts":[[0.195,0.3569,1122]],"dwell":0.198,"keyframe":null},{"kind":"cursor","t0":1320,"t1":1386,"pts":[[0.1939,0.3545,1320]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1386,"t1":1452,"pts":[[0.1844,0.3489,1386]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1452,"t1":1518,"pts":[[0.1624,0.3441,1452]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1518,"t1":1584,"pts":[[0.1415,0.3427,1518]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1584,"t1":1650,"pts":[[0.1339,0.3459,1584]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1650,"t1":1716,"pts":[[0.1508,0.3412,1650]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1716,"t1":1782,"pts":[[0.1861,0.3384,1716]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1782,"t1":1848,"pts":[[0.2231,0.339,1782]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1848,"t1":1914,"pts":[[0.2512,0.3459,1848]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1914,"t1":1980,"pts":[[0.2654,0.3551,1914]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":1980,"t1":2046,"pts":[[0.2624,0.365,1980]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.2442,0.3732,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.2153,0.3792,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.18,0.3785,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.1483,0.3747,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2376,"pts":[[0.1227,0.3675,2310]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2376,"t1":2442,"pts":[[0.12,0.3574,2376]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2442,"t1":2508,"pts":[[0.1326,0.3481,2442]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2508,"t1":4026,"pts":[[0.1947,0.3624,2508]],"dwell":1.518,"keyframe":null},{"kind":"cursor","t0":4026,"t1":4092,"pts":[[0.196,0.363,4026]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4092,"t1":4158,"pts":[[0.1975,0.3615,4092]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4158,"t1":4290,"pts":[[0.1984,0.3602,4158]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":4290,"t1":4356,"pts":[[0.1986,0.3675,4290]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4356,"t1":4422,"pts":[[0.2123,0.3848,4356]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4422,"t1":4488,"pts":[[0.2549,0.4015,4422]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4488,"t1":4554,"pts":[[0.3239,0.4155,4488]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4554,"t1":4620,"pts":[[0.4051,0.4244,4554]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4620,"t1":4686,"pts":[[0.4798,0.4263,4620]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4686,"t1":4752,"pts":[[0.5317,0.4202,4686]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4752,"t1":4818,"pts":[[0.5541,0.4073,4752]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4818,"t1":4884,"pts":[[0.5707,0.394,4818]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4884,"t1":4950,"pts":[[0.611,0.3878,4884]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4950,"t1":5016,"pts":[[0.6599,0.3895,4950]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5016,"t1":5082,"pts":[[0.7053,0.3939,5016]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5082,"t1":5148,"pts":[[0.7274,0.4038,5082]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5148,"t1":5214,"pts":[[0.7202,0.4144,5148]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5214,"t1":5280,"pts":[[0.6901,0.4227,5214]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5280,"t1":5346,"pts":[[0.6442,0.4264,5280]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5346,"t1":5412,"pts":[[0.5963,0.4244,5346]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5412,"t1":5478,"pts":[[0.5609,0.4174,5412]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5478,"t1":5544,"pts":[[0.5416,0.4078,5478]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5544,"t1":5610,"pts":[[0.5554,0.3973,5544]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5610,"t1":6468,"pts":[[0.6264,0.4013,5610]],"dwell":0.858,"keyframe":null},{"kind":"cursor","t0":6468,"t1":6534,"pts":[[0.6232,0.4014,6468]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6534,"t1":7260,"pts":[[0.6209,0.4023,6534]],"dwell":0.726,"keyframe":null},{"kind":"cursor","t0":7260,"t1":7326,"pts":[[0.6216,0.4038,7260]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7326,"t1":7590,"pts":[[0.6229,0.4041,7326]],"dwell":0.264,"keyframe":null},{"kind":"cursor","t0":7590,"t1":7656,"pts":[[0.6239,0.4049,7590]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7656,"t1":7722,"pts":[[0.6274,0.4063,7656]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7722,"t1":7788,"pts":[[0.6315,0.4071,7722]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7788,"t1":7854,"pts":[[0.6331,0.4067,7788]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7854,"t1":7920,"pts":[[0.6048,0.3905,7854]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7920,"t1":7986,"pts":[[0.5703,0.3715,7920]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7986,"t1":8052,"pts":[[0.5357,0.355,7986]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8052,"t1":8118,"pts":[[0.5011,0.3421,8052]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8184,"pts":[[0.4666,0.3324,8118]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8184,"t1":8250,"pts":[[0.432,0.3282,8184]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8250,"t1":8316,"pts":[[0.3975,0.3277,8250]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8316,"t1":8382,"pts":[[0.3629,0.3317,8316]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8382,"t1":8448,"pts":[[0.3284,0.3399,8382]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.2938,0.3501,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8580,"pts":[[0.2672,0.3581,8514]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8580,"t1":8646,"pts":[[0.2702,0.3541,8580]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.2732,0.3501,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":8778,"pts":[[0.2763,0.3462,8712]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8778,"t1":8844,"pts":[[0.2748,0.3481,8778]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8844,"t1":8910,"pts":[[0.2673,0.3581,8844]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8910,"t1":8976,"pts":[[0.2752,0.363,8910]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8976,"t1":9042,"pts":[[0.2178,0.3491,8976]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9042,"t1":9834,"pts":[[0.1631,0.3619,9042]],"dwell":0.792,"keyframe":null},{"kind":"cursor","t0":9834,"t1":9900,"pts":[[0.1643,0.3617,9834]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9900,"t1":10032,"pts":[[0.1662,0.3607,9900]],"dwell":0.132,"keyframe":null},{"kind":"cursor","t0":10032,"t1":10692,"pts":[[0.1672,0.3597,10032]],"dwell":0.66,"keyframe":null},{"kind":"cursor","t0":10692,"t1":10758,"pts":[[0.1652,0.3576,10692]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10758,"t1":12276,"pts":[[0.1638,0.3567,10758]],"dwell":1.518,"keyframe":null},{"kind":"cursor","t0":12276,"t1":12870,"pts":[[0.1622,0.3569,12276]],"dwell":0.594,"keyframe":null},{"kind":"cursor","t0":12870,"t1":12936,"pts":[[0.1641,0.3602,12870]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":12936,"t1":13000,"pts":[[0.1664,0.3615,12936]],"dwell":0.064,"keyframe":null},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.4375,0.2439,2420],[0.875,0.2439,2420],[0.4375,0.6829,2420],[0.875,0.6829,2420],[0.6563,0.4634,2420]],"dwell":0.4383,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.1094,0.2439,2420],[0.3281,0.2439,2420],[0.1094,0.3902,2420],[0.3281,0.3902,2420],[0.2188,0.3171,2420]],"dwell":0.1789,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.5,0.2439,2420],[0.6094,0.2439,2420],[0.5,0.5122,2420],[0.6094,0.5122,2420],[0.5547,0.378,2420]],"dwell":0.1713,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.7969,0.3171,2420],[0.8281,0.3171,2420],[0.7969,0.4634,2420],[0.8281,0.4634,2420],[0.8125,0.3902,2420]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.3281,0.9268,2420],[0.5156,0.9268,2420],[0.3281,0.9512,2420],[0.5156,0.9512,2420],[0.4219,0.939,2420]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.625,0.9268,2420],[0.6875,0.9268,2420],[0.625,0.9512,2420],[0.6875,0.9512,2420],[0.6563,0.939,2420]],"dwell":0.039,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.625,0.9756,2420],[0.6875,0.9756,2420],[0.625,1.0,2420],[0.6875,1.0,2420],[0.6563,0.9878,2420]],"dwell":0.039,"keyframe":0},{"kind":"change","t0":2420,"t1":2420,"pts":[[0.4219,0.9756,2420],[0.4688,0.9756,2420],[0.4219,1.0,2420],[0.4688,1.0,2420],[0.4453,0.9878,2420]],"dwell":0.0338,"keyframe":0}],"snapshots":[[0,[{"id":4101,"app":"Finder","rect":[0.4398,0.2328,0.8796,0.6804],"layer":0,"regular":true,"frontmost":true},{"id":4100,"app":"Notes","rect":[0.0984,0.2059,0.4109,0.5998],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false},{"id":38,"app":"Dock","rect":[0.0,0.9338,1.0,1.0],"layer":20,"regular":false,"frontmost":false}]]]}},{"name":"window-drag","truth":{"scenario":"window-drag","description":"Dragging the Notes window across the screen, then inking in it","screen":[1728.0,1117.0],"whole":[60.0,160.0,1260.0,800.0],"spans":[{"t0":0,"t1":1900,"subject":"notes","ideal":[60.0,160.0,520.0,420.0],"targets":[]},{"t0":1900,"t1":3900,"subject":"move","ideal":[60.0,160.0,1260.0,800.0],"targets":[]},{"t0":3900,"t1":13000,"subject":"notes","ideal":[800.0,540.0,520.0,420.0],"targets":[[844.0,650.0,125.0,16.0]]}]},"pxPerPt":1.4814814,"keyframes":[{"index":1,"t":3520,"must":[]},{"index":2,"t":5280,"must":[[0.4817,0.5708,0.5725,0.6071]]},{"index":3,"t":8800,"must":[[0.485,0.624,0.6095,0.6275]]}],"input":{"screen":[1728.0,1117.0],"duration":13000,"keyframes":[3520,5280,8800],"items":[{"kind":"stroke","t0":4400,"t1":5200,"pts":[[0.4977,0.5763,4400],[0.5023,0.5746,4418],[0.5075,0.5729,4436],[0.513,0.5709,4453],[0.5199,0.5708,4471],[0.5267,0.5714,4489],[0.5337,0.5708,4507],[0.5395,0.5725,4524],[0.5453,0.5737,4542],[0.5523,0.5743,4560],[0.558,0.5759,4578],[0.562,0.5781,4596],[0.5636,0.5809,4613],[0.5685,0.583,4631],[0.5699,0.5857,4649],[0.5725,0.5883,4667],[0.5721,0.5911,4684],[0.5686,0.5936,4702],[0.5688,0.5965,4720],[0.5663,0.5991,4738],[0.5618,0.6012,4756],[0.5548,0.6021,4773],[0.5502,0.6038,4791],[0.5466,0.6066,4809],[0.5399,0.6071,4827],[0.533,0.6065,4844],[0.5269,0.6069,4862],[0.5206,0.6069,4880],[0.5146,0.6061,4898],[0.5084,0.6056,4916],[0.5037,0.6038,4933],[0.4962,0.6035,4951],[0.4915,0.6015,4969],[0.491,0.5984,4987],[0.4883,0.5961,5004],[0.4827,0.5941,5022],[0.4856,0.5912,5040],[0.4832,0.5886,5058],[0.4817,0.5858,5076],[0.4842,0.5832,5093],[0.4897,0.5812,5111],[0.491,0.5784,5129],[0.4951,0.5762,5147],[0.5014,0.575,5164],[0.5061,0.5729,5182],[0.5139,0.5734,5200]],"dwell":0.0,"keyframe":null},{"kind":"stroke","t0":8400,"t1":8900,"pts":[[0.485,0.6241,8400],[0.4923,0.6246,8429],[0.4996,0.624,8459],[0.5069,0.6252,8488],[0.5143,0.6262,8518],[0.5216,0.6242,8547],[0.5289,0.6259,8576],[0.5363,0.6243,8606],[0.5436,0.6268,8635],[0.5509,0.6247,8665],[0.5582,0.6259,8694],[0.5656,0.6266,8724],[0.5729,0.6271,8753],[0.5802,0.6265,8782],[0.5876,0.6253,8812],[0.5949,0.6275,8841],[0.6022,0.6259,8871],[0.6095,0.6265,8900]],"dwell":0.0,"keyframe":null},{"kind":"drag","t0":2000,"t1":3300,"pts":[[0.1852,0.1567,2000],[0.6134,0.4969,3300]],"dwell":0.0,"keyframe":null},{"kind":"cursor","t0":0,"t1":2046,"pts":[[0.1852,0.1567,0]],"dwell":2.046,"keyframe":null},{"kind":"cursor","t0":2046,"t1":2112,"pts":[[0.1894,0.16,2046]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2112,"t1":2178,"pts":[[0.1955,0.1649,2112]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2178,"t1":2244,"pts":[[0.2092,0.1757,2178]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2244,"t1":2310,"pts":[[0.2257,0.1889,2244]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2310,"t1":2376,"pts":[[0.2482,0.2067,2310]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2376,"t1":2442,"pts":[[0.2725,0.226,2376]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2442,"t1":2508,"pts":[[0.301,0.2487,2442]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2508,"t1":2574,"pts":[[0.3305,0.2721,2508]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2574,"t1":2640,"pts":[[0.3623,0.2973,2574]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2640,"t1":2706,"pts":[[0.3944,0.3229,2640]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2706,"t1":2772,"pts":[[0.4266,0.3485,2706]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2772,"t1":2838,"pts":[[0.4588,0.374,2772]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2838,"t1":2904,"pts":[[0.4887,0.3978,2838]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2904,"t1":2970,"pts":[[0.5182,0.4212,2904]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":2970,"t1":3036,"pts":[[0.5431,0.441,2970]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3036,"t1":3102,"pts":[[0.5674,0.4603,3036]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3102,"t1":3168,"pts":[[0.5844,0.4738,3102]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3168,"t1":3234,"pts":[[0.6009,0.4869,3168]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3234,"t1":3300,"pts":[[0.6073,0.492,3234]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3300,"t1":3366,"pts":[[0.6134,0.4969,3300]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3366,"t1":3432,"pts":[[0.6065,0.5016,3366]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3432,"t1":3498,"pts":[[0.5995,0.5064,3432]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3498,"t1":3564,"pts":[[0.5926,0.5112,3498]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3564,"t1":3630,"pts":[[0.5856,0.5159,3564]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3630,"t1":3696,"pts":[[0.5787,0.5207,3630]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3696,"t1":3762,"pts":[[0.5718,0.5255,3696]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3762,"t1":3828,"pts":[[0.5648,0.5302,3762]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3828,"t1":3894,"pts":[[0.5579,0.535,3828]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3894,"t1":3960,"pts":[[0.5509,0.5398,3894]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":3960,"t1":4026,"pts":[[0.544,0.5445,3960]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4026,"t1":4092,"pts":[[0.537,0.5493,4026]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4092,"t1":4158,"pts":[[0.5301,0.5541,4092]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4158,"t1":4224,"pts":[[0.5231,0.5588,4158]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4224,"t1":4290,"pts":[[0.5162,0.5636,4224]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4290,"t1":4356,"pts":[[0.5093,0.5684,4290]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4356,"t1":4422,"pts":[[0.5023,0.5731,4356]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4422,"t1":4488,"pts":[[0.5036,0.5742,4422]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4488,"t1":4554,"pts":[[0.5264,0.5713,4488]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4554,"t1":4620,"pts":[[0.5499,0.5741,4554]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4620,"t1":4686,"pts":[[0.5654,0.5817,4620]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4686,"t1":4752,"pts":[[0.5718,0.5914,4686]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4752,"t1":4818,"pts":[[0.5627,0.6008,4752]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4818,"t1":4884,"pts":[[0.5432,0.6068,4818]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4884,"t1":4950,"pts":[[0.5193,0.6067,4884]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":4950,"t1":5016,"pts":[[0.4967,0.6035,4950]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5016,"t1":5082,"pts":[[0.4846,0.5948,5016]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5082,"t1":5148,"pts":[[0.4826,0.5849,5082]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5148,"t1":5214,"pts":[[0.4956,0.5761,5148]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":5214,"t1":6534,"pts":[[0.5244,0.587,5214]],"dwell":1.32,"keyframe":null},{"kind":"cursor","t0":6534,"t1":6600,"pts":[[0.5268,0.5887,6534]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6600,"t1":6666,"pts":[[0.5304,0.5899,6600]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":6666,"t1":7260,"pts":[[0.5323,0.5899,6666]],"dwell":0.594,"keyframe":null},{"kind":"cursor","t0":7260,"t1":7326,"pts":[[0.5304,0.5896,7260]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":7326,"t1":7920,"pts":[[0.5282,0.5903,7326]],"dwell":0.594,"keyframe":null},{"kind":"cursor","t0":7920,"t1":8118,"pts":[[0.5277,0.5874,7920]],"dwell":0.198,"keyframe":null},{"kind":"cursor","t0":8118,"t1":8184,"pts":[[0.5272,0.5858,8118]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8184,"t1":8250,"pts":[[0.5205,0.5876,8184]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8250,"t1":8316,"pts":[[0.5046,0.5999,8250]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8316,"t1":8382,"pts":[[0.4894,0.6148,8316]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8382,"t1":8448,"pts":[[0.4847,0.6231,8382]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8448,"t1":8514,"pts":[[0.4969,0.6242,8448]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8514,"t1":8580,"pts":[[0.5134,0.6261,8514]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8580,"t1":8646,"pts":[[0.5298,0.6257,8580]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8646,"t1":8712,"pts":[[0.5462,0.626,8646]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8712,"t1":8778,"pts":[[0.5627,0.6263,8712]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8778,"t1":8844,"pts":[[0.5791,0.6266,8778]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8844,"t1":8910,"pts":[[0.5956,0.6274,8844]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8910,"t1":8976,"pts":[[0.6091,0.6238,8910]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":8976,"t1":9042,"pts":[[0.5467,0.6116,8976]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":9042,"t1":10362,"pts":[[0.5399,0.6166,9042]],"dwell":1.32,"keyframe":null},{"kind":"cursor","t0":10362,"t1":10428,"pts":[[0.5372,0.6136,10362]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10428,"t1":10494,"pts":[[0.5338,0.6115,10428]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":10494,"t1":11946,"pts":[[0.5323,0.6112,10494]],"dwell":1.452,"keyframe":null},{"kind":"cursor","t0":11946,"t1":12870,"pts":[[0.5333,0.6122,11946]],"dwell":0.924,"keyframe":null},{"kind":"cursor","t0":12870,"t1":12936,"pts":[[0.5348,0.6143,12870]],"dwell":0.066,"keyframe":null},{"kind":"cursor","t0":12936,"t1":13000,"pts":[[0.536,0.6156,12936]],"dwell":0.064,"keyframe":null},{"kind":"change","t0":3520,"t1":3520,"pts":[[0.0156,0.0488,3520],[0.9375,0.0488,3520],[0.0156,0.5854,3520],[0.9375,0.5854,3520],[0.4766,0.3171,3520]],"dwell":0.7033,"keyframe":0},{"kind":"change","t0":3520,"t1":3520,"pts":[[0.2969,0.439,3520],[0.9844,0.439,3520],[0.2969,0.9268,3520],[0.9844,0.9268,3520],[0.6406,0.6829,3520]],"dwell":0.5791,"keyframe":0},{"kind":"change","t0":3520,"t1":3520,"pts":[[0.8438,0.7561,3520],[0.9688,0.7561,3520],[0.8438,0.8049,3520],[0.9688,0.8049,3520],[0.9063,0.7805,3520]],"dwell":0.0781,"keyframe":0},{"kind":"change","t0":3520,"t1":3520,"pts":[[0.4375,0.2439,3520],[0.4844,0.2439,3520],[0.4375,0.3659,3520],[0.4844,0.3659,3520],[0.4609,0.3049,3520]],"dwell":0.0756,"keyframe":0},{"kind":"change","t0":3520,"t1":3520,"pts":[[0.9219,0.0976,3520],[0.9844,0.0976,3520],[0.9219,0.1707,3520],[0.9844,0.1707,3520],[0.9531,0.1341,3520]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":3520,"t1":3520,"pts":[[0.625,0.2683,3520],[0.6875,0.2683,3520],[0.625,0.3415,3520],[0.6875,0.3415,3520],[0.6563,0.3049,3520]],"dwell":0.0676,"keyframe":0},{"kind":"change","t0":3520,"t1":3520,"pts":[[0.7188,0.3659,3520],[0.8125,0.3659,3520],[0.7188,0.3902,3520],[0.8125,0.3902,3520],[0.7656,0.378,3520]],"dwell":0.0478,"keyframe":0},{"kind":"change","t0":3520,"t1":3520,"pts":[[0.0469,0.0,3520],[0.125,0.0,3520],[0.0469,0.0244,3520],[0.125,0.0244,3520],[0.0859,0.0122,3520]],"dwell":0.0437,"keyframe":0}],"snapshots":[[0,[{"id":4100,"app":"Notes","rect":[0.0347,0.1432,0.3356,0.5192],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.3009,0.0537,0.9838,0.9132],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[2000,[{"id":4100,"app":"Notes","rect":[0.0347,0.1432,0.3356,0.5192],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.3009,0.0537,0.9838,0.9132],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[2260,[{"id":4100,"app":"Notes","rect":[0.1042,0.188,0.4051,0.564],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.3009,0.0537,0.9838,0.9132],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[2520,[{"id":4100,"app":"Notes","rect":[0.2431,0.2954,0.544,0.6714],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.3009,0.0537,0.9838,0.9132],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[2780,[{"id":4100,"app":"Notes","rect":[0.3704,0.4029,0.6713,0.7789],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.3009,0.0537,0.9838,0.9132],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[3040,[{"id":4100,"app":"Notes","rect":[0.4398,0.4655,0.7407,0.8415],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.3009,0.0537,0.9838,0.9132],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]],[3300,[{"id":4100,"app":"Notes","rect":[0.463,0.4834,0.7639,0.8594],"layer":0,"regular":true,"frontmost":true},{"id":4101,"app":"Safari","rect":[0.3009,0.0537,0.9838,0.9132],"layer":0,"regular":true,"frontmost":false},{"id":6,"app":"Window Server","rect":[0.0,0.0,1.0,0.0295],"layer":24,"regular":false,"frontmost":false}]]]}}] \ No newline at end of file diff --git a/apps/web/src/scrawl/autocrop.test.ts b/apps/web/src/scrawl/autocrop.test.ts new file mode 100644 index 0000000000000..bd59ed82484de --- /dev/null +++ b/apps/web/src/scrawl/autocrop.test.ts @@ -0,0 +1,541 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + activityItems, + computeCrops, + cropLabel, + FULL, + pixelRect, + windowSnapshots, + type ActivityItem, + type AutocropInput, + type Crop, + type Rect, +} from "./autocrop.ts"; +import { changeRegions, contentMap, type RgbaImage } from "./autocropContent.ts"; +import corpusData from "./autocrop-corpus.json"; + +/** `ideal` and `targets` are `[x, y, width, height]` in points. */ +interface CorpusSpan { + readonly t0: number; + readonly t1: number; + readonly subject: string; + readonly ideal: Rect; + readonly targets: ReadonlyArray; +} + +interface CorpusCase { + readonly name: string; + readonly pxPerPt: number; + readonly truth: { + readonly screen: readonly [number, number]; + readonly spans: ReadonlyArray; + }; + readonly keyframes: ReadonlyArray<{ readonly t: number; readonly must: ReadonlyArray }>; + readonly input: { + readonly screen: readonly [number, number]; + readonly duration: number; + readonly keyframes: ReadonlyArray; + readonly items: ReadonlyArray<{ + readonly kind: ActivityItem["kind"]; + readonly t0: number; + readonly t1: number; + readonly pts: ActivityItem["points"]; + readonly dwell: number; + readonly keyframe: number | null; + }>; + readonly snapshots: ReadonlyArray< + readonly [ + number, + ReadonlyArray<{ + readonly id: number; + readonly app: string; + readonly rect: Rect; + readonly layer: number; + readonly regular: boolean; + readonly frontmost: boolean; + }>, + ] + >; + }; +} + +// Simulated captures with ground truth, from Scrawl (src/autocrop-corpus.json). +const corpus = corpusData as unknown as ReadonlyArray; + +const inputOf = ( + { input }: CorpusCase, + { windows = true, cursor = true }: { windows?: boolean; cursor?: boolean } = {}, +): AutocropInput => ({ + screen: input.screen, + durationMs: input.duration, + keyframes: input.keyframes, + items: input.items.flatMap((item): Array => { + if (item.kind === "cursor" && !cursor) return []; + const base = { + kind: item.kind, + t0: item.t0, + t1: item.t1, + points: item.pts, + dwell: item.dwell, + }; + return [item.keyframe === null ? base : { ...base, keyframe: item.keyframe }]; + }), + snapshots: windows + ? input.snapshots.map(([t, snapshotWindows]) => ({ t, windows: snapshotWindows })) + : [], +}); + +const normalized = (box: Rect, [width, height]: readonly [number, number]): Rect => [ + box[0] / width, + box[1] / height, + (box[0] + box[2]) / width, + (box[1] + box[3]) / height, +]; + +const areaOf = (r: Rect) => Math.max(0, r[2] - r[0]) * Math.max(0, r[3] - r[1]); +const common = (a: Rect, b: Rect): Rect => [ + Math.max(a[0], b[0]), + Math.max(a[1], b[1]), + Math.min(a[2], b[2]), + Math.min(a[3], b[3]), +]; + +const overlap = (a: Rect, b: Rect) => { + const w = Math.max(0, Math.min(a[2], b[2]) - Math.max(a[0], b[0])); + const h = Math.max(0, Math.min(a[3], b[3]) - Math.max(a[1], b[1])); + const shared = w * h; + return shared / ((a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - shared); +}; + +/** Scrawl's score: the crop keeps what matters, sits close to the ideal, and shows it large. */ +const scoreCase = (testCase: CorpusCase, rects: ReadonlyArray) => { + const { screen, spans } = testCase.truth; + const [sw, sh] = screen; + const scored = testCase.keyframes.map((keyframe, index) => { + const span = spans.find(({ t0, t1 }) => t0 <= keyframe.t && keyframe.t < t1) ?? spans.at(-1)!; + const rect = rects[index]!; + const ideal = normalized(span.ideal, screen); + const targets = span.targets.map((target) => normalized(target, screen)); + const holds = (box: Rect) => + box[0] >= rect[0] - 2 / sw && + box[1] >= rect[1] - 2 / sh && + box[2] <= rect[2] + 2 / sw && + box[3] <= rect[3] + 2 / sh; + const covered = [...keyframe.must, ...targets].every(holds); + const contexts = targets.map((target) => { + const margin = Math.max( + 0.2 * Math.max((target[2] - target[0]) * sw, (target[3] - target[1]) * sh), + 24, + ); + const grown = [ + Math.max(target[0] - margin / sw, 0), + Math.max(target[1] - margin / sh, 0), + Math.min(target[2] + margin / sw, 1), + Math.min(target[3] + margin / sh, 1), + ] as const; + return areaOf(common(grown, rect)) / Math.max(areaOf(grown), 1e-9); + }); + const context = + contexts.length === 0 ? 1 : contexts.reduce((sum, value) => sum + value, 0) / contexts.length; + const precision = areaOf(common(rect, ideal)) / Math.max(areaOf(rect), 1e-9); + const w = (rect[2] - rect[0]) * sw * testCase.pxPerPt; + const h = (rect[3] - rect[1]) * sh * testCase.pxPerPt; + const scale = Math.min((h > w ? 440 : 640) / w, 960 / h); + const resolution = Math.min( + 1, + Math.max( + 0, + Math.min((Math.min(scale, 1) * testCase.pxPerPt) / 0.8, 1) - 0.5 * Math.max(scale - 1, 0), + ), + ); + const score = covered + ? 0.45 * overlap(rect, ideal) + 0.2 * context + 0.15 * resolution + 0.2 * precision + : 0; + return { covered, score, rect, subject: span.subject }; + }); + const pairs = scored.flatMap((entry, index) => { + const before = scored[index - 1]; + return before?.subject === entry.subject ? [overlap(entry.rect, before.rect)] : []; + }); + const stability = + pairs.length === 0 ? 1 : pairs.reduce((sum, value) => sum + value, 0) / pairs.length; + const mean = scored.reduce((sum, { score }) => sum + score, 0) / scored.length; + return { + covered: scored.every(({ covered }) => covered), + score: 0.85 * mean + 0.15 * stability, + }; +}; + +const SCREEN = [1000, 1000] as const; + +const click = (t: number, x: number, y: number): ActivityItem => ({ + kind: "click", + t0: t, + t1: t, + dwell: 0, + points: [[x, y, t]], +}); + +const crops = (keyframes: ReadonlyArray, items: ReadonlyArray) => + computeCrops({ screen: SCREEN, durationMs: 40_000, keyframes, items }); + +const inside = (rect: Rect, x: number, y: number) => + x >= rect[0] && x <= rect[2] && y >= rect[1] && y <= rect[3]; + +describe("computeCrops on Scrawl's corpus", () => { + it("keeps every keyframe's activity and beats the full frame", () => { + expect(corpus.length).toBeGreaterThanOrEqual(12); + let total = 0; + for (const testCase of corpus) { + const result = scoreCase( + testCase, + computeCrops(inputOf(testCase)).map((crop) => crop.rect), + ); + expect(result.covered, testCase.name).toBe(true); + const uncropped = scoreCase( + testCase, + testCase.keyframes.map(() => FULL), + ); + expect(result.score, testCase.name).toBeGreaterThan(uncropped.score); + total += result.score; + } + expect(total / corpus.length).toBeGreaterThanOrEqual(0.85); + }); + + it("snaps to the window in use", () => { + for (const name of ["side-by-side", "overlapping-front", "two-windows", "three-window-hop"]) { + const testCase = corpus.find((candidate) => candidate.name === name)!; + const found = computeCrops(inputOf(testCase)); + testCase.keyframes.forEach((keyframe, index) => { + const span = testCase.truth.spans.find( + ({ t0, t1 }) => t0 <= keyframe.t && keyframe.t < t1, + )!; + const ideal = normalized(span.ideal, testCase.truth.screen); + expect(overlap(found[index]!.rect, ideal), `${name} #${index + 1}`).toBeGreaterThan(0.9); + }); + } + }); + + it("keeps all deliberate input with neither a window log nor a cursor log", () => { + for (const testCase of corpus) { + const { items, keyframes, duration } = testCase.input; + const found = computeCrops(inputOf(testCase, { windows: false, cursor: false })); + found.forEach(({ rect }, index) => { + const from = index === 0 ? 0 : (keyframes[index - 1]! + keyframes[index]!) / 2; + const to = + index + 1 === keyframes.length + ? Math.max(duration, keyframes[index]!) + : (keyframes[index]! + keyframes[index + 1]!) / 2; + for (const item of items) { + if (item.kind === "cursor" || item.kind === "change") continue; + if (item.t1 < from || item.t0 > to) continue; + for (const [x, y] of item.pts) { + expect( + x >= rect[0] - 1e-3 && + x <= rect[2] + 1e-3 && + y >= rect[1] - 1e-3 && + y <= rect[3] + 1e-3, + `${testCase.name} #${index + 1}`, + ).toBe(true); + } + } + }); + } + }); + + it("does not let a wandering cursor grow the crop", () => { + const testCase = corpus.find((candidate) => candidate.name === "stray-cursor")!; + const withCursor = computeCrops(inputOf(testCase)); + const without = computeCrops(inputOf(testCase, { cursor: false })); + withCursor.forEach((crop, index) => { + expect(overlap(crop.rect, without[index]!.rect)).toBeGreaterThan(0.6); + }); + }); +}); + +describe("computeCrops", () => { + it("gives each keyframe its own crop when the activity moves", () => { + const [first, second] = crops( + [5_000, 30_000], + [click(5_000, 0.1, 0.1), click(30_000, 0.9, 0.9)], + ); + expect(inside(first!.rect, 0.1, 0.1)).toBe(true); + expect(inside(first!.rect, 0.9, 0.9)).toBe(false); + expect(inside(second!.rect, 0.9, 0.9)).toBe(true); + expect(inside(second!.rect, 0.1, 0.1)).toBe(false); + expect(first!.kind).toBe("activity"); + }); + + it("shares one crop between neighbours looking at the same place", () => { + const [first, second] = crops( + [5_000, 30_000], + [click(5_000, 0.5, 0.5), click(30_000, 0.52, 0.5)], + ); + expect(first!.rect).toEqual(second!.rect); + }); + + it("keeps the full frame when activity covers most of the screen", () => { + const [crop] = crops([5_000], [click(4_000, 0.02, 0.02), click(5_000, 0.98, 0.98)]); + expect(crop!.kind).toBe("full"); + expect(crop!.rect).toEqual(FULL); + }); + + it("gives a keyframe with no activity the nearest crop", () => { + const [first, second, third] = crops([10_000, 20_000, 30_000], [click(30_000, 0.8, 0.8)]); + expect(third!.kind).toBe("activity"); + expect(first!.kind).toBe("fallback"); + expect(second!.rect).toEqual(third!.rect); + }); + + it("takes the full frame when nothing happened at all", () => { + expect(crops([5_000], [])[0]).toMatchObject({ kind: "full", rect: FULL }); + }); + + it("gives a keyframe the ink it owns however long ago it was drawn, and no one else", () => { + const items = activityItems({ + durationMs: 40_000, + keyframes: [10_000, 30_000], + strokes: [{ t0: 1_000, t1: 2_000, points: [0.1, 0.1, 0.15, 0.15] }], + events: [{ kind: "click", t: 10_000, x: 0.8, y: 0.8 }], + inkOwners: new Map([[0, 1]]), + }); + const [first, second] = crops([10_000, 30_000], items); + expect(inside(first!.rect, 0.1, 0.1)).toBe(false); + expect(inside(second!.rect, 0.1, 0.1)).toBe(true); + }); + + it("keeps a click in the crop of the keyframe whose text names it", () => { + const events = [ + { kind: "click", t: 2_000, x: 0.1, y: 0.1 }, + { kind: "click", t: 25_000, x: 0.5, y: 0.5 }, + { kind: "click", t: 30_000, x: 0.9, y: 0.9 }, + ] as const; + const source = { durationMs: 40_000, keyframes: [25_000, 30_000], strokes: [], events }; + const bySlice = crops([25_000, 30_000], activityItems(source)); + expect(inside(bySlice[0]!.rect, 0.1, 0.1)).toBe(true); + expect(inside(bySlice[1]!.rect, 0.1, 0.1)).toBe(false); + const owned = crops( + [25_000, 30_000], + activityItems({ ...source, inputOwner: (t) => (t === 2_000 ? 1 : undefined) }), + ); + expect(inside(owned[0]!.rect, 0.1, 0.1)).toBe(false); + expect(inside(owned[1]!.rect, 0.1, 0.1)).toBe(true); + }); + + it("counts what a click changed nearby as part of the activity, not other changes", () => { + const items = activityItems({ + durationMs: 40_000, + keyframes: [5_000], + strokes: [], + events: [{ kind: "click", t: 5_000, x: 0.3, y: 0.3 }], + changes: [ + [ + [0.28, 0.28, 0.55, 0.55], + [0.9, 0.9, 0.95, 0.95], + ], + ], + }); + const [crop] = crops([5_000], items); + expect(inside(crop!.rect, 0.55, 0.55)).toBe(true); + expect(inside(crop!.rect, 0.95, 0.95)).toBe(false); + }); + + it("crops to what changed when nothing was touched", () => { + const items = activityItems({ + durationMs: 40_000, + keyframes: [5_000], + strokes: [], + events: [], + changes: [[[0.7, 0.7, 0.8, 0.8]]], + }); + const [crop] = crops([5_000], items); + expect(crop!.kind).toBe("activity"); + expect(inside(crop!.rect, 0.75, 0.75)).toBe(true); + expect(inside(crop!.rect, 0.1, 0.1)).toBe(false); + }); + + it("ignores a scroll with no position, and crops around one with a position", () => { + const place = (x?: number, y?: number) => + activityItems({ + durationMs: 40_000, + keyframes: [5_000], + strokes: [], + events: [ + { kind: "scroll", t: 5_000, ...(x === undefined || y === undefined ? {} : { x, y }) }, + ], + }); + expect(place()).toEqual([]); + expect(inside(crops([5_000], place(0.7, 0.7))[0]!.rect, 0.7, 0.7)).toBe(true); + }); + + it("reads the window log in the shape scrawl.json keeps it", () => { + const snapshots = windowSnapshots( + [ + { + t: 100, + windows: [ + { + id: 1, + app: "Notes", + bounds: [100, 100, 400, 400], + layer: 0, + frontmost: true, + }, + ], + }, + ], + SCREEN, + (t) => t - 100, + ); + expect(snapshots[0]).toEqual({ + t: 0, + windows: [ + { + id: 1, + app: "Notes", + rect: [0.1, 0.1, 0.5, 0.5], + layer: 0, + regular: true, + frontmost: true, + }, + ], + }); + const [crop] = computeCrops({ + screen: SCREEN, + durationMs: 10_000, + keyframes: [5_000], + items: [click(5_000, 0.3, 0.3)], + snapshots, + }); + expect(crop).toMatchObject({ kind: "window", apps: ["Notes"] }); + crop!.rect.forEach((value, index) => { + expect(value).toBeCloseTo([0.1, 0.1, 0.5, 0.5][index]!); + }); + }); +}); + +describe("cropLabel", () => { + const crop = (kind: Crop["kind"], apps: ReadonlyArray): Crop => ({ + rect: FULL, + kind, + reason: "", + apps, + }); + + it.each([ + [[crop("full", [])], null], + [[crop("activity", [])], "the activity"], + [[crop("focus", ["Helium"]), crop("full", ["Other"])], "the activity in Helium"], + [[crop("window", ["Helium"]), crop("focus", ["Code"])], "Helium + Code"], + ])("names %j", (found, label) => { + expect(cropLabel(found)).toBe(label); + }); +}); + +describe("pixelRect", () => { + it("stays on the frame, takes no extra column, and is never empty", () => { + expect(pixelRect([0.6, 0, 1, 1], 100, 100)).toEqual({ x: 60, y: 0, width: 40, height: 100 }); + expect(pixelRect([1, 1, 1, 1], 100, 100)).toEqual({ x: 99, y: 99, width: 1, height: 1 }); + }); +}); + +/** A white screen with dark rows of "text": `rows` is [y0, y1, x0, x1] in points. */ +const textScreen = ( + size: number, + rows: ReadonlyArray, +): RgbaImage => { + const data = new Uint8ClampedArray(size * size * 4).fill(255); + for (const [y0, y1, x0, x1] of rows) { + for (let y = y0; y < y1; y++) { + // Alternating columns, so every 3 pt cell holds an edge. + for (let x = x0; x < x1; x += 2) { + const at = (y * size + x) * 4; + data[at] = data[at + 1] = data[at + 2] = 0; + } + } + } + return { width: size, height: size, data }; +}; + +const refined = (image: RgbaImage, at: readonly [number, number]) => + computeCrops({ + screen: [image.width, image.height], + durationMs: 10_000, + keyframes: [5_000], + items: [click(5_000, at[0], at[1])], + content: [contentMap(image, [image.width, image.height])], + })[0]!.rect; + +describe("moving crop edges off content", () => { + const SIZE = 1200; + const blank = computeCrops({ + screen: [SIZE, SIZE], + durationMs: 10_000, + keyframes: [5_000], + items: [click(5_000, 0.5, 0.5)], + })[0]!.rect; + + it("finishes a line of text the crop would cut", () => { + const edge = Math.round(blank[2] * SIZE); + const end = edge + 100; + const rect = refined(textScreen(SIZE, [[590, 610, 400, end]]), [0.5, 0.5]); + expect(rect[2] * SIZE).toBeGreaterThanOrEqual(end); + }); + + it("does not cut a block of text across its middle", () => { + const edge = Math.round(blank[3] * SIZE); + const rows = Array.from({ length: 12 }, (_, line) => { + const y = edge - 96 + line * 16; + return [y, y + 10, 300, 900] as const; + }); + const rect = refined(textScreen(SIZE, rows), [0.5, 0.5]); + const bottom = rect[3] * SIZE; + const cutsRow = rows.some(([y0, y1]) => bottom > y0 + 1 && bottom < y1 - 1); + expect(cutsRow).toBe(false); + }); + + it("never moves an edge past the activity", () => { + const rows = Array.from({ length: 60 }, (_, line) => { + const y = line * 20; + return [y, y + 18, 0, SIZE] as const; + }); + const rect = refined(textScreen(SIZE, rows), [0.3, 0.7]); + expect(inside(rect, 0.3, 0.7)).toBe(true); + }); +}); + +describe("changeRegions", () => { + const SIZE = 640; + const solid = (paint?: (x: number, y: number) => boolean): RgbaImage => { + const data = new Uint8ClampedArray(SIZE * SIZE * 4).fill(255); + if (paint !== undefined) { + for (let y = 0; y < SIZE; y++) { + for (let x = 0; x < SIZE; x++) { + if (!paint(x, y)) continue; + const at = (y * SIZE + x) * 4; + data[at] = data[at + 1] = data[at + 2] = 0; + } + } + } + return { width: SIZE, height: SIZE, data }; + }; + + it("boxes what changed, largest first, and drops a lone cell", () => { + const after = solid( + (x, y) => + (x >= 100 && x < 300 && y >= 100 && y < 200) || + (x >= 500 && x < 560 && y >= 500 && y < 520) || + (x >= 10 && x < 20 && y >= 600 && y < 610), + ); + const regions = changeRegions(solid(), after); + expect(regions).toEqual([ + [100 / SIZE, 100 / SIZE, 300 / SIZE, 200 / SIZE], + [500 / SIZE, 500 / SIZE, 560 / SIZE, 520 / SIZE], + ]); + }); + + it("finds nothing on an unchanged screen", () => { + expect(changeRegions(solid(), solid())).toEqual([]); + }); +}); diff --git a/apps/web/src/scrawl/autocrop.ts b/apps/web/src/scrawl/autocrop.ts new file mode 100644 index 0000000000000..46f20067374f3 --- /dev/null +++ b/apps/web/src/scrawl/autocrop.ts @@ -0,0 +1,691 @@ +// Crops each keyframe to where its moment happened, from the ink, clicks, what changed on screen +// and, when logged, the cursor and windows. Rules and numbers are Scrawl's (src/autocrop.rs), tuned +// there on a corpus and real captures. Rects are fractions of the capture; sizes are its points. + +import { refineCropEdges, type ContentMap } from "./autocropContent"; + +export type Rect = readonly [number, number, number, number]; +export const FULL: Rect = [0, 0, 1, 1]; + +const SNAP_THRESHOLD = 0.8; +const MAX_WINDOWS = 2; +const MIN_SHARE = 0.1; +// A window bigger than this is no crop; focus on the activity inside it. +const BIG_WINDOW = 0.6; +const CONTEXT = 0.15; +const MIN_SIZE: readonly [number, number] = [320, 200]; +const MAX_ZOOM = 4; +// Resting the cursor after drawing must not outvote the drawing. +const PASSIVE_CAP = 0.5; +const MIN_DWELL_S = 0.15; +const HIT_SLOP = 24; +const OUTLIER = 0.12; +const MERGE_IOU = 0.6; +// Crops covering more than this aren't worth it: keep the full frame. +const FULL_ABOVE = 0.85; +const CAUSE_REACH = 40; +const OVERLAY_AREA = 0.85; +const MENU_BAR_LAYER = 24; +const DOCK_LAYER = 20; +const CHROME_MARGIN = 40; + +export type ActivityKind = "stroke" | "click" | "drag" | "scroll" | "cursor" | "change"; + +const WEIGHTS: Record, number> = { + stroke: 1, + click: 0.6, + drag: 0.6, + scroll: 0.3, +}; +const CURSOR_WEIGHT = 0.15; +const CHANGE_WEIGHT = 0.1; + +const isDeliberate = (kind: ActivityKind) => kind !== "cursor" && kind !== "change"; + +export interface ActivityItem { + readonly kind: ActivityKind; + readonly t0: number; + readonly t1: number; + readonly points: ReadonlyArray; + /** Seconds the cursor rested here; for a change, the square root of its area. */ + readonly dwell: number; + /** Belongs to this keyframe alone, whenever it happened. */ + readonly keyframe?: number; +} + +export interface CaptureWindow { + readonly id: number; + readonly app: string; + readonly rect: Rect; + readonly layer: number; + readonly regular: boolean; + readonly frontmost: boolean; +} + +export interface WindowSnapshot { + readonly t: number; + /** Front to back. */ + readonly windows: ReadonlyArray; +} + +export interface AutocropInput { + /** The capture's size in points. */ + readonly screen: readonly [number, number]; + readonly durationMs: number; + readonly keyframes: ReadonlyArray; + readonly items: ReadonlyArray; + readonly snapshots?: ReadonlyArray; + /** One per keyframe: where its screen has something on it, for moving crop edges. */ + readonly content?: ReadonlyArray; +} + +export type CropKind = "window" | "windows" | "focus" | "activity" | "fallback" | "full"; + +export interface Crop { + readonly rect: Rect; + readonly kind: CropKind; + /** Why, for people (no window titles). */ + readonly reason: string; + readonly apps: ReadonlyArray; + /** The activity itself, which moving an edge must keep. */ + readonly need?: Rect; + /** The window a focus crop sits in. */ + readonly within?: Rect; +} + +const area = (r: Rect) => Math.max(0, r[2] - r[0]) * Math.max(0, r[3] - r[1]); +const union = (a: Rect, b: Rect): Rect => [ + Math.min(a[0], b[0]), + Math.min(a[1], b[1]), + Math.max(a[2], b[2]), + Math.max(a[3], b[3]), +]; +const intersect = (a: Rect, b: Rect): Rect => [ + Math.max(a[0], b[0]), + Math.max(a[1], b[1]), + Math.min(a[2], b[2]), + Math.min(a[3], b[3]), +]; +const clipToScreen = (r: Rect) => intersect(r, FULL); +const unionWith = (a: Rect | null, b: Rect) => (a === null ? b : union(a, b)); +const overlap = (a: Rect, b: Rect) => { + const shared = area(intersect(a, b)); + const total = area(a) + area(b) - shared; + return total <= 0 ? 0 : shared / total; +}; +const containsRect = (outer: Rect, inner: Rect) => + inner[0] >= outer[0] - 1e-4 && + inner[1] >= outer[1] - 1e-4 && + inner[2] <= outer[2] + 1e-4 && + inner[3] <= outer[3] + 1e-4; +const isFull = (r: Rect) => r[0] <= 0 && r[1] <= 0 && r[2] >= 1 && r[3] >= 1; +const boxOf = (points: ReadonlyArray): Rect | null => { + let box: Rect | null = null; + for (const [x, y] of points) box = unionWith(box, [x, y, x, y]); + return box; +}; + +const isChrome = (win: CaptureWindow) => + (win.layer === MENU_BAR_LAYER && win.app === "Window Server") || + (win.layer === DOCK_LAYER && win.app === "Dock"); +const chromeLabel = (win: CaptureWindow) => (win.layer === MENU_BAR_LAYER ? "menu bar" : "dock"); + +/** A snapshot in the shape `scrawl.json` keeps it: bounds are points, `[x, y, width, height]`. */ +export interface LoggedWindows { + readonly t: number; + readonly windows: ReadonlyArray<{ + readonly id: number; + readonly app: string; + readonly bounds: readonly [number, number, number, number]; + readonly layer: number; + readonly regular?: boolean; + readonly frontmost?: boolean; + }>; +} + +export const windowSnapshots = ( + logged: ReadonlyArray, + [width, height]: readonly [number, number], + shift: (t: number) => number, +): Array => + logged.map((snapshot) => ({ + t: shift(snapshot.t), + windows: snapshot.windows.map((win) => { + const [x, y, w, h] = win.bounds; + return { + id: win.id, + app: win.app, + rect: [x / width, y / height, (x + w) / width, (y + h) / height], + layer: win.layer, + regular: win.regular ?? true, + frontmost: win.frontmost ?? false, + }; + }), + })); + +/** Evenly spaced picks, so a long stroke does not cost more than its shape needs. */ +const subsample = (values: ReadonlyArray, max: number): ReadonlyArray => { + if (values.length <= max || max < 2) return values; + const spread = values.length - 1; + return Array.from( + { length: max }, + (_, i) => values[Math.trunc((i * spread + Math.trunc((max - 1) / 2)) / (max - 1))]!, + ); +}; + +const weightOf = (item: ActivityItem) => { + if (item.kind === "cursor") return CURSOR_WEIGHT * item.dwell; + if (item.kind === "change") { + return CHANGE_WEIGHT * Math.min(1, Math.max(0.2, item.dwell * 3)); + } + return WEIGHTS[item.kind]; +}; + +interface WeightedPoint { + readonly x: number; + readonly y: number; + readonly t: number; + weight: number; + readonly dwell: number; + readonly kind: ActivityKind; + outlier: boolean; +} + +function weightedMedian(points: ReadonlyArray, key: "x" | "y") { + const sorted = points.toSorted((a, b) => a[key] - b[key]); + const half = sorted.reduce((sum, point) => sum + point.weight, 0) / 2; + let total = 0; + for (const point of sorted) { + total += point.weight; + if (total >= half) return point[key]; + } + return sorted.at(-1)?.[key] ?? 0.5; +} + +const appsOf = (windows: ReadonlyArray) => [ + ...new Set(windows.map((win) => win.app)), +]; + +export function computeCrops(input: AutocropInput): Array { + const count = input.keyframes.length; + if (count === 0) return []; + const sw = Math.max(1, input.screen[0]); + const sh = Math.max(1, input.screen[1]); + const snapshots = input.snapshots ?? []; + + const windowsAt = (t: number): ReadonlyArray => { + const first = snapshots[0]; + if (first === undefined) return []; + let snapshot = first; + for (const candidate of snapshots) { + if (candidate.t > t) break; + snapshot = candidate; + } + // Agent apps' overlays cover the screen without being what the capture is about. + return snapshot.windows.filter( + (win) => win.regular || win.layer !== 0 || area(clipToScreen(win.rect)) <= OVERLAY_AREA, + ); + }; + + const pad = (r: Rect, points: number): Rect => [ + r[0] - points / sw, + r[1] - points / sh, + r[2] + points / sw, + r[3] + points / sh, + ]; + const withMargin = (core: Rect, within: Rect) => + pad(core, CONTEXT * Math.sqrt((within[2] - within[0]) * sw * ((within[3] - within[1]) * sh))); + + /** Minimum size and maximum zoom (only ever growing), kept on screen and, if it fits, in `within`. */ + const shape = (r: Rect, within: Rect = FULL): Rect => { + let w = Math.max(r[2] - r[0], 1e-4) * sw; + let h = Math.max(r[3] - r[1], 1e-4) * sh; + w = Math.max(w, Math.min(MIN_SIZE[0], sw)); + h = Math.max(h, Math.min(MIN_SIZE[1], sh)); + const need = 1 / MAX_ZOOM; + const cover = Math.max(w / sw, h / sh); + if (cover < need) { + w *= need / cover; + h *= need / cover; + } + const width = Math.min(w / sw, 1); + const height = Math.min(h / sh, 1); + const keep = (center: number, half: number, low: number, high: number) => { + const fits = high - low >= 2 * half; + return Math.min(Math.max(center, (fits ? low : 0) + half), (fits ? high : 1) - half); + }; + const cx = keep((r[0] + r[2]) / 2, width / 2, Math.max(within[0], 0), Math.min(within[2], 1)); + const cy = keep((r[1] + r[3]) / 2, height / 2, Math.max(within[1], 0), Math.min(within[3], 1)); + return [cx - width / 2, cy - height / 2, cx + width / 2, cy + height / 2]; + }; + const fullIfBig = (r: Rect) => (area(r) >= FULL_ABOVE ? FULL : r); + + const sliceOf = (k: number): readonly [number, number] => { + const t = input.keyframes[k]!; + const before = k === 0 ? 0 : (input.keyframes[k - 1]! + t) / 2; + const after = + k + 1 === count ? Math.max(input.durationMs, t) : (t + input.keyframes[k + 1]!) / 2; + return [before, after]; + }; + + const cropAround = (k: number): Crop | null => { + const slice = sliceOf(k); + const atKeyframe = windowsAt(input.keyframes[k]!); + const points: Array = []; + for (const item of input.items) { + const inSlice = item.t1 >= slice[0] && item.t0 <= slice[1]; + if (item.keyframe === undefined ? !inSlice : item.keyframe !== k) continue; + const weight = weightOf(item); + if (weight <= 0 || item.points.length === 0) continue; + for (const [x, y, t] of item.points) { + points.push({ + x, + y, + t, + weight: weight / item.points.length, + dwell: item.dwell, + kind: item.kind, + outlier: false, + }); + } + } + const anchors = subsample( + points.filter((point) => isDeliberate(point.kind)), + 300, + ); + const diagonal = Math.hypot(sw, sh); + const distance = (a: { x: number; y: number }, x: number, y: number) => + Math.hypot((a.x - x) * sw, (a.y - y) * sh) / diagonal; + const cursor = points.filter((point) => point.kind === "cursor"); + const middle = + anchors.length === 0 && cursor.length > 0 + ? { x: weightedMedian(cursor, "x"), y: weightedMedian(cursor, "y") } + : null; + for (const point of cursor) { + const away = + middle !== null + ? distance(point, middle.x, middle.y) + : anchors.reduce((best, a) => Math.min(best, distance(point, a.x, a.y)), Infinity); + point.outlier = away > OUTLIER; + } + const sumOf = (test: (point: WeightedPoint) => boolean) => + points.reduce((sum, point) => sum + (test(point) ? point.weight : 0), 0); + const deliberate = sumOf((point) => isDeliberate(point.kind)); + const passive = sumOf((point) => !isDeliberate(point.kind) && !point.outlier); + if (deliberate > 0 && passive > PASSIVE_CAP * deliberate) { + const factor = (PASSIVE_CAP * deliberate) / passive; + for (const point of points) { + if (!isDeliberate(point.kind)) point.weight *= factor; + } + } + + const slopX = HIT_SLOP / sw; + const slopY = HIT_SLOP / sh; + const shares = new Map(); + const chromeHits = new Map< + number, + { win: CaptureWindow; hits: Array } + >(); + const addShare = (win: CaptureWindow, weight: number) => { + const now = atKeyframe.find((other) => other.id === win.id) ?? win; + const share = shares.get(win.id) ?? { weight: 0, win: now }; + share.weight += weight; + shares.set(win.id, share); + }; + const kept: Array = []; + let panels: Rect | null = null; + let total = 0; + for (const point of points) { + if (point.outlier) continue; + const windows = windowsAt(point.t); + const inside = (win: CaptureWindow, mx: number, my: number) => + point.x >= win.rect[0] - mx && + point.x <= win.rect[2] + mx && + point.y >= win.rect[1] - my && + point.y <= win.rect[3] + my; + const hit = + windows.find((win) => inside(win, 0, 0)) ?? + windows.find((win) => !isChrome(win) && inside(win, slopX, slopY)); + if (hit !== undefined && isChrome(hit)) { + if (!isDeliberate(point.kind)) continue; + const entry = chromeHits.get(hit.id) ?? { win: hit, hits: [] }; + entry.hits.push([point.x, point.y]); + chromeHits.set(hit.id, entry); + total += point.weight; + // The menu bar is the frontmost app's: it counts for that app's window. + const front = windows.find((win) => win.frontmost && !isChrome(win)); + if (hit.layer === MENU_BAR_LAYER && front !== undefined) addShare(front, point.weight); + continue; + } + total += point.weight; + kept.push(point); + if (hit === undefined) continue; + // A menu or panel belongs to its app's window, and the crop grows to show it. + const owner = + hit.layer !== 0 ? windows.find((win) => win.layer === 0 && win.app === hit.app) : undefined; + if (owner !== undefined && (isDeliberate(point.kind) || point.dwell >= MIN_DWELL_S)) { + panels = unionWith(panels, clipToScreen(hit.rect)); + } + addShare(owner ?? hit, point.weight); + } + if (total <= 0) return null; + + const ranked = [...shares.values()].toSorted( + (a, b) => b.weight - a.weight || a.win.id - b.win.id, + ); + const positions = (test: (point: WeightedPoint) => boolean) => + kept.filter(test).map((point) => [point.x, point.y] as const); + // Deliberate input and where the cursor rested; what changed only when there is nothing else. + const ink = boxOf(positions((point) => isDeliberate(point.kind))); + const rested = boxOf( + positions( + (point) => + isDeliberate(point.kind) || (point.kind === "cursor" && point.dwell >= MIN_DWELL_S), + ), + ); + // Changes near a click, drag or scroll are what it did: the scrolled page, a popover. + const pokes = points.filter( + (point) => point.kind === "click" || point.kind === "drag" || point.kind === "scroll", + ); + const reachX = CAUSE_REACH / sw; + const reachY = CAUSE_REACH / sh; + let caused: Rect | null = null; + for (const item of input.items) { + if (item.kind !== "change" || item.keyframe !== k) continue; + const box = boxOf(item.points.map(([x, y]) => [x, y] as const))!; + const near = pokes.some( + ({ x, y }) => + x >= box[0] - reachX && + x <= box[2] + reachX && + y >= box[1] - reachY && + y <= box[3] + reachY, + ); + if (near) caused = unionWith(caused, box); + } + const activity = + (rested !== null && caused !== null ? union(rested, caused) : (rested ?? caused)) ?? + boxOf(positions(() => true)); + + let snapped: ReadonlyArray = []; + let held = 0; + for (const [index, { weight }] of ranked.slice(0, MAX_WINDOWS).entries()) { + if (index > 0 && weight / total < MIN_SHARE) break; + held += weight / total; + if (held >= SNAP_THRESHOLD) { + snapped = ranked.slice(0, index + 1).map(({ win }) => win); + break; + } + } + const percent = Math.round(held * 100); + let within = FULL; + let rect: Rect; + let kind: CropKind; + let reason: string; + let apps: ReadonlyArray; + if (snapped.length > 0) { + const box = snapped.reduce( + (r, win) => unionWith(r, clipToScreen(win.rect)), + null, + )!; + apps = appsOf(snapped); + if (area(box) <= BIG_WINDOW) { + rect = box; + kind = snapped.length === 1 ? "window" : "windows"; + reason = + snapped.length === 1 + ? `${snapped[0]!.app} (${percent}% of the activity)` + : `${snapped.length} windows: ${apps.join(" + ")} (${percent}% of the activity)`; + } else { + rect = intersect(withMargin(activity ?? box, box), box); + within = box; + kind = "focus"; + reason = `around the activity in ${apps.join(" + ")}`; + } + } else { + rect = withMargin(activity ?? ink ?? FULL, FULL); + kind = "activity"; + reason = "around the activity"; + apps = appsOf(ranked.slice(0, 2).map(({ win }) => win)); + } + if (ink !== null && !containsRect(rect, ink)) { + rect = union(rect, ink); + reason += " + input outside it"; + } + if (panels !== null && !containsRect(rect, panels)) { + rect = union(rect, panels); + reason += " + its menu"; + } + for (const { win, hits } of chromeHits.values()) { + const r = clipToScreen(win.rect); + const box = boxOf(hits)!; + const horizontal = (r[2] - r[0]) * sw >= (r[3] - r[1]) * sh; + rect = union( + rect, + horizontal + ? [box[0] - CHROME_MARGIN / sw, r[1], box[2] + CHROME_MARGIN / sw, r[3]] + : [r[0], box[1] - CHROME_MARGIN / sh, r[2], box[3] + CHROME_MARGIN / sh], + ); + reason += ` + ${chromeLabel(win)}`; + } + rect = fullIfBig(shape(rect, within)); + return { + rect, + kind: isFull(rect) ? "full" : kind, + reason, + apps, + ...(activity === null ? {} : { need: activity }), + ...(kind === "focus" ? { within } : {}), + }; + }; + + const frontmostWindow = (t: number) => { + const normal = windowsAt(t).filter((win) => !isChrome(win)); + return normal.find((win) => win.frontmost) ?? normal.find((win) => win.regular); + }; + + // Nothing happened in this keyframe's slice: the nearest crop, else the frontmost window. + const fallback = (k: number, nearest: Crop | undefined): Crop => { + if (nearest !== undefined) { + return { + rect: nearest.rect, + kind: isFull(nearest.rect) ? "full" : "fallback", + reason: "talk only: the previous crop", + apps: nearest.apps, + }; + } + const front = frontmostWindow(input.keyframes[k]!); + if (front !== undefined) { + const rect = fullIfBig(shape(clipToScreen(front.rect))); + if (!isFull(rect) && area(clipToScreen(front.rect)) <= BIG_WINDOW) { + return { + rect, + kind: "fallback", + reason: `talk only: the frontmost window (${front.app})`, + apps: [front.app], + }; + } + } + return { rect: FULL, kind: "full", reason: "talk only: the whole screen", apps: [] }; + }; + + const raw = input.keyframes.map((_t, k) => cropAround(k)); + const crops = raw.map( + (crop, k): Crop => + crop ?? + fallback( + k, + raw.slice(0, k).findLast((other) => other !== null) ?? + raw.slice(k + 1).find((other) => other !== null) ?? + undefined, + ), + ); + + // Neighbours looking at the same place share one crop, so the images do not jump around. + let start = 0; + for (let k = 1; k <= count; k++) { + const merged = crops.slice(start, k).reduce((r, c) => unionWith(r, c.rect), null)!; + if (k < count && overlap(merged, crops[k]!.rect) >= MERGE_IOU) continue; + if (k - start > 1) { + const rect = fullIfBig(shape(merged)); + for (let at = start; at < k; at++) crops[at] = { ...crops[at]!, rect }; + } + start = k; + } + + return crops.map((crop, k) => { + const content = input.content?.[k]; + let rect = crop.rect; + if ((crop.kind === "focus" || crop.kind === "activity") && !isFull(rect) && content) { + const middle = [(rect[0] + rect[2]) / 2, (rect[1] + rect[3]) / 2] as const; + const keep = + crop.need === undefined + ? ([middle[0], middle[1], middle[0], middle[1]] as const) + : intersect(crop.need, rect); + rect = fullIfBig( + refineCropEdges( + { content, screen: [sw, sh], minSize: MIN_SIZE, maxZoom: MAX_ZOOM }, + rect, + keep, + crop.within ?? FULL, + ), + ); + } + return { ...crop, rect, kind: isFull(rect) ? "full" : crop.kind }; + }); +} + +/** The summary's name for what the capture was cropped to (no window titles); null if it wasn't. */ +export function cropLabel(crops: ReadonlyArray): string | null { + const cropped = crops.filter((crop) => crop.kind !== "full"); + if (cropped.length === 0) return null; + const apps = [...new Set(cropped.flatMap((crop) => crop.apps))]; + if (apps.length === 0) return "the activity"; + const onlyFocus = cropped.every((crop) => crop.kind === "focus" || crop.kind === "activity"); + return onlyFocus ? `the activity in ${apps.join(" + ")}` : apps.join(" + "); +} + +/** The pixel box `[x, y, width, height]` of `r` in a frame, at least one pixel on each side. */ +export function pixelRect(r: Rect, width: number, height: number) { + // A hair of tolerance so 0.6 × 100 = 60.000004 doesn't take an extra column. + const hair = 1e-3; + const x0 = Math.min(Math.max(Math.floor(r[0] * width + hair), 0), width - 1); + const y0 = Math.min(Math.max(Math.floor(r[1] * height + hair), 0), height - 1); + const x1 = Math.min(Math.max(Math.ceil(r[2] * width - hair), x0 + 1), width); + const y1 = Math.min(Math.max(Math.ceil(r[3] * height - hair), y0 + 1), height); + return { x: x0, y: y0, width: x1 - x0, height: y1 - y0 }; +} + +const MAX_STROKE_POINTS = 48; +const CURSOR_REST_CAP_MS = 4_000; + +export type CursorSample = readonly [t: number, x: number, y: number]; + +export interface ActivitySource { + readonly durationMs: number; + readonly keyframes: ReadonlyArray; + /** Each stroke's points are x, y, x, y, ... */ + readonly strokes: ReadonlyArray<{ + readonly t0: number; + readonly t1: number; + readonly points: ReadonlyArray; + }>; + readonly events: ReadonlyArray< + | { readonly kind: "click"; readonly t: number; readonly x: number; readonly y: number } + | { + readonly kind: "drag"; + readonly t: number; + readonly t1: number; + readonly x0: number; + readonly y0: number; + readonly x1: number; + readonly y1: number; + } + | { readonly kind: "scroll"; readonly t: number; readonly x?: number; readonly y?: number } + >; + readonly cursor?: ReadonlyArray; + /** `changes[k]` are the regions keyframe `k` changed since keyframe `k - 1`. */ + readonly changes?: ReadonlyArray>; + /** Strokes (by index) whose ink a keyframe draws, however long before it they were made. */ + readonly inkOwners?: ReadonlyMap; + /** The keyframe whose text names the input at this moment: its crop has to show it. */ + readonly inputOwner?: (t: number) => number | undefined; +} + +export function activityItems(source: ActivitySource): Array { + const items: Array = []; + source.strokes.forEach((stroke, index) => { + const count = Math.trunc(stroke.points.length / 2); + if (count === 0) return; + const span = stroke.t1 - stroke.t0; + const owner = source.inkOwners?.get(index); + items.push({ + kind: "stroke", + t0: stroke.t0, + t1: stroke.t1, + dwell: 0, + points: subsample( + Array.from({ length: count }, (_, at) => { + const t = stroke.t0 + (count > 1 ? (span * at) / (count - 1) : 0); + return [stroke.points[at * 2]!, stroke.points[at * 2 + 1]!, t] as const; + }), + MAX_STROKE_POINTS, + ), + ...(owner === undefined ? {} : { keyframe: owner }), + }); + }); + for (const event of source.events) { + const owner = source.inputOwner?.(event.t); + const owned = owner === undefined ? {} : { keyframe: owner }; + if (event.kind === "click") { + const { t, x, y } = event; + items.push({ kind: "click", t0: t, t1: t, dwell: 0, points: [[x, y, t]], ...owned }); + } else if (event.kind === "drag") { + items.push({ + ...owned, + kind: "drag", + t0: event.t, + t1: event.t1, + dwell: 0, + points: [ + [event.x0, event.y0, event.t], + [event.x1, event.y1, event.t1], + ], + }); + } else if (event.x !== undefined && event.y !== undefined) { + const { t, x, y } = event; + items.push({ kind: "scroll", t0: t, t1: t, dwell: 0, points: [[x, y, t]], ...owned }); + } + } + const cursor = source.cursor ?? []; + cursor.forEach(([t, x, y], index) => { + const next = cursor[index + 1]?.[0] ?? Math.max(source.durationMs, t); + items.push({ + kind: "cursor", + t0: t, + t1: next, + dwell: Math.min(next - t, CURSOR_REST_CAP_MS) / 1000, + points: [[x, y, t]], + }); + }); + source.changes?.forEach((regions, keyframe) => { + const t = source.keyframes[keyframe]; + if (t === undefined) return; + for (const [x0, y0, x1, y1] of regions) { + items.push({ + kind: "change", + t0: t, + t1: t, + keyframe, + dwell: Math.sqrt(area([x0, y0, x1, y1])), + points: [ + [x0, y0, t], + [x1, y0, t], + [x0, y1, t], + [x1, y1, t], + [(x0 + x1) / 2, (y0 + y1) / 2, t], + ], + }); + } + }); + return items; +} diff --git a/apps/web/src/scrawl/autocropContent.ts b/apps/web/src/scrawl/autocropContent.ts new file mode 100644 index 0000000000000..f95cff02082d4 --- /dev/null +++ b/apps/web/src/scrawl/autocropContent.ts @@ -0,0 +1,351 @@ +// What a keyframe's pixels tell the crop: which regions changed since the last keyframe, and where +// the screen is empty, so crop edges land in white space rather than through a line of text. Rules +// and numbers are Scrawl's (src/autocrop.rs). Sizes are in capture points. + +import type { Rect } from "./autocrop"; + +export interface RgbaImage { + readonly width: number; + readonly height: number; + readonly data: Uint8ClampedArray; +} + +const clip = (r: Rect): Rect => [ + Math.max(r[0], 0), + Math.max(r[1], 0), + Math.min(r[2], 1), + Math.min(r[3], 1), +]; + +const CELL_PT = 3; +const EDGE_CONTRAST = 28; + +// Which 3 pt cells of a frame have edges in them (text, borders, icons), as an integral image. +export interface ContentMap { + readonly cols: number; + readonly rows: number; + readonly sum: Uint32Array; +} + +const lumaAt = (image: RgbaImage, x: number, y: number) => { + const at = (y * image.width + x) * 4; + return Math.trunc((image.data[at]! * 3 + image.data[at + 1]! * 6 + image.data[at + 2]!) / 10); +}; + +export function contentMap(image: RgbaImage, screen: readonly [number, number]): ContentMap { + const cols = Math.max(1, Math.round(screen[0] / CELL_PT)); + const rows = Math.max(1, Math.round(screen[1] / CELL_PT)); + const sum = new Uint32Array((cols + 1) * (rows + 1)); + const { width, height } = image; + if (width < 2 || height < 2) return { cols, rows, sum }; + const step = Math.max(1, Math.trunc(Math.trunc(width / cols) / 3)); + for (let row = 0; row < rows; row++) { + const y0 = Math.trunc((row * height) / rows); + const y1 = Math.max(y0 + 1, Math.min(Math.trunc(((row + 1) * height) / rows), height - 1)); + let along = 0; + for (let col = 0; col < cols; col++) { + const x0 = Math.trunc((col * width) / cols); + const x1 = Math.max(x0 + 1, Math.min(Math.trunc(((col + 1) * width) / cols), width - 1)); + let busy = false; + for (let y = y0; y < y1 && !busy; y += step) { + for (let x = x0; x < x1 && !busy; x += step) { + const luma = lumaAt(image, x, y); + busy = + Math.abs(luma - lumaAt(image, x + 1, y)) >= EDGE_CONTRAST || + Math.abs(luma - lumaAt(image, x, y + 1)) >= EDGE_CONTRAST; + } + } + if (busy) along++; + sum[(row + 1) * (cols + 1) + col + 1] = sum[row * (cols + 1) + col + 1]! + along; + } + } + return { cols, rows, sum }; +} + +/** Busy cells in columns `c0..c1` and rows `r0..r1`. */ +function busyCells(content: ContentMap, c0: number, r0: number, c1: number, r1: number) { + const right = Math.min(c1, content.cols); + const bottom = Math.min(r1, content.rows); + const left = Math.max(c0, 0); + const top = Math.max(r0, 0); + if (left >= right || top >= bottom) return 0; + const at = (c: number, r: number) => content.sum[r * (content.cols + 1) + c]!; + return at(right, bottom) + at(left, top) - at(left, bottom) - at(right, top); +} + +const CHANGE_COLUMNS = 64; +const CHANGE_SAMPLES = 8; +const CHANGE_LUMA = 10; +const MAX_CHANGE_REGIONS = 8; + +/** Coarse regions that differ between two frames, largest first. A lone cell is noise. */ +export function changeRegions(previous: RgbaImage, current: RgbaImage): Array { + if ( + previous.width < CHANGE_COLUMNS || + current.width < CHANGE_COLUMNS || + previous.height < 8 || + current.height < 8 + ) { + return []; + } + const cell = Math.trunc(current.width / CHANGE_COLUMNS); + const rows = Math.max(1, Math.trunc(current.height / cell)); + const sample = (image: RgbaImage, nx: number, ny: number) => + lumaAt( + image, + Math.min(Math.trunc(nx * image.width), image.width - 1), + Math.min(Math.trunc(ny * image.height), image.height - 1), + ); + const changed = new Uint8Array(CHANGE_COLUMNS * rows); + for (let row = 0; row < rows; row++) { + for (let col = 0; col < CHANGE_COLUMNS; col++) { + let difference = 0; + for (let sy = 0; sy < CHANGE_SAMPLES; sy++) { + for (let sx = 0; sx < CHANGE_SAMPLES; sx++) { + const nx = (col + (sx + 0.5) / CHANGE_SAMPLES) / CHANGE_COLUMNS; + const ny = (row + (sy + 0.5) / CHANGE_SAMPLES) / rows; + difference += Math.abs(sample(previous, nx, ny) - sample(current, nx, ny)); + } + } + if (difference / (CHANGE_SAMPLES * CHANGE_SAMPLES) >= CHANGE_LUMA) { + changed[row * CHANGE_COLUMNS + col] = 1; + } + } + } + const boxes: Array<{ readonly size: number; readonly rect: Rect }> = []; + for (let start = 0; start < changed.length; start++) { + if (changed[start] !== 1) continue; + changed[start] = 2; + const stack = [start]; + let [c0, r0, c1, r1, count] = [CHANGE_COLUMNS, rows, 0, 0, 0]; + while (stack.length > 0) { + const at = stack.pop()!; + const row = Math.trunc(at / CHANGE_COLUMNS); + const col = at % CHANGE_COLUMNS; + c0 = Math.min(c0, col); + r0 = Math.min(r0, row); + c1 = Math.max(c1, col); + r1 = Math.max(r1, row); + count++; + for (let dr = -1; dr <= 1; dr++) { + for (let dc = -1; dc <= 1; dc++) { + const nr = row + dr; + const nc = col + dc; + if (nr < 0 || nc < 0 || nr >= rows || nc >= CHANGE_COLUMNS) continue; + const next = nr * CHANGE_COLUMNS + nc; + if (changed[next] === 1) { + changed[next] = 2; + stack.push(next); + } + } + } + } + if (count >= 2) { + boxes.push({ + size: (c1 - c0 + 1) * (r1 - r0 + 1), + rect: [c0 / CHANGE_COLUMNS, r0 / rows, (c1 + 1) / CHANGE_COLUMNS, (r1 + 1) / rows], + }); + } + } + return boxes + .toSorted((a, b) => b.size - a.size) + .slice(0, MAX_CHANGE_REGIONS) + .map((box) => box.rect); +} + +const SNAP_OUT = 120; +const SNAP_OUT_SHARE = 0.6; +const SNAP_IN = 60; +const COLUMN_GAP = 12; +const TRIM_MARGIN = 24; +const CONTENT_REACH = 360; +const CONTENT_EXTRA = 160; + +interface Frame { + readonly content: ContentMap; + readonly screen: readonly [number, number]; + readonly minSize: readonly [number, number]; + readonly maxZoom: number; +} + +/** Moves each edge to the nearest stretch of empty screen, never into `keep`, then trims margins. */ +function snapToEmptySpace({ content, screen, minSize, maxZoom }: Frame, r: Rect, keep: Rect): Rect { + const { cols, rows } = content; + const reach = (size: number) => Math.round(Math.max(SNAP_OUT, SNAP_OUT_SHARE * size) / CELL_PT); + const outX = reach((r[2] - r[0]) * screen[0]); + const outY = reach((r[3] - r[1]) * screen[1]); + const inward = Math.round(SNAP_IN / CELL_PT); + const snapEdge = ( + position: number, + limit: number, + count: number, + gap: number, + direction: -1 | 1, + out: number, + empty: (line: number) => boolean, + ) => { + // The `gap` lines just inside a boundary are empty, so nothing straddles it. + const clean = (boundary: number) => { + for (let k = 0; k < gap; k++) { + const line = direction < 0 ? boundary + k : boundary - 1 - k; + if (line >= 0 && line < count && !empty(line)) return false; + } + return true; + }; + if (clean(position)) return position; + for (let distance = 1; distance <= Math.max(out, inward); distance++) { + const outer = position + direction * distance; + if (distance <= out && outer >= 0 && outer <= count && clean(outer)) return outer; + const inner = position - direction * distance; + const insideKeep = direction < 0 ? inner > limit : inner < limit; + if (distance <= inward && !insideKeep && inner >= 0 && inner <= count && clean(inner)) { + return inner; + } + } + return position; + }; + let c0 = Math.floor(r[0] * cols); + let c1 = Math.ceil(r[2] * cols); + let r0 = Math.floor(r[1] * rows); + let r1 = Math.ceil(r[3] * rows); + const k0 = Math.floor(keep[0] * cols); + const k1 = Math.floor(keep[1] * rows); + const k2 = Math.ceil(keep[2] * cols); + const k3 = Math.ceil(keep[3] * rows); + // Rules and borders that cross the whole crop don't make a gutter look full. + const tenth = (span: number) => Math.max(0, Math.trunc(span / 10)); + const columnEmpty = (c: number) => busyCells(content, c, r0, c + 1, r1) <= tenth(r1 - r0); + const columnGap = Math.round(COLUMN_GAP / CELL_PT); + c0 = snapEdge(c0, k0, cols, columnGap, -1, outX, columnEmpty); + c1 = snapEdge(c1, k2, cols, columnGap, 1, outX, columnEmpty); + const rowEmpty = (row: number) => busyCells(content, c0, row, c1, row + 1) <= tenth(c1 - c0); + r0 = snapEdge(r0, k1, rows, 1, -1, outY, rowEmpty); + r1 = snapEdge(r1, k3, rows, 1, 1, outY, rowEmpty); + + const margin = Math.round(TRIM_MARGIN / CELL_PT); + const few = (span: number) => Math.max(0, Math.trunc(span / 30)); + const blankRun = ( + from: number, + step: 1 | -1, + limit: number, + empty: (line: number) => boolean, + ) => { + let run = 0; + while ((from + step * run - limit) * step < 0 && empty(from + step * run)) run++; + return run; + }; + const clamp = (value: number, high: number) => Math.min(Math.max(value, 0), high); + const minCols = Math.round(Math.max(minSize[0], screen[0] / maxZoom) / CELL_PT); + const sparseColumn = (c: number) => busyCells(content, c, r0, c + 1, r1) <= few(r1 - r0); + const spareCols = Math.max(0, c1 - c0 - minCols); + const cutLeft = clamp(blankRun(c0, 1, k0, sparseColumn) - margin, spareCols); + const cutRight = clamp(blankRun(c1 - 1, -1, k2 - 1, sparseColumn) - margin, spareCols - cutLeft); + c0 += cutLeft; + c1 -= cutRight; + const minRows = Math.round(minSize[1] / CELL_PT); + const sparseRow = (row: number) => busyCells(content, c0, row, c1, row + 1) <= few(c1 - c0); + const spareRows = Math.max(0, r1 - r0 - minRows); + const cutTop = clamp(blankRun(r0, 1, k1, sparseRow) - margin, spareRows); + const cutBottom = clamp(blankRun(r1 - 1, -1, k3 - 1, sparseRow) - margin, spareRows - cutTop); + r0 += cutTop; + r1 -= cutBottom; + return clip([c0 / cols, r0 / rows, c1 / cols, r1 / rows]); +} + +/** A prompt at the bottom of an empty terminal: when most of the crop is blank, reach for content. */ +function reachForContent({ content }: Frame, r: Rect, keep: Rect): Rect | null { + const { cols, rows } = content; + const c0 = Math.max(0, Math.floor(r[0] * cols)); + const c1 = Math.ceil(r[2] * cols); + const r0 = Math.max(0, Math.floor(r[1] * rows)); + const r1 = Math.min(rows, Math.ceil(r[3] * rows)); + const tolerance = Math.trunc(Math.max(0, c1 - c0) / 60); + const empty = (row: number) => busyCells(content, c0, row, c1, row + 1) <= tolerance; + let bestStart = 0; + let bestEnd = 0; + let run = 0; + for (let row = r0; row < r1; row++) { + run = empty(row) ? run + 1 : 0; + if (run > bestEnd - bestStart) { + bestStart = row + 1 - run; + bestEnd = row + 1; + } + } + if ((bestEnd - bestStart) * 2 < Math.max(0, r1 - r0)) return null; + const reach = Math.trunc(CONTENT_REACH / CELL_PT); + const extra = CONTENT_EXTRA / CELL_PT; + const keepTop = Math.trunc(keep[1] * rows); + const keepBottom = Math.ceil(keep[3] * rows); + if (bestEnd <= keepTop + 1) { + for (let row = r0 - 1; row >= Math.max(0, r0 - reach); row--) { + if (!empty(row)) return [r[0], Math.max(0, row - extra) / rows, r[2], r[3]]; + } + return null; + } + if (bestStart + 1 >= keepBottom) { + for (let row = r1; row < Math.min(r1 + reach, rows); row++) { + if (!empty(row)) return [r[0], r[1], r[2], Math.min(rows, row + 1 + extra) / rows]; + } + } + return null; +} + +/** Follows each line of text that crosses a side edge to its end, so no line is cut mid-word. */ +function finishLines({ content, screen }: Frame, r: Rect, within: Rect): Rect { + const { cols, rows } = content; + const reach = Math.round( + Math.max(SNAP_OUT, SNAP_OUT_SHARE * (r[2] - r[0]) * screen[0]) / CELL_PT, + ); + const gap = Math.round(COLUMN_GAP / CELL_PT); + if (reach <= 0) return r; + const busy = (c: number, row: number) => + c >= 0 && c < cols && busyCells(content, c, row, c + 1, row + 1) > 0; + const c0 = Math.floor(r[0] * cols); + const c1 = Math.ceil(r[2] * cols); + const r0 = Math.max(0, Math.floor(r[1] * rows)); + const r1 = Math.min(rows, Math.ceil(r[3] * rows)); + const w0 = Math.max(0, Math.floor(Math.max(0, within[0]) * cols)); + const w1 = Math.min(cols, Math.ceil(Math.min(1, within[2]) * cols)); + // Where the line on `row` crossing boundary `edge` ends going `direction`, within `limit`. + const lineEnd = (row: number, edge: number, direction: 1 | -1, limit: number) => { + const outside = direction > 0 ? edge : edge - 1; + const inside = outside - direction; + const near = (from: number, step: number) => { + for (let k = 0; k < gap; k++) if (busy(from + step * k, row)) return true; + return false; + }; + if (!near(inside, -direction) || !near(outside, direction)) return null; + let c = outside; + let last = inside; + let blank = 0; + while (blank < gap) { + if ((c - limit) * direction > 0 || c < 0 || c >= cols) return null; + if (busy(c, row)) { + last = c; + blank = 0; + } else { + blank++; + } + c += direction; + } + // Ending right at the window's edge is a border, not the end of a line. + return (limit - last) * direction >= 2 * gap ? last : null; + }; + let right = c1; + let left = c0; + for (let row = r0; row < r1; row++) { + const end = lineEnd(row, c1, 1, Math.min(c1 + reach, w1) - 1); + if (end !== null) right = Math.max(right, end + 1 + gap); + const start = lineEnd(row, c0, -1, Math.max(c0 - reach, w0)); + if (start !== null) left = Math.min(left, start - gap); + } + return clip([Math.max(left, w0) / cols, r[1], Math.min(right, w1) / cols, r[3]]); +} + +/** Edges of a crop around activity, moved off text and UI. `within` caps how far lines may run. */ +export function refineCropEdges(frame: Frame, rect: Rect, keep: Rect, within: Rect): Rect { + let snapped = snapToEmptySpace(frame, rect, keep); + const grown = reachForContent(frame, snapped, keep); + if (grown !== null) snapped = snapToEmptySpace(frame, grown, keep); + return finishLines(frame, snapped, within); +} diff --git a/apps/web/src/scrawl/contactSheet.test.ts b/apps/web/src/scrawl/contactSheet.test.ts new file mode 100644 index 0000000000000..04f2b34c93f7a --- /dev/null +++ b/apps/web/src/scrawl/contactSheet.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { layoutSheet, planSheetRows } from "./contactSheet.ts"; + +const sizes = (dims: ReadonlyArray) => + dims.map(([width, height]) => ({ width, height })); + +describe("planSheetRows", () => { + it("stacks two frames only when side by side would be too wide", () => { + expect(planSheetRows([1.6, 1.6])).toEqual([[0, 2]]); + expect(planSheetRows([2, 2])).toEqual([ + [0, 1], + [1, 2], + ]); + }); + + it("keeps tall crops from being crushed next to wide ones", () => { + const aspects = [ + [685, 648], + [1183, 837], + [1183, 837], + [694, 726], + [1183, 1095], + [694, 1416], + [898, 1416], + [898, 1416], + ].map(([width, height]) => width! / height!); + const areas = planSheetRows(aspects) + .flatMap(([start, end]) => { + const row = aspects.slice(start, end); + const total = row.reduce((sum, aspect) => sum + aspect, 0); + return row.map((aspect) => aspect / total ** 2); + }) + .toSorted((a, b) => a - b); + const median = (areas[3]! + areas[4]!) / 2; + expect(areas[0]! / median).toBeGreaterThanOrEqual(0.55); + }); + + it("never gives a narrow frame a row of its own among three or more", () => { + for (const row of planSheetRows([1.4, 1.4, 1.4])) { + expect(row[1] - row[0]).toBeGreaterThan(1); + } + }); + + it("places every frame in order however many marks a capture has", () => { + const rows = planSheetRows(Array.from({ length: 40 }, () => 1.6)); + expect( + rows.flatMap(([start, end]) => Array.from({ length: end - start }, (_, i) => start + i)), + ).toEqual(Array.from({ length: 40 }, (_, i) => i)); + }); +}); + +describe("layoutSheet", () => { + const within = (dims: ReadonlyArray) => { + const layout = layoutSheet(sizes(dims)); + expect(Math.max(layout.width, layout.height)).toBeLessThanOrEqual(1600); + expect(layout.width * layout.height).toBeLessThanOrEqual(1_565_000); + return layout; + }; + + it("fills every row to both sides, with 6 px gutters", () => { + const layout = within([ + [1054, 327], + [1183, 837], + [1183, 837], + [685, 648], + [694, 1416], + [898, 1416], + [898, 1416], + ]); + const tops = [...new Set(layout.cells.map((cell) => cell.y))]; + let bottom = 6; + for (const y of tops) { + const cells = layout.cells.filter((cell) => cell.y === y); + expect(y).toBe(bottom); + expect(cells[0]!.x).toBe(6); + const last = cells.at(-1)!; + expect(last.x + last.width).toBe(layout.width - 6); + cells.slice(1).forEach((cell, index) => { + const before = cells[index]!; + expect(cell.x).toBe(before.x + before.width + 6); + }); + bottom = y + cells[0]!.height + 6; + } + expect(layout.height).toBe(bottom); + }); + + it("keeps each frame's shape", () => { + const dims = [ + [1200, 600], + [800, 800], + [1920, 1080], + [640, 900], + ] as const; + within(dims).cells.forEach((cell, index) => { + const [width, height] = dims[index]!; + expect(cell.width / cell.height).toBeCloseTo(width / height, 1); + }); + }); + + it("lays out one to twelve frames of a screen within the caps", () => { + for (let count = 1; count <= 12; count++) { + const layout = within(Array.from({ length: count }, () => [2560, 1440] as const)); + expect(layout.cells).toHaveLength(count); + } + }); + + it("does not blow a small lone frame up past its minimum sheet", () => { + const layout = within([[320, 200]]); + expect(layout.width).toBe(640 + 12); + }); +}); diff --git a/apps/web/src/scrawl/contactSheet.ts b/apps/web/src/scrawl/contactSheet.ts new file mode 100644 index 0000000000000..e1202c08330bd --- /dev/null +++ b/apps/web/src/scrawl/contactSheet.ts @@ -0,0 +1,176 @@ +// Lays keyframes of any shape into justified rows in time order, so a tall crop and a wide one both +// stay readable. The cost terms are Scrawl's (src/render.rs), tuned with blind reading tests. + +const SHEET_PAD = 6; +const SHEET_GAP = 6; +const MAX_EDGE = 1600; +// Past about this, a vision model's API scales the image down again. +const MAX_PIXELS = 1_560_000; +const MAX_ROW = 6; +// Every split of up to 12 frames is a few thousand candidates; more would never finish. +const MAX_SEARCHED = 12; +const WIDE_FRAME = 1.85; + +type Rows = ReadonlyArray; + +const sumOf = (values: ReadonlyArray) => values.reduce((sum, value) => sum + value, 0); + +function costOf(rows: Rows, aspects: ReadonlyArray, targetSheet: number) { + const count = aspects.length; + const average = sumOf(aspects) / count; + const targetRow = Math.max(Math.sqrt(sumOf(aspects) * targetSheet), average); + const rowAspects = rows.map(([start, end]) => sumOf(aspects.slice(start, end))); + const sheetAspect = 1 / sumOf(rowAspects.map((aspect) => 1 / aspect)); + + // Relative displayed area of a frame in a row is its aspect over the row's squared. + const areas = rows.flatMap(([start, end], row) => + aspects.slice(start, end).map((aspect) => aspect / rowAspects[row]! ** 2), + ); + const logAreas = areas.map(Math.log); + const meanLogArea = sumOf(logAreas) / count; + const areaVariance = sumOf(logAreas.map((value) => (value - meanLogArea) ** 2)) / count; + const sorted = areas.toSorted((a, b) => a - b); + const medianArea = + Math.max(sorted[Math.trunc((count - 1) / 2)]! + sorted[Math.trunc(count / 2)]!, 1e-6) * 0.5; + let cost = Math.log(sheetAspect / targetSheet) ** 2 * 1.8 + areaVariance * 8; + for (const area of areas) { + const ratio = area / medianArea; + if (ratio < 0.58) cost += ((0.58 - ratio) / 0.58) ** 2 * 18; + if (ratio > 2.3) cost += ((ratio - 2.3) / 2.3) ** 2 * 4; + } + + const meanLogRow = sumOf(rowAspects.map(Math.log)) / rows.length; + let rowVariance = 0; + let rowOff = 0; + if (rows.length > 4) cost += (rows.length - 4) * 2.5; + rows.forEach(([start, end], row) => { + const aspect = rowAspects[row]!; + rowVariance += (Math.log(aspect) - meanLogRow) ** 2; + rowOff += Math.log(aspect / targetRow) ** 2; + for (const frame of aspects.slice(start, end)) { + const share = (frame / aspect) * Math.min(sheetAspect, 1); + if (share < 0.16) cost += ((0.16 - share) / 0.16) ** 2 * 8; + } + const columns = end - start; + if (columns > 4 && average > 0.75) cost += (columns - 4) * 2; + if (columns > 3 && average > 1.15) cost += (columns - 3) * 2; + cost += row * 1e-4 * columns; + }); + return cost + (rowVariance / rows.length) * 0.6 + (rowOff / rows.length) * 0.4; +} + +/** Splits frames of these aspect ratios, in time order, into rows of one to six. */ +export function planSheetRows(aspects: ReadonlyArray): Rows { + const count = aspects.length; + if (count === 0) return []; + if (count === 1) return [[0, 1]]; + if (count === 2) { + return aspects[0]! + aspects[1]! > 3.6 + ? [ + [0, 1], + [1, 2], + ] + : [[0, 2]]; + } + if (count > MAX_SEARCHED) { + const columns = Math.ceil(Math.sqrt(count)); + return Array.from( + { length: Math.ceil(count / columns) }, + (_, row) => [row * columns, Math.min((row + 1) * columns, count)] as const, + ); + } + const targetSheet = sumOf(aspects) / count < 0.7 ? 1.05 : 1.25; + let best: Rows = [[0, count]]; + let bestCost = Infinity; + const current: Array = []; + const search = (start: number) => { + if (start === count) { + const cost = costOf(current, aspects, targetSheet); + if (cost < bestCost) { + bestCost = cost; + best = [...current]; + } + return; + } + for (let size = 1; size <= Math.min(MAX_ROW, count - start); size++) { + // A frame gets a row of its own only when it is wide enough to use it. + if (size === 1 && aspects[start]! < WIDE_FRAME) continue; + current.push([start, start + size]); + search(start + size); + current.pop(); + } + }; + search(0); + return best; +} + +export interface SheetCell { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface SheetLayout { + readonly width: number; + readonly height: number; + readonly cells: ReadonlyArray; +} + +/** Whole-pixel places for frames of these sizes, within the sheet's size caps. */ +export function layoutSheet( + frames: ReadonlyArray<{ readonly width: number; readonly height: number }>, +): SheetLayout { + const aspects = frames.map(({ width, height }) => + Math.min(4.5, Math.max(0.25, Math.max(width, 1) / Math.max(height, 1))), + ); + const rows = planSheetRows(aspects); + const rowAspects = rows.map(([start, end]) => Math.max(sumOf(aspects.slice(start, end)), 0.1)); + const sheetAspect = 1 / Math.max(sumOf(rowAspects.map((aspect) => 1 / aspect)), 0.01); + const nativeRow = rows.reduce((widest, [start, end]) => { + const width = sumOf(frames.slice(start, end).map((frame) => frame.width)); + return Math.max(widest, width + SHEET_GAP * (end - start - 1)); + }, 0); + const desired = Math.max(nativeRow, frames.length === 1 ? 640 : 960); + const edge = MAX_EDGE - 2 * SHEET_PAD; + let inner = sheetAspect >= 1 ? Math.min(edge, desired) : Math.min(edge * sheetAspect, desired); + const innerHeight = + sumOf( + rows.map(([start, end], row) => { + const available = Math.max(inner - SHEET_GAP * (end - start - 1), 32); + return available / rowAspects[row]!; + }), + ) + + SHEET_GAP * Math.max(0, rows.length - 1); + const totalWidth = inner + 2 * SHEET_PAD; + const totalHeight = innerHeight + 2 * SHEET_PAD; + const scale = Math.min( + 1, + MAX_EDGE / Math.max(totalWidth, totalHeight), + Math.sqrt(MAX_PIXELS / (totalWidth * totalHeight)), + ); + if (scale < 1) inner = Math.max(totalWidth * scale - 2 * SHEET_PAD, 64); + const innerWidth = Math.max(Math.round(inner + 2 * SHEET_PAD), 64) - 2 * SHEET_PAD; + + const cells: Array = []; + let y = SHEET_PAD; + rows.forEach(([start, end], row) => { + const available = Math.max(innerWidth - SHEET_GAP * (end - start - 1), 1); + const rowAspect = rowAspects[row]!; + const height = Math.max(Math.round(available / rowAspect), 1); + let along = 0; + for (let index = start; index < end; index++) { + const left = Math.round(available * (along / rowAspect)); + along += aspects[index]!; + const right = index + 1 === end ? available : Math.round(available * (along / rowAspect)); + cells.push({ + x: SHEET_PAD + left + (index - start) * SHEET_GAP, + y, + width: Math.max(right - left, 1), + height, + }); + } + y += height + SHEET_GAP; + }); + return { width: innerWidth + 2 * SHEET_PAD, height: y - SHEET_GAP + SHEET_PAD, cells }; +} diff --git a/apps/web/src/scrawl/frameAnnotations.test.ts b/apps/web/src/scrawl/frameAnnotations.test.ts new file mode 100644 index 0000000000000..edecedee4ce84 --- /dev/null +++ b/apps/web/src/scrawl/frameAnnotations.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { chipDodgesRight, clicksToRing, frameChipScale } from "./frameAnnotations.ts"; + +describe("clicksToRing", () => { + it("shows the click nearest the frame's moment, and only one within reach", () => { + expect( + clicksToRing({ frameT: 1_000, actions: [] }, [ + { t: 500, x: 0.1, y: 0.1 }, + { t: 900, x: 0.2, y: 0.2 }, + { t: 5_000, x: 0.3, y: 0.3 }, + ]), + ).toEqual([[0.2, 0.2]]); + }); + + it("shows no click from another moment, whose screen may be gone", () => { + expect(clicksToRing({ frameT: 10_000, actions: [] }, [{ t: 2_000, x: 0.1, y: 0.1 }])).toEqual( + [], + ); + }); + + it("keeps a hand pointing on this screen, and one ring for a double click", () => { + const rings = clicksToRing( + { + frameT: 5_000, + actions: [ + { + kind: "clicks", + t0: 1_000, + t1: 1_200, + count: 2, + points: [ + [0.5, 0.5], + [0.501, 0.5], + ], + changed: false, + }, + { kind: "clicks", t0: 3_000, t1: 3_000, count: 1, points: [[0.7, 0.7]], changed: false }, + ], + }, + [], + ); + expect(rings).toEqual([ + [0.5, 0.5], + [0.7, 0.7], + ]); + }); + + it("drops pointing from before the screen changed", () => { + const rings = clicksToRing( + { + frameT: 5_000, + actions: [ + { kind: "clicks", t0: 1_000, t1: 1_000, count: 1, points: [[0.5, 0.5]], changed: false }, + { kind: "scene", t0: 2_000, t1: 2_000 }, + { kind: "clicks", t0: 3_000, t1: 3_000, count: 1, points: [[0.7, 0.7]], changed: false }, + ], + }, + [], + ); + expect(rings).toEqual([[0.7, 0.7]]); + }); + + it("leaves clicks the screen answered to their own moment", () => { + const rings = clicksToRing( + { + frameT: 5_000, + actions: [ + { kind: "clicks", t0: 1_000, t1: 1_000, count: 1, points: [[0.5, 0.5]], changed: true }, + ], + }, + [{ t: 1_000, x: 0.5, y: 0.5 }], + ); + expect(rings).toEqual([]); + }); +}); + +describe("chipDodgesRight", () => { + it.each([ + ["clear", [], [], false], + ["ink under the chip", [[0.1, 0.1]], [], true], + ["a click under the chip", [], [[0.1, 0.1]], true], + ["ink elsewhere", [[0.5, 0.5]], [], false], + ["a click where it would dodge to", [[0.1, 0.1]], [[0.9, 0.1]], false], + ["ink outside the crop", [[-0.1, 0.1]], [], false], + ] as const)("%s", (_name, ink, focus, expected) => { + expect(chipDodgesRight(ink, focus)).toBe(expected); + }); +}); + +describe("frameChipScale", () => { + it("grows with the frame and stays readable on a small crop", () => { + expect(frameChipScale(2560, 1440)).toBe(3); + expect(frameChipScale(320, 200)).toBe(0.45); + }); +}); diff --git a/apps/web/src/scrawl/frameAnnotations.ts b/apps/web/src/scrawl/frameAnnotations.ts new file mode 100644 index 0000000000000..a99bd53a6f2b1 --- /dev/null +++ b/apps/web/src/scrawl/frameAnnotations.ts @@ -0,0 +1,132 @@ +// What gets drawn on a keyframe: a subtle ring where the click landed, and the keyframe's number. +// Sizes and rules are Scrawl's (src/render.rs). Blind reading tests there found a 20 px chip failed +// on a sheet shown at half size, a bare digit read as screen text, and `#6` read as `8`. + +import type { NarrationLine } from "./narration"; + +const CLICK_NEAR_MS = 600; +// A double click is one place. +const SAME_PLACE = 0.01; + +type Point = readonly [x: number, y: number]; + +const nearSamePlace = (a: Point, b: Point) => Math.hypot(a[0] - b[0], a[1] - b[1]) < SAME_PLACE; + +/** + * The click the frame shows, nearest its moment. Bursts the screen did not answer are a hand pointing + * at things on this very screen, so all of those stay, back to the last time the screen changed. + */ +export function clicksToRing( + line: Pick, + clicks: ReadonlyArray<{ readonly t: number; readonly x: number; readonly y: number }>, +): Array { + let nearest: { readonly delta: number; readonly point: Point } | null = null; + for (const click of clicks) { + const delta = Math.abs(click.t - line.frameT); + if (delta <= CLICK_NEAR_MS && (nearest === null || delta < nearest.delta)) { + nearest = { delta, point: [click.x, click.y] }; + } + } + const changedAt = line.actions.reduce((latest, action) => { + const changed = + action.kind === "scene" || (action.kind === "clicks" && action.changed === true); + return changed && action.t0 <= line.frameT ? Math.max(latest, action.t0) : latest; + }, -Infinity); + const points: Array = nearest === null ? [] : [nearest.point]; + for (const action of line.actions) { + if (action.kind !== "clicks" || action.changed !== false) continue; + if (action.t0 < changedAt || action.t0 > line.frameT) continue; + for (const point of action.points) { + if (!points.some((seen) => nearSamePlace(seen, point))) points.push(point); + } + } + return points; +} + +const TOP_EDGE = 0.18; +const LEFT_EDGE = 0.3; +const RIGHT_EDGE = 0.7; + +/** Whether the number moves to the top right: ink, a click or the cursor sits under top left. */ +export function chipDodgesRight(busy: ReadonlyArray, focus: ReadonlyArray): boolean { + const inTopLeft = ([x, y]: Point) => x >= 0 && x < LEFT_EDGE && y >= 0 && y < TOP_EDGE; + const inTopRight = ([x, y]: Point) => x > RIGHT_EDGE && x <= 1 && y >= 0 && y < TOP_EDGE; + return (busy.some(inTopLeft) || focus.some(inTopLeft)) && !focus.some(inTopRight); +} + +/** The scale of a chip on a keyframe file. On a sheet every tile uses `SHEET_CHIP_SCALE`. */ +export const frameChipScale = (width: number, height: number) => + Math.max(Math.min(height / 480, width / 280), 0.45); +export const SHEET_CHIP_SCALE = 0.9; + +const annotationScale = (width: number, height: number) => + Math.min(2, Math.max(1, Math.min(width, height) / 800, Math.max(width, height) / 1280)); + +const CLICK_BLUE = "0, 136, 255"; +const WHITE = "255, 255, 255"; + +/** A thin ring and small dot, no fill, so the clicked thing stays readable; never red like ink. */ +export function drawClickRing( + context: CanvasRenderingContext2D, + [x, y]: Point, + width: number, + height: number, +) { + const k = annotationScale(width, height); + const ring = (radius: number, thickness: number, color: string) => { + context.lineWidth = thickness; + context.strokeStyle = color; + context.beginPath(); + context.arc(x * width, y * height, radius, 0, Math.PI * 2); + context.stroke(); + }; + const dot = (radius: number, color: string) => { + context.fillStyle = color; + context.beginPath(); + context.arc(x * width, y * height, radius, 0, Math.PI * 2); + context.fill(); + }; + const radius = 9 * k; + ring(radius + 1.1 * k, 0.9 * k, `rgba(${WHITE}, 0.4)`); + ring(radius, 1.3 * k, `rgba(${CLICK_BLUE}, 0.8)`); + dot(2 * k, `rgba(${WHITE}, 0.5)`); + dot(1.3 * k, `rgba(${CLICK_BLUE}, 0.9)`); +} + +/** The number alone, bold on a small white square with a thin dark ring, inset from a top corner. */ +export function drawChip( + context: CanvasRenderingContext2D, + label: string, + cell: { + readonly x: number; + readonly y: number; + readonly width: number; + }, + scale: number, + dodgeRight: boolean, +) { + const size = 16 * scale; + context.font = `700 ${size}px Arial, system-ui, sans-serif`; + const textWidth = context.measureText(label).width; + const height = size * 1.5; + const width = Math.max(textWidth + size * 0.62, height); + const inset = 5 * scale; + const ring = Math.max(1.4 * scale, 1); + const radius = size * 0.3; + const x = dodgeRight + ? Math.max(cell.x + cell.width - width - inset, cell.x + inset) + : cell.x + inset; + const y = cell.y + inset; + context.fillStyle = "rgba(16, 17, 20, 0.9)"; + context.beginPath(); + context.roundRect(x - ring, y - ring, width + 2 * ring, height + 2 * ring, radius + ring); + context.fill(); + context.fillStyle = "#fafafc"; + context.beginPath(); + context.roundRect(x, y, width, height, radius); + context.fill(); + context.fillStyle = "#101114"; + context.textAlign = "center"; + context.textBaseline = "middle"; + context.fillText(label, x + width / 2, y + height / 2 + scale * 0.5); +} diff --git a/apps/web/src/scrawl/narration.test.ts b/apps/web/src/scrawl/narration.test.ts index 2c82b0b07cf7b..af169ae375ce9 100644 --- a/apps/web/src/scrawl/narration.test.ts +++ b/apps/web/src/scrawl/narration.test.ts @@ -143,4 +143,46 @@ describe("buildNarration", () => { expect(lines).toHaveLength(9); expect(lines[0]!.text).toBe("Erster Satz ohne Aktion. Das hier markiert."); }); + + const actedSentence = (t: number) => say("Hier klicke ich jetzt.", t, 100); + + it.each([39_999, 40_000, 139_999, 600_000])( + "lets ordinary clicks fill only the base budget in %i ms", + (durationMs) => { + const times = Array.from({ length: 12 }, (_, index) => 500 + index * 3_000); + const lines = buildNarration({ + words: times.flatMap(actedSentence), + events: times.map((t) => ({ kind: "click" as const, t: t + 100, x: 0.5, y: 0.5 })), + marks: [], + durationMs, + }); + expect(lines).toHaveLength(6); + }, + ); + + it("spends the budget past the base on long stretches that show something", () => { + const early = Array.from({ length: 6 }, (_, index) => 500 + index * 2_000); + const late = [50_000, 90_000, 130_000]; + const lines = buildNarration({ + words: [...early, ...late].flatMap(actedSentence), + events: [...early, ...late].map((t) => ({ + kind: "click" as const, + t: t + 100, + x: 0.5, + y: 0.5, + })), + marks: [], + durationMs: 140_000, + }); + expect(lines.map((line) => line.t0)).toEqual(expect.arrayContaining(late)); + }); + + it("takes no keyframe for long speech over a screen where nothing happens", () => { + const words = Array.from({ length: 6 }, (_, index) => + say("Ich rede hier einfach weiter ohne etwas zu tun.", index * 20_000, 200), + ).flat(); + const lines = buildNarration({ words, events: [], marks: [], durationMs: 120_000 }); + expect(lines).toHaveLength(1); + expect(lines[0]!.text.split(" ")).toHaveLength(words.length); + }); }); diff --git a/apps/web/src/scrawl/narration.ts b/apps/web/src/scrawl/narration.ts index a9b67540af783..cebf9ea99be8f 100644 --- a/apps/web/src/scrawl/narration.ts +++ b/apps/web/src/scrawl/narration.ts @@ -73,8 +73,9 @@ const SHORT_DRAG = 0.15; const ATTACH_MS = 3_000; const SETTLE_MS = 300; // Scrawl's keyframe budget: each image costs a reading model about 1.5k tokens. +const BASE_KEYFRAMES = 6; const budgetFor = (durationMs: number) => - Math.min(12, 6 + Math.max(0, Math.ceil((durationMs - 40_000) / 20_000))); + Math.min(12, BASE_KEYFRAMES + Math.trunc(Math.max(0, durationMs - 20_000) / 20_000)); const COVERAGE_MS = 35_000; const COVERAGE_WORDS = 100; @@ -198,7 +199,10 @@ const attach = (sentences: Array, actions: ReadonlyArray sentence.actions.reduce((best, action) => Math.max(best, SCORES[action.kind]), 0); -/** The sentences that own a keyframe: every mark, then the strongest actions, then long stretches. */ +/** + * The sentences that own a keyframe: every mark, then long stretches that show something, then the + * strongest actions. Ordinary actions fill only the base budget; the rest is for marks and coverage. + */ const chooseFrameOwners = (sentences: ReadonlyArray, durationMs: number) => { const budget = budgetFor(durationMs); const owners = new Set(); @@ -207,26 +211,34 @@ const chooseFrameOwners = (sentences: ReadonlyArray, durationMs: numbe .filter((entry) => entry.score > 0) .sort((a, b) => b.score - a.score || a.index - b.index); for (const { index, score } of ranked) { - if (score >= SCORES.mark || owners.size < budget) owners.add(index); + if (score >= SCORES.mark) owners.add(index); } - if (sentences.length > 0) { - let stretchStart = 0; - let stretchWords = 0; - for (let index = 0; index < sentences.length; index++) { - const sentence = sentences[index]!; - if (owners.has(index)) { - stretchStart = sentence.t0; - stretchWords = sentence.words.length; - continue; - } - stretchWords += sentence.words.length; - const long = sentence.t1 - stretchStart > COVERAGE_MS || stretchWords > COVERAGE_WORDS; - if (long && owners.size < budget) { - owners.add(index); - stretchStart = sentence.t0; - stretchWords = sentence.words.length; - } + let stretchStart = 0; + let stretchWords = 0; + let best = -1; + for (let index = 0; index < sentences.length; index++) { + const sentence = sentences[index]!; + if (owners.has(index)) { + stretchStart = sentence.t0; + stretchWords = sentence.words.length; + best = -1; + continue; } + stretchWords += sentence.words.length; + // The latest of equals splits the stretch most evenly. + const score = scoreOf(sentence); + if (score > 0 && score >= (best < 0 ? 0 : scoreOf(sentences[best]!))) best = index; + const long = sentence.t1 - stretchStart > COVERAGE_MS || stretchWords > COVERAGE_WORDS; + if (long && best >= 0 && owners.size < budget) { + owners.add(best); + stretchStart = sentences[best]!.t0; + stretchWords = sentences.slice(best, index + 1).flatMap((s) => s.words).length; + best = -1; + } + } + for (const { index } of ranked) { + if (owners.size >= Math.min(budget, BASE_KEYFRAMES)) break; + owners.add(index); } // Speech with nothing on screen still gets one frame to be read against. if (owners.size === 0 && sentences.length > 0) owners.add(0); diff --git a/apps/web/src/scrawl/scrawlBundle.ts b/apps/web/src/scrawl/scrawlBundle.ts index f90df615ecd55..8d407775c421f 100644 --- a/apps/web/src/scrawl/scrawlBundle.ts +++ b/apps/web/src/scrawl/scrawlBundle.ts @@ -9,6 +9,28 @@ import { type ScrawlStroke, } from "@t3tools/contracts"; +import { + activityItems, + computeCrops, + cropLabel, + FULL, + pixelRect, + windowSnapshots, + type Crop, + type CursorSample, + type LoggedWindows, + type Rect, +} from "./autocrop"; +import { changeRegions, contentMap, type ContentMap, type RgbaImage } from "./autocropContent"; +import { layoutSheet } from "./contactSheet"; +import { + chipDodgesRight, + clicksToRing, + drawChip, + drawClickRing, + frameChipScale, + SHEET_CHIP_SCALE, +} from "./frameAnnotations"; import { buildNarration, formatNarrationLine, @@ -18,13 +40,8 @@ import { } from "./narration"; import { withoutInkChanges } from "./sceneChanges"; -const SHEET_LONG_EDGE = 1600; -const SHEET_GUTTER = 6; const STROKE_GROUP_MS = 1_500; -const CROP_PADDING = 0.08; -// Cropping to most of the screen only loses context. -const MAX_CROP_AREA = 0.7; -const MIN_CROP_SIDE = 0.35; +const SHEET_BACKGROUND = "rgb(28, 29, 34)"; export interface CaptureRecording { readonly session: ScrawlStartResult; @@ -41,6 +58,9 @@ export interface CaptureRecording { readonly durationMs: number; readonly sceneChanges: ReadonlyArray; readonly wav: Uint8Array; + /** Scrawl's window log and cursor samples, on the input clock. Crops are better with them. */ + readonly windows?: ReadonlyArray; + readonly cursor?: ReadonlyArray; } interface ManifestMark extends CaptureMark { @@ -107,45 +127,6 @@ const strokesToMarks = (strokes: ReadonlyArray): Array): { - readonly x: number; - readonly y: number; - readonly width: number; - readonly height: number; -} | null { - const xs: Array = []; - const ys: Array = []; - for (const event of recording.input.events) { - if (event.kind === "click") { - xs.push(event.x); - ys.push(event.y); - } else { - xs.push(event.x0, event.x1); - ys.push(event.y0, event.y1); - } - } - for (const stroke of recording.input.strokes) { - stroke.points.forEach((value, index) => (index % 2 === 0 ? xs : ys).push(value)); - } - if (xs.length === 0) return null; - const widen = (low: number, high: number) => { - let start = low - CROP_PADDING; - let end = high + CROP_PADDING; - if (end - start < MIN_CROP_SIDE) { - const middle = (start + end) / 2; - start = middle - MIN_CROP_SIDE / 2; - end = middle + MIN_CROP_SIDE / 2; - } - start = Math.max(0, start); - end = Math.min(1, end); - return [start, end - start] as const; - }; - const [x, width] = widen(Math.min(...xs), Math.max(...xs)); - const [y, height] = widen(Math.min(...ys), Math.max(...ys)); - return width * height > MAX_CROP_AREA ? null : { x, y, width, height }; -} - const seekTo = (video: HTMLVideoElement, seconds: number) => new Promise((resolve, reject) => { const onSeeked = () => { @@ -188,70 +169,82 @@ const toBlob = (canvas: HTMLCanvasElement) => ); }); -const drawClickRings = ( - context: CanvasRenderingContext2D, - line: NarrationLine, - width: number, - height: number, -) => { - context.lineWidth = Math.max(2, width / 700); - context.strokeStyle = "rgba(239, 68, 68, 0.85)"; - for (const action of line.actions) { - if (action.kind !== "clicks") continue; - for (const [x, y] of action.points) { - context.beginPath(); - context.arc(x * width, y * height, Math.max(10, width / 90), 0, Math.PI * 2); - context.stroke(); - } - } -}; - -const drawChip = (context: CanvasRenderingContext2D, label: string, height: number) => { - // Blind reading tests in Scrawl: the number needs its own white square to be read as a number. - const size = Math.max(24, height / 18); - context.fillStyle = "#ffffff"; - context.strokeStyle = "#111111"; - context.lineWidth = 2; - context.beginPath(); - context.roundRect(5, 5, size, size, 4); - context.fill(); - context.stroke(); - context.fillStyle = "#000000"; - context.font = `700 ${Math.round(size * 0.68)}px system-ui`; - context.textAlign = "center"; - context.textBaseline = "middle"; - context.fillText(label, 5 + size / 2, 5 + size / 2 + 1); -}; +interface KeyframeImage { + readonly canvas: HTMLCanvasElement; + readonly dodgeRight: boolean; +} -/** Frames in time order in justified rows, each with its number. */ -const drawContactSheet = (frames: ReadonlyArray) => { - const columns = Math.max(1, Math.ceil(Math.sqrt(frames.length))); - const rows = Math.ceil(frames.length / columns); - const first = frames[0]!; - const aspect = first.width / first.height; - const landscape = aspect >= 1; - const cellWidth = landscape - ? Math.floor((SHEET_LONG_EDGE - SHEET_GUTTER * (columns + 1)) / columns) - : Math.floor((SHEET_LONG_EDGE / aspect - SHEET_GUTTER * (columns + 1)) / columns); - const cellHeight = Math.round(cellWidth / aspect); +/** Frames in time order in justified rows, each with its number at one size. */ +const drawContactSheet = (frames: ReadonlyArray) => { + const layout = layoutSheet(frames.map(({ canvas }) => canvas)); const sheet = document.createElement("canvas"); - sheet.width = columns * cellWidth + (columns + 1) * SHEET_GUTTER; - sheet.height = rows * cellHeight + (rows + 1) * SHEET_GUTTER; + sheet.width = layout.width; + sheet.height = layout.height; const context = sheet.getContext("2d")!; - context.fillStyle = "#101014"; + context.fillStyle = SHEET_BACKGROUND; context.fillRect(0, 0, sheet.width, sheet.height); - frames.forEach((frame, index) => { - const x = SHEET_GUTTER + (index % columns) * (cellWidth + SHEET_GUTTER); - const y = SHEET_GUTTER + Math.floor(index / columns) * (cellHeight + SHEET_GUTTER); - context.save(); - context.translate(x, y); - context.drawImage(frame, 0, 0, cellWidth, cellHeight); - drawChip(context, String(index + 1), cellHeight); - context.restore(); + context.imageSmoothingQuality = "high"; + layout.cells.forEach((cell, index) => { + const frame = frames[index]!; + context.drawImage(frame.canvas, cell.x, cell.y, cell.width, cell.height); + drawChip(context, String(index + 1), cell, SHEET_CHIP_SCALE, frame.dodgeRight); }); return sheet; }; +/** One keyframe cut to `rect`, with its click rings; the number goes on after the sheet is drawn. */ +const renderKeyframe = ( + raw: HTMLCanvasElement, + rect: Rect, + rings: ReadonlyArray, + ink: ReadonlyArray, +): KeyframeImage => { + const box = pixelRect(rect, raw.width, raw.height); + const canvas = document.createElement("canvas"); + canvas.width = box.width; + canvas.height = box.height; + const context = canvas.getContext("2d")!; + context.save(); + context.translate(-box.x, -box.y); + context.drawImage(raw, 0, 0); + context.restore(); + const inFrame = ([x, y]: readonly [number, number]) => + [(x * raw.width - box.x) / box.width, (y * raw.height - box.y) / box.height] as const; + const ringsInFrame = rings.map(inFrame); + for (const ring of ringsInFrame) { + if (ring[0] >= 0 && ring[0] <= 1 && ring[1] >= 0 && ring[1] <= 1) { + drawClickRing(context, ring, box.width, box.height); + } + } + return { canvas, dodgeRight: chipDodgesRight(ink.map(inFrame), ringsInFrame) }; +}; + +const stampFrameChip = ({ canvas, dodgeRight }: KeyframeImage, index: number) => { + const { width, height } = canvas; + drawChip( + canvas.getContext("2d")!, + String(index + 1), + { x: 0, y: 0, width }, + frameChipScale(width, height), + dodgeRight, + ); +}; + +/** A crop bug must never cost the capture: the full frames are always right. */ +const cropsOrFull = (compute: () => Array, count: number): ReadonlyArray => { + try { + return compute(); + } catch (error) { + console.warn("Scrawl auto-crop failed", error); + return Array.from({ length: count }, () => ({ + rect: FULL, + kind: "full" as const, + reason: "not cropped", + apps: [], + })); + } +}; + const formatDuration = (ms: number) => { const seconds = Math.round(ms / 1000); return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; @@ -260,14 +253,14 @@ const formatDuration = (ms: number) => { const summaryText = ( recording: Pick, lines: ReadonlyArray, - cropped: boolean, + croppedTo: string | null, ) => { const { app, windowTitle, bundlePath } = recording.session; const source = [app, windowTitle ? `“${windowTitle}”` : ""].filter(Boolean).join(" ") || "Screen"; return [ `[scrawl ${formatDuration(recording.durationMs)} · ${source} · ${bundlePath}]`, "Open contact-sheet.png and frames/ in that folder before changing anything: the marks and words only make sense on the images, which can show more than the text does.", - ...(cropped ? ["Cropped to: the activity (full frames in frames-full/)"] : []), + ...(croppedTo === null ? [] : [`Cropped to: ${croppedTo} (full frames in frames-full/)`]), ...lines.map(formatNarrationLine), ].join("\n"); }; @@ -295,58 +288,119 @@ export async function buildBundle( durationMs: recording.durationMs, sceneChanges: withoutInkChanges(recording.sceneChanges, strokes, SCRAWL_INK_HOLD_MS), }); - const box = activityBox({ input: { events, strokes } }); + const { width, height } = recording.session; + const screen = [width, height] as const; const { video, dispose } = await loadVideo(recording.video); - const frames: Array = []; - const fullFrames: Array = []; - const bytesOf = async (blob: Blob) => new Uint8Array(await blob.arrayBuffer()); + const raws: Array = []; + const contents: Array = []; + const changes: Array> = []; + const grabFrame = async (t: number) => { + const seconds = Math.max(0, (t - recording.videoOffsetMs) / 1000); + await seekTo(video, Math.min(seconds, Math.max(0, video.duration - 0.05))); + const canvas = document.createElement("canvas"); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + const context = canvas.getContext("2d", { willReadFrequently: true })!; + context.drawImage(video, 0, 0); + return { canvas, image: context.getImageData(0, 0, canvas.width, canvas.height) }; + }; try { - for (const [index, line] of lines.entries()) { - const seconds = Math.max(0, (line.frameT - recording.videoOffsetMs) / 1000); - await seekTo(video, Math.min(seconds, Math.max(0, video.duration - 0.05))); - const full = document.createElement("canvas"); - full.width = video.videoWidth; - full.height = video.videoHeight; - const context = full.getContext("2d")!; - context.drawImage(video, 0, 0); - drawClickRings(context, line, full.width, full.height); - const name = `frame-${String(index + 1).padStart(2, "0")}.png`; - if (box === null) { - await write(`frames/${name}`, await bytesOf(await toBlob(full))); - frames.push(full); - continue; - } - await write(`frames-full/${name}`, await bytesOf(await toBlob(full))); - fullFrames.push(full); - const cropped = document.createElement("canvas"); - cropped.width = Math.round(box.width * full.width); - cropped.height = Math.round(box.height * full.height); - cropped - .getContext("2d")! - .drawImage(full, -Math.round(box.x * full.width), -Math.round(box.y * full.height)); - await write(`frames/${name}`, await bytesOf(await toBlob(cropped))); - frames.push(cropped); + let previous: RgbaImage | null = + lines[0]!.frameT > recording.videoOffsetMs ? (await grabFrame(0)).image : null; + for (const line of lines) { + const { canvas, image } = await grabFrame(line.frameT); + raws.push(canvas); + contents.push(contentMap(image, screen)); + changes.push(previous === null ? [] : changeRegions(previous, image)); + previous = image; } } finally { dispose(); } - const sheet = drawContactSheet(frames); - const contactSheet = await toBlob(sheet); + const keyframeTimes = lines.map((line) => line.frameT); + const inkOwners = new Map(); + lines.forEach((line, index) => { + for (const action of line.actions) { + if (action.kind !== "mark") continue; + const mark = marks.find((candidate) => candidate.id === action.id); + for (const stroke of mark?.strokes ?? []) inkOwners.set(stroke, index); + } + }); + const crops = cropsOrFull( + () => + computeCrops({ + screen, + durationMs: recording.durationMs, + keyframes: keyframeTimes, + items: activityItems({ + durationMs: recording.durationMs, + keyframes: keyframeTimes, + strokes, + events, + changes, + inkOwners, + inputOwner: (t) => { + const owner = lines.findIndex((line) => + line.actions.some( + (action) => action.kind !== "mark" && t >= action.t0 && t <= action.t1, + ), + ); + return owner < 0 ? undefined : owner; + }, + cursor: (recording.cursor ?? []).map(([t, x, y]): CursorSample => [shift(t), x, y]), + }), + snapshots: windowSnapshots(recording.windows ?? [], screen, shift), + content: contents, + }), + lines.length, + ); + const label = cropLabel(crops); + + const clicks = events.filter((event) => event.kind === "click"); + const frames: Array = []; + const fullFrames: Array = []; + lines.forEach((line, index) => { + const raw = raws[index]!; + const rings = clicksToRing(line, clicks); + const ink = strokes.flatMap((stroke, at) => { + const shown = + inkOwners.get(at) === index || + (stroke.t0 <= line.frameT && line.frameT <= stroke.t1 + SCRAWL_INK_HOLD_MS); + if (!shown) return []; + return Array.from( + { length: Math.trunc(stroke.points.length / 2) }, + (_, point) => [stroke.points[point * 2]!, stroke.points[point * 2 + 1]!] as const, + ); + }); + frames.push(renderKeyframe(raw, crops[index]!.rect, rings, ink)); + if (label !== null) fullFrames.push(renderKeyframe(raw, FULL, rings, ink)); + }); + + const bytesOf = async (blob: Blob) => new Uint8Array(await blob.arrayBuffer()); + const contactSheet = await toBlob(drawContactSheet(frames)); await write("contact-sheet.png", await bytesOf(contactSheet)); - if (fullFrames.length > 0) { + if (label !== null) { await write( "contact-sheet-full.png", await bytesOf(await toBlob(drawContactSheet(fullFrames))), ); } + const writeFrames = async (folder: string, images: ReadonlyArray) => { + for (const [index, image] of images.entries()) { + stampFrameChip(image, index); + const name = `frame-${String(index + 1).padStart(2, "0")}.png`; + await write(`${folder}/${name}`, await bytesOf(await toBlob(image.canvas))); + } + }; + await writeFrames("frames", frames); + await writeFrames("frames-full", fullFrames); await write("audio.wav", recording.wav); - const text = summaryText(recording, lines, box !== null); + const text = summaryText(recording, lines, label); const encoder = new TextEncoder(); await write("summary.md", encoder.encode(`${text}\n`)); - const { width, height } = recording.session; const manifest = { version: 1, id: recording.session.id, @@ -384,25 +438,26 @@ export async function buildBundle( video: recording.videoName, videoOffsetMs: recording.videoOffsetMs, audio: "audio.wav", - windows: [], - cursor: [], - ...(box === null + windows: recording.windows ?? [], + cursor: recording.cursor ?? [], + ...(label === null ? {} : { autocrop: { - label: "the activity", + label, fullFrames: "frames-full", fullContactSheet: "contact-sheet-full.png", - crops: lines.map((_line, index) => ({ + crops: crops.map(({ rect, kind, reason, apps }, index) => ({ index: index + 1, - x: Math.round(box.x * width), - y: Math.round(box.y * height), - width: Math.round(box.width * width), - height: Math.round(box.height * height), - kind: "focus", - reason: "around the activity", - apps: [recording.session.app], + x: Math.round(rect[0] * width), + y: Math.round(rect[1] * height), + width: Math.round((rect[2] - rect[0]) * width), + height: Math.round((rect[3] - rect[1]) * height), + kind, + reason, + apps, })), + changes, }, }), }; diff --git a/vite.config.ts b/vite.config.ts index 468837c255686..cd5c7e99008a0 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -168,6 +168,8 @@ export default defineConfig({ "pnpm-lock.yaml", "*.tsbuildinfo", "**/routeTree.gen.ts", + // Copied whole from Scrawl; one line, since pretty-printing makes it 50,000. + "apps/web/src/scrawl/autocrop-corpus.json", "**/*.graphql.ts", "packages/source-control-github/schema/**", "apps/server/schema/**", From fec7228df49deb7c5c962a5cc37d65bed74d22ca Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:27:17 +0100 Subject: [PATCH 4/9] feat(scrawl): captures log the wheel, cursor, windows, app switches, shortcuts, flags and what was clicked Co-Authored-By: Claude Opus 5.5 --- FORK.md | 12 + apps/desktop/src/scrawl/DesktopScrawl.ts | 125 ++++- .../desktop/src/scrawl/ScrawlContextWorker.ts | 472 ++++++++++++++++++ apps/desktop/src/scrawl/contextLog.test.ts | 99 ++++ apps/desktop/src/scrawl/contextLog.ts | 146 ++++++ apps/desktop/src/scrawl/elementLookup.test.ts | 138 +++++ apps/desktop/src/scrawl/elementLookup.ts | 299 +++++++++++ apps/desktop/src/scrawl/rawInput.test.ts | 85 ++++ apps/desktop/src/scrawl/rawInput.ts | 91 ++++ .../src/scrawl/scrawlContextProcess.ts | 70 +++ apps/desktop/src/scrawl/windowAtPoint.ts | 15 + apps/desktop/src/scrawl/windowList.test.ts | 87 ++++ apps/desktop/src/scrawl/windowList.ts | 143 ++++++ apps/desktop/vite.config.ts | 1 + apps/web/src/scrawl/narration.test.ts | 118 +++++ apps/web/src/scrawl/narration.ts | 176 +++++-- apps/web/src/scrawl/scrawlBundle.test.ts | 27 + apps/web/src/scrawl/scrawlBundle.ts | 59 ++- apps/web/src/scrawl/windowChanges.test.ts | 42 ++ apps/web/src/scrawl/windowChanges.ts | 39 ++ packages/contracts/src/scrawl.ts | 54 ++ 21 files changed, 2242 insertions(+), 56 deletions(-) create mode 100644 apps/desktop/src/scrawl/ScrawlContextWorker.ts create mode 100644 apps/desktop/src/scrawl/contextLog.test.ts create mode 100644 apps/desktop/src/scrawl/contextLog.ts create mode 100644 apps/desktop/src/scrawl/elementLookup.test.ts create mode 100644 apps/desktop/src/scrawl/elementLookup.ts create mode 100644 apps/desktop/src/scrawl/rawInput.test.ts create mode 100644 apps/desktop/src/scrawl/rawInput.ts create mode 100644 apps/desktop/src/scrawl/scrawlContextProcess.ts create mode 100644 apps/desktop/src/scrawl/windowAtPoint.ts create mode 100644 apps/desktop/src/scrawl/windowList.test.ts create mode 100644 apps/desktop/src/scrawl/windowList.ts create mode 100644 apps/web/src/scrawl/scrawlBundle.test.ts create mode 100644 apps/web/src/scrawl/windowChanges.test.ts create mode 100644 apps/web/src/scrawl/windowChanges.ts diff --git a/FORK.md b/FORK.md index cc27d2eb494b2..0b10889db4784 100644 --- a/FORK.md +++ b/FORK.md @@ -160,6 +160,18 @@ the contact sheet land in the composer on screen. Esc in T3 Code discards a capt - Desktop app only: a browser cannot see the whole screen, and mobile has no such thing. Clicks and ink need Windows; elsewhere a capture has voice and video. +- **Alt+Shift+F** flags a moment while recording (`⚑`, always its own keyframe). The pill shows the + flag for a moment. +- Besides clicks and ink, a capture logs the wheel, the cursor, which windows are on screen, app + switches, Ctrl and Win shortcuts, and what a click or stroke landed on (`clicked button “Area”`). + Only the cursor and flags work outside Windows. Typed text is never logged, AltGr counts as + typing, passwords are never read, and window titles stay in `scrawl.json`: they do not reach the + text an agent reads. Fields say only that they were clicked, not what they are called. +- This runs in two helper processes, for the length of a capture: one reads Raw Input (no system-wide + hook, so no program's mouse waits on it) and the window list, the other is a PowerShell asking UI + Automation (about 80 MB). The first lookup in a Chromium-based app turns on that app's accessibility + tree, which can slow a heavy page. T3 Code's own windows are not looked up, so clicks on them stay + unnamed. A precision touchpad's two-finger scroll may not show as Raw Input wheel events (unchecked). - While the Scrawl app runs it holds Alt+Shift+S. T3 Code logs that and takes the shortcut within 15 seconds of Scrawl closing. The composer button works either way. - Speech and pointing do not run in step, so sentences are never cut at a click: whole sentences diff --git a/apps/desktop/src/scrawl/DesktopScrawl.ts b/apps/desktop/src/scrawl/DesktopScrawl.ts index f63fe2e36f6c7..4a6c6c132daeb 100644 --- a/apps/desktop/src/scrawl/DesktopScrawl.ts +++ b/apps/desktop/src/scrawl/DesktopScrawl.ts @@ -27,7 +27,18 @@ import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import { loadWindowsForegroundApi } from "../electron/WindowsForeground.ts"; import { activeWindow } from "../snapShot/ActiveWindow.ts"; import * as DesktopWindow from "../window/DesktopWindow.ts"; -import { ScrawlInputTracker } from "./inputTracker.ts"; +import { + mergeCaptureInput, + pointerLookupKey, + ScrawlContextLog, + strokeLookupKey, + strokeLookupPoint, +} from "./contextLog.ts"; +import { startElementLookups, type ElementHit, type ElementLookups } from "./elementLookup.ts"; +import { ScrawlInputTracker, type CursorPoint } from "./inputTracker.ts"; +import { startScrawlContextProcess, type ScrawlContextProcess } from "./scrawlContextProcess.ts"; +import { loadWindowAtPoint } from "./windowAtPoint.ts"; +import { taskbarStrip } from "./windowList.ts"; const DEFAULT_SHORTCUT = "Alt+Shift+S"; // Another program (the Scrawl app uses the same keys) may hold the shortcut and let go of it later. @@ -37,6 +48,8 @@ const SHORTCUT_RETRY_MS = 15_000; const INPUT_POLL_MS = 10; const VK_LBUTTON = 0x01; const VK_SHIFT = 0x10; +const FLAG_SHORTCUT = "Alt+Shift+F"; +const LOOKUP_DRAIN_MS = 400; // Like Scrawl: frames, text and contact sheet stay, the heavy media goes. const MEDIA_KEPT_MS = 7 * 24 * 60 * 60 * 1000; const MEDIA_FILES = ["recording.mp4", "recording.webm", "audio.wav"]; @@ -115,6 +128,11 @@ interface RunningCapture { readonly bundlePath: string; readonly startedAtMs: number; readonly tracker: ScrawlInputTracker; + readonly context: ScrawlContextLog; + readonly watcher: ScrawlContextProcess | null; + readonly lookups: ElementLookups | null; + readonly flagShortcut: boolean; + readonly windowTitle: string; readonly display: Electron.Display; readonly ink: Electron.BrowserWindow; readonly pill: Electron.BrowserWindow; @@ -137,8 +155,11 @@ const make = Effect.gen(function* () { if (!capture.ink.isDestroyed()) void capture.ink.webContents.executeJavaScript(script, true); }; - /** Clicks and strokes need Windows' key state; elsewhere a capture has voice and video only. */ - const startInputPoll = (isKeyDown: ((virtualKey: number) => boolean) | null) => { + /** Clicks and strokes need Windows' key state; elsewhere a capture has voice, video and cursor. */ + const startInputPoll = ( + isKeyDown: ((virtualKey: number) => boolean) | null, + lookAt: (capture: RunningCapture, key: string, at: CursorPoint) => void, + ) => { let left = false; let shift = false; return setInterval(() => { @@ -149,7 +170,11 @@ const make = Effect.gen(function* () { if (isKeyDown !== null) { const nextLeft = isKeyDown(VK_LBUTTON); const nextShift = isKeyDown(VK_SHIFT); - if (nextLeft && !left) current.tracker.leftDown(t, at); + if (nextLeft && !left) { + current.tracker.leftDown(t, at); + current.watcher?.requestSnapshot(); + lookAt(current, pointerLookupKey(t), at); + } if (!nextLeft && left) current.tracker.leftUp(t, at); if (nextShift && !shift) { current.tracker.shiftDown(t, at); @@ -157,7 +182,11 @@ const make = Effect.gen(function* () { current.inkTimer = null; } if (!nextShift && shift) { - current.tracker.shiftUp(); + if (current.tracker.shiftUp()) { + const stroke = current.tracker.result().strokes.at(-1)!; + const point = strokeLookupPoint(stroke, current.display.bounds); + lookAt(current, strokeLookupKey(stroke.t0), point); + } ink(current, "inkLift()"); // Fades on a clock, not on the next click, so a click's effect on the screen stays its own. current.inkTimer = setTimeout(() => ink(current, "inkClear()"), SCRAWL_INK_HOLD_MS); @@ -165,6 +194,7 @@ const make = Effect.gen(function* () { left = nextLeft; shift = nextShift; } + current.context.cursor(t, at); const point = current.tracker.moved(t, at); if (point !== null) { ink( @@ -210,6 +240,13 @@ const make = Effect.gen(function* () { catch: toError, }); const tracker = new ScrawlInputTracker(display.bounds); + const context = new ScrawlContextLog(display.bounds); + const windowAtPoint = + keys === null + ? null + : yield* Effect.tryPromise({ try: () => loadWindowAtPoint(), catch: toError }).pipe( + Effect.orElseSucceed(() => null), + ); const inkWindow = new Electron.BrowserWindow(overlayOptions(display.bounds)); inkWindow.setIgnoreMouseEvents(true); @@ -234,15 +271,71 @@ const make = Effect.gen(function* () { void pill.loadURL(dataUrl(PILL_PAGE)); pill.showInactive(); + const nativeId = (window: Electron.BrowserWindow) => + Number(window.getNativeWindowHandle().readBigUInt64LE() & 0xffff_ffffn); + const relative = (at: number) => at - startedAtMs; + const watcher = + keys === null + ? null + : startScrawlContextProcess( + NodePath.join(__dirname, "scrawl", "ScrawlContextWorker.cjs"), + { + frame: Electron.screen.dipToScreenRect(null, display.bounds), + dock: taskbarStrip( + Electron.screen.dipToScreenRect(null, display.bounds), + Electron.screen.dipToScreenRect(null, display.workArea), + ), + ignoredWindowIds: [nativeId(inkWindow), nativeId(pill)], + }, + (message) => { + if (message.type === "wheel") { + const at = Electron.screen.getCursorScreenPoint(); + context.scrolled(relative(message.at), at, message.dx, message.dy); + } else if (message.type === "chord") { + context.pressed(relative(message.at), message.shortcut); + } else if (message.type === "app") { + context.switchedTo(relative(message.at), message.app); + } else if (message.type === "windows") { + context.windows({ t: relative(message.at), windows: message.windows }); + } + }, + ); + const lookups = keys === null ? null : startElementLookups(); + const flagShortcut = Electron.globalShortcut.register(FLAG_SHORTCUT, () => { + const current = running; + if (current === null) return; + current.context.flagged(Date.now() - current.startedAtMs); + if (!current.pill.isDestroyed()) { + void current.pill.webContents.executeJavaScript('setClock("⚑")', true); + } + }); + const lookAt = (current: RunningCapture, key: string, at: CursorPoint) => { + const { x, y, width, height } = current.display.bounds; + if (at.x < x || at.y < y || at.x >= x + width || at.y >= y + height) return; + if (keys === null || windowAtPoint === null) return; + const point = Electron.screen.dipToScreenPoint(at); + // Asking about our own window would turn on this renderer's accessibility tree. + const window = windowAtPoint(point.x, point.y); + if (window === 0n || keys.getWindowThreadAndProcessId(window).processId === process.pid) { + return; + } + current.lookups?.look(key, point.x, point.y); + }; + const capture: RunningCapture = { id, bundlePath, startedAtMs, tracker, + context, + watcher, + lookups, + flagShortcut, + windowTitle: front?.title ?? "", display, ink: inkWindow, pill, - inputTimer: startInputPoll(keys === null ? null : keys.isKeyDown), + inputTimer: startInputPoll(keys === null ? null : keys.isKeyDown, lookAt), inkTimer: null, clockTimer: setInterval(() => { const current = running; @@ -267,7 +360,12 @@ const make = Effect.gen(function* () { } satisfies ScrawlStartResult; }); - const tearDown = (capture: RunningCapture) => { + const tearDown = (capture: RunningCapture, keepHelpers = false) => { + if (!keepHelpers) { + capture.watcher?.kill(); + void capture.lookups?.finish(0); + } + if (capture.flagShortcut) Electron.globalShortcut.unregister(FLAG_SHORTCUT); clearInterval(capture.inputTimer); if (capture.inkTimer !== null) clearTimeout(capture.inkTimer); clearInterval(capture.clockTimer); @@ -282,9 +380,16 @@ const make = Effect.gen(function* () { } const capture = running; capture.tracker.shiftUp(); - tearDown(capture); - const { events, strokes } = capture.tracker.result(); - return { events: [...events], strokes: [...strokes] }; + tearDown(capture, true); + const [hits] = yield* Effect.promise(() => + Promise.all([ + capture.lookups?.finish(LOOKUP_DRAIN_MS) ?? new Map(), + capture.watcher?.stop(), + ]), + ); + return mergeCaptureInput(capture.tracker.result(), capture.context.result(), hits, [ + capture.windowTitle, + ]); }); const cancel = Effect.fn("DesktopScrawl.cancel")(function* (input: ScrawlCaptureRef) { diff --git a/apps/desktop/src/scrawl/ScrawlContextWorker.ts b/apps/desktop/src/scrawl/ScrawlContextWorker.ts new file mode 100644 index 0000000000000..e38172488228d --- /dev/null +++ b/apps/desktop/src/scrawl/ScrawlContextWorker.ts @@ -0,0 +1,472 @@ +// @effect-diagnostics globalTimers:off -- A plain Node child; the poll runs outside any Effect fiber. +// @effect-diagnostics globalDate:off -- Times share the wall clock with the capture. +// Watches what the desktop does during a capture, Windows only, in a process of its own: a failed +// native call here cannot take the recording with it, and the app's main thread never does this. +// +// The wheel cannot be polled, and a low-level hook would make every mouse move in every program +// wait on this process. Raw Input delivers to one hidden window and nobody waits for it. Messages +// queue until the timer reads them in one call, however fast the mouse reports. + +import { DataType, load, open } from "ffi-rs"; + +import { chordLabel, parseRawInput } from "./rawInput.ts"; +import { AppSwitchWatcher, filterWindows, type PixelRect, type RawWindow } from "./windowList.ts"; + +export interface ScrawlContextInit { + readonly frame: PixelRect; + readonly dock: PixelRect | null; + /** Ink and pill: they are the capture's own, not something on screen. */ + readonly ignoredWindowIds: ReadonlyArray; +} + +export type ScrawlContextRequest = + | ({ readonly type: "init" } & ScrawlContextInit) + | { readonly type: "snapshot" } + | { readonly type: "stop" }; + +export type ScrawlContextMessage = + | { readonly type: "ready" } + | { readonly type: "wheel"; readonly at: number; readonly dx: number; readonly dy: number } + | { readonly type: "chord"; readonly at: number; readonly shortcut: string } + | { readonly type: "app"; readonly at: number; readonly app: string } + | { + readonly type: "windows"; + readonly at: number; + readonly windows: ReturnType; + } + | { readonly type: "stopped" }; + +const READ_MS = 25; +const LOOK_EVERY_READS = 3; +// Scrawl's window log: a look at most this often while the hand moves, and now and then anyway. +const MOVING_SNAPSHOT_MS = 500; +const IDLE_SNAPSHOT_MS = 2_000; +const MAX_WINDOWS = 4_096; + +const USER = "scrawl-user32"; +const KERNEL = "scrawl-kernel32"; +const DWM = "scrawl-dwmapi"; +const VERSION = "scrawl-version"; + +const PER_MONITOR_AWARE_V2 = -4n; +const HWND_MESSAGE = -3n; +const RIDEV_INPUTSINK = 0x100; +const HEADER_BYTES = 24; +const GW_HWNDNEXT = 2; +const GWL_STYLE = -16; +const GWL_EXSTYLE = -20; +const GW_OWNER = 4; +const GA_ROOTOWNER = 3; +const WS_CHILD = 0x40000000; +const DWMWA_EXTENDED_FRAME_BOUNDS = 9; +const DWMWA_CLOAKED = 14; +const PROCESS_QUERY_LIMITED_INFORMATION = 0x1000; + +const VK_SHIFT = 0x10; +const VK_CONTROL = 0x11; +const VK_MENU = 0x12; +const VK_LWIN = 0x5b; +const VK_RWIN = 0x5c; +const VK_RMENU = 0xa5; + +const wide = (text: string) => Buffer.from(`${text}\0`, "utf16le"); +const fromWide = (buffer: Buffer, chars: number) => + buffer.subarray(0, chars * 2).toString("utf16le"); + +const user = ( + funcName: string, + retType: DataType, + paramsType: Array, + paramsValue: Array, +) => load({ library: USER, funcName, retType, paramsType, paramsValue }) as T; + +const isDown = (virtualKey: number) => + (user("GetAsyncKeyState", DataType.I16, [DataType.I32], [virtualKey]) & 0x8000) !== 0; + +const nextWindow = (handle: bigint) => + handle === 0n + ? user("GetTopWindow", DataType.BigInt, [DataType.BigInt], [0n]) + : user( + "GetWindow", + DataType.BigInt, + [DataType.BigInt, DataType.U32], + [handle, GW_HWNDNEXT], + ); + +const windowLong = (handle: bigint, index: number) => + Number( + user( + "GetWindowLongPtrW", + DataType.BigInt, + [DataType.BigInt, DataType.I32], + [handle, index], + ) & 0xffff_ffffn, + ); + +const processOf = (handle: bigint) => { + const out = Buffer.alloc(4); + user( + "GetWindowThreadProcessId", + DataType.U32, + [DataType.BigInt, DataType.U8Array], + [handle, out], + ); + return out.readUInt32LE(0); +}; + +const classOf = (handle: bigint) => { + const buffer = Buffer.alloc(512); + const chars = user( + "GetClassNameW", + DataType.I32, + [DataType.BigInt, DataType.U8Array, DataType.I32], + [handle, buffer, 256], + ); + return chars > 0 ? fromWide(buffer, chars) : ""; +}; + +const titleOf = (handle: bigint) => { + const buffer = Buffer.alloc(2048); + const chars = user( + "GetWindowTextW", + DataType.I32, + [DataType.BigInt, DataType.U8Array, DataType.I32], + [handle, buffer, 1024], + ); + return chars > 0 ? fromWide(buffer, chars) : ""; +}; + +const dwmAttribute = (handle: bigint, attribute: number, out: Buffer) => + load({ + library: DWM, + funcName: "DwmGetWindowAttribute", + retType: DataType.I32, + paramsType: [DataType.BigInt, DataType.U32, DataType.U8Array, DataType.U32], + paramsValue: [handle, attribute, out, out.byteLength], + }) === 0; + +const readMemory = (address: bigint, bytes: number) => { + const out = Buffer.alloc(bytes); + load({ + library: KERNEL, + funcName: "RtlMoveMemory", + retType: DataType.Void, + paramsType: [DataType.U8Array, DataType.BigInt, DataType.U64], + paramsValue: [out, address, bytes], + }); + return out; +}; + +const processPath = (processId: number) => { + const handle = load({ + library: KERNEL, + funcName: "OpenProcess", + retType: DataType.BigInt, + paramsType: [DataType.U32, DataType.Boolean, DataType.U32], + paramsValue: [PROCESS_QUERY_LIMITED_INFORMATION, false, processId], + }) as bigint; + if (handle === 0n) return ""; + const buffer = Buffer.alloc(2 * 32_768); + const size = Buffer.alloc(4); + size.writeUInt32LE(32_768, 0); + const ok = load({ + library: KERNEL, + funcName: "QueryFullProcessImageNameW", + retType: DataType.Boolean, + paramsType: [DataType.BigInt, DataType.U32, DataType.U8Array, DataType.U8Array], + paramsValue: [handle, 0, buffer, size], + }); + load({ + library: KERNEL, + funcName: "CloseHandle", + retType: DataType.Boolean, + paramsType: [DataType.BigInt], + paramsValue: [handle], + }); + return ok ? fromWide(buffer, size.readUInt32LE(0)) : ""; +}; + +/** A pointer into the version block that VerQueryValueW found, with its length in `unit`s. */ +const versionValue = (block: Buffer, query: string, unit: number) => { + const address = Buffer.alloc(8); + const length = Buffer.alloc(4); + const found = load({ + library: VERSION, + funcName: "VerQueryValueW", + retType: DataType.Boolean, + paramsType: [DataType.U8Array, DataType.U8Array, DataType.U8Array, DataType.U8Array], + paramsValue: [block, wide(query), address, length], + }); + const pointer = address.readBigUInt64LE(0); + const size = length.readUInt32LE(0); + return found && pointer !== 0n && size > 0 ? readMemory(pointer, size * unit) : null; +}; + +/** The name people know an app by: its FileDescription, such as "Visual Studio Code". */ +const fileDescription = (path: string) => { + const name = wide(path); + const size = load({ + library: VERSION, + funcName: "GetFileVersionInfoSizeW", + retType: DataType.U32, + paramsType: [DataType.U8Array, DataType.BigInt], + paramsValue: [name, 0n], + }) as number; + if (size === 0) return ""; + const block = Buffer.alloc(size); + const read = load({ + library: VERSION, + funcName: "GetFileVersionInfoW", + retType: DataType.Boolean, + paramsType: [DataType.U8Array, DataType.U32, DataType.U32, DataType.U8Array], + paramsValue: [name, 0, size, block], + }); + if (!read) return ""; + const translation = versionValue(block, "\\VarFileInfo\\Translation", 1); + const language = translation === null ? 0x0409 : translation.readUInt16LE(0); + const codePage = translation === null ? 0x04b0 : translation.readUInt16LE(2); + const hex = (value: number) => value.toString(16).padStart(4, "0"); + const value = versionValue( + block, + `\\StringFileInfo\\${hex(language)}${hex(codePage)}\\FileDescription`, + 2, + ); + return value === null ? "" : value.toString("utf16le").replace(/\0.*$/su, "").trim(); +}; + +const identities = new Map(); +const identityOf = (processId: number) => { + const known = identities.get(processId); + if (known !== undefined) return known; + const path = processPath(processId); + const exe = path.split("\\").at(-1) ?? ""; + const identity = { + app: (path === "" ? "" : fileDescription(path)) || exe.replace(/\.exe$/iu, ""), + bundleId: exe.toLowerCase(), + }; + identities.set(processId, identity); + return identity; +}; + +const readWindows = (): Array => { + const windows: Array = []; + const rect = Buffer.alloc(16); + const cloaked = Buffer.alloc(4); + let handle = nextWindow(0n); + for (let seen = 0; handle !== 0n && seen < MAX_WINDOWS; seen++) { + const visible = user("IsWindowVisible", DataType.Boolean, [DataType.BigInt], [handle]); + if (visible && (windowLong(handle, GWL_STYLE) & WS_CHILD) === 0) { + const processId = processOf(handle); + const framed = dwmAttribute(handle, DWMWA_EXTENDED_FRAME_BOUNDS, rect); + cloaked.writeUInt32LE(0, 0); + dwmAttribute(handle, DWMWA_CLOAKED, cloaked); + const left = rect.readInt32LE(0); + const top = rect.readInt32LE(4); + windows.push({ + id: Number(handle & 0xffff_ffffn), + processId, + ...identityOf(processId), + title: titleOf(handle), + className: classOf(handle), + rect: framed + ? { + x: left, + y: top, + width: rect.readInt32LE(8) - left, + height: rect.readInt32LE(12) - top, + } + : { x: 0, y: 0, width: 0, height: 0 }, + exStyle: windowLong(handle, GWL_EXSTYLE), + owned: + user( + "GetWindow", + DataType.BigInt, + [DataType.BigInt, DataType.U32], + [handle, GW_OWNER], + ) !== 0n, + minimized: user("IsIconic", DataType.Boolean, [DataType.BigInt], [handle]), + cloaked: cloaked.readUInt32LE(0) !== 0, + }); + } + handle = nextWindow(handle); + } + return windows; +}; + +const frontWindow = () => { + const front = user("GetForegroundWindow", DataType.BigInt, [], []); + if (front === 0n) return null; + const root = user( + "GetAncestor", + DataType.BigInt, + [DataType.BigInt, DataType.U32], + [front, GA_ROOTOWNER], + ); + const handle = root === 0n ? front : root; + const processId = processOf(handle); + return { processId, app: identityOf(processId).app, className: classOf(handle) }; +}; + +const cursorNow = () => { + const out = Buffer.alloc(8); + return user("GetCursorPos", DataType.Boolean, [DataType.U8Array], [out]) + ? out.readBigInt64LE(0) + : 0n; +}; + +const startRawInput = () => { + const window = user( + "CreateWindowExW", + DataType.BigInt, + [ + DataType.U32, + DataType.U8Array, + DataType.U8Array, + DataType.U32, + DataType.I32, + DataType.I32, + DataType.I32, + DataType.I32, + DataType.BigInt, + DataType.BigInt, + DataType.BigInt, + DataType.BigInt, + ], + [0, wide("Message"), wide(""), 0, 0, 0, 0, 0, HWND_MESSAGE, 0n, 0n, 0n], + ); + if (window === 0n) throw new Error("No window for raw input."); + const devices = Buffer.alloc(32); + // Generic desktop page: usage 2 is the mouse, 6 the keyboard. + for (const [index, usage] of [2, 6].entries()) { + devices.writeUInt16LE(1, index * 16); + devices.writeUInt16LE(usage, index * 16 + 2); + devices.writeUInt32LE(RIDEV_INPUTSINK, index * 16 + 4); + devices.writeBigUInt64LE(window, index * 16 + 8); + } + const registered = user( + "RegisterRawInputDevices", + DataType.Boolean, + [DataType.U8Array, DataType.U32, DataType.U32], + [devices, 2, 16], + ); + if (!registered) throw new Error("Raw input was refused."); +}; + +const send = (message: ScrawlContextMessage) => process.send?.(message); + +const run = (init: ScrawlContextInit) => { + startRawInput(); + const buffer = Buffer.alloc(64 * 1024); + const size = Buffer.alloc(4); + const keysDown = new Set(); + const ignored = new Set(init.ignoredWindowIds); + const switches = new AppSwitchWatcher(); + let reads = 0; + let wanted = true; + let lastSnapshot = 0; + let lastCursor = 0n; + let cursorAtSnapshot = 0n; + let lastFrontProcess: number | null = null; + + const readInput = () => { + size.writeUInt32LE(buffer.byteLength, 0); + const count = user( + "GetRawInputBuffer", + DataType.I32, + [DataType.U8Array, DataType.U8Array, DataType.U32], + [buffer, size, HEADER_BYTES], + ); + if (count <= 0) return; + const at = Date.now(); + for (const input of parseRawInput(buffer, count)) { + if (input.kind === "wheel") { + send({ type: "wheel", at, dx: input.dx, dy: input.dy }); + } else if (!input.down) { + keysDown.delete(input.virtualKey); + } else if (!keysDown.has(input.virtualKey)) { + keysDown.add(input.virtualKey); + // Asked of Windows, not tracked from the stream: a Ctrl whose release was lost to the lock + // screen would otherwise turn all later typing into chords. + const ctrl = isDown(VK_CONTROL); + const win = isDown(VK_LWIN) || isDown(VK_RWIN); + if (!ctrl && !win) continue; + const shortcut = chordLabel(input.virtualKey, { + ctrl, + win, + alt: isDown(VK_MENU), + shift: isDown(VK_SHIFT), + rightAlt: isDown(VK_RMENU), + }); + if (shortcut !== null) { + send({ type: "chord", at, shortcut }); + wanted = true; + } + } + } + }; + + const look = () => { + const at = Date.now(); + const front = frontWindow(); + const switched = switches.look(at, front); + if (switched !== null) send({ type: "app", at: switched.t, app: switched.app }); + lastCursor = cursorNow(); + const since = at - lastSnapshot; + const due = + wanted || + (front?.processId ?? null) !== lastFrontProcess || + (lastCursor !== cursorAtSnapshot && since >= MOVING_SNAPSHOT_MS) || + since >= IDLE_SNAPSHOT_MS; + if (!due) return; + wanted = false; + lastSnapshot = at; + lastFrontProcess = front?.processId ?? null; + cursorAtSnapshot = lastCursor; + send({ + type: "windows", + at, + windows: filterWindows( + readWindows(), + init.frame, + init.dock, + front?.processId ?? null, + ignored, + ), + }); + }; + + const timer = setInterval(() => { + readInput(); + if (++reads % LOOK_EVERY_READS === 0) look(); + }, READ_MS); + look(); + send({ type: "ready" }); + + process.on("message", (request: ScrawlContextRequest) => { + if (request.type === "snapshot") wanted = true; + if (request.type !== "stop") return; + clearInterval(timer); + readInput(); + send({ type: "stopped" }); + setImmediate(() => process.exit(0)); + }); +}; + +const main = () => { + process.once("disconnect", () => process.exit(0)); + open({ library: USER, path: "user32.dll" }); + open({ library: KERNEL, path: "kernel32.dll" }); + open({ library: DWM, path: "dwmapi.dll" }); + open({ library: VERSION, path: "version.dll" }); + user( + "SetProcessDpiAwarenessContext", + DataType.Boolean, + [DataType.BigInt], + [PER_MONITOR_AWARE_V2], + ); + process.once("message", (request: ScrawlContextRequest) => { + if (request.type === "init") run(request); + }); +}; + +main(); diff --git a/apps/desktop/src/scrawl/contextLog.test.ts b/apps/desktop/src/scrawl/contextLog.test.ts new file mode 100644 index 0000000000000..4d37eb6c8da92 --- /dev/null +++ b/apps/desktop/src/scrawl/contextLog.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + mergeCaptureInput, + pointerLookupKey, + ScrawlContextLog, + strokeLookupKey, + strokeLookupPoint, +} from "./contextLog.ts"; + +const AREA = { x: 1000, y: 0, width: 1000, height: 1000 }; + +describe("ScrawlContextLog", () => { + it("thins the cursor to real movement inside the display", () => { + const log = new ScrawlContextLog(AREA); + log.cursor(0, { x: 1500, y: 500 }); + log.cursor(10, { x: 1600, y: 500 }); + log.cursor(100, { x: 1501, y: 500 }); + log.cursor(200, { x: 1600, y: 500 }); + log.cursor(300, { x: 10, y: 10 }); + expect(log.result().cursor).toEqual([ + [0, 0.5, 0.5], + [200, 0.6, 0.5], + ]); + }); + + it("drops a scroll off this display and stores a window list only when it changed", () => { + const log = new ScrawlContextLog(AREA); + log.scrolled(1, { x: 10, y: 10 }, 0, 1); + log.scrolled(2, { x: 1500, y: 250 }, 0, -1); + const windows = [{ id: 1, app: "A", bounds: [0, 0, 9, 9], layer: 0 }] as const; + log.windows({ t: 1, windows }); + log.windows({ t: 2, windows }); + log.windows({ t: 3, windows: [] }); + const result = log.result(); + expect(result.events).toEqual([{ kind: "scroll", t: 2, x: 0.5, y: 0.25, dx: 0, dy: -1 }]); + expect(result.windows.map((snapshot) => snapshot.t)).toEqual([1, 3]); + }); +}); + +describe("mergeCaptureInput", () => { + const log = () => new ScrawlContextLog(AREA); + + it("names presses and strokes, in time order, and never by a window's title", () => { + const context = log(); + context.pressed(50, "Ctrl+C"); + context.windows({ + t: 0, + windows: [ + { + id: 1, + app: "Mail", + title: "Inbox of a private person - Mail", + bounds: [0, 0, 9, 9], + layer: 0, + }, + ], + }); + const merged = mergeCaptureInput( + { + events: [ + { kind: "click", t: 100, x: 0.5, y: 0.5 }, + { kind: "click", t: 10, x: 0.5, y: 0.5 }, + { kind: "drag", t: 200, t1: 300, x0: 0, y0: 0, x1: 1, y1: 1 }, + ], + strokes: [{ t0: 400, t1: 500, points: [0, 0, 1, 1] }], + }, + context.result(), + new Map([ + [pointerLookupKey(100), { role: "button", label: "“Area”" }], + [pointerLookupKey(10), { role: "tab item", label: "“Inbox of a private person”" }], + [pointerLookupKey(200), { role: "slider", label: "“Volume”" }], + [strokeLookupKey(400), { role: "button", label: "“Save” #save" }], + ]), + ["Quiet title"], + ); + expect(merged.events).toMatchObject([ + { kind: "click", t: 10, role: "tab item" }, + { kind: "key", t: 50 }, + { kind: "click", t: 100, role: "button", label: "“Area”" }, + { kind: "drag", t: 200, label: "slider “Volume”" }, + ]); + expect(merged.events[0]).not.toHaveProperty("label"); + expect(merged.strokes[0]!.on).toBe("button “Save” #save"); + }); +}); + +describe("strokeLookupPoint", () => { + it("asks about the middle of a stroke, and above a flat one, which underlines the thing", () => { + expect(strokeLookupPoint({ t0: 0, t1: 1, points: [0.1, 0.1, 0.3, 0.3] }, AREA)).toEqual({ + x: 1200, + y: 200, + }); + expect(strokeLookupPoint({ t0: 0, t1: 1, points: [0.1, 0.5, 0.3, 0.505] }, AREA)).toEqual({ + x: 1200, + y: 492, + }); + }); +}); diff --git a/apps/desktop/src/scrawl/contextLog.ts b/apps/desktop/src/scrawl/contextLog.ts new file mode 100644 index 0000000000000..9ded4223c3ecd --- /dev/null +++ b/apps/desktop/src/scrawl/contextLog.ts @@ -0,0 +1,146 @@ +// What a capture logs besides clicks and ink: the cursor, the wheel, shortcuts, app switches, the +// windows on screen and flags. All of it is plain data, so it can be tested without a desktop. + +import type { + ScrawlInputEvent, + ScrawlStopResult, + ScrawlStroke, + ScrawlWindowSnapshot, +} from "@t3tools/contracts"; + +import { elementOn, namesWindowTitle, type ElementHit } from "./elementLookup.ts"; +import type { CursorPoint, DisplayArea } from "./inputTracker.ts"; + +// Scrawl's log: about 15 samples a second at most, and only when the hand really moved. +const CURSOR_TICK_MS = 66; +const CURSOR_MIN_MOVE = 2; + +const round4 = (value: number) => Math.round(value * 10_000) / 10_000; + +export class ScrawlContextLog { + readonly #area: DisplayArea; + readonly #events: Array = []; + readonly #cursor: Array = []; + readonly #windows: Array = []; + readonly #flags: Array = []; + #lastWindows = ""; + #cursorAt: { readonly t: number; readonly at: CursorPoint } | null = null; + + constructor(area: DisplayArea) { + this.#area = area; + } + + #normalized(at: CursorPoint): { readonly x: number; readonly y: number } | null { + const { x, y, width, height } = this.#area; + if (at.x < x || at.y < y || at.x >= x + width || at.y >= y + height) return null; + return { x: round4((at.x - x) / width), y: round4((at.y - y) / height) }; + } + + cursor(t: number, at: CursorPoint): void { + const last = this.#cursorAt; + if (last !== null) { + if (t - last.t < CURSOR_TICK_MS) return; + if (Math.hypot(at.x - last.at.x, at.y - last.at.y) < CURSOR_MIN_MOVE) return; + } + const point = this.#normalized(at); + if (point === null) return; + this.#cursorAt = { t, at }; + this.#cursor.push([t, point.x, point.y]); + } + + scrolled(t: number, at: CursorPoint, dx: number, dy: number): void { + const point = this.#normalized(at); + if (point !== null) this.#events.push({ kind: "scroll", t, ...point, dx, dy }); + } + + pressed(t: number, shortcut: string): void { + this.#events.push({ kind: "key", t, shortcut }); + } + + switchedTo(t: number, app: string): void { + this.#events.push({ kind: "app_change", t, app }); + } + + flagged(t: number): void { + this.#flags.push(t); + } + + /** Stored only when something differs: most looks find the same windows in the same order. */ + windows(snapshot: ScrawlWindowSnapshot): void { + const key = JSON.stringify(snapshot.windows); + if (key === this.#lastWindows) return; + this.#lastWindows = key; + this.#windows.push(snapshot); + } + + result() { + return { + events: this.#events, + cursor: this.#cursor, + windows: this.#windows, + flags: this.#flags, + }; + } +} + +export const pointerLookupKey = (t: number) => `pointer:${t}`; +export const strokeLookupKey = (t0: number) => `stroke:${t0}`; + +/** Where to ask what a stroke marks: its middle, or just above a flat one, which underlines it. */ +export function strokeLookupPoint(stroke: ScrawlStroke, area: DisplayArea): CursorPoint { + const xs = stroke.points.filter((_, index) => index % 2 === 0); + const ys = stroke.points.filter((_, index) => index % 2 === 1); + const x0 = Math.min(...xs); + const x1 = Math.max(...xs); + const y0 = Math.min(...ys); + const width = (x1 - x0) * area.width; + const height = (Math.max(...ys) - y0) * area.height; + const top = area.y + y0 * area.height; + return { + x: area.x + ((x0 + x1) / 2) * area.width, + y: height < 24 && width > 3 * height ? top - 8 : top + height / 2, + }; +} + +/** One capture's whole input, in time order, with what each press and stroke landed on. */ +export function mergeCaptureInput( + tracked: { + readonly events: ReadonlyArray; + readonly strokes: ReadonlyArray; + }, + log: ReturnType, + hits: ReadonlyMap, + knownTitles: ReadonlyArray, +): ScrawlStopResult { + const titles = [ + ...knownTitles, + ...log.windows.flatMap((snapshot) => snapshot.windows.map((window) => window.title ?? "")), + ].filter((title) => title !== ""); + const allowed = (hit: ElementHit | undefined): ElementHit | undefined => { + if (hit?.label === undefined || !namesWindowTitle(hit.label, titles)) return hit; + return { role: hit.role }; + }; + const events = [...tracked.events, ...log.events] + .map((event): ScrawlInputEvent => { + if (event.kind !== "click" && event.kind !== "drag") return event; + const hit = allowed(hits.get(pointerLookupKey(event.t))); + if (hit === undefined) return event; + if (event.kind === "drag") { + const on = elementOn(hit); + return on === undefined ? event : { ...event, label: on }; + } + return { ...event, role: hit.role, ...(hit.label === undefined ? {} : { label: hit.label }) }; + }) + .toSorted((a, b) => a.t - b.t); + const strokes = tracked.strokes.map((stroke) => { + const on = elementOn(allowed(hits.get(strokeLookupKey(stroke.t0)))); + return on === undefined ? stroke : { ...stroke, on }; + }); + return { + events, + strokes, + cursor: [...log.cursor], + windows: [...log.windows], + flags: [...log.flags], + }; +} diff --git a/apps/desktop/src/scrawl/elementLookup.test.ts b/apps/desktop/src/scrawl/elementLookup.test.ts new file mode 100644 index 0000000000000..75cdf3e9162ee --- /dev/null +++ b/apps/desktop/src/scrawl/elementLookup.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { elementHit, elementOn, ElementLookups, namesWindowTitle } from "./elementLookup.ts"; + +const BUTTON = 50000; +const EDIT = 50004; +const TEXT = 50020; +const DOCUMENT = 50030; +const WINDOW = 50032; +const PANE = 50033; + +describe("elementHit", () => { + it.each([ + [ + "a button", + { type: BUTTON, role: "button", name: "Area", automationId: "area" }, + { role: "button", label: "“Area” #area" }, + ], + [ + "a numbered Win32 id", + { type: BUTTON, role: "button", name: "OK", automationId: "1001" }, + { role: "button", label: "“OK”" }, + ], + [ + "help text when there is no name", + { type: BUTTON, role: "button", help: "Saves" }, + { role: "button", label: "“Saves”" }, + ], + [ + "a field keeps its role, not its name", + { type: EDIT, role: "edit", name: "whatever it says" }, + { role: "edit" }, + ], + ["a password field the helper read nothing of", { type: EDIT, role: "edit" }, { role: "edit" }], + ["a page, named after its title", { type: DOCUMENT, role: "document", name: "My bank" }, null], + ["a window, named after its title", { type: WINDOW, role: "window", name: "My bank" }, null], + ["a pane", { type: PANE, role: "pane", name: "My bank" }, null], + ["text, which may be what was typed", { type: TEXT, role: "text", name: "hunter2" }, null], + ["a failed lookup", {}, null], + ])("%s", (_name, answer, expected) => { + expect(elementHit(answer)).toEqual(expected); + }); + + it("keeps a label to one short line", () => { + const label = elementHit({ + type: BUTTON, + role: "button", + name: `a\n b ${"c".repeat(90)}`, + })!.label!; + expect(label).toMatch(/^“a b c+…”$/u); + expect([...label].length).toBe(62); + }); +}); + +describe("elementOn", () => { + it("needs something more than a role", () => { + expect(elementOn({ role: "button", label: "“Area”" })).toBe("button “Area”"); + expect(elementOn({ role: "edit" })).toBeUndefined(); + expect(elementOn(undefined)).toBeUndefined(); + }); +}); + +describe("namesWindowTitle", () => { + const titles = ["Inbox of someone private - Mail", "Save"]; + + it("catches a tab named after the page, and leaves ordinary captions alone", () => { + expect(namesWindowTitle("“Inbox of someone private”", titles)).toBe(true); + expect(namesWindowTitle("“Inbox of someone…”", titles)).toBe(true); + expect(namesWindowTitle("“save” #s", titles)).toBe(true); + expect(namesWindowTitle("“Mail”", titles)).toBe(false); + expect(namesWindowTitle("#only-an-id", titles)).toBe(false); + }); +}); + +describe("ElementLookups", () => { + const setup = () => { + let now = 0; + const written: Array = []; + let closed = 0; + const lookups = new ElementLookups( + { + write: (line) => written.push(line), + close: () => { + closed++; + }, + }, + () => now, + ); + return { + lookups, + written, + closes: () => closed, + advance: (ms: number) => { + now += ms; + }, + }; + }; + const answer = (id: number) => JSON.stringify({ id, type: BUTTON, role: "button", name: "Area" }); + + it("waits for the child, asks one at a time, and skips a click too old to describe", async () => { + const { lookups, written, advance } = setup(); + lookups.look("old", 1, 1); + advance(600); + lookups.look("a", 2, 2); + lookups.look("b", 3, 3); + expect(written).toEqual([]); + lookups.answer("ready"); + expect(written).toEqual(["1 2 2\n"]); + lookups.answer("not json"); + lookups.answer(answer(9)); + expect(written).toHaveLength(1); + lookups.answer(answer(1)); + expect(written).toEqual(["1 2 2\n", "2 3 3\n"]); + lookups.answer(answer(2)); + const hits = await lookups.finish(0); + expect([...hits.keys()]).toEqual(["a", "b"]); + }); + + it("finishes at once when nothing runs, and ends the child either way", async () => { + const quiet = setup(); + quiet.lookups.answer("ready"); + expect((await quiet.lookups.finish(10_000)).size).toBe(0); + expect(quiet.closes()).toBe(1); + + const neverReady = setup(); + neverReady.lookups.look("a", 1, 1); + expect((await neverReady.lookups.finish(10_000)).size).toBe(0); + expect(neverReady.closes()).toBe(1); + }); + + it("gives up on a target app that does not answer", async () => { + const { lookups, closes } = setup(); + lookups.answer("ready"); + lookups.look("hung", 1, 1); + expect((await lookups.finish(0)).size).toBe(0); + expect(closes()).toBe(1); + }); +}); diff --git a/apps/desktop/src/scrawl/elementLookup.ts b/apps/desktop/src/scrawl/elementLookup.ts new file mode 100644 index 0000000000000..1f58ae634565b --- /dev/null +++ b/apps/desktop/src/scrawl/elementLookup.ts @@ -0,0 +1,299 @@ +// @effect-diagnostics globalTimers:off -- Lookups are answered at native callback boundaries outside Effect fibers. +// @effect-diagnostics globalDate:off -- A lookup's age is measured on the wall clock. +// @effect-diagnostics nodeBuiltinImport:off -- This desktop-only helper owns a child process. +// What a click or stroke landed on, from UI Automation. COM cannot be called through ffi-rs, so this +// is a PowerShell child, started with the capture: about 80 MB while one runs, no CPU while it waits, +// and a target app that hangs holds up only that child. Only names, control types and automation ids +// are ever read, never a control's value, so nothing typed into a field leaves it. +// +// It compiles a little C# because PowerShell's own System.Windows.Automation stops at coarse panes: +// on the taskbar it found a Pane where IUIAutomation, as Scrawl uses it, finds the Button. +// +// The first lookup in a Chromium-based app turns on that app's accessibility tree, which is the real +// cost of naming things. T3 Code's own windows are skipped to keep it out of this renderer. + +import * as NodeChildProcess from "node:child_process"; + +// A lookup this late describes whatever the click already opened, not what was clicked. +const STALE_MS = 500; +const MAX_CHARS = 60; +const TITLE_MATCH_MIN = 12; + +// Vtable order is the contract here: the numbered slots stand for methods this never calls. +const ELEMENT_SCRIPT = String.raw` +Add-Type -TypeDefinition @' +using System; +using System.Runtime.InteropServices; +using System.Text; +public static class ScrawlElement { + [StructLayout(LayoutKind.Sequential)] public struct P { public int x; public int y; } + [ComImport, Guid("30cbe57d-d9d0-452a-ab13-7ac5ac4825ee"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + interface IA { void a0(); void a1(); void a2(); void a3(); E ElementFromPoint(P pt); void a5(); void a6(); void a7(); void a8(); void a9(); void a10(); W ControlViewWalker { get; } } + [ComImport, Guid("4042c624-389c-4afc-a630-e0c9ee12fe0b"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + interface W { E GetParentElement(E e); } + [ComImport, Guid("d22108aa-8ac5-49a5-837b-37bbb3d7591e"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + interface E { + void e0(); void e1(); void e2(); void e3(); void e4(); void e5(); void e6(); void e7(); void e8(); void e9(); void e10(); void e11(); void e12(); void e13(); void e14(); void e15(); void e16(); + int ProcessId { get; } int ControlType { get; } + [return: MarshalAs(UnmanagedType.BStr)] string Localized(); + [return: MarshalAs(UnmanagedType.BStr)] string Name(); + void e21(); void e22(); void e23(); void e24(); void e25(); + [return: MarshalAs(UnmanagedType.BStr)] string AutomationId(); + void e27(); + [return: MarshalAs(UnmanagedType.BStr)] string HelpText(); + void e29(); void e30(); void e31(); + int IsPassword { get; } + } + [ComImport, Guid("ff48dba4-60ef-4201-aa87-54103eef594e")] class C {} + [DllImport("user32.dll")] static extern bool SetProcessDpiAwarenessContext(IntPtr value); + static string Q(string name, string value) { + var o = new StringBuilder(",\"" + name + "\":\""); + foreach (var c in value ?? "") { + if (c < 32 || c > 126 || c == '"' || c == '\\') o.Append("\\u").Append(((int)c).ToString("x4")); + else o.Append(c); + } + return o.Append('"').ToString(); + } + // Text, Image, Group and Custom are usually a piece of a control, which then names them. + static bool Part(int t) { return t == 50020 || t == 50006 || t == 50026 || t == 50025; } + static bool Control(int t) { return t == 50000 || t == 50031 || t == 50005 || t == 50011 || t == 50002 || t == 50013 || t == 50019 || t == 50007 || t == 50024 || t == 50029; } + static string Look(IA a, int x, int y) { + P p; p.x = x; p.y = y; + E e = a.ElementFromPoint(p); + if (Part(e.ControlType)) { + W w = a.ControlViewWalker; + E c = e; + for (int i = 0; i < 2 && c != null; i++) { + c = w.GetParentElement(c); + if (c != null && Control(c.ControlType)) { e = c; break; } + } + } + var o = ",\"type\":" + e.ControlType + Q("role", e.Localized()); + if (e.IsPassword != 0) return o; + return o + Q("name", e.Name()) + Q("help", e.HelpText()) + Q("automationId", e.AutomationId()); + } + public static void Run() { + SetProcessDpiAwarenessContext(new IntPtr(-4)); + var a = (IA)new C(); + Console.Out.WriteLine("ready"); + string line; + while ((line = Console.In.ReadLine()) != null) { + var f = line.Split(' '); + string o = ""; + try { o = Look(a, (int)double.Parse(f[1]), (int)double.Parse(f[2])); } catch (Exception) {} + Console.Out.WriteLine("{\"id\":" + f[0] + o + "}"); + } + } +} +'@ +[ScrawlElement]::Run() +`; + +export interface ElementHit { + readonly role: string; + /** `“Area” #area`, for a click's label. */ + readonly label?: string; +} + +export interface ElementAnswer { + readonly type?: number; + readonly role?: string; + readonly name?: string; + readonly help?: string; + readonly automationId?: string; +} + +const clip = (raw: string | undefined): string | undefined => { + const text = raw?.split(/\s+/u).filter(Boolean).join(" "); + if (text === undefined || text === "") return undefined; + return text.length <= MAX_CHARS + ? text + : `${[...text] + .slice(0, MAX_CHARS - 1) + .join("") + .trimEnd()}…`; +}; + +// UI Automation's control type ids. A control's name is its caption. Everything else can hold what +// the user typed or the window title: a Document's name is the page title, a Pane's or Window's the +// window title, and a Text inside a rich editor is the typed text itself. +const NAMED_TYPES = new Set( + Object.values({ + Button: 50000, + CheckBox: 50002, + Hyperlink: 50005, + Image: 50006, + ListItem: 50007, + List: 50008, + Menu: 50009, + MenuBar: 50010, + MenuItem: 50011, + ProgressBar: 50012, + RadioButton: 50013, + ScrollBar: 50014, + Slider: 50015, + Spinner: 50016, + StatusBar: 50017, + Tab: 50018, + TabItem: 50019, + ToolBar: 50021, + Tree: 50023, + TreeItem: 50024, + Custom: 50025, + Group: 50026, + Thumb: 50027, + DataItem: 50029, + SplitButton: 50031, + Header: 50034, + HeaderItem: 50035, + Table: 50036, + }), +); +// Worth saying that a field was clicked, without saying anything about what it holds. +const ROLE_ONLY_TYPES = new Set(Object.values({ ComboBox: 50003, Edit: 50004 })); + +/** What may be said about an element, or null when its role and name say nothing. */ +export function elementHit(answer: ElementAnswer): ElementHit | null { + const type = answer.type ?? 0; + const named = NAMED_TYPES.has(type); + if (!named && !ROLE_ONLY_TYPES.has(type)) return null; + const role = clip(answer.role) ?? "element"; + const name = named ? (clip(answer.name) ?? clip(answer.help)) : undefined; + // Win32 controls number their ids; only a name helps an agent. + const automationId = /^\d*$/u.test(answer.automationId ?? "") + ? undefined + : clip(answer.automationId); + const parts = [ + name === undefined ? null : `“${name}”`, + automationId === undefined ? null : `#${automationId}`, + ].filter((part) => part !== null); + return parts.length === 0 ? { role } : { role, label: parts.join(" ") }; +} + +/** `button “Area”`, for what a drag moved or a stroke marks. Null when only its role is known. */ +export const elementOn = (hit: ElementHit | undefined): string | undefined => + hit?.label === undefined ? undefined : `${hit.role} ${hit.label}`; + +/** A browser tab is named after its page, which is the window's title. Short names only if equal. */ +export function namesWindowTitle(label: string, titles: ReadonlyArray): boolean { + const name = /“([^”]*)”/u.exec(label)?.[1]?.replace(/…$/u, "").toLowerCase(); + if (name === undefined || name === "") return false; + return titles.some((raw) => { + const title = raw.toLowerCase(); + return title === name || (name.length >= TITLE_MATCH_MIN && title.includes(name)); + }); +} + +interface Job { + readonly key: string; + readonly x: number; + readonly y: number; + readonly at: number; +} + +export interface LookupTransport { + readonly write: (line: string) => void; + readonly close: () => void; +} + +/** One lookup at a time, none written before the child is ready, none sent once stale. */ +export class ElementLookups { + readonly #transport: LookupTransport; + readonly #now: () => number; + readonly #hits = new Map(); + readonly #queue: Array = []; + #ready = false; + #inFlight: { readonly id: number; readonly key: string } | null = null; + #nextId = 1; + #onIdle: (() => void) | null = null; + + constructor(transport: LookupTransport, now: () => number = Date.now) { + this.#transport = transport; + this.#now = now; + } + + look(key: string, x: number, y: number): void { + this.#queue.push({ key, x, y, at: this.#now() }); + this.#pump(); + } + + #pump(): void { + if (!this.#ready || this.#inFlight !== null) return; + let job = this.#queue.shift(); + while (job !== undefined && this.#now() - job.at > STALE_MS) job = this.#queue.shift(); + if (job === undefined) { + this.#onIdle?.(); + return; + } + const id = this.#nextId++; + this.#inFlight = { id, key: job.key }; + this.#transport.write(`${id} ${job.x} ${job.y}\n`); + } + + answer(line: string): void { + if (line === "ready") { + this.#ready = true; + this.#pump(); + return; + } + let parsed: ElementAnswer & { readonly id?: number }; + try { + parsed = JSON.parse(line) as ElementAnswer & { readonly id?: number }; + } catch { + return; + } + const flight = this.#inFlight; + if (flight === null || parsed.id !== flight.id) return; + this.#inFlight = null; + const hit = elementHit(parsed); + if (hit !== null) this.#hits.set(flight.key, hit); + this.#pump(); + } + + /** Waits for a running lookup at most `waitMs`, then ends the child. */ + finish(waitMs: number): Promise> { + return new Promise((resolve) => { + const done = () => { + clearTimeout(timer); + this.#onIdle = null; + this.#transport.close(); + resolve(this.#hits); + }; + const timer = setTimeout(done, waitMs); + if (!this.#ready || (this.#inFlight === null && this.#queue.length === 0)) done(); + else this.#onIdle = done; + }); + } +} + +/** Starts the child. On any failure there are simply no names. */ +export function startElementLookups(): ElementLookups { + let child: NodeChildProcess.ChildProcess | null = null; + const lookups = new ElementLookups({ + write: (line) => { + child?.stdin?.write(line, () => undefined); + }, + close: () => { + child?.kill(); + }, + }); + try { + child = NodeChildProcess.spawn( + "powershell.exe", + ["-NoLogo", "-NoProfile", "-NonInteractive", "-MTA", "-Command", ELEMENT_SCRIPT], + { stdio: ["pipe", "pipe", "ignore"], windowsHide: true }, + ); + } catch { + return lookups; + } + child.on("error", () => undefined); + child.stdin?.on("error", () => undefined); + let pending = ""; + child.stdout?.setEncoding("utf8").on("data", (chunk: string) => { + pending += chunk; + const lines = pending.split(/\r?\n/u); + pending = lines.pop() ?? ""; + for (const line of lines) lookups.answer(line); + }); + return lookups; +} diff --git a/apps/desktop/src/scrawl/rawInput.test.ts b/apps/desktop/src/scrawl/rawInput.test.ts new file mode 100644 index 0000000000000..16a6af0dcb93a --- /dev/null +++ b/apps/desktop/src/scrawl/rawInput.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { chordLabel, parseRawInput } from "./rawInput.ts"; + +const block = (type: number, device: bigint, fill: (body: Buffer) => void, bodyBytes: number) => { + const buffer = Buffer.alloc(24 + bodyBytes); + buffer.writeUInt32LE(type, 0); + buffer.writeUInt32LE(buffer.length, 4); + buffer.writeBigUInt64LE(device, 8); + fill(buffer.subarray(24)); + return buffer; +}; +const mouse = (flags: number, data: number) => + block( + 0, + 1n, + (body) => { + body.writeUInt16LE(flags, 4); + body.writeInt16LE(data, 6); + }, + 24, + ); +const key = (virtualKey: number, flags: number, device = 1n) => + block( + 1, + device, + (body) => { + body.writeUInt16LE(flags, 2); + body.writeUInt16LE(virtualKey, 6); + }, + 16, + ); +// Blocks start on 8-byte boundaries; a real keyboard block is exactly 40 bytes, a mouse one 48. +const unaligned = (...blocks: Array) => + Buffer.concat( + blocks.flatMap((part) => [part, Buffer.alloc(((part.length + 7) & ~7) - part.length)]), + ); + +describe("parseRawInput", () => { + it("keeps the wheel and keys, in order, and drops moves and injected keys", () => { + const buffer = unaligned( + mouse(0, 0), + mouse(0x0400, 120), + mouse(0x0400, -240), + mouse(0x0800, 120), + key(0x43, 0), + key(0x43, 1), + key(0x56, 0, 0n), + ); + expect(parseRawInput(buffer, 7)).toEqual([ + { kind: "wheel", dx: 0, dy: 1 }, + { kind: "wheel", dx: 0, dy: -2 }, + { kind: "wheel", dx: 1, dy: 0 }, + { kind: "key", virtualKey: 0x43, down: true }, + { kind: "key", virtualKey: 0x43, down: false }, + ]); + }); + + it("stops at a block it cannot read instead of walking on through garbage", () => { + expect(parseRawInput(Buffer.alloc(96), 3)).toEqual([]); + }); +}); + +describe("chordLabel", () => { + const held = { ctrl: false, win: false, alt: false, shift: false, rightAlt: false }; + + it.each([ + ["typing", 0x41, held, null], + ["a capital", 0x41, { ...held, shift: true }, null], + ["Alt alone", 0x46, { ...held, alt: true }, null], + [ + "AltGr, typing @ on a German keyboard", + 0x51, + { ...held, ctrl: true, alt: true, rightAlt: true }, + null, + ], + ["a key not worth logging", 0xbe, { ...held, ctrl: true }, null], + ["Ctrl+C", 0x43, { ...held, ctrl: true }, "Ctrl+C"], + ["Ctrl+Shift+Enter", 0x0d, { ...held, ctrl: true, shift: true }, "Ctrl+Shift+Enter"], + ["the left Ctrl+Alt", 0x54, { ...held, ctrl: true, alt: true }, "Ctrl+Alt+T"], + ["Win+F5", 0x74, { ...held, win: true }, "Win+F5"], + ])("%s", (_name, virtualKey, modifiers, expected) => { + expect(chordLabel(virtualKey, modifiers)).toBe(expected); + }); +}); diff --git a/apps/desktop/src/scrawl/rawInput.ts b/apps/desktop/src/scrawl/rawInput.ts new file mode 100644 index 0000000000000..14b5ae5235212 --- /dev/null +++ b/apps/desktop/src/scrawl/rawInput.ts @@ -0,0 +1,91 @@ +// Reads what GetRawInputBuffer hands back. Keys are reduced to chords right here, so ordinary typing +// never leaves this file. + +const HEADER_BYTES = 24; +const TYPE_MOUSE = 0; +const TYPE_KEYBOARD = 1; +const WHEEL = 0x0400; +const HORIZONTAL_WHEEL = 0x0800; +const WHEEL_NOTCH = 120; +const KEY_BREAK = 1; + +export type RawInput = + | { readonly kind: "wheel"; readonly dx: number; readonly dy: number } + | { readonly kind: "key"; readonly virtualKey: number; readonly down: boolean }; + +/** Mouse moves are most of what arrives and are dropped here. Injected keys carry no device. */ +export function parseRawInput(buffer: Buffer, count: number): Array { + const inputs: Array = []; + let offset = 0; + for (let index = 0; index < count && offset + HEADER_BYTES <= buffer.length; index++) { + const type = buffer.readUInt32LE(offset); + const size = buffer.readUInt32LE(offset + 4); + if (size < HEADER_BYTES) break; + const body = offset + HEADER_BYTES; + if (type === TYPE_MOUSE) { + const flags = buffer.readUInt16LE(body + 4); + const delta = buffer.readInt16LE(body + 6) / WHEEL_NOTCH; + if (flags & WHEEL) inputs.push({ kind: "wheel", dx: 0, dy: delta }); + else if (flags & HORIZONTAL_WHEEL) inputs.push({ kind: "wheel", dx: delta, dy: 0 }); + } else if (type === TYPE_KEYBOARD && buffer.readBigUInt64LE(offset + 8) !== 0n) { + inputs.push({ + kind: "key", + virtualKey: buffer.readUInt16LE(body + 6), + down: (buffer.readUInt16LE(body + 2) & KEY_BREAK) === 0, + }); + } + offset += (size + 7) & ~7; + } + return inputs; +} + +export interface HeldModifiers { + readonly ctrl: boolean; + readonly win: boolean; + readonly alt: boolean; + readonly shift: boolean; + /** The right Alt. On many layouts it is AltGr, which Windows reports as Ctrl+Alt. */ + readonly rightAlt: boolean; +} + +/** A key's name in a logged chord, for the keys worth logging. */ +const chordKeyName = (virtualKey: number): string | null => { + if ((virtualKey >= 0x30 && virtualKey <= 0x39) || (virtualKey >= 0x41 && virtualKey <= 0x5a)) { + return String.fromCharCode(virtualKey); + } + if (virtualKey >= 0x70 && virtualKey <= 0x87) return `F${virtualKey - 0x6f}`; + switch (virtualKey) { + case 0x0d: + return "Enter"; + case 0x09: + return "Tab"; + case 0x20: + return "Space"; + case 0x08: + return "Backspace"; + case 0x2e: + return "Delete"; + default: + return null; + } +}; + +/** + * `Ctrl+Shift+C`, or null for anything that may be typing. AltGr is typing: on a German keyboard + * AltGr+Q is an @, and logging it as Ctrl+Alt+Q would spell out part of a password. + */ +export function chordLabel(virtualKey: number, held: HeldModifiers): string | null { + if (!held.ctrl && !held.win) return null; + if (held.rightAlt && !held.win) return null; + const key = chordKeyName(virtualKey); + if (key === null) return null; + return [ + held.ctrl ? "Ctrl" : null, + held.win ? "Win" : null, + held.alt ? "Alt" : null, + held.shift ? "Shift" : null, + key, + ] + .filter((part) => part !== null) + .join("+"); +} diff --git a/apps/desktop/src/scrawl/scrawlContextProcess.ts b/apps/desktop/src/scrawl/scrawlContextProcess.ts new file mode 100644 index 0000000000000..110b70fd98192 --- /dev/null +++ b/apps/desktop/src/scrawl/scrawlContextProcess.ts @@ -0,0 +1,70 @@ +// @effect-diagnostics globalTimers:off -- Helper timeouts run at child-process callback boundaries outside Effect fibers. +// @effect-diagnostics nodeBuiltinImport:off -- This desktop-only helper owns a Node child process. + +import * as NodeChildProcess from "node:child_process"; + +import type { + ScrawlContextInit, + ScrawlContextMessage, + ScrawlContextRequest, +} from "./ScrawlContextWorker.ts"; + +const STOP_WAIT_MS = 300; + +export interface ScrawlContextProcess { + readonly requestSnapshot: () => void; + /** Resolves once what the child still held has arrived, or soon after it stopped answering. */ + readonly stop: () => Promise; + readonly kill: () => void; +} + +/** Without it a capture still has its clicks, ink and cursor: this only adds to them. */ +export function startScrawlContextProcess( + workerPath: string, + init: ScrawlContextInit, + onMessage: (message: ScrawlContextMessage) => void, +): ScrawlContextProcess { + let worker: NodeChildProcess.ChildProcess | null = null; + try { + worker = NodeChildProcess.fork(workerPath, [], { + env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" }, + execArgv: [], + stdio: ["ignore", "ignore", "ignore", "ipc"], + }); + } catch { + worker = null; + } + const child = worker; + const send = (request: ScrawlContextRequest) => { + if (child?.connected !== true) return; + child.send(request, () => undefined); + }; + child?.on("error", () => undefined); + child?.on("message", (message) => onMessage(message as ScrawlContextMessage)); + send({ type: "init", ...init }); + + return { + requestSnapshot: () => send({ type: "snapshot" }), + stop: () => + new Promise((resolve) => { + if (child === null || child.exitCode !== null) { + resolve(); + return; + } + const finish = () => { + clearTimeout(timer); + child.kill(); + resolve(); + }; + const timer = setTimeout(finish, STOP_WAIT_MS); + child.on("message", (message) => { + if ((message as ScrawlContextMessage).type === "stopped") finish(); + }); + child.once("exit", finish); + send({ type: "stop" }); + }), + kill: () => { + child?.kill(); + }, + }; +} diff --git a/apps/desktop/src/scrawl/windowAtPoint.ts b/apps/desktop/src/scrawl/windowAtPoint.ts new file mode 100644 index 0000000000000..71c8523003a18 --- /dev/null +++ b/apps/desktop/src/scrawl/windowAtPoint.ts @@ -0,0 +1,15 @@ +// WindowFromPoint takes its POINT by value, which on x64 travels as one 64-bit integer. + +export async function loadWindowAtPoint(): Promise<(x: number, y: number) => bigint> { + const { DataType, load, open } = await import("ffi-rs"); + const library = "scrawl-point-user32"; + open({ library, path: "user32.dll" }); + return (x, y) => + load({ + library, + funcName: "WindowFromPoint", + retType: DataType.BigInt, + paramsType: [DataType.BigInt], + paramsValue: [(BigInt(Math.round(y)) << 32n) | (BigInt(Math.round(x)) & 0xffff_ffffn)], + }) as bigint; +} diff --git a/apps/desktop/src/scrawl/windowList.test.ts b/apps/desktop/src/scrawl/windowList.test.ts new file mode 100644 index 0000000000000..7f957c0cf9e31 --- /dev/null +++ b/apps/desktop/src/scrawl/windowList.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { AppSwitchWatcher, filterWindows, taskbarStrip, type RawWindow } from "./windowList.ts"; + +const FRAME = { x: 1920, y: 0, width: 2560, height: 1440 }; +const raw = (over: Partial & { readonly id: number }): RawWindow => ({ + processId: 7, + app: "Helium", + bundleId: "chrome.exe", + title: "Some page", + className: "Chrome_WidgetWin_1", + rect: { x: 2000, y: 100, width: 800, height: 600 }, + exStyle: 0, + owned: false, + minimized: false, + cloaked: false, + ...over, +}); + +describe("filterWindows", () => { + it("keeps what Scrawl's log keeps, relative to the display, and marks the front window", () => { + const entries = filterWindows( + [ + raw({ id: 1, processId: 9, app: "Other", bundleId: "other.exe" }), + raw({ id: 2 }), + raw({ id: 3 }), + raw({ id: 4, rect: { x: 0, y: 0, width: 800, height: 600 } }), + raw({ id: 5, minimized: true }), + raw({ id: 6, cloaked: true }), + raw({ id: 7, rect: { x: 2000, y: 100, width: 20, height: 600 } }), + raw({ id: 8, className: "Progman" }), + raw({ id: 9 }), + raw({ id: 10, processId: 11, app: "Tool", exStyle: 0x80 }), + raw({ id: 11, exStyle: 0x8 }), + ], + FRAME, + null, + 7, + new Set([9]), + ); + expect(entries.map((entry) => [entry.id, entry.layer, entry.frontmost ?? false])).toEqual([ + [1, 0, false], + [2, 0, true], + [3, 0, false], + [11, 3, false], + ]); + expect(entries[1]).toMatchObject({ bounds: [80, 100, 800, 600], title: "Some page" }); + }); + + it("gives the taskbar the strip it reserves on this display, or drops it", () => { + const taskbar = raw({ + id: 1, + app: "Windows Explorer", + className: "Shell_TrayWnd", + rect: { ...FRAME }, + exStyle: 0x80, + }); + const dock = taskbarStrip(FRAME, { x: 1920, y: 0, width: 2560, height: 1400 }); + expect(dock).toEqual({ x: 1920, y: 1400, width: 2560, height: 40 }); + expect(filterWindows([taskbar], FRAME, dock, null, new Set())).toMatchObject([ + { app: "Dock", layer: 20, bounds: [0, 1400, 2560, 40] }, + ]); + expect(filterWindows([taskbar], FRAME, null, null, new Set())).toEqual([]); + }); +}); + +describe("AppSwitchWatcher", () => { + const front = (app: string, className = "Window") => ({ app, className }); + + it("reports an app that stayed, when it came, and says nothing of the shell in between", () => { + const watcher = new AppSwitchWatcher(); + expect(watcher.look(0, front("Helium"))).toBeNull(); + expect(watcher.look(100, front("Explorer", "XamlExplorerHostIslandWindow"))).toBeNull(); + expect(watcher.look(150, null)).toBeNull(); + expect(watcher.look(200, front("Code"))).toBeNull(); + expect(watcher.look(450, front("Code"))).toEqual({ app: "Code", t: 200 }); + expect(watcher.look(500, front("Code"))).toBeNull(); + }); + + it("ignores an app that was in front only for a moment", () => { + const watcher = new AppSwitchWatcher(); + watcher.look(0, front("Helium")); + expect(watcher.look(100, front("Code"))).toBeNull(); + expect(watcher.look(150, front("Helium"))).toBeNull(); + expect(watcher.look(1_000, front("Helium"))).toBeNull(); + }); +}); diff --git a/apps/desktop/src/scrawl/windowList.ts b/apps/desktop/src/scrawl/windowList.ts new file mode 100644 index 0000000000000..db346b919a0b7 --- /dev/null +++ b/apps/desktop/src/scrawl/windowList.ts @@ -0,0 +1,143 @@ +// Scrawl's rules for which windows the log keeps and how an app switch is told from the shell +// passing through. + +import type { ScrawlWindowEntry } from "@t3tools/contracts"; + +export interface PixelRect { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface RawWindow { + readonly id: number; + readonly processId: number; + readonly app: string; + readonly bundleId: string; + readonly title: string; + readonly className: string; + readonly rect: PixelRect; + readonly exStyle: number; + readonly owned: boolean; + readonly minimized: boolean; + readonly cloaked: boolean; +} + +const WS_EX_TOPMOST = 0x8; +const WS_EX_TRANSPARENT = 0x20; +const WS_EX_TOOLWINDOW = 0x80; +const WS_EX_APPWINDOW = 0x40000; +const WS_EX_NOACTIVATE = 0x8000000; + +const SHELL_CLASSES = new Set(["Progman", "WorkerW", "Shell_TrayWnd", "Shell_SecondaryTrayWnd"]); +const TASKBAR_CLASSES = new Set(["Shell_TrayWnd", "Shell_SecondaryTrayWnd"]); + +const isAppWindow = (window: RawWindow) => { + const tool = (window.exStyle & WS_EX_TOOLWINDOW) !== 0; + const app = (window.exStyle & WS_EX_APPWINDOW) !== 0; + const passThrough = (window.exStyle & (WS_EX_TRANSPARENT | WS_EX_NOACTIVATE)) !== 0; + return (app || (!tool && !window.owned)) && !passThrough && !SHELL_CLASSES.has(window.className); +}; + +const overlaps = (a: PixelRect, b: PixelRect) => + a.x < b.x + b.width && a.x + a.width > b.x && a.y < b.y + b.height && a.y + a.height > b.y; + +/** The strip the taskbar reserves on a display: the part of it outside the work area. */ +export function taskbarStrip(display: PixelRect, workArea: PixelRect): PixelRect | null { + const left = workArea.x - display.x; + const top = workArea.y - display.y; + const right = display.x + display.width - (workArea.x + workArea.width); + const bottom = display.y + display.height - (workArea.y + workArea.height); + const MIN = 8; + if (bottom >= MIN) return { ...display, y: workArea.y + workArea.height, height: bottom }; + if (top >= MIN) return { ...display, height: top }; + if (left >= MIN) return { ...display, width: left }; + if (right >= MIN) return { ...display, x: workArea.x + workArea.width, width: right }; + return null; +} + +/** `windows` runs front to back. Bounds come out relative to the captured display. */ +export function filterWindows( + windows: ReadonlyArray, + frame: PixelRect, + dock: PixelRect | null, + frontProcessId: number | null, + ignoredIds: ReadonlySet, +): Array { + const regularProcesses = new Set(windows.filter(isAppWindow).map((w) => w.processId)); + const entries: Array = []; + let frontMarked = false; + for (const window of windows) { + if (ignoredIds.has(window.id) || window.minimized || window.cloaked) continue; + const taskbar = TASKBAR_CLASSES.has(window.className); + if (taskbar ? dock === null : SHELL_CLASSES.has(window.className)) continue; + const rect = taskbar ? dock! : window.rect; + if (rect.width < 24 || rect.height < 16 || !overlaps(rect, frame)) continue; + const layer = taskbar + ? 20 + : isAppWindow(window) && (window.exStyle & WS_EX_TOPMOST) === 0 + ? 0 + : 3; + const regular = regularProcesses.has(window.processId); + if (layer === 3 && !regular) continue; + const frontmost: boolean = !frontMarked && layer === 0 && window.processId === frontProcessId; + frontMarked ||= frontmost; + entries.push({ + id: window.id, + app: taskbar ? "Dock" : window.app, + ...(window.bundleId === "" ? {} : { bundleId: window.bundleId }), + ...(regular || taskbar ? {} : { regular: false }), + ...(window.title === "" ? {} : { title: window.title }), + bounds: [ + Math.round(rect.x - frame.x), + Math.round(rect.y - frame.y), + Math.round(rect.width), + Math.round(rect.height), + ], + layer, + ...(frontmost ? { frontmost: true } : {}), + }); + } + return entries; +} + +// Alt+Tab and the taskbar hold the foreground for a moment; the app that follows is the switch. +const PASSING_CLASSES = new Set([ + "ForegroundStaging", + "LauncherTipWnd", + "MultitaskingViewFrame", + "Shell_SecondaryTrayWnd", + "Shell_TrayWnd", + "TaskSwitcherWnd", + "XamlExplorerHostIslandWindow", +]); +const SETTLE_MS = 200; + +/** Reports an app once it has stayed in front, so passing through the shell says nothing. */ +export class AppSwitchWatcher { + #current: string | null = null; + #pending: { readonly app: string; readonly t: number } | null = null; + + /** Returns the app and when it came to the front, once that is settled. */ + look( + t: number, + front: { readonly app: string; readonly className: string } | null, + ): { readonly app: string; readonly t: number } | null { + if (front === null || front.app === "" || PASSING_CLASSES.has(front.className)) return null; + if (this.#current === null) { + this.#current = front.app; + return null; + } + if (front.app === this.#current) { + this.#pending = null; + return null; + } + if (this.#pending?.app !== front.app) this.#pending = { app: front.app, t }; + if (t - this.#pending.t < SETTLE_MS) return null; + const settled = this.#pending; + this.#current = settled.app; + this.#pending = null; + return settled; + } +} diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index 55c3aa03b6490..39d1432cb935c 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -75,6 +75,7 @@ export default defineConfig({ define: publicConfigDefine, entry: [ "src/electron/WindowsForegroundFocusWorker.ts", + "src/scrawl/ScrawlContextWorker.ts", "src/snapShot/GlobalShiftShortcutWorker.ts", "src/snapShot/RegionSnapShotWorker.ts", "src/snapShot/SnapShotAccessibilityWorker.ts", diff --git a/apps/web/src/scrawl/narration.test.ts b/apps/web/src/scrawl/narration.test.ts index 2c82b0b07cf7b..e35c0e933f1b2 100644 --- a/apps/web/src/scrawl/narration.test.ts +++ b/apps/web/src/scrawl/narration.test.ts @@ -129,6 +129,124 @@ describe("buildNarration", () => { expect(formatNarrationLine(lines.at(-1)!, 1)).toBe("- #2 (0:20) clicked"); }); + it("names what a click landed on, and still says pointed when the screen did not answer", () => { + const lines = buildNarration({ + words: say("Das hier und das hier.", 0), + events: [ + { kind: "click", t: 100, x: 0.1, y: 0.1, role: "button", label: "“Area”" }, + { kind: "click", t: 200, x: 0.1, y: 0.1, role: "button", label: "“Area”" }, + { kind: "click", t: 300, x: 0.2, y: 0.1 }, + { kind: "click", t: 9_000, x: 0.5, y: 0.5, role: "tab item", label: "“Files”" }, + ], + marks: [{ id: "m1", t0: 400, t1: 500, on: "button “Save”" }], + durationMs: 12_000, + sceneChanges: [9_400], + }); + expect(formatNarrationLine(lines[0]!, 0)).toContain( + "[marked button “Save”, pointed at button “Area” ×2, pointed]", + ); + expect(formatNarrationLine(lines[1]!, 1)).toBe("- #2 (0:09) clicked tab item “Files”"); + }); + + it("says shortcuts, app switches and windows at their place, without an image or a cut", () => { + const sentence = "Bitte diese Boxen hier entfernen und den Text darunter auch."; + const lines = buildNarration({ + words: say(sentence, 0), + events: [ + { kind: "click", t: 300, x: 0.1, y: 0.1 }, + { kind: "key", t: 600, shortcut: "Ctrl+C" }, + { kind: "key", t: 900, shortcut: "Ctrl+C" }, + { kind: "app_change", t: 1_200, app: "Helium" }, + { kind: "window", t: 1_500, text: "Helium window opened" }, + ], + marks: [], + durationMs: 4_000, + }); + expect(lines.map((line) => line.text)).toEqual([sentence]); + expect(lines[0]!.frameT).toBe(600); + expect(formatNarrationLine(lines[0]!, 0)).toContain( + "[clicked, pressed Ctrl+C ×2, switched to Helium, Helium window opened]", + ); + }); + + it("gives no frame to a shortcut in a silence, and folds it into the keyframe before", () => { + const lines = buildNarration({ + words: say("Hallo Welt.", 0), + events: [{ kind: "key", t: 20_000, shortcut: "Ctrl+S" }], + marks: [{ id: "m1", t0: 100, t1: 300 }], + durationMs: 25_000, + }); + expect(lines).toHaveLength(1); + expect(lines[0]!.frameT).toBe(300); + expect(formatNarrationLine(lines[0]!, 0)).toContain("pressed Ctrl+S"); + }); + + it("says which way the wheel went, and the screen answering it is not a scene of its own", () => { + const lines = buildNarration({ + words: [], + events: [ + { kind: "scroll", t: 1_000, dx: 0, dy: -1 }, + { kind: "scroll", t: 1_200, dx: 0, dy: -1 }, + { kind: "scroll", t: 9_000, dx: 0, dy: 1 }, + { kind: "scroll", t: 9_200, dx: 0, dy: -1 }, + ], + marks: [], + durationMs: 12_000, + sceneChanges: [1_100, 1_900, 9_300], + }); + expect(lines.map(formatNarrationLine)).toEqual([ + "- #1 (0:02) scrolled down", + "- #2 (0:10) scrolled up and down", + ]); + }); + + it("gives a flag its own keyframe and marker, and tells it from a mark", () => { + const flag = { id: "f1", t0: 20_000, t1: 20_000, shape: "flag" }; + const said = buildNarration({ + words: say("Hallo Welt.", 19_800), + events: [], + marks: [flag], + durationMs: 25_000, + }); + expect(formatNarrationLine(said[0]!, 0)).toBe("- #1 (0:20) ⚑ “Hallo Welt.”"); + const silent = buildNarration({ words: [], events: [], marks: [flag], durationMs: 25_000 }); + expect(formatNarrationLine(silent[0]!, 0)).toBe("- #1 (0:20) ⚑ flagged this moment"); + }); + + it("says a named drag however short, and a scrub without its direction", () => { + const line = buildNarration({ + words: [], + events: [ + { + kind: "drag", + t: 100, + t1: 200, + x0: 0.5, + y0: 0.5, + x1: 0.51, + y1: 0.5, + label: "slider “Volume”", + }, + { + kind: "drag", + t: 400, + t1: 500, + x0: 0.5, + y0: 0.5, + x1: 0.6, + y1: 0.5, + label: "button “Tab”", + }, + { kind: "drag", t: 700, t1: 800, x0: 0.5, y0: 0.5, x1: 0.51, y1: 0.5 }, + ], + marks: [], + durationMs: 1_000, + })[0]!; + expect(formatNarrationLine(line, 0)).toBe( + "- #1 (0:01) scrubbed slider “Volume”, dragged button “Tab” right", + ); + }); + it("keeps every mark over budget, and speech ahead of the first keyframe with it", () => { const marks = Array.from({ length: 9 }, (_, index) => ({ id: `m${index}`, diff --git a/apps/web/src/scrawl/narration.ts b/apps/web/src/scrawl/narration.ts index a9b67540af783..5190ea8178295 100644 --- a/apps/web/src/scrawl/narration.ts +++ b/apps/web/src/scrawl/narration.ts @@ -10,7 +10,14 @@ export interface TimedWord { } export type CaptureEvent = - | { readonly kind: "click"; readonly t: number; readonly x: number; readonly y: number } + | { + readonly kind: "click"; + readonly t: number; + readonly x: number; + readonly y: number; + readonly role?: string; + readonly label?: string; + } | { readonly kind: "drag"; readonly t: number; @@ -19,14 +26,20 @@ export type CaptureEvent = readonly y0: number; readonly x1: number; readonly y1: number; + readonly label?: string; } - | { readonly kind: "scroll"; readonly t: number }; + | { readonly kind: "scroll"; readonly t: number; readonly dx?: number; readonly dy?: number } + | { readonly kind: "key"; readonly t: number; readonly shortcut: string } + | { readonly kind: "app_change"; readonly t: number; readonly app: string } + | { readonly kind: "window"; readonly t: number; readonly text: string }; export interface CaptureMark { readonly id: string; readonly t0: number; readonly t1: number; readonly shape?: string; + /** What it was drawn on, such as `button “Area”`. */ + readonly on?: string; } export type NarrationAction = @@ -36,13 +49,41 @@ export type NarrationAction = readonly t1: number; readonly count: number; readonly points: ReadonlyArray; + /** What each click landed on, in step with `points`, where something names it. */ + readonly targets: ReadonlyArray; /** False when the screen did not react: the hand was pointing, not operating anything. */ readonly changed?: boolean; } - | { readonly kind: "drag"; readonly t0: number; readonly t1: number; readonly direction: string } - | { readonly kind: "scroll"; readonly t0: number; readonly t1: number } - | { readonly kind: "mark"; readonly t0: number; readonly t1: number; readonly id: string } - | { readonly kind: "scene"; readonly t0: number; readonly t1: number }; + | { + readonly kind: "drag"; + readonly t0: number; + readonly t1: number; + readonly direction: string; + readonly target?: string; + } + | { + readonly kind: "scroll"; + readonly t0: number; + readonly t1: number; + readonly directions: ReadonlyArray; + } + | { + readonly kind: "mark"; + readonly t0: number; + readonly t1: number; + readonly id: string; + readonly on?: string; + readonly flag?: boolean; + } + | { readonly kind: "scene"; readonly t0: number; readonly t1: number } + /** Said at its place in the speech, but never worth an image of its own. */ + | { + readonly kind: "note"; + readonly t0: number; + readonly t1: number; + readonly text: string; + readonly count: number; + }; export interface NarrationLine { readonly t0: number; @@ -67,8 +108,11 @@ const MAX_SENTENCE_WORDS = 60; const BURST_GAP_MS = 1_500; const REACTION_MS = 1_000; const SCROLL_GAP_MS = 1_500; +const NOTE_REPEAT_MS = 3_000; // Scrawl's rule: a drag this short (as a share of the screen) is the hand moving, not an act. const SHORT_DRAG = 0.15; +const NO_DIRECTION_DRAG = 0.04; +const SCRUBBED = /slider|scroll bar|scrubber|timeline|progress/iu; // An action in a speech pause goes to the sentence it is nearest to, up to this far away. const ATTACH_MS = 3_000; const SETTLE_MS = 300; @@ -84,6 +128,7 @@ const SCORES: Record = { drag: 50, scene: 40, scroll: 30, + note: 0, }; const endsSentence = (word: string) => /[.!?;:]["”’)\]]*$/.test(word); @@ -134,43 +179,83 @@ export function foldActions( t0: mark.t0, t1: mark.t1, id: mark.id, + ...(mark.on === undefined ? {} : { on: mark.on }), + ...(mark.shape === "flag" ? { flag: true } : {}), })); const bursts: Array<{ t0: number; t1: number; count: number; points: Array; + targets: Array; }> = []; - const scrolls: Array<{ t0: number; t1: number }> = []; + const scrolls: Array<{ t0: number; t1: number; directions: Set }> = []; + const notes: Array<{ t0: number; t1: number; text: string; count: number }> = []; + const addNote = (t: number, text: string) => { + const same = notes.findLast((note) => note.text === text); + if (same !== undefined && t - same.t1 < NOTE_REPEAT_MS) { + same.t1 = t; + same.count++; + } else { + notes.push({ t0: t, t1: t, text, count: 1 }); + } + }; for (const event of [...events].sort((a, b) => a.t - b.t)) { if (event.kind === "click") { + const target = [event.role, event.label].filter(Boolean).join(" ") || null; const burst = bursts.at(-1); if (burst !== undefined && event.t - burst.t1 <= BURST_GAP_MS) { burst.t1 = event.t; burst.count++; burst.points.push([event.x, event.y]); + burst.targets.push(target); } else { - bursts.push({ t0: event.t, t1: event.t, count: 1, points: [[event.x, event.y]] }); + bursts.push({ + t0: event.t, + t1: event.t, + count: 1, + points: [[event.x, event.y]], + targets: [target], + }); } } else if (event.kind === "scroll") { - const scroll = scrolls.at(-1); + let scroll = scrolls.at(-1); if (scroll !== undefined && event.t - scroll.t1 <= SCROLL_GAP_MS) scroll.t1 = event.t; - else scrolls.push({ t0: event.t, t1: event.t }); + else { + scroll = { t0: event.t, t1: event.t, directions: new Set() }; + scrolls.push(scroll); + } + // A wheel turned away from the hand goes up the page, which Windows counts as positive. + if ((event.dy ?? 0) > 0) scroll.directions.add("up"); + if ((event.dy ?? 0) < 0) scroll.directions.add("down"); + if ((event.dx ?? 0) > 0) scroll.directions.add("right"); + if ((event.dx ?? 0) < 0) scroll.directions.add("left"); + } else if (event.kind === "key") { + addNote(event.t, `pressed ${event.shortcut}`); + } else if (event.kind === "app_change") { + addNote(event.t, `switched to ${event.app}`); + } else if (event.kind === "window") { + addNote(event.t, event.text); } else { const dx = event.x1 - event.x0; const dy = event.y1 - event.y0; - if (Math.hypot(dx, dy) >= SHORT_DRAG) { + const length = Math.hypot(dx, dy); + if (length >= SHORT_DRAG || event.label !== undefined) { actions.push({ kind: "drag", t0: event.t, t1: event.t1, - direction: directionOf(dx, dy), + direction: length < NO_DIRECTION_DRAG ? "" : directionOf(dx, dy), + ...(event.label === undefined ? {} : { target: event.label }), }); } } } for (const burst of bursts) actions.push({ kind: "clicks", ...burst }); - for (const scroll of scrolls) actions.push({ kind: "scroll", ...scroll }); + for (const scroll of scrolls) { + actions.push({ kind: "scroll", ...scroll, directions: [...scroll.directions] }); + } + for (const note of notes) actions.push({ kind: "note", ...note }); return actions.sort((a, b) => a.t0 - b.t0); } @@ -235,7 +320,7 @@ const chooseFrameOwners = (sentences: ReadonlyArray, durationMs: numbe const frameTimeOf = (t0: number, actions: ReadonlyArray, durationMs: number) => { const marks = actions.filter((action) => action.kind === "mark"); - const moment = marks.at(-1) ?? actions.at(-1); + const moment = marks.at(-1) ?? actions.findLast((action) => SCORES[action.kind] > 0); if (moment === undefined) return t0; const settle = moment.kind === "mark" ? 0 : SETTLE_MS; return Math.min(moment.t1 + settle, durationMs); @@ -246,7 +331,10 @@ const withReactions = ( actions: ReadonlyArray, sceneChanges: ReadonlyArray, ): Array => { - const clickTimes = actions.filter((action) => action.kind === "clicks"); + // The screen answering the wheel is no news of its own, like the screen answering a click. + const clickTimes = actions.filter( + (action) => action.kind === "clicks" || action.kind === "scroll", + ); const reacted = (action: { readonly t0: number; readonly t1: number }) => sceneChanges.some((t) => t >= action.t0 && t <= action.t1 + REACTION_MS); const scenes: Array = sceneChanges @@ -307,27 +395,49 @@ const formatClock = (ms: number) => { return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; }; +const dragText = (drag: Extract) => { + if (drag.target === undefined) return `dragged ${drag.direction}`; + // A scrub's direction says nothing: the frame shows where it landed. + if (SCRUBBED.test(drag.target)) return `scrubbed ${drag.target}`; + return [`dragged ${drag.target}`, drag.direction].filter(Boolean).join(" "); +}; + const describeActions = (actions: ReadonlyArray) => { - let marks = 0; - let clicks = 0; - let pointed = 0; - let scrolled = false; + const marks = new Map(); + const clicks = new Map(); + const pointed = new Map(); + const notes = new Map(); + const add = (into: Map, text: string, count = 1) => + into.set(text, (into.get(text) ?? 0) + count); const drags = new Set(); + const directions = new Set(); + let scrolled = false; for (const action of actions) { - if (action.kind === "mark") marks++; - else if (action.kind === "clicks") { - if (action.changed === false) pointed += action.count; - else clicks += action.count; - } else if (action.kind === "scroll") scrolled = true; - else if (action.kind === "drag") drags.add(action.direction); + if (action.kind === "mark") { + if (action.flag !== true) + add(marks, action.on === undefined ? "marked" : `marked ${action.on}`); + } else if (action.kind === "clicks") { + for (const target of action.targets) { + if (action.changed === false) { + add(pointed, target === null ? "pointed" : `pointed at ${target}`); + } else { + add(clicks, target === null ? "clicked" : `clicked ${target}`); + } + } + } else if (action.kind === "scroll") { + scrolled = true; + for (const direction of action.directions) directions.add(direction); + } else if (action.kind === "drag") drags.add(dragText(action)); + else if (action.kind === "note") add(notes, action.text, action.count); } - const counted = (word: string, count: number) => (count > 1 ? `${word} ×${count}` : word); + const counted = ([text, count]: [string, number]) => (count > 1 ? `${text} ×${count}` : text); return [ - marks > 0 ? counted("marked", marks) : null, - clicks > 0 ? counted("clicked", clicks) : null, - pointed > 0 ? counted("pointed", pointed) : null, - ...[...drags].map((direction) => `dragged ${direction}`), - scrolled ? "scrolled" : null, + ...[...marks].map(counted), + ...[...clicks].map(counted), + ...[...pointed].map(counted), + ...drags, + scrolled ? ["scrolled", [...directions].join(" and ")].filter(Boolean).join(" ") : null, + ...[...notes].map(counted), ] .filter((part) => part !== null) .join(", "); @@ -335,9 +445,11 @@ const describeActions = (actions: ReadonlyArray) => { /** `- #2 (0:09) [clicked ×3] “…”`: the shape Scrawl pastes, which agents and skills already read. */ export function formatNarrationLine(line: NarrationLine, index: number): string { - const head = `- #${index + 1} (${formatClock(line.frameT)})`; + const flagged = line.actions.some((action) => action.kind === "mark" && action.flag === true); + const head = `- #${index + 1} (${formatClock(line.frameT)})${flagged ? " ⚑" : ""}`; const actions = describeActions(line.actions); if (line.text === "") { + if (flagged && actions === "") return `${head} flagged this moment`; const shown = line.actions.length > 0 ? "screen changed" : "nothing said or done"; return `${head} ${actions === "" ? shown : actions}`; } diff --git a/apps/web/src/scrawl/scrawlBundle.test.ts b/apps/web/src/scrawl/scrawlBundle.test.ts new file mode 100644 index 0000000000000..a64d7c761c292 --- /dev/null +++ b/apps/web/src/scrawl/scrawlBundle.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { summaryText } from "./scrawlBundle.ts"; + +describe("summaryText", () => { + it("names the app and keeps the window title out of what an agent reads", () => { + const text = summaryText( + { + durationMs: 25_000, + session: { + id: "scrawl-a", + sourceId: "screen:1", + width: 100, + height: 100, + startedAtMs: 0, + app: "Helium", + windowTitle: "A private title", + bundlePath: "C:/captures/scrawl-a.scrawl", + }, + }, + [], + false, + ); + expect(text).toContain("[scrawl 0:25 · Helium · C:/captures/scrawl-a.scrawl]"); + expect(text).not.toContain("A private title"); + }); +}); diff --git a/apps/web/src/scrawl/scrawlBundle.ts b/apps/web/src/scrawl/scrawlBundle.ts index f90df615ecd55..1322d1c8794ce 100644 --- a/apps/web/src/scrawl/scrawlBundle.ts +++ b/apps/web/src/scrawl/scrawlBundle.ts @@ -17,6 +17,7 @@ import { type TimedWord, } from "./narration"; import { withoutInkChanges } from "./sceneChanges"; +import { windowChanges } from "./windowChanges"; const SHEET_LONG_EDGE = 1600; const SHEET_GUTTER = 6; @@ -63,7 +64,14 @@ const manifestStroke = (stroke: ScrawlStroke, width: number, height: number) => count > 1 ? Math.round((span * index) / (count - 1)) : 0, ); } - return { t0: stroke.t0, t1: stroke.t1, fw: width, fh: height, pts }; + return { + t0: stroke.t0, + t1: stroke.t1, + fw: width, + fh: height, + pts, + ...(stroke.on === undefined ? {} : { on: stroke.on }), + }; }; const KEYFRAME_REASONS: Record = { @@ -72,6 +80,7 @@ const KEYFRAME_REASONS: Record drag: "click", scene: "scene_change", scroll: "scene_change", + note: "scene_change", }; export interface BuiltBundle { @@ -80,6 +89,18 @@ export interface BuiltBundle { readonly lines: ReadonlyArray; } +// Scrawl's time-only marker: no ink, the whole screen, and always a keyframe of its own. +const flagsToMarks = (flags: ReadonlyArray): Array => + flags.map((t, index) => ({ + id: `f${index + 1}`, + t0: t, + t1: t, + strokes: [], + boxNorm: [0, 0, 1, 1], + shape: "flag", + description: "flagged this moment", + })); + const strokesToMarks = (strokes: ReadonlyArray): Array => { const groups: Array<{ t0: number; t1: number; strokes: Array }> = []; strokes.forEach((stroke, index) => { @@ -97,9 +118,11 @@ const strokesToMarks = (strokes: ReadonlyArray): Array (at % 2 === 0 ? xs : ys).push(value)); } + const on = group.strokes.map((strokeIndex) => strokes[strokeIndex]!.on).find(Boolean); return { id: `m${index + 1}`, ...group, + ...(on === undefined ? {} : { on }), boxNorm: [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)], shape: "mark", description: `marked (${group.strokes.length} ${group.strokes.length === 1 ? "stroke" : "strokes"})`, @@ -108,7 +131,7 @@ const strokesToMarks = (strokes: ReadonlyArray): Array): { +function activityBox(recording: { readonly input: Pick }): { readonly x: number; readonly y: number; readonly width: number; @@ -120,7 +143,7 @@ function activityBox(recording: Pick): { if (event.kind === "click") { xs.push(event.x); ys.push(event.y); - } else { + } else if (event.kind === "drag") { xs.push(event.x0, event.x1); ys.push(event.y0, event.y1); } @@ -257,13 +280,14 @@ const formatDuration = (ms: number) => { return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; }; -const summaryText = ( +export const summaryText = ( recording: Pick, lines: ReadonlyArray, cropped: boolean, ) => { - const { app, windowTitle, bundlePath } = recording.session; - const source = [app, windowTitle ? `“${windowTitle}”` : ""].filter(Boolean).join(" ") || "Screen"; + const { app, bundlePath } = recording.session; + // The title stays in scrawl.json: a window's title can hold anything. + const source = app || "Screen"; return [ `[scrawl ${formatDuration(recording.durationMs)} · ${source} · ${bundlePath}]`, "Open contact-sheet.png and frames/ in that folder before changing anything: the marks and words only make sense on the images, which can show more than the text does.", @@ -278,19 +302,23 @@ export async function buildBundle( ): Promise { const shift = (t: number) => Math.max(0, t - recording.inputOffsetMs); const events = recording.input.events.map((event) => - event.kind === "click" - ? { ...event, t: shift(event.t) } - : { ...event, t: shift(event.t), t1: shift(event.t1) }, + event.kind === "drag" + ? { ...event, t: shift(event.t), t1: shift(event.t1) } + : { ...event, t: shift(event.t) }, ); + const windows = recording.input.windows.map((snapshot) => ({ + ...snapshot, + t: shift(snapshot.t), + })); const strokes = recording.input.strokes.map((stroke) => ({ ...stroke, t0: shift(stroke.t0), t1: shift(stroke.t1), })); - const marks = strokesToMarks(strokes); + const marks = [...strokesToMarks(strokes), ...flagsToMarks(recording.input.flags.map(shift))]; const lines = buildNarration({ words: recording.words, - events, + events: [...events, ...windowChanges(windows)], marks, durationMs: recording.durationMs, sceneChanges: withoutInkChanges(recording.sceneChanges, strokes, SCRAWL_INK_HOLD_MS), @@ -370,13 +398,16 @@ export async function buildBundle( marks, events: events.map((event) => (event.kind === "click" ? { ...event, button: "left" } : event)), keyframes: lines.map((line, index) => { - const main = line.actions.find((action) => action.kind === "mark") ?? line.actions.at(-1); + const main = + line.actions.find((action) => action.kind === "mark") ?? + line.actions.findLast((action) => action.kind !== "note"); return { index: index + 1, t: Math.round(line.frameT), reason: main === undefined ? "speech" : KEYFRAME_REASONS[main.kind], file: `frames/frame-${String(index + 1).padStart(2, "0")}.png`, event: main?.kind ?? "speech", + ...(main?.kind === "mark" ? { markId: main.id } : {}), said: line.text, }; }), @@ -384,8 +415,8 @@ export async function buildBundle( video: recording.videoName, videoOffsetMs: recording.videoOffsetMs, audio: "audio.wav", - windows: [], - cursor: [], + windows, + cursor: recording.input.cursor.map(([t, x, y]) => [shift(t), x, y]), ...(box === null ? {} : { diff --git a/apps/web/src/scrawl/windowChanges.test.ts b/apps/web/src/scrawl/windowChanges.test.ts new file mode 100644 index 0000000000000..aa69b13f820fb --- /dev/null +++ b/apps/web/src/scrawl/windowChanges.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { windowChanges } from "./windowChanges.ts"; + +const window = (id: number, app: string, over: { layer?: number; regular?: boolean } = {}) => ({ + id, + app, + title: "A private title", + bounds: [0, 0, 100, 100] as const, + layer: 0, + ...over, +}); + +describe("windowChanges", () => { + it("names apps, never titles, and only ordinary windows", () => { + const changes = windowChanges([ + { t: 0, windows: [window(1, "Helium"), window(2, "Code"), window(9, "Tool", { layer: 3 })] }, + { + t: 500, + windows: [ + window(1, "Helium"), + window(3, "Notepad"), + window(8, "Popup", { regular: false }), + ], + }, + ]); + expect(changes).toEqual([ + { kind: "window", t: 500, text: "Code window closed" }, + { kind: "window", t: 500, text: "Notepad window opened" }, + ]); + }); + + it("makes one line of a virtual desktop switch", () => { + const changes = windowChanges([ + { t: 0, windows: [1, 2, 3].map((id) => window(id, "A")) }, + { t: 500, windows: [4, 5, 6].map((id) => window(id, "B")) }, + ]); + expect(changes).toEqual([ + { kind: "window", t: 500, text: "on-screen windows changed (3 opened, 3 closed)" }, + ]); + }); +}); diff --git a/apps/web/src/scrawl/windowChanges.ts b/apps/web/src/scrawl/windowChanges.ts new file mode 100644 index 0000000000000..51ef85f240b46 --- /dev/null +++ b/apps/web/src/scrawl/windowChanges.ts @@ -0,0 +1,39 @@ +// Ordinary windows opening and closing between two looks at the screen, by app name. Titles stay in +// scrawl.json: this is what reaches the agent. + +import type { ScrawlWindowSnapshot } from "@t3tools/contracts"; + +import type { CaptureEvent } from "./narration"; + +// More than this changing at once is a virtual desktop switch, not several things happening. +const WINDOW_BURST = 4; + +export function windowChanges( + snapshots: ReadonlyArray, +): Array { + const normal = (snapshot: ScrawlWindowSnapshot) => + snapshot.windows.filter((window) => window.layer === 0 && window.regular !== false); + const changes: Array = []; + for (let index = 1; index < snapshots.length; index++) { + const before = normal(snapshots[index - 1]!); + const after = normal(snapshots[index]!); + const t = snapshots[index]!.t; + const opened = after.filter((window) => !before.some((other) => other.id === window.id)); + const closed = before.filter((window) => !after.some((other) => other.id === window.id)); + if (opened.length + closed.length > WINDOW_BURST) { + changes.push({ + kind: "window", + t, + text: `on-screen windows changed (${opened.length} opened, ${closed.length} closed)`, + }); + continue; + } + for (const window of closed) { + changes.push({ kind: "window", t, text: `${window.app} window closed` }); + } + for (const window of opened) { + changes.push({ kind: "window", t, text: `${window.app} window opened` }); + } + } + return changes; +} diff --git a/packages/contracts/src/scrawl.ts b/packages/contracts/src/scrawl.ts index 665833f19d28b..dce1ec59f0262 100644 --- a/packages/contracts/src/scrawl.ts +++ b/packages/contracts/src/scrawl.ts @@ -27,6 +27,10 @@ export const ScrawlInputEvent = Schema.Union([ t: Schema.Number, x: Schema.Number, y: Schema.Number, + /** What UI Automation calls the element under the press, such as `button`. */ + role: Schema.optionalKey(Schema.String), + /** Its name and id, such as `“Area” #area`. Never a value, password or window title. */ + label: Schema.optionalKey(Schema.String), }), Schema.Struct({ kind: Schema.Literal("drag"), @@ -36,6 +40,27 @@ export const ScrawlInputEvent = Schema.Union([ y0: Schema.Number, x1: Schema.Number, y1: Schema.Number, + label: Schema.optionalKey(Schema.String), + }), + /** Wheel notches, positive away from the user and to the right, as Windows counts them. */ + Schema.Struct({ + kind: Schema.Literal("scroll"), + t: Schema.Number, + x: Schema.Number, + y: Schema.Number, + dx: Schema.Number, + dy: Schema.Number, + }), + /** Only chords with Ctrl or Win: typing never gets here. */ + Schema.Struct({ + kind: Schema.Literal("key"), + t: Schema.Number, + shortcut: Schema.String, + }), + Schema.Struct({ + kind: Schema.Literal("app_change"), + t: Schema.Number, + app: Schema.String, }), ]); export type ScrawlInputEvent = typeof ScrawlInputEvent.Type; @@ -45,12 +70,41 @@ export const ScrawlStroke = Schema.Struct({ t1: Schema.Number, /** x, y, x, y, ... */ points: Schema.Array(Schema.Number), + /** What it was drawn on, such as `button “Area”`. */ + on: Schema.optionalKey(Schema.String), }); export type ScrawlStroke = typeof ScrawlStroke.Type; +/** One on-screen window. Its title stays in scrawl.json and never reaches agent text. */ +export const ScrawlWindowEntry = Schema.Struct({ + id: Schema.Number, + app: Schema.String, + bundleId: Schema.optionalKey(Schema.String), + /** Only ever false: a tool or overlay that is rarely what a capture is about. */ + regular: Schema.optionalKey(Schema.Boolean), + title: Schema.optionalKey(Schema.String), + /** x, y, width, height in pixels from the captured display's top left. */ + bounds: Schema.Tuple([Schema.Number, Schema.Number, Schema.Number, Schema.Number]), + /** 0 for ordinary windows, 3 for tool and always-on-top ones, 20 for the taskbar. */ + layer: Schema.Number, + frontmost: Schema.optionalKey(Schema.Boolean), +}); +export type ScrawlWindowEntry = typeof ScrawlWindowEntry.Type; + +export const ScrawlWindowSnapshot = Schema.Struct({ + t: Schema.Number, + windows: Schema.Array(ScrawlWindowEntry), +}); +export type ScrawlWindowSnapshot = typeof ScrawlWindowSnapshot.Type; + export const ScrawlStopResult = Schema.Struct({ events: Schema.Array(ScrawlInputEvent), strokes: Schema.Array(ScrawlStroke), + /** t, x, y with positions as fractions of the display, thinned to real movement. */ + cursor: Schema.Array(Schema.Tuple([Schema.Number, Schema.Number, Schema.Number])), + windows: Schema.Array(ScrawlWindowSnapshot), + /** Moments the user flagged on purpose. */ + flags: Schema.Array(Schema.Number), }); export type ScrawlStopResult = typeof ScrawlStopResult.Type; From 7a8de8aea3ffb0c59a822a05038f5a113c2f9591 Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:27:38 +0100 Subject: [PATCH 5/9] fix(scrawl): the manifest's window and cursor logs share one clock with its events Co-Authored-By: Claude Opus 5.5 --- apps/web/src/scrawl/autocrop.test.ts | 3 +-- apps/web/src/scrawl/autocrop.ts | 3 +-- apps/web/src/scrawl/scrawlBundle.ts | 21 ++++++++++++++++----- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/apps/web/src/scrawl/autocrop.test.ts b/apps/web/src/scrawl/autocrop.test.ts index bd59ed82484de..ca97c3e0a229b 100644 --- a/apps/web/src/scrawl/autocrop.test.ts +++ b/apps/web/src/scrawl/autocrop.test.ts @@ -386,10 +386,9 @@ describe("computeCrops", () => { }, ], SCREEN, - (t) => t - 100, ); expect(snapshots[0]).toEqual({ - t: 0, + t: 100, windows: [ { id: 1, diff --git a/apps/web/src/scrawl/autocrop.ts b/apps/web/src/scrawl/autocrop.ts index 46f20067374f3..1f9c95ee8ab3f 100644 --- a/apps/web/src/scrawl/autocrop.ts +++ b/apps/web/src/scrawl/autocrop.ts @@ -146,10 +146,9 @@ export interface LoggedWindows { export const windowSnapshots = ( logged: ReadonlyArray, [width, height]: readonly [number, number], - shift: (t: number) => number, ): Array => logged.map((snapshot) => ({ - t: shift(snapshot.t), + t: snapshot.t, windows: snapshot.windows.map((win) => { const [x, y, w, h] = win.bounds; return { diff --git a/apps/web/src/scrawl/scrawlBundle.ts b/apps/web/src/scrawl/scrawlBundle.ts index 8d407775c421f..c00a033073faf 100644 --- a/apps/web/src/scrawl/scrawlBundle.ts +++ b/apps/web/src/scrawl/scrawlBundle.ts @@ -58,7 +58,7 @@ export interface CaptureRecording { readonly durationMs: number; readonly sceneChanges: ReadonlyArray; readonly wav: Uint8Array; - /** Scrawl's window log and cursor samples, on the input clock. Crops are better with them. */ + /** Scrawl's window log and cursor samples, on the clock of `input`. Crops are better with them. */ readonly windows?: ReadonlyArray; readonly cursor?: ReadonlyArray; } @@ -290,6 +290,15 @@ export async function buildBundle( }); const { width, height } = recording.session; const screen = [width, height] as const; + const windows = (recording.windows ?? []).map((snapshot) => ({ + ...snapshot, + t: Math.round(shift(snapshot.t)), + })); + const cursor = (recording.cursor ?? []).map(([t, x, y]): CursorSample => [ + Math.round(shift(t)), + x, + y, + ]); const { video, dispose } = await loadVideo(recording.video); const raws: Array = []; @@ -349,9 +358,9 @@ export async function buildBundle( ); return owner < 0 ? undefined : owner; }, - cursor: (recording.cursor ?? []).map(([t, x, y]): CursorSample => [shift(t), x, y]), + cursor, }), - snapshots: windowSnapshots(recording.windows ?? [], screen, shift), + snapshots: windowSnapshots(windows, screen), content: contents, }), lines.length, @@ -376,6 +385,8 @@ export async function buildBundle( }); frames.push(renderKeyframe(raw, crops[index]!.rect, rings, ink)); if (label !== null) fullFrames.push(renderKeyframe(raw, FULL, rings, ink)); + // Twelve full screens add up: each is released as soon as it is cut. + raw.width = 0; }); const bytesOf = async (blob: Blob) => new Uint8Array(await blob.arrayBuffer()); @@ -438,8 +449,8 @@ export async function buildBundle( video: recording.videoName, videoOffsetMs: recording.videoOffsetMs, audio: "audio.wav", - windows: recording.windows ?? [], - cursor: recording.cursor ?? [], + windows, + cursor, ...(label === null ? {} : { From 389b22a97b4bad0031fc51299b0f11bc98d3cf84 Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:29:02 +0100 Subject: [PATCH 6/9] feat(scrawl): agents can zoom, pull frames, inspect and export a capture without the Scrawl app Co-Authored-By: Claude Opus 5.5 --- FORK.md | 8 +- apps/desktop/package.json | 1 + apps/desktop/src/boot.ts | 3 +- apps/desktop/src/scrawl/DesktopScrawl.ts | 29 +- apps/desktop/src/scrawl/scrawlCli.ts | 449 ++++++++++++++ .../src/scrawl/scrawlLookCloser.test.ts | 160 +++++ apps/desktop/src/scrawl/scrawlLookCloser.ts | 550 ++++++++++++++++++ apps/desktop/src/scrawl/scrawlRenderJob.ts | 80 +++ apps/desktop/src/scrawl/scrawlRenderPage.ts | 440 ++++++++++++++ apps/desktop/src/scrawl/scrawlShim.test.ts | 23 + apps/desktop/src/scrawl/scrawlShim.ts | 62 ++ apps/desktop/vite.config.ts | 19 +- apps/web/src/scrawl/scrawlBundle.ts | 12 +- knip.jsonc | 2 + packages/contracts/src/scrawl.ts | 2 + pnpm-lock.yaml | 23 + 16 files changed, 1854 insertions(+), 9 deletions(-) create mode 100644 apps/desktop/src/scrawl/scrawlCli.ts create mode 100644 apps/desktop/src/scrawl/scrawlLookCloser.test.ts create mode 100644 apps/desktop/src/scrawl/scrawlLookCloser.ts create mode 100644 apps/desktop/src/scrawl/scrawlRenderJob.ts create mode 100644 apps/desktop/src/scrawl/scrawlRenderPage.ts create mode 100644 apps/desktop/src/scrawl/scrawlShim.test.ts create mode 100644 apps/desktop/src/scrawl/scrawlShim.ts diff --git a/FORK.md b/FORK.md index cc27d2eb494b2..a4ee4da60f74b 100644 --- a/FORK.md +++ b/FORK.md @@ -169,5 +169,11 @@ the contact sheet land in the composer on screen. Esc in T3 Code discards a capt - The recording pill in the corner shows in the video and keyframes; `setContentProtection` does not keep it out on Windows 10 22H2. - Bundles are in `/scrawl/captures` in Scrawl's format, so the Scrawl skill and the - `scrawl` CLI (`inspect`, `zoom`, `frames`) read them. Video and audio are deleted after 7 days. + `scrawl` CLI read them too. Video, audio and `zoom/` are deleted after 7 days. +- Agents look closer with `/bin/t3-scrawl.cmd` (`t3-scrawl` elsewhere), which the app writes + at every start and the pasted text names: `zoom --at 0:21|"a phrase"|--mark N`, `frames + --from --to`, `inspect [--json|--marks]`, `export [--out] [--zoom] [--crop] [--size]`. It needs no + Scrawl app and no running T3 Code. The app's exe does the work with `--scrawl`: Chromium is the + only video decoder in the app, so plain Node (`t3.cmd`) and a shipped ffmpeg lost to a hidden page + running mediabunny on WebCodecs. Agents on another machine cannot reach the bundle either way. - Code: `apps/desktop/src/scrawl/`, `apps/web/src/scrawl/`, `packages/contracts/src/scrawl.ts`. diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 886ccf3cd798f..5dfcec7d788cd 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -27,6 +27,7 @@ "electron-updater": "^6.8.9", "electron-webauthn": "catalog:", "ffi-rs": "1.3.2", + "mediabunny": "1.61.6", "playwright-core": "1.60.0", "react-grab": "^0.1.32", "tldts": "catalog:" diff --git a/apps/desktop/src/boot.ts b/apps/desktop/src/boot.ts index f2524f246c664..c4ead7f0e300c 100644 --- a/apps/desktop/src/boot.ts +++ b/apps/desktop/src/boot.ts @@ -1,2 +1,3 @@ require("./compileCache.cjs"); -require("./main.cjs"); +// Kept off the app's main: no lock, no server, no windows. See scrawl/scrawlCli.ts. +require(process.argv.includes("--scrawl") ? "./scrawlCli.cjs" : "./main.cjs"); diff --git a/apps/desktop/src/scrawl/DesktopScrawl.ts b/apps/desktop/src/scrawl/DesktopScrawl.ts index f63fe2e36f6c7..350d664c1767b 100644 --- a/apps/desktop/src/scrawl/DesktopScrawl.ts +++ b/apps/desktop/src/scrawl/DesktopScrawl.ts @@ -28,6 +28,7 @@ import { loadWindowsForegroundApi } from "../electron/WindowsForeground.ts"; import { activeWindow } from "../snapShot/ActiveWindow.ts"; import * as DesktopWindow from "../window/DesktopWindow.ts"; import { ScrawlInputTracker } from "./inputTracker.ts"; +import { renderScrawlShim, scrawlShimName } from "./scrawlShim.ts"; const DEFAULT_SHORTCUT = "Alt+Shift+S"; // Another program (the Scrawl app uses the same keys) may hold the shortcut and let go of it later. @@ -39,7 +40,7 @@ const VK_LBUTTON = 0x01; const VK_SHIFT = 0x10; // Like Scrawl: frames, text and contact sheet stay, the heavy media goes. const MEDIA_KEPT_MS = 7 * 24 * 60 * 60 * 1000; -const MEDIA_FILES = ["recording.mp4", "recording.webm", "audio.wav"]; +const MEDIA_FILES = ["recording.mp4", "recording.webm", "audio.wav", "zoom"]; const PILL_WIDTH = 96; const PILL_HEIGHT = 30; @@ -86,7 +87,9 @@ const dropOldMedia = async (capturesDir: string, now: number) => { const stat = await NodeFSP.stat(folder).catch(() => null); if (stat === null || now - stat.mtimeMs < MEDIA_KEPT_MS) continue; await Promise.all( - MEDIA_FILES.map((file) => NodeFSP.rm(NodePath.join(folder, file), { force: true })), + MEDIA_FILES.map((file) => + NodeFSP.rm(NodePath.join(folder, file), { force: true, recursive: true }), + ), ).catch(() => undefined); } }; @@ -132,6 +135,27 @@ const make = Effect.gen(function* () { // Finished captures keep writing their frames and summary after the watcher stopped. const bundles = new Map(); let nextIndex = 0; + const lookCloserCommand = NodePath.join( + environment.baseDir, + "bin", + scrawlShimName(environment.platform), + ); + // Written on every start: the executable's path changes when the app moves or the channel does. + const shimWritten = environment.isPackaged + ? yield* Effect.promise(async () => { + await NodeFSP.mkdir(NodePath.dirname(lookCloserCommand), { recursive: true }); + await NodeFSP.writeFile( + lookCloserCommand, + renderScrawlShim({ + platform: environment.platform, + executable: process.execPath, + t3Home: environment.baseDir, + }), + { mode: 0o755 }, + ); + return true; + }).pipe(Effect.catchDefect(() => Effect.succeed(false))) + : false; const ink = (capture: RunningCapture, script: string) => { if (!capture.ink.isDestroyed()) void capture.ink.webContents.executeJavaScript(script, true); @@ -264,6 +288,7 @@ const make = Effect.gen(function* () { app: front?.owner.name.replace(/\.exe$/i, "") ?? "", windowTitle: front?.title ?? "", bundlePath, + ...(shimWritten ? { lookCloserCommand } : {}), } satisfies ScrawlStartResult; }); diff --git a/apps/desktop/src/scrawl/scrawlCli.ts b/apps/desktop/src/scrawl/scrawlCli.ts new file mode 100644 index 0000000000000..f624bcdf1316f --- /dev/null +++ b/apps/desktop/src/scrawl/scrawlCli.ts @@ -0,0 +1,449 @@ +// @effect-diagnostics nodeBuiltinImport:off -- A one-shot command before and outside the app's Effect runtime. +// @effect-diagnostics globalDate:off -- Only ages leftover temp profiles. +// `t3-scrawl`: look closer at a capture and export it. boot.ts lands here for `--scrawl`, ahead of +// the app's main, because a video can only be decoded inside Chromium: no single-instance lock, no +// server, no window on screen, so it runs beside an open app or with none open at all. + +import * as Electron from "electron"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeUtil from "node:util"; + +import { + FULL_RECT, + type BundleManifest, + type MarkEntry, + type Rect, + cameraPath, + cropAt, + cropOfKeyframe, + findMark, + markEntries, + markJson, + markLine, + markWindow, + momentWindow, + outputSize, + parseClockMs, + preciseClock, + rangeTimes, + resolveCrop, + resolveMoment, + saidBetween, + sampleTimes, + shortClock, +} from "./scrawlLookCloser.ts"; +import { + SCRAWL_RENDER_PROGRESS_CHANNEL, + type Annotations, + type ExportResult, + type ProbeResult, + type RenderJob, +} from "./scrawlRenderJob.ts"; +import { SCRAWL_CLI_FLAG } from "./scrawlShim.ts"; + +const INK_HOLD_MS = 5_000; +const MAX_RANGE_FRAMES = 2_000; +const DEFAULT_EXPORT_LONG_EDGE = 1920; +const STALE_PROFILE_MS = 60 * 60 * 1000; +const PROFILE_PREFIX = "t3-scrawl-"; + +const USAGE = `t3-scrawl: look closer at a Scrawl capture and export it. is a .scrawl folder, or latest. + + t3-scrawl zoom --at 0:21 a PNG sheet of frames around a time + t3-scrawl zoom --at "a phrase" or around something they said + t3-scrawl zoom --mark 2 [--frame N] or around one mark, with only its ink + [--span 2] seconds covered [--frames 12] cells [--full] the whole screen, not the crop + t3-scrawl frames --from 0:04 --to 0:06 [--fps 4] plain PNGs of a range + t3-scrawl inspect [--json | --marks [--json]] the text, the manifest, the marks + t3-scrawl export [--out demo.mp4] [--zoom] [--crop auto|full|x,y,w,h] [--size 1920] + [--fps N] [--no-clicks] [--no-ink] one mp4 with the audio in sync + +zoom and frames write into /zoom/ and print the paths: open the PNGs and look at them. +Video and audio are deleted 7 days after a capture; frames/, the contact sheet and scrawl.json stay.`; + +class UsageError extends Error {} + +const capturesDir = () => { + const home = process.env.T3CODE_HOME?.trim() || NodePath.join(NodeOS.homedir(), ".t3"); + return NodePath.join(home, "scrawl", "captures"); +}; + +const latestBundle = () => { + const dir = capturesDir(); + const newest = (NodeFS.existsSync(dir) ? NodeFS.readdirSync(dir) : []) + .filter((name) => name.endsWith(".scrawl")) + .map((name) => NodePath.join(dir, name)) + .filter((path) => NodeFS.existsSync(NodePath.join(path, "scrawl.json"))) + .map((path) => ({ path, at: NodeFS.statSync(NodePath.join(path, "scrawl.json")).mtimeMs })) + .toSorted((left, right) => right.at - left.at)[0]; + if (newest === undefined) throw new Error(`No captures in ${dir}`); + return newest.path; +}; + +const loadBundle = (arg: string | undefined) => { + const named = arg === undefined || arg === "latest" ? latestBundle() : NodePath.resolve(arg); + const dir = NodePath.basename(named) === "scrawl.json" ? NodePath.dirname(named) : named; + const manifestPath = NodePath.join(dir, "scrawl.json"); + if (!NodeFS.existsSync(manifestPath)) throw new Error(`No capture at ${dir}`); + const manifest = JSON.parse(NodeFS.readFileSync(manifestPath, "utf8")) as BundleManifest; + return { dir, manifest }; +}; + +const videoOf = (dir: string, manifest: BundleManifest) => { + const path = manifest.video ? NodePath.join(dir, manifest.video) : null; + if (path !== null && NodeFS.existsSync(path)) return path; + throw new Error( + manifest.video + ? "The capture's video is gone: video and audio are deleted 7 days after a capture. frames/, contact-sheet.png and scrawl.json remain." + : "The capture has no video.", + ); +}; + +const annotationsFor = ( + manifest: BundleManifest, + options: { readonly ink: boolean; readonly clicks: boolean }, +): Annotations => ({ + strokes: + options.ink && manifest.inkInVideo !== true + ? (manifest.strokes ?? []).map((stroke) => ({ + fromMs: stroke.t0, + untilMs: stroke.t1 + INK_HOLD_MS, + whole: false, + points: stroke.pts, + })) + : [], + clicks: options.clicks + ? (manifest.events ?? []).flatMap((event) => + event.kind === "click" && "x" in event ? [{ t: event.t, x: event.x, y: event.y }] : [], + ) + : [], +}); + +/** Only this mark's ink, whole from the moment it starts, also where the video holds every mark. */ +const markAnnotations = (manifest: BundleManifest, entry: MarkEntry): Annotations => ({ + strokes: entry.mark.strokes.flatMap((index) => { + const stroke = manifest.strokes?.[index]; + return stroke === undefined + ? [] + : [ + { + fromMs: entry.mark.t0, + untilMs: Number.MAX_SAFE_INTEGER, + whole: true, + points: stroke.pts, + }, + ]; + }), + clicks: annotationsFor(manifest, { ink: false, clicks: true }).clicks, +}); + +const makeProfile = () => { + const tmp = NodeOS.tmpdir(); + for (const name of NodeFS.readdirSync(tmp)) { + if (!name.startsWith(PROFILE_PREFIX)) continue; + const path = NodePath.join(tmp, name); + try { + if (Date.now() - NodeFS.statSync(path).mtimeMs > STALE_PROFILE_MS) { + NodeFS.rmSync(path, { recursive: true, force: true }); + } + } catch { + // Another run may be using it or removing it. + } + } + return NodeFS.mkdtempSync(NodePath.join(tmp, PROFILE_PREFIX)); +}; + +const openRenderer = async (profile: string) => { + await Electron.app.whenReady(); + const page = NodePath.join(profile, "render.html"); + NodeFS.writeFileSync(page, ""); + const window = new Electron.BrowserWindow({ + show: false, + webPreferences: { + // Only this file's own script runs here, on local files. WebCodecs needs a secure context, + // so a file: page and not about:blank or data:. + nodeIntegration: true, + contextIsolation: false, + sandbox: false, + backgroundThrottling: false, + }, + }); + await window.loadFile(page); + const pageScript = NodePath.join(__dirname, "scrawlRenderPage.cjs"); + return (job: RenderJob) => + new Promise((resolve, reject) => { + window.webContents.once("render-process-gone", (_event, details) => + reject(new Error(`The renderer stopped: ${details.reason}`)), + ); + window.webContents + .executeJavaScript(`require(${JSON.stringify(pageScript)}).run(${JSON.stringify(job)})`) + .then(resolve, reject); + }); +}; + +type Render = Awaited>; + +const runZoom = async (render: Render, args: ReadonlyArray) => { + const { values, positionals } = NodeUtil.parseArgs({ + args: [...args], + allowPositionals: true, + options: { + at: { type: "string" }, + mark: { type: "string" }, + frame: { type: "string" }, + span: { type: "string", default: "2" }, + frames: { type: "string", default: "12" }, + full: { type: "boolean", default: false }, + }, + }); + if ((values.at === undefined) === (values.mark === undefined)) { + throw new UsageError("Pass one of --at or --mark"); + } + const span = Number(values.span); + const count = Number(values.frames); + if (!Number.isFinite(span) || span <= 0) throw new UsageError("--span must be positive"); + if (!Number.isInteger(count) || count < 1) throw new UsageError("--frames must be 1 or more"); + const { dir, manifest } = loadBundle(positionals[0]); + const video = videoOf(dir, manifest); + const spanMs = Math.round(span * 1000); + const lines: Array = []; + let window: readonly [number, number]; + let mark: MarkEntry | null = null; + if (values.mark !== undefined) { + const keyframe = values.frame === undefined ? undefined : Number(values.frame); + mark = findMark(manifest, values.mark, keyframe); + lines.push(markLine(mark)); + window = markWindow(mark.mark.t0, mark.mark.t1, spanMs, manifest.durationMs); + } else { + const at = resolveMoment(manifest, values.at!); + if (at > manifest.durationMs) { + throw new Error(`${preciseClock(at)} is past the end (${preciseClock(manifest.durationMs)})`); + } + window = momentWindow(at, spanMs, manifest.durationMs); + } + const [from, to] = window; + const times = sampleTimes(from, to, count); + const markCrop = + mark !== null && mark.keyframe !== null ? cropOfKeyframe(manifest, mark.keyframe) : null; + const viewAt = (t: number): Rect => + values.full ? FULL_RECT : ((mark !== null ? markCrop : cropAt(manifest, t)) ?? FULL_RECT); + const out = NodePath.join(dir, "zoom", `zoom-${from}-${to}.png`); + NodeFS.mkdirSync(NodePath.dirname(out), { recursive: true }); + await render({ + kind: "sheet", + video, + videoOffsetMs: manifest.videoOffsetMs ?? 0, + out, + annotations: + mark !== null + ? markAnnotations(manifest, mark) + : annotationsFor(manifest, { ink: true, clicks: true }), + cells: times.map((t) => ({ t, label: preciseClock(t), view: viewAt(t) })), + }); + lines.push(out); + lines.push( + `${preciseClock(from)} → ${preciseClock(to)}, ${times.length} frames: ${times + .map((t, index) => `#${index + 1} ${preciseClock(t)}`) + .join(", ")}`, + ); + const said = saidBetween(manifest, from, to); + if (said !== "") lines.push(`Said: “${said}”`); + return lines.join("\n"); +}; + +const GENERATED_FRAME = /^frame-\d+\.png$/; + +const runFrames = async (render: Render, args: ReadonlyArray) => { + const { values, positionals } = NodeUtil.parseArgs({ + args: [...args], + allowPositionals: true, + options: { + from: { type: "string" }, + to: { type: "string" }, + fps: { type: "string", default: "4" }, + }, + }); + if (values.from === undefined || values.to === undefined) { + throw new UsageError("Pass --from and --to"); + } + const from = parseClockMs(values.from); + const to = parseClockMs(values.to); + const fps = Number(values.fps); + if (from === null) throw new UsageError(`invalid --from time: ${values.from}`); + if (to === null) throw new UsageError(`invalid --to time: ${values.to}`); + if (from > to) throw new UsageError("The range ends before it starts"); + if (!Number.isFinite(fps) || fps <= 0 || fps > 240) { + throw new UsageError("--fps must be between 1 and 240"); + } + const { dir, manifest } = loadBundle(positionals[0]); + if ( + manifest.video === undefined || + !NodeFS.existsSync(NodePath.join(dir, manifest.video ?? "")) + ) { + // The keyframes outlive the video, and may already answer the question. + const kept = (manifest.keyframes ?? []) + .filter((keyframe) => keyframe.t >= from && keyframe.t <= to) + .map((keyframe) => NodePath.join(dir, keyframe.file)) + .filter((path) => NodeFS.existsSync(path)); + if (kept.length === 0) videoOf(dir, manifest); + return kept.join("\n"); + } + const times = rangeTimes(from, to, fps); + if (times.length > MAX_RANGE_FRAMES) { + throw new UsageError(`That is ${times.length} frames; use a shorter range or a lower --fps`); + } + const outDir = NodePath.join(dir, "zoom", `range-${from}-${to}`); + NodeFS.mkdirSync(outDir, { recursive: true }); + // Repeating at a lower fps must not return leftovers from the last run. + for (const name of NodeFS.readdirSync(outDir)) { + if (GENERATED_FRAME.test(name)) NodeFS.rmSync(NodePath.join(outDir, name)); + } + const frames = times.map((t, index) => ({ + t, + out: NodePath.join(outDir, `frame-${String(index + 1).padStart(3, "0")}.png`), + })); + await render({ + kind: "frames", + video: videoOf(dir, manifest), + videoOffsetMs: manifest.videoOffsetMs ?? 0, + frames, + }); + return frames.map((frame) => frame.out).join("\n"); +}; + +const runInspect = (args: ReadonlyArray) => { + const { values, positionals } = NodeUtil.parseArgs({ + args: [...args], + allowPositionals: true, + options: { + json: { type: "boolean", default: false }, + marks: { type: "boolean", default: false }, + }, + }); + const { dir, manifest } = loadBundle(positionals[0]); + if (values.marks) { + const entries = markEntries(manifest); + if (values.json) return JSON.stringify(entries.map(markJson), null, 2); + return entries.length === 0 ? "No marks" : entries.map(markLine).join("\n"); + } + if (values.json) return JSON.stringify(manifest, null, 2); + const summary = NodePath.join(dir, "summary.md"); + if (!NodeFS.existsSync(summary)) throw new Error(`No summary.md in ${dir}`); + return NodeFS.readFileSync(summary, "utf8").trimEnd(); +}; + +const runExport = async (render: Render, args: ReadonlyArray) => { + const { values, positionals } = NodeUtil.parseArgs({ + args: [...args], + allowPositionals: true, + options: { + out: { type: "string" }, + crop: { type: "string", default: "auto" }, + size: { type: "string", default: String(DEFAULT_EXPORT_LONG_EDGE) }, + zoom: { type: "boolean", default: false }, + fps: { type: "string" }, + "no-clicks": { type: "boolean", default: false }, + "no-ink": { type: "boolean", default: false }, + }, + }); + const { dir, manifest } = loadBundle(positionals[0]); + const video = videoOf(dir, manifest); + const offset = manifest.videoOffsetMs ?? 0; + const probe = await render({ kind: "probe", video }); + const crop = resolveCrop(values.crop, manifest, probe.width, probe.height); + const size = Number(values.size); + if (!Number.isFinite(size)) throw new UsageError("--size must be a number"); + const { width, height } = outputSize( + (crop[2] - crop[0]) * probe.width, + (crop[3] - crop[1]) * probe.height, + size, + ); + const fps = values.fps === undefined ? (probe.packetRate >= 45 ? 60 : 30) : Number(values.fps); + if (!Number.isInteger(fps) || fps < 1 || fps > 120) { + throw new UsageError("--fps must be between 1 and 120"); + } + const frameCount = Math.max(1, Math.ceil(probe.seconds * fps)); + const out = NodePath.resolve( + values.out ?? + NodePath.join( + Electron.app.getPath("videos"), + "Scrawl", + `${NodePath.basename(dir, ".scrawl")}.mp4`, + ), + ); + NodeFS.mkdirSync(NodePath.dirname(out), { recursive: true }); + const audio = manifest.audio ? NodePath.join(dir, manifest.audio) : null; + const result = await render({ + kind: "export", + video, + videoOffsetMs: offset, + audio: audio !== null && NodeFS.existsSync(audio) ? audio : null, + out, + partial: `${out}.${process.pid}.partial`, + crop, + width, + height, + fps, + frameCount, + annotations: annotationsFor(manifest, { + ink: !values["no-ink"], + clicks: !values["no-clicks"], + }), + views: values.zoom ? cameraPath(manifest, crop, offset, frameCount, fps) : null, + }); + return [ + out, + `${width}×${height} @ ${fps} fps`, + shortClock(result.seconds * 1000), + `${(result.bytes / 1_048_576).toFixed(1)} MB`, + result.audio === null ? "no audio" : `${result.audio} audio`, + ...(values.zoom ? ["auto-zoom"] : []), + ].join(" · "); +}; + +const finish = (text: string, code: number) => { + (code === 0 ? process.stdout : process.stderr).write(`${text}\n`, () => Electron.app.exit(code)); +}; + +const main = async () => { + const args = process.argv.slice(process.argv.indexOf(SCRAWL_CLI_FLAG) + 1); + const [command, ...rest] = args; + if (command === undefined || command === "help" || args.includes("--help")) { + return USAGE; + } + if (command === "inspect") return runInspect(rest); + if (command !== "zoom" && command !== "frames" && command !== "export") { + throw new UsageError(`Unknown command “${command}”.`); + } + // Chromium writes its own logs to the same stderr an agent reads the error from. + Electron.app.commandLine.appendSwitch("log-level", "3"); + Electron.app.commandLine.appendSwitch("disk-cache-size", "1"); + // The running app holds its own profile, and two agents may run this at once. + const profile = makeProfile(); + Electron.app.setPath("userData", profile); + Electron.ipcMain.on(SCRAWL_RENDER_PROGRESS_CHANNEL, (_event, percent: number) => { + process.stderr.write(`Exporting ${percent}%\n`); + }); + const render = await openRenderer(profile); + if (command === "zoom") return runZoom(render, rest); + return command === "frames" ? runFrames(render, rest) : runExport(render, rest); +}; + +// A hidden window keeps the app alive, so nothing may leave this with the process still running. +process.on("uncaughtException", (cause) => finish(cause.stack ?? String(cause), 1)); +process.on("unhandledRejection", (cause) => finish(String(cause), 1)); +main().then( + (text) => finish(text, 0), + (cause: unknown) => { + if ( + cause instanceof UsageError || + (cause as { code?: string }).code?.startsWith("ERR_PARSE_ARGS") + ) { + finish(`${(cause as Error).message}\n\n${USAGE}`, 2); + } else { + finish(cause instanceof Error ? cause.message : String(cause), 1); + } + }, +); diff --git a/apps/desktop/src/scrawl/scrawlLookCloser.test.ts b/apps/desktop/src/scrawl/scrawlLookCloser.test.ts new file mode 100644 index 0000000000000..8356af5b4823c --- /dev/null +++ b/apps/desktop/src/scrawl/scrawlLookCloser.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + type BundleManifest, + type BundleMark, + autoCrop, + cameraPath, + cropAt, + findMark, + markEntries, + markLine, + markWindow, + momentWindow, + outputSize, + parseClockMs, + preciseClock, + rangeTimes, + resolveCrop, + resolveMoment, + sampleTimes, +} from "./scrawlLookCloser.ts"; + +const mark = (id: string, t0: number, frame?: string): BundleMark => ({ + id, + t0, + t1: t0 + 200, + strokes: [0], + ...(frame === undefined ? {} : { frame }), +}); + +const manifest = (overrides: Partial = {}): BundleManifest => ({ + durationMs: 25_000, + target: { frame: { width: 2000, height: 1000 } }, + transcript: { + words: [ + { t0: 9000, t1: 9200, text: "but," }, + { t0: 9300, t1: 9500, text: "the" }, + { t0: 9500, t1: 9700, text: "Width" }, + { t0: 9800, t1: 10_000, text: "changes." }, + ], + }, + ...overrides, +}); + +describe("scrawl look closer", () => { + // These vectors are the Scrawl CLI's own tests, so both tools pick the same frames. + it("resolves phrases and times", () => { + expect(resolveMoment(manifest(), "width changes")).toBe(9500); + expect(resolveMoment(manifest(), "0:12")).toBe(12_000); + expect(() => resolveMoment(manifest(), "tabular numbers")).toThrow(/neither a time/); + expect(parseClockMs("1500ms")).toBe(1500); + expect(parseClockMs("12.5s")).toBe(12_500); + expect(parseClockMs("a:b")).toBeNull(); + expect(parseClockMs("")).toBeNull(); + }); + + it("leans a window after the moment and stays inside the recording", () => { + expect(momentWindow(10_000, 2000, 25_000)).toEqual([9500, 11_500]); + expect(momentWindow(200, 2000, 25_000)).toEqual([0, 2000]); + expect(momentWindow(24_800, 2000, 25_000)).toEqual([23_000, 25_000]); + expect(sampleTimes(1000, 2000, 3)).toEqual([1000, 1500, 2000]); + expect(preciseClock(61_234)).toBe("1:01.234"); + expect(rangeTimes(1000, 2000, 4)).toEqual([1000, 1250, 1500, 1750, 2000]); + }); + + it("centres a mark's window and holds all of it", () => { + expect(markWindow(9322, 10_908, 2000, 13_400)).toEqual([8822, 11_408]); + expect(markWindow(6661, 7182, 2000, 13_400)).toEqual([5921, 7921]); + expect(markWindow(200, 400, 2000, 13_400)).toEqual([0, 2000]); + expect(markWindow(13_000, 13_300, 2000, 13_400)).toEqual([11_400, 13_400]); + }); + + it("numbers marks the way the text does and finds them by number or id", () => { + const bundle = manifest({ + marks: [ + mark("m1", 1000, "frames/frame-01.png"), + mark("m2", 2000, "frames/frame-02.png"), + mark("m3", 2500, "frames/frame-02.png"), + ], + keyframes: [ + { index: 1, t: 1200, file: "frames/frame-01.png" }, + { index: 2, t: 2700, file: "frames/frame-02.png" }, + ], + }); + expect(markEntries(bundle).map(markLine)).toEqual([ + "#1 ✎ 0:01 marked · m1", + "#2 ✎① 0:02 marked · m2", + "#2 ✎② 0:02 marked · m3", + ]); + expect(findMark(bundle, "m1", undefined).mark.id).toBe("m1"); + expect(findMark(bundle, "2", undefined).mark.id).toBe("m3"); + expect(findMark(bundle, "②", undefined).mark.id).toBe("m3"); + expect(findMark(bundle, "1", 1).mark.id).toBe("m1"); + expect(findMark(bundle, "1", undefined).mark.id).toBe("m2"); + expect(() => findMark(bundle, "9", undefined)).toThrow(/No mark ⑨/); + expect(() => findMark(bundle, "nope", undefined)).toThrow(/neither a mark number/); + }); + + it("still tells marks apart when no keyframe names them", () => { + const bundle = manifest({ marks: [mark("m1", 1000), mark("m2", 2000)], keyframes: [] }); + expect(findMark(bundle, "2", undefined).mark.id).toBe("m2"); + }); + + it("reads crops as fractions of the capture", () => { + const crop = { x: 200, y: 100, width: 1000, height: 500, kind: "focus" }; + const bundle = manifest({ + keyframes: [ + { index: 1, t: 1000, file: "a" }, + { index: 2, t: 9000, file: "b" }, + ], + autocrop: { crops: [crop, { ...crop, x: 600, kind: "focus" }] }, + }); + expect(cropAt(bundle, 1100)).toEqual([0.1, 0.1, 0.6, 0.6]); + expect(cropAt(bundle, 8000)).toEqual([0.3, 0.1, 0.8, 0.6]); + expect(autoCrop(bundle)).toEqual([0.1, 0.1, 0.8, 0.6]); + expect(autoCrop(manifest({ autocrop: { crops: [{ ...crop, kind: "full" }] } }))).toEqual([ + 0, 0, 1, 1, + ]); + }); + + it("parses --crop as fractions or pixels and refuses what falls outside", () => { + expect(resolveCrop("full", manifest(), 1000, 500)).toEqual([0, 0, 1, 1]); + expect(resolveCrop("0.1,0.2,0.5,0.6", manifest(), 1000, 500)).toEqual([ + 0.1, + 0.2, + 0.6, + expect.closeTo(0.8), + ]); + expect(resolveCrop("100,100,500,250", manifest(), 1000, 500)).toEqual([0.1, 0.2, 0.6, 0.7]); + expect(() => resolveCrop("0.5,0.5,0.9,0.9", manifest(), 1000, 500)).toThrow(/past the edge/); + expect(() => resolveCrop("1,2,3", manifest(), 1000, 500)).toThrow(/four positive/); + expect(() => resolveCrop("0,0,0.001,0.5", manifest(), 1000, 500)).toThrow(/16 px/); + }); + + it("keeps the crop's aspect on even sides and never scales up", () => { + expect(outputSize(2560, 1440, 1920)).toEqual({ width: 1920, height: 1080 }); + expect(outputSize(999.4, 500.2, 1920)).toEqual({ width: 1000, height: 500 }); + expect(() => outputSize(100, 100, 10)).toThrow(/at least 64/); + }); + + it("moves the camera to activity and back, on the bundle's clock", () => { + const bundle = manifest({ + events: [{ kind: "click", t: 3000, x: 0.8, y: 0.25 }], + }); + const offsetMs = 1000; + const rects = cameraPath(bundle, [0, 0, 1, 1], offsetMs, 10 * 30, 30); + expect(rects).toHaveLength(300); + expect(rects[0]).toEqual([0, 0, 1, 1]); + // Bundle time 5 s, two after the click: settled on it at 1.8x. + const zoomed = rects[120]!; + expect(zoomed[2] - zoomed[0]).toBeCloseTo(1 / 1.8, 2); + expect((zoomed[0] + zoomed[2]) / 2).toBeCloseTo(0.72, 1); + for (const rect of rects) { + expect(rect[0]).toBeGreaterThanOrEqual(-1e-9); + expect(rect[2]).toBeLessThanOrEqual(1 + 1e-9); + } + // The zoom is released 2.5 s after the click: back out by the end. + expect(rects[299]![2] - rects[299]![0]).toBeCloseTo(1, 1); + }); +}); diff --git a/apps/desktop/src/scrawl/scrawlLookCloser.ts b/apps/desktop/src/scrawl/scrawlLookCloser.ts new file mode 100644 index 0000000000000..3cf9258d4cd7a --- /dev/null +++ b/apps/desktop/src/scrawl/scrawlLookCloser.ts @@ -0,0 +1,550 @@ +// What `t3-scrawl` decides before any pixel moves: which moment, mark, crop and camera path. Pure, +// and read against the same scrawl.json as the Scrawl app's own CLI, so numbers and windows agree. + +export type Rect = readonly [x0: number, y0: number, x1: number, y1: number]; + +export const FULL_RECT: Rect = [0, 0, 1, 1]; + +export interface BundleWord { + readonly t0: number; + readonly t1: number; + readonly text: string; +} + +export interface BundleMark { + readonly id: string; + readonly strokes: ReadonlyArray; + readonly t0: number; + readonly t1: number; + readonly shape?: string; + readonly on?: string; + readonly frame?: string; + readonly pin?: number; + readonly boxNorm?: readonly [number, number, number, number]; +} + +export interface BundleKeyframe { + readonly index: number; + readonly t: number; + readonly file: string; + readonly markId?: string; +} + +export interface BundleStroke { + readonly t0: number; + readonly t1: number; + /** Flat x, y, dt triples with positions in 0..=10000. */ + readonly pts: ReadonlyArray; +} + +export type BundleEvent = + | { + readonly kind: "click"; + readonly t: number; + readonly x: number; + readonly y: number; + } + | { + readonly kind: "drag"; + readonly t: number; + readonly t1: number; + readonly x0: number; + readonly y0: number; + readonly x1: number; + readonly y1: number; + } + | { readonly kind: string; readonly t?: number }; + +export interface BundleCrop { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; + readonly kind?: string; +} + +export interface BundleManifest { + readonly durationMs: number; + readonly target: { + readonly mode?: string; + readonly frame: { readonly width: number; readonly height: number }; + }; + readonly transcript: { + readonly words?: ReadonlyArray; + readonly segments?: ReadonlyArray; + }; + readonly strokes?: ReadonlyArray; + readonly marks?: ReadonlyArray; + readonly events?: ReadonlyArray; + readonly keyframes?: ReadonlyArray; + readonly video?: string | null; + readonly videoOffsetMs?: number; + readonly audio?: string | null; + /** Native captures record the screen with the ink on it; the Scrawl app's video has none. */ + readonly inkInVideo?: boolean; + readonly autocrop?: { readonly crops?: ReadonlyArray } | null; +} + +/** `0:12`, `12.5`, `12s` or `1500ms`; null when it is none of those. */ +export const parseClockMs = (input: string): number | null => { + const text = input.trim(); + const seconds = (value: string) => { + const parsed = value.trim() === "" ? Number.NaN : Number(value); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; + }; + const colon = text.indexOf(":"); + if (colon >= 0) { + const minutes = text.slice(0, colon); + const rest = seconds(text.slice(colon + 1)); + if (!/^\d+$/.test(minutes) || rest === null) return null; + return Number(minutes) * 60_000 + Math.round(rest * 1000); + } + if (text.endsWith("ms")) { + const ms = text.slice(0, -2).trim(); + return /^\d+$/.test(ms) ? Number(ms) : null; + } + const rest = seconds(text.endsWith("s") ? text.slice(0, -1) : text); + return rest === null ? null : Math.round(rest * 1000); +}; + +const pad = (value: number, width: number) => String(value).padStart(width, "0"); + +/** `0:09` */ +export const shortClock = (ms: number) => + `${Math.floor(ms / 60_000)}:${pad(Math.floor(ms / 1000) % 60, 2)}`; + +/** `1:01.234` */ +export const preciseClock = (ms: number) => `${shortClock(ms)}.${pad(Math.round(ms) % 1000, 3)}`; + +const normalize = (text: string) => + text + .replace(/[^\p{L}\p{N}\s]/gu, "") + .toLowerCase() + .split(/\s+/) + .filter(Boolean); + +const findPhraseMs = (manifest: BundleManifest, phrase: string): number | null => { + const needle = normalize(phrase); + if (needle.length === 0) return null; + const words = manifest.transcript.words ?? []; + const tokens = words.map((word) => normalize(word.text).join("")); + const start = tokens.findIndex((_token, index) => + needle.every((part, offset) => tokens[index + offset] === part), + ); + if (start >= 0) return words[start]!.t0; + const joined = needle.join(" "); + const segment = (manifest.transcript.segments ?? []).find((entry) => + normalize(entry.text).join(" ").includes(joined), + ); + return segment?.t0 ?? null; +}; + +/** `--at`: a time or a phrase from the narration. */ +export const resolveMoment = (manifest: BundleManifest, at: string): number => { + const moment = parseClockMs(at) ?? findPhraseMs(manifest, at); + if (moment === null) throw new Error(`“${at}” is neither a time nor a phrase in the narration`); + return moment; +}; + +const clamp = (value: number, low: number, high: number) => Math.min(Math.max(value, low), high); + +const fitWindow = (from: number, span: number, durationMs: number): readonly [number, number] => { + const to = from + span; + return to > durationMs ? [Math.max(0, durationMs - span), durationMs] : [from, to]; +}; + +/** A quarter before the moment and the rest after: what is said is usually about to happen. */ +export const momentWindow = (atMs: number, spanMs: number, durationMs: number) => { + const span = clamp(spanMs, 200, Math.max(durationMs, 200)); + return fitWindow(Math.max(0, atMs - Math.floor(span / 4)), span, durationMs); +}; + +/** Centred on a mark, at least `spanMs`, and the whole mark with half a second either side. */ +export const markWindow = (t0: number, t1: number, spanMs: number, durationMs: number) => { + const span = clamp(Math.max(spanMs, t1 - t0 + 1000), 200, Math.max(durationMs, 200)); + const middle = Math.floor((t0 + t1) / 2); + return fitWindow(Math.max(0, middle - Math.floor(span / 2)), span, durationMs); +}; + +const MAX_SHEET_FRAMES = 48; + +/** Evenly spaced across the window, both ends included. */ +export const sampleTimes = (from: number, to: number, count: number): ReadonlyArray => { + const cells = clamp(Math.trunc(count), 1, MAX_SHEET_FRAMES); + if (cells === 1 || to <= from) return [from]; + return Array.from( + { length: cells }, + (_unused, index) => from + Math.floor(((to - from) * index) / (cells - 1)), + ); +}; + +/** One PNG per `1000 / fps` milliseconds, from the first moment on. */ +export const rangeTimes = (from: number, to: number, fps: number): ReadonlyArray => { + const step = Math.max(1, Math.floor(1000 / fps)); + const times: Array = []; + for (let t = from; t <= to; t += step) times.push(t); + return times; +}; + +export const saidBetween = (manifest: BundleManifest, from: number, to: number) => { + const overlapping = (entry: BundleWord) => entry.t1 >= from && entry.t0 <= to; + const words = (manifest.transcript.words ?? []).filter(overlapping); + const source = + words.length > 0 ? words : (manifest.transcript.segments ?? []).filter(overlapping); + return source.map((entry) => entry.text.trim()).join(" "); +}; + +const CIRCLED = "①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳"; +const circled = (number: number) => CIRCLED[number - 1] ?? `(${number})`; + +export interface MarkEntry { + readonly mark: BundleMark; + readonly keyframe: number | null; + readonly number: number; + readonly numbered: boolean; +} + +const markOnKeyframe = (mark: BundleMark, keyframe: BundleKeyframe) => + keyframe.markId === mark.id || (keyframe.file !== "" && mark.frame === keyframe.file); + +/** Every mark in time order, numbered the way the text numbers them. */ +export const markEntries = (manifest: BundleManifest): ReadonlyArray => { + const marks = manifest.marks ?? []; + const keyframes = manifest.keyframes ?? []; + const entries = marks.map((mark) => { + const keyframe = keyframes.find((candidate) => markOnKeyframe(mark, candidate)); + const shown = keyframe + ? marks + .filter((other) => other.strokes.length > 0 && markOnKeyframe(other, keyframe)) + .toSorted((left, right) => left.t0 - right.t0) + : []; + const position = shown.indexOf(mark); + return { + mark, + keyframe: keyframe?.index ?? null, + number: position >= 0 ? position + 1 : 1, + numbered: position >= 0 && shown.length >= 2, + }; + }); + const sorted = entries.toSorted((left, right) => left.mark.t0 - right.mark.t0); + // With no keyframe naming its marks, one running count is what still tells them apart. + return sorted.length > 1 && sorted.every((entry) => entry.keyframe === null) + ? sorted.map((entry, index) => ({ ...entry, number: index + 1, numbered: true })) + : sorted; +}; + +const markVerbs: Record = { + circle: "circled", + underline: "underlined", + line: "drew a line", + arrow: "drew an arrow", + tap: "tapped", + mark: "marked", + flag: "flagged this moment", +}; + +const markText = (mark: BundleMark) => { + const shape = mark.shape ?? "mark"; + const verb = shape === "pin" ? `pin ${mark.pin ?? 1}` : (markVerbs[shape] ?? "marked"); + const on = mark.on?.trim(); + const joiner = + shape === "arrow" ? " pointing to " : shape === "pin" || shape === "line" ? " on " : " "; + const strokes = mark.strokes.length > 1 ? ` (${mark.strokes.length} strokes)` : ""; + return `${verb}${on ? `${joiner}“${on}”` : ""}${strokes}`; +}; + +/** `#2 ✎② 0:09 marked (2 strokes) · m2` */ +export const markLine = (entry: MarkEntry) => { + const frame = entry.keyframe === null ? "#-" : `#${entry.keyframe}`; + const number = entry.numbered ? circled(entry.number) : ""; + return `${frame} ✎${number} ${shortClock(entry.mark.t0)} ${markText(entry.mark)} · ${entry.mark.id}`; +}; + +export const markJson = (entry: MarkEntry) => ({ + keyframe: entry.keyframe, + number: entry.number, + numbered: entry.numbered, + id: entry.mark.id, + t0: entry.mark.t0, + t1: entry.mark.t1, + time: shortClock(entry.mark.t0), + shape: entry.mark.shape ?? "mark", + on: entry.mark.on ?? null, + strokes: entry.mark.strokes.length, + text: markText(entry.mark), + frame: entry.mark.frame ?? null, +}); + +/** The mark `query` names: its id, or its number as the text shows it, on `keyframe` when given. */ +export const findMark = ( + manifest: BundleManifest, + query: string, + keyframe: number | undefined, +): MarkEntry => { + const entries = markEntries(manifest); + const wanted = query.trim(); + const byId = entries.find((entry) => entry.mark.id === wanted); + if (byId !== undefined) return byId; + const circledAt = CIRCLED.indexOf(wanted); + const number = /^\d+$/.test(wanted) ? Number(wanted) : circledAt >= 0 ? circledAt + 1 : null; + if (number === null) throw new Error(`“${wanted}” is neither a mark number nor a mark id`); + const onFrame = entries.filter( + (entry) => entry.number === number && (keyframe === undefined || entry.keyframe === keyframe), + ); + const shown = onFrame.filter((entry) => entry.numbered); + const candidates = shown.length > 0 ? shown : onFrame; + if (candidates.length === 1) return candidates[0]!; + if (candidates.length === 0) { + throw new Error( + keyframe === undefined + ? `No mark ${circled(number)} in this bundle` + : `Keyframe #${keyframe} has no mark ${circled(number)}`, + ); + } + throw new Error( + `Mark ${circled(number)} is on several keyframes (${candidates.map(markLine).join("; ")}); add --frame or use the mark id`, + ); +}; + +const cropRect = (manifest: BundleManifest, crop: BundleCrop | undefined): Rect | null => { + if (crop === undefined || crop.kind === "full") return null; + const { width, height } = manifest.target.frame; + return [ + crop.x / width, + crop.y / height, + (crop.x + crop.width) / width, + (crop.y + crop.height) / height, + ]; +}; + +/** The auto-crop of the keyframe nearest `t`; keyframe crops meet halfway between keyframes. */ +export const cropAt = (manifest: BundleManifest, t: number): Rect | null => { + const keyframes = manifest.keyframes ?? []; + let nearest = -1; + keyframes.forEach((keyframe, index) => { + if (nearest < 0 || Math.abs(keyframe.t - t) < Math.abs(keyframes[nearest]!.t - t)) { + nearest = index; + } + }); + return cropRect(manifest, manifest.autocrop?.crops?.[nearest]); +}; + +/** The auto-crop of the keyframe a mark belongs to, whose crop holds its ink. */ +export const cropOfKeyframe = (manifest: BundleManifest, keyframeIndex: number): Rect | null => { + const position = (manifest.keyframes ?? []).findIndex((entry) => entry.index === keyframeIndex); + return position < 0 ? null : cropRect(manifest, manifest.autocrop?.crops?.[position]); +}; + +/** `--crop auto`: every keyframe's crop together, or the whole capture without any. */ +export const autoCrop = (manifest: BundleManifest): Rect => { + const crops = (manifest.autocrop?.crops ?? []).map((crop) => cropRect(manifest, crop)); + if (crops.length === 0 || crops.includes(null)) return FULL_RECT; + const rects = crops as ReadonlyArray; + return [ + clamp(Math.min(...rects.map((rect) => rect[0])), 0, 1), + clamp(Math.min(...rects.map((rect) => rect[1])), 0, 1), + clamp(Math.max(...rects.map((rect) => rect[2])), 0, 1), + clamp(Math.max(...rects.map((rect) => rect[3])), 0, 1), + ]; +}; + +/** `full`, `auto`, or `x,y,w,h` as fractions of the capture (all ≤ 1) or pixels of the video. */ +export const resolveCrop = ( + arg: string, + manifest: BundleManifest, + videoWidth: number, + videoHeight: number, +): Rect => { + const text = arg.trim(); + if (text === "full") return FULL_RECT; + if (text === "auto") return autoCrop(manifest); + const parts = text.split(",").map((part) => (part.trim() === "" ? Number.NaN : Number(part))); + const [x, y, width, height] = parts; + if ( + parts.length !== 4 || + x === undefined || + y === undefined || + width === undefined || + height === undefined || + parts.some((part) => !Number.isFinite(part) || part < 0) || + width <= 0 || + height <= 0 + ) { + throw new Error(`A crop is full, auto, or x,y,w,h (four positive numbers); got “${text}”`); + } + const inPixels = parts.some((part) => part > 1); + const rect: Rect = inPixels + ? [x / videoWidth, y / videoHeight, (x + width) / videoWidth, (y + height) / videoHeight] + : [x, y, x + width, y + height]; + if (rect[2] > 1.0001 || rect[3] > 1.0001) { + throw new Error("The crop runs past the edge of the recording"); + } + const clipped: Rect = [rect[0], rect[1], Math.min(rect[2], 1), Math.min(rect[3], 1)]; + if ((clipped[2] - clipped[0]) * videoWidth < 16 || (clipped[3] - clipped[1]) * videoHeight < 16) { + throw new Error("The crop is smaller than 16 px"); + } + return clipped; +}; + +const even = (value: number) => Math.max(Math.round(value / 2) * 2, 2); + +/** The crop's aspect, even sides for H.264, long edge at most `cap`, and never scaled up. */ +export const outputSize = (cropWidth: number, cropHeight: number, cap: number) => { + if (cap < 64) throw new Error("--size must be at least 64"); + const scale = Math.min(cap / Math.max(cropWidth, cropHeight), 1); + return { width: even(cropWidth * scale), height: even(cropHeight * scale) }; +}; + +const ZOOM_SCALE = 1.8; +const MIN_ZOOM_SCALE = 1.5; +const FIT_FRACTION = 0.7; +const LEAD_MS = 350; +const HOLD_MS = 2500; +const SPRING_OMEGA = (2 * Math.PI) / (1.2 * 0.9); +const CAMERA_STEP_MS = 1000 / 60; + +interface Activity { + readonly t0: number; + readonly t1: number; + readonly box: Rect; +} + +const activities = (manifest: BundleManifest, crop: Rect): ReadonlyArray => { + const width = crop[2] - crop[0]; + const height = crop[3] - crop[1]; + const toCrop = (x0: number, y0: number, x1: number, y1: number): Rect => [ + clamp((Math.min(x0, x1) - crop[0]) / width, 0, 1), + clamp((Math.min(y0, y1) - crop[1]) / height, 0, 1), + clamp((Math.max(x0, x1) - crop[0]) / width, 0, 1), + clamp((Math.max(y0, y1) - crop[1]) / height, 0, 1), + ]; + const found: Array = []; + for (const event of manifest.events ?? []) { + if (event.kind === "click" && "x" in event) { + found.push({ t0: event.t, t1: event.t, box: toCrop(event.x, event.y, event.x, event.y) }); + } else if (event.kind === "drag" && "x0" in event) { + found.push({ + t0: event.t, + t1: event.t1, + box: toCrop(event.x0, event.y0, event.x1, event.y1), + }); + } + } + for (const mark of manifest.marks ?? []) { + if (mark.boxNorm === undefined) continue; + const [x0, y0, x1, y1] = mark.boxNorm; + found.push({ t0: mark.t0, t1: mark.t1, box: toCrop(x0, y0, x1, y1) }); + } + return found.toSorted((left, right) => left.t0 - right.t0); +}; + +interface CameraSegment { + readonly start: number; + readonly release: number; + readonly scale: number; + readonly center: readonly [number, number]; +} + +const cameraSegments = (found: ReadonlyArray): ReadonlyArray => { + const clusters: Array<{ t0: number; t1: number; box: [number, number, number, number] }> = []; + for (const activity of found) { + const open = clusters.at(-1); + if (open !== undefined && activity.t0 - open.t1 <= HOLD_MS) { + open.t1 = Math.max(open.t1, activity.t1); + open.box = [ + Math.min(open.box[0], activity.box[0]), + Math.min(open.box[1], activity.box[1]), + Math.max(open.box[2], activity.box[2]), + Math.max(open.box[3], activity.box[3]), + ]; + } else { + clusters.push({ t0: activity.t0, t1: activity.t1, box: [...activity.box] }); + } + } + return clusters.map(({ t0, t1, box }) => { + const span = Math.max(box[2] - box[0], box[3] - box[1]); + const fitted = span > 0 ? Math.min(ZOOM_SCALE, FIT_FRACTION / span) : ZOOM_SCALE; + return { + start: Math.max(0, t0 - LEAD_MS), + release: t1 + HOLD_MS, + // Activity wider than this leaves the view as it is: a small zoom only looks like a mistake. + scale: fitted < MIN_ZOOM_SCALE ? 1 : fitted, + center: [(box[0] + box[2]) / 2, (box[1] + box[3]) / 2], + }; + }); +}; + +const spring = (position: number, velocity: number, target: number, dt: number) => { + const distance = position - target; + const pull = velocity + SPRING_OMEGA * distance; + const decay = Math.exp(-SPRING_OMEGA * dt); + return [ + target + (distance + pull * dt) * decay, + (velocity - SPRING_OMEGA * pull * dt) * decay, + ] as const; +}; + +/** + * `--zoom`: eases toward clicks, drags and marks, holds while activity stays nearby and eases out. + * Critically damped springs on the view's scale and offset, so a zoom travels as one move with no + * bounce. One rectangle of the capture per output frame. + */ +export const cameraPath = ( + manifest: BundleManifest, + crop: Rect, + videoOffsetMs: number, + frameCount: number, + fps: number, +): ReadonlyArray => { + const segments = cameraSegments(activities(manifest, crop)); + let scale = 1; + let scaleSpeed = 0; + let offsetX = 0; + let offsetXSpeed = 0; + let offsetY = 0; + let offsetYSpeed = 0; + const rects: Array = []; + const width = crop[2] - crop[0]; + const height = crop[3] - crop[1]; + let clockMs = 0; + const dt = CAMERA_STEP_MS / 1000; + for (let frame = 0; frame < frameCount; frame++) { + const untilMs = videoOffsetMs + (frame * 1000) / fps; + while (clockMs <= untilMs) { + const active = segments.findLast( + (segment) => segment.start <= clockMs && clockMs < segment.release && segment.scale > 1, + ); + const targetScale = active?.scale ?? 1; + const targetOffset = (focus: number) => clamp(0.5 - targetScale * focus, 1 - targetScale, 0); + [scale, scaleSpeed] = spring(scale, scaleSpeed, targetScale, dt); + [offsetX, offsetXSpeed] = spring( + offsetX, + offsetXSpeed, + targetOffset(active?.center[0] ?? 0.5), + dt, + ); + [offsetY, offsetYSpeed] = spring( + offsetY, + offsetYSpeed, + targetOffset(active?.center[1] ?? 0.5), + dt, + ); + if (scale < 1) { + scale = 1; + scaleSpeed = 0; + } + offsetX = clamp(offsetX, 1 - scale, 0); + offsetY = clamp(offsetY, 1 - scale, 0); + clockMs += CAMERA_STEP_MS; + } + const viewX = -offsetX / scale; + const viewY = -offsetY / scale; + rects.push([ + crop[0] + viewX * width, + crop[1] + viewY * height, + crop[0] + (viewX + 1 / scale) * width, + crop[1] + (viewY + 1 / scale) * height, + ]); + } + return rects; +}; diff --git a/apps/desktop/src/scrawl/scrawlRenderJob.ts b/apps/desktop/src/scrawl/scrawlRenderJob.ts new file mode 100644 index 0000000000000..7a8839d5f3dc0 --- /dev/null +++ b/apps/desktop/src/scrawl/scrawlRenderJob.ts @@ -0,0 +1,80 @@ +// What the command side hands the hidden page: everything decided, only pixels left. + +export const SCRAWL_RENDER_PROGRESS_CHANNEL = "scrawl-render-progress"; + +export type RenderRect = readonly [number, number, number, number]; + +export interface Annotations { + readonly strokes: ReadonlyArray<{ + readonly fromMs: number; + readonly untilMs: number; + /** Drawn entire from `fromMs`, instead of growing along its own time. */ + readonly whole: boolean; + /** Flat x, y, dt triples with positions in 0..=10000. */ + readonly points: ReadonlyArray; + }>; + readonly clicks: ReadonlyArray<{ + readonly t: number; + readonly x: number; + readonly y: number; + }>; +} + +export interface SheetJob { + readonly kind: "sheet"; + readonly video: string; + readonly videoOffsetMs: number; + readonly out: string; + readonly annotations: Annotations; + readonly cells: ReadonlyArray<{ + readonly t: number; + readonly label: string; + readonly view: RenderRect; + }>; +} + +export interface FramesJob { + readonly kind: "frames"; + readonly video: string; + readonly videoOffsetMs: number; + readonly frames: ReadonlyArray<{ readonly t: number; readonly out: string }>; +} + +export interface ExportJob { + readonly kind: "export"; + readonly video: string; + readonly videoOffsetMs: number; + readonly audio: string | null; + readonly out: string; + /** Written here and renamed on success, so a failed run never leaves a broken mp4 behind. */ + readonly partial: string; + readonly crop: RenderRect; + readonly width: number; + readonly height: number; + readonly fps: number; + readonly frameCount: number; + readonly annotations: Annotations; + /** A view per output frame when the camera moves. */ + readonly views: ReadonlyArray | null; +} + +export interface ProbeJob { + readonly kind: "probe"; + readonly video: string; +} + +export interface ProbeResult { + readonly width: number; + readonly height: number; + readonly seconds: number; + readonly packetRate: number; +} + +export type RenderJob = ProbeJob | SheetJob | FramesJob | ExportJob; + +export interface ExportResult { + readonly frames: number; + readonly seconds: number; + readonly audio: "aac" | "opus" | null; + readonly bytes: number; +} diff --git a/apps/desktop/src/scrawl/scrawlRenderPage.ts b/apps/desktop/src/scrawl/scrawlRenderPage.ts new file mode 100644 index 0000000000000..72fd6d1dc043a --- /dev/null +++ b/apps/desktop/src/scrawl/scrawlRenderPage.ts @@ -0,0 +1,440 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Runs in a hidden Chromium page with direct file access, outside any Effect runtime. +// The one place that touches pixels for `t3-scrawl`. Chromium is the only codec in the app, so this +// runs in a hidden page: mediabunny drives WebCodecs for decoding, encoding and the mp4 container. + +import { ipcRenderer } from "electron"; +import { + ALL_FORMATS, + AudioBufferSource, + CanvasSource, + CustomSource, + Input, + Mp4OutputFormat, + Output, + QUALITY_HIGH, + StreamTarget, + VideoSampleSink, + canEncodeAudio, + canEncodeVideo, + type InputVideoTrack, +} from "mediabunny"; +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; + +import { + SCRAWL_RENDER_PROGRESS_CHANNEL, + type Annotations, + type ExportJob, + type ExportResult, + type FramesJob, + type ProbeJob, + type ProbeResult, + type RenderJob, + type RenderRect, + type SheetJob, +} from "./scrawlRenderJob.ts"; + +const SHEET_PAD = 12; +const SHEET_GAP = 10; +const LANDSCAPE_CELL = 480; +const LANDSCAPE_COLUMNS = 3; +const PORTRAIT_CELL = 300; +const PORTRAIT_COLUMNS = 4; +const INK_COLOR = "#ff2d2d"; +const RING_COLOR = "rgba(239, 68, 68, 0.85)"; +const CLICK_RING_MS = 600; +const AUDIO_RATE = 48_000; +const AUDIO_CHUNK_SECONDS = 1; + +const openVideo = async (path: string) => { + const file = await NodeFSP.open(path, "r"); + const { size } = await file.stat(); + const input = new Input({ + formats: ALL_FORMATS, + source: new CustomSource({ + getSize: () => size, + read: async (start, end) => { + const bytes = new Uint8Array(end - start); + await file.read(bytes, 0, bytes.length, start); + return bytes; + }, + prefetchProfile: "fileSystem", + }), + }); + const close = async () => { + input.dispose(); + await file.close(); + }; + try { + const track = await input.getPrimaryVideoTrack(); + if (track === null) throw new Error("The recording has no video track"); + if (!(await track.canDecode())) { + throw new Error(`This app cannot decode the recording's ${track.codec ?? "unknown"} video`); + } + return { input, track, close }; + } catch (cause) { + await close(); + throw cause; + } +}; + +/** Times on the video's clock; a sink returns nothing for a time before the first frame. */ +const videoSeconds = async (track: InputVideoTrack, bundleMs: number, offsetMs: number) => + Math.max(await track.getFirstTimestamp(), (bundleMs - offsetMs) / 1000); + +const writePng = async (canvas: HTMLCanvasElement, path: string) => { + const blob = await new Promise((resolve, reject) => { + canvas.toBlob( + (result) => (result ? resolve(result) : reject(new Error("A frame cannot be encoded."))), + "image/png", + ); + }); + await NodeFSP.writeFile(path, new Uint8Array(await blob.arrayBuffer())); +}; + +const makeCanvas = (width: number, height: number) => { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d", { alpha: false, willReadFrequently: true }); + if (context === null) throw new Error("No canvas to draw on."); + context.imageSmoothingQuality = "high"; + return { canvas, context }; +}; + +/** Paints what the video does not hold, under a transform from capture fractions to the canvas. */ +const drawAnnotations = ( + context: CanvasRenderingContext2D, + annotations: Annotations, + bundleMs: number, + view: RenderRect, + target: { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; + }, +) => { + const scaleX = target.width / (view[2] - view[0]); + const scaleY = target.height / (view[3] - view[1]); + const toX = (x: number) => target.x + (x - view[0]) * scaleX; + const toY = (y: number) => target.y + (y - view[1]) * scaleY; + context.save(); + context.beginPath(); + context.rect(target.x, target.y, target.width, target.height); + context.clip(); + context.lineCap = "round"; + context.lineJoin = "round"; + const unit = Math.max(1.5, target.width / 480); + context.strokeStyle = INK_COLOR; + context.lineWidth = unit * 2; + for (const stroke of annotations.strokes) { + if (bundleMs < stroke.fromMs || bundleMs > stroke.untilMs) continue; + context.beginPath(); + for (let index = 0; index + 2 < stroke.points.length; index += 3) { + if (!stroke.whole && stroke.fromMs + stroke.points[index + 2]! > bundleMs) break; + const x = toX(stroke.points[index]! / 10_000); + const y = toY(stroke.points[index + 1]! / 10_000); + if (index === 0) context.moveTo(x, y); + else context.lineTo(x, y); + } + context.stroke(); + } + context.strokeStyle = RING_COLOR; + context.lineWidth = unit; + for (const click of annotations.clicks) { + const age = bundleMs - click.t; + if (age < 0 || age > CLICK_RING_MS) continue; + context.beginPath(); + context.arc(toX(click.x), toY(click.y), unit * (4 + (age / CLICK_RING_MS) * 5), 0, Math.PI * 2); + context.stroke(); + } + context.restore(); +}; + +const drawCellLabels = ( + context: CanvasRenderingContext2D, + label: string, + number: number, + cell: { readonly x: number; readonly y: number; readonly height: number }, +) => { + context.save(); + context.font = "600 13px system-ui"; + context.textBaseline = "middle"; + const pillWidth = context.measureText(label).width + 14; + const pillY = cell.y + cell.height - 28; + context.fillStyle = "rgba(18, 18, 20, 0.85)"; + context.beginPath(); + context.roundRect(cell.x + 6, pillY, pillWidth, 22, 11); + context.fill(); + context.fillStyle = "#ffffff"; + context.textAlign = "left"; + context.fillText(label, cell.x + 13, pillY + 12); + // The white square keeps a number readable as a number, as on the contact sheet. + context.strokeStyle = "#111111"; + context.lineWidth = 2; + context.beginPath(); + context.roundRect(cell.x + 5, cell.y + 5, 24, 24, 4); + context.fill(); + context.stroke(); + context.fillStyle = "#000000"; + context.font = "700 15px system-ui"; + context.textAlign = "center"; + context.fillText(String(number), cell.x + 17, cell.y + 18); + context.restore(); +}; + +const renderSheet = async (job: SheetJob) => { + const video = await openVideo(job.video); + try { + const sourceWidth = video.track.displayWidth; + const sourceHeight = video.track.displayHeight; + const aspects = job.cells.map(({ view }) => { + const width = (view[2] - view[0]) * sourceWidth; + const height = (view[3] - view[1]) * sourceHeight; + return { width, height }; + }); + const portrait = aspects.every(({ width, height }) => height > width); + const cellWidth = portrait ? PORTRAIT_CELL : LANDSCAPE_CELL; + const columns = Math.min(job.cells.length, portrait ? PORTRAIT_COLUMNS : LANDSCAPE_COLUMNS); + const heights = aspects.map(({ width, height }) => (cellWidth * height) / width); + const rows: Array = []; + for (let index = 0; index < heights.length; index += columns) { + rows.push(Math.max(...heights.slice(index, index + columns))); + } + const { canvas, context } = makeCanvas( + Math.ceil(SHEET_PAD * 2 + cellWidth * columns + SHEET_GAP * (columns - 1)), + Math.ceil( + SHEET_PAD * 2 + + rows.reduce((total, row) => total + row, 0) + + SHEET_GAP * Math.max(rows.length - 1, 0), + ), + ); + context.fillStyle = "#141414"; + context.fillRect(0, 0, canvas.width, canvas.height); + const sink = new VideoSampleSink(video.track); + const times = await Promise.all( + job.cells.map(({ t }) => videoSeconds(video.track, t, job.videoOffsetMs)), + ); + let index = 0; + for await (const sample of sink.samplesAtTimestamps(times)) { + const cell = job.cells[index]!; + const column = index % columns; + const row = Math.floor(index / columns); + const x = SHEET_PAD + column * (cellWidth + SHEET_GAP); + const y = + SHEET_PAD + rows.slice(0, row).reduce((total, height) => total + height + SHEET_GAP, 0); + const height = heights[index]!; + if (sample !== null) { + sample.draw( + context, + cell.view[0] * sourceWidth, + cell.view[1] * sourceHeight, + aspects[index]!.width, + aspects[index]!.height, + x, + y, + cellWidth, + height, + ); + sample.close(); + } + drawAnnotations(context, job.annotations, cell.t, cell.view, { + x, + y, + width: cellWidth, + height, + }); + drawCellLabels(context, cell.label, index + 1, { x, y, height }); + index++; + } + await writePng(canvas, job.out); + } finally { + await video.close(); + } +}; + +const renderFrames = async (job: FramesJob) => { + const video = await openVideo(job.video); + try { + const { canvas, context } = makeCanvas(video.track.displayWidth, video.track.displayHeight); + const sink = new VideoSampleSink(video.track); + const times = await Promise.all( + job.frames.map(({ t }) => videoSeconds(video.track, t, job.videoOffsetMs)), + ); + let index = 0; + for await (const sample of sink.samplesAtTimestamps(times)) { + if (sample !== null) { + sample.draw(context, 0, 0, canvas.width, canvas.height); + sample.close(); + await writePng(canvas, job.frames[index]!.out); + } + index++; + } + } finally { + await video.close(); + } +}; + +const probe = async (job: ProbeJob): Promise => { + const video = await openVideo(job.video); + try { + const first = await video.track.getFirstTimestamp(); + const stats = await video.track.computePacketStats(100); + return { + width: video.track.displayWidth, + height: video.track.displayHeight, + seconds: (await video.track.computeDuration()) - first, + packetRate: stats.averagePacketRate, + }; + } finally { + await video.close(); + } +}; + +/** Resampled to what AAC encoders take: the recording's audio is 16 kHz for the dictation model. */ +const readAudio = async (path: string, skipSeconds: number, seconds: number) => { + const decoder = new OfflineAudioContext(1, AUDIO_RATE, AUDIO_RATE); + const wav = await NodeFSP.readFile(path); + // A silent capture has a WAV with nothing in it, which is no reason to fail the video. + const decoded = await decoder + .decodeAudioData( + wav.buffer.slice(wav.byteOffset, wav.byteOffset + wav.byteLength) as ArrayBuffer, + ) + .catch(() => null); + if (decoded === null) return null; + const samples = decoded.getChannelData(0); + const first = Math.round(skipSeconds * decoded.sampleRate); + const count = Math.min(samples.length - first, Math.ceil(seconds * decoded.sampleRate)); + return count > 0 ? samples.subarray(first, first + count) : null; +}; + +const audioCodecFor = async () => { + const options = { numberOfChannels: 1, sampleRate: AUDIO_RATE, bitrate: QUALITY_HIGH }; + if (await canEncodeAudio("aac", options)) return "aac" as const; + return (await canEncodeAudio("opus", options)) ? ("opus" as const) : null; +}; + +const renderExport = async (job: ExportJob): Promise => { + const video = await openVideo(job.video); + const out = await NodeFSP.open(job.partial, "w"); + try { + if (!(await canEncodeVideo("avc", { width: job.width, height: job.height }))) { + throw new Error("This app has no H.264 encoder on this machine."); + } + const first = await video.track.getFirstTimestamp(); + const { frameCount } = job; + const views = job.views ?? [job.crop]; + const output = new Output({ + format: new Mp4OutputFormat(), + target: new StreamTarget( + new WritableStream({ + write: async (chunk: { data: Uint8Array; position: number }) => { + await out.write(chunk.data, 0, chunk.data.length, chunk.position); + }, + }), + ), + }); + const { canvas, context } = makeCanvas(job.width, job.height); + const videoSource = new CanvasSource(canvas, { + codec: "avc", + bitrate: QUALITY_HIGH, + keyFrameInterval: 2, + }); + output.addVideoTrack(videoSource, { frameRate: job.fps }); + + const pcm = + job.audio === null + ? null + : await readAudio(job.audio, job.videoOffsetMs / 1000, frameCount / job.fps); + const audioCodec = pcm === null ? null : await audioCodecFor(); + const audioSource = + audioCodec === null ? null : new AudioBufferSource({ codec: audioCodec, bitrate: 128_000 }); + if (audioSource !== null) output.addAudioTrack(audioSource); + await output.start(); + + let audioAt = 0; + const addAudioUntil = async (seconds: number) => { + if (pcm === null || audioSource === null) return; + const until = Math.min(pcm.length, Math.ceil(seconds * AUDIO_RATE)); + if (until <= audioAt) return; + const chunk = new AudioBuffer({ + length: until - audioAt, + numberOfChannels: 1, + sampleRate: AUDIO_RATE, + }); + chunk.copyToChannel(pcm.subarray(audioAt, until) as Float32Array, 0); + audioAt = until; + await audioSource.add(chunk); + }; + + const sink = new VideoSampleSink(video.track); + const times = Array.from({ length: frameCount }, (_unused, index) => first + index / job.fps); + const sourceWidth = video.track.displayWidth; + const sourceHeight = video.track.displayHeight; + let index = 0; + let lastProgress = 0; + for await (const sample of sink.samplesAtTimestamps(times)) { + const view = views[Math.min(index, views.length - 1)]!; + const seconds = index / job.fps; + if (sample !== null) { + sample.draw( + context, + view[0] * sourceWidth, + view[1] * sourceHeight, + (view[2] - view[0]) * sourceWidth, + (view[3] - view[1]) * sourceHeight, + 0, + 0, + job.width, + job.height, + ); + sample.close(); + } + drawAnnotations(context, job.annotations, job.videoOffsetMs + seconds * 1000, view, { + x: 0, + y: 0, + width: job.width, + height: job.height, + }); + await addAudioUntil(seconds + AUDIO_CHUNK_SECONDS); + await videoSource.add(seconds, 1 / job.fps); + index++; + const progress = Math.floor((index / frameCount) * 10); + if (progress > lastProgress) { + lastProgress = progress; + ipcRenderer.send(SCRAWL_RENDER_PROGRESS_CHANNEL, progress * 10); + } + } + await addAudioUntil(Number.POSITIVE_INFINITY); + await output.finalize(); + await out.close(); + await NodeFSP.rename(job.partial, job.out); + return { + frames: index, + seconds: frameCount / job.fps, + audio: audioCodec, + bytes: NodeFS.statSync(job.out).size, + }; + } catch (cause) { + await out.close().catch(() => undefined); + await NodeFSP.rm(job.partial, { force: true }).catch(() => undefined); + throw cause; + } finally { + await video.close(); + } +}; + +/** @public Called by name from the hidden page. */ +export const run = async (job: RenderJob): Promise => { + if (job.kind === "probe") return probe(job); + if (job.kind === "sheet") { + await renderSheet(job); + return null; + } + if (job.kind === "frames") { + await renderFrames(job); + return null; + } + return renderExport(job); +}; diff --git a/apps/desktop/src/scrawl/scrawlShim.test.ts b/apps/desktop/src/scrawl/scrawlShim.test.ts new file mode 100644 index 0000000000000..9e59fb8edc84f --- /dev/null +++ b/apps/desktop/src/scrawl/scrawlShim.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { renderScrawlShim } from "./scrawlShim.ts"; + +describe("renderScrawlShim", () => { + it("clears run-as-node, which an agent inherits and which has no video decoder", () => { + const windows = renderScrawlShim({ + platform: "win32", + executable: "C:\\Program Files (x86)\\T3 100%\\T3 Code.exe", + t3Home: "C:\\Users\\me\\.t3", + }); + expect(windows).toContain('set "ELECTRON_RUN_AS_NODE="\r\n'); + expect(windows).toContain('"C:\\Program Files (x86)\\T3 100%%\\T3 Code.exe" --scrawl %*'); + const unix = renderScrawlShim({ + platform: "linux", + executable: "/opt/it's/t3", + t3Home: "/home/me/.t3", + }); + expect(unix).toContain("unset ELECTRON_RUN_AS_NODE\n"); + expect(unix).toContain(`app='/opt/it'"'"'s/t3'`); + expect(unix).toContain('exec "$app" --scrawl "$@"'); + }); +}); diff --git a/apps/desktop/src/scrawl/scrawlShim.ts b/apps/desktop/src/scrawl/scrawlShim.ts new file mode 100644 index 0000000000000..a4124a76687f3 --- /dev/null +++ b/apps/desktop/src/scrawl/scrawlShim.ts @@ -0,0 +1,62 @@ +// The command an agent runs. `t3.cmd` cannot serve here: it starts the app as plain Node, which has +// no video decoder. Not called `scrawl`, so the real Scrawl app stays what that name finds on PATH. + +export const SCRAWL_CLI_FLAG = "--scrawl"; + +const MOVED = "T3 Code has moved or been removed. Open the app once to update this command."; +const MARKER = "Written by T3 Code: looks closer at and exports Scrawl captures."; + +export const scrawlShimName = (platform: NodeJS.Platform) => + platform === "win32" ? "t3-scrawl.cmd" : "t3-scrawl"; + +const cmdText = (value: string) => value.replaceAll("%", "%%"); +const cmdWord = (value: string) => `"${cmdText(value)}"`; +const shellWord = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`; + +export const renderScrawlShim = (input: { + readonly platform: NodeJS.Platform; + readonly executable: string; + readonly t3Home: string; +}) => { + if (input.platform !== "win32") { + return [ + "#!/bin/sh", + `# ${MARKER}`, + `home=${shellWord(input.t3Home)}`, + 'export T3CODE_HOME="${T3CODE_HOME:-$home}"', + "unset ELECTRON_RUN_AS_NODE", + `app=${shellWord(input.executable)}`, + 'if [ ! -x "$app" ]; then', + ` echo "${MOVED}" >&2`, + " exit 127", + "fi", + `exec "$app" ${SCRAWL_CLI_FLAG} "$@"`, + "", + ].join("\n"); + } + const utf8 = [input.executable, input.t3Home].some((value) => + [...value].some((character) => character.codePointAt(0)! > 0x7f), + ); + const restore = utf8 ? ["chcp %t3_codepage% >nul"] : []; + return [ + "@echo off", + `rem ${MARKER}`, + "setlocal EnableExtensions DisableDelayedExpansion", + ...(utf8 + ? [`for /f "tokens=2 delims=:." %%c in ('chcp') do set "t3_codepage=%%c"`, "chcp 65001 >nul"] + : []), + `if not defined T3CODE_HOME set "T3CODE_HOME=${cmdText(input.t3Home)}"`, + // Agents can inherit this from a server the app runs as Node, and it would make this Node too. + 'set "ELECTRON_RUN_AS_NODE="', + `if exist ${cmdWord(input.executable)} goto run`, + ...restore, + `echo ${MOVED} 1>&2`, + "exit /b 127", + ":run", + `${cmdWord(input.executable)} ${SCRAWL_CLI_FLAG} %*`, + 'set "t3_exit=%ERRORLEVEL%"', + ...restore, + "exit /b %t3_exit%", + "", + ].join("\r\n"); +}; diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index 55c3aa03b6490..db79ca7d3e368 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -87,7 +87,21 @@ export default defineConfig({ }, }, { - // boot.cjs requires the other two at runtime, so all three stay separate files. + format: "cjs", + outDir: "dist-electron", + dts: false, + sourcemap: true, + outExtensions: () => ({ js: ".cjs" }), + entry: ["src/scrawl/scrawlCli.ts", "src/scrawl/scrawlRenderPage.ts"], + clean: false, + deps: { + alwaysBundle: (id) => !id.startsWith("node:") && id !== "electron", + neverBundle: (id) => id === "electron", + onlyBundle: false, + }, + }, + { + // boot.cjs requires the others at runtime, so they all stay separate files. format: "cjs", outDir: "dist-electron", dts: false, @@ -96,7 +110,8 @@ export default defineConfig({ entry: ["src/boot.ts", "src/compileCache.ts"], clean: false, deps: { - neverBundle: (id) => id === "./main.cjs" || id === "./compileCache.cjs", + neverBundle: (id) => + id === "./main.cjs" || id === "./compileCache.cjs" || id === "./scrawlCli.cjs", }, }, { diff --git a/apps/web/src/scrawl/scrawlBundle.ts b/apps/web/src/scrawl/scrawlBundle.ts index f90df615ecd55..ba44b34f5f816 100644 --- a/apps/web/src/scrawl/scrawlBundle.ts +++ b/apps/web/src/scrawl/scrawlBundle.ts @@ -1,6 +1,6 @@ // Builds what an agent reads from a finished recording: keyframes taken from the video, the contact -// sheet, the summary text and the manifest. The folder keeps Scrawl's shape, so the Scrawl skill and -// the `scrawl` CLI (zoom, frames, export) read these bundles as well. +// sheet, the summary text and the manifest. The folder keeps Scrawl's shape, so `t3-scrawl`, the Scrawl +// skill and the `scrawl` CLI read these bundles alike. import { SCRAWL_INK_HOLD_MS, @@ -262,11 +262,16 @@ const summaryText = ( lines: ReadonlyArray, cropped: boolean, ) => { - const { app, windowTitle, bundlePath } = recording.session; + const { app, windowTitle, bundlePath, lookCloserCommand } = recording.session; const source = [app, windowTitle ? `“${windowTitle}”` : ""].filter(Boolean).join(" ") || "Screen"; return [ `[scrawl ${formatDuration(recording.durationMs)} · ${source} · ${bundlePath}]`, "Open contact-sheet.png and frames/ in that folder before changing anything: the marks and words only make sense on the images, which can show more than the text does.", + ...(lookCloserCommand === undefined + ? [] + : [ + `To look closer (motion, a flicker, what a mark points at), run: "${lookCloserCommand}" zoom "${bundlePath}" --at 0:21 (or --at "a phrase", or --mark m1). It prints a PNG to open. Also: frames --from --to, inspect --marks, export.`, + ]), ...(cropped ? ["Cropped to: the activity (full frames in frames-full/)"] : []), ...lines.map(formatNarrationLine), ].join("\n"); @@ -350,6 +355,7 @@ export async function buildBundle( const manifest = { version: 1, id: recording.session.id, + inkInVideo: true, createdAtMs: Date.now(), durationMs: recording.durationMs, target: { diff --git a/knip.jsonc b/knip.jsonc index 3cec1656d0cc3..5fca36015c380 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -69,6 +69,8 @@ "gnome-extension/extension.js!", "src/electron/WindowsForegroundFocusWorker.ts!", "src/snapShot/GlobalShiftShortcutWorker.ts!", + "src/scrawl/scrawlCli.ts!", + "src/scrawl/scrawlRenderPage.ts!", "src/snapShot/RegionSnapShotWorker.ts!", "src/snapShot/SnapShotAccessibilityWorker.ts!", "src/boot.ts!", diff --git a/packages/contracts/src/scrawl.ts b/packages/contracts/src/scrawl.ts index 665833f19d28b..c10bfbf373ce7 100644 --- a/packages/contracts/src/scrawl.ts +++ b/packages/contracts/src/scrawl.ts @@ -17,6 +17,8 @@ export const ScrawlStartResult = Schema.Struct({ app: Schema.String, windowTitle: Schema.String, bundlePath: Schema.String, + /** The command agents look closer with. Only an installed app writes one. */ + lookCloserCommand: Schema.optional(Schema.String), }); export type ScrawlStartResult = typeof ScrawlStartResult.Type; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ff8c13115670d..d3a6bc091e963 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -469,6 +469,9 @@ importers: ffi-rs: specifier: 1.3.2 version: 1.3.2 + mediabunny: + specifier: 1.61.6 + version: 1.61.6 playwright-core: specifier: 1.60.0 version: 1.60.0 @@ -7078,6 +7081,12 @@ packages: '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/dom-mediacapture-transform@0.1.12': + resolution: {integrity: sha512-d7/QsLRwF864A5mgIM/YrfiglHoYn7zgCcAoJgW404r+2DwnNr7EBbLnCWpmOMgH8y0te73L1AV6H1bmauaWFw==} + + '@types/dom-webcodecs@0.1.13': + resolution: {integrity: sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==} + '@types/emscripten@1.41.5': resolution: {integrity: sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==} @@ -10109,6 +10118,9 @@ packages: resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} engines: {node: '>= 0.8'} + mediabunny@1.61.6: + resolution: {integrity: sha512-VHEZ53/7jGl4dQ+uOdndsZufFYwEUFWv9UXgvrGun3OapyzoqI9iaPOD7z3xxY/nAttLZBNwswgJzMtGJAquOQ==} + memoize-one@5.2.1: resolution: {integrity: sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==} @@ -16706,6 +16718,12 @@ snapshots: '@types/deep-eql@4.0.2': {} + '@types/dom-mediacapture-transform@0.1.12': + dependencies: + '@types/dom-webcodecs': 0.1.13 + + '@types/dom-webcodecs@0.1.13': {} + '@types/emscripten@1.41.5': {} '@types/estree-jsx@1.0.5': @@ -20164,6 +20182,11 @@ snapshots: media-typer@1.1.0: {} + mediabunny@1.61.6: + dependencies: + '@types/dom-mediacapture-transform': 0.1.12 + '@types/dom-webcodecs': 0.1.13 + memoize-one@5.2.1: {} merge-descriptors@2.0.0: {} From aa251adb5208a251ca47fb74609b48ebcfde3efb Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:29:56 +0100 Subject: [PATCH 7/9] fix(scrawl): a crash leaves a readable video, not just a header The MP4 muxer writes nothing until the next keyframe, so a capture that crashed had only a header on disk. A 2-second keyframe interval puts a fragment there every 2 seconds. Co-Authored-By: Claude Opus 5.5 --- FORK.md | 2 ++ apps/web/src/components/scrawl/ScrawlRuntime.tsx | 5 +++-- apps/web/src/scrawl/scrawlCapture.ts | 9 ++++++++- docs/README.md | 1 + 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/FORK.md b/FORK.md index 33112eff9ce73..0cd886e08d0af 100644 --- a/FORK.md +++ b/FORK.md @@ -175,6 +175,8 @@ the contact sheet land in the composer on screen. Esc in T3 Code discards a capt them back. A crash or reload during a capture loses at most the last seconds: the next launch rebuilds it once from `unfinished.json`, `timing.json`, `audio.pcm` and the video. A capture that fails is kept and offers a manual rebuild. Scene changes come from sampling the video again. +- The recorder sets a 2-second keyframe interval on purpose: without it the MP4 muxer put nothing but + a header on disk for over 11 seconds, so a crash left no video. Measured in Chromium. - Past captures (Alt+Shift+H, palette, Settings) find, insert, pin, or bin a capture and show the date retention acts on. Same presets and `.pinned` marker as Scrawl: by default video goes to the Recycle Bin after 7 days and the capture after 30. The list and the hourly pass share one plan diff --git a/apps/web/src/components/scrawl/ScrawlRuntime.tsx b/apps/web/src/components/scrawl/ScrawlRuntime.tsx index c0413432a6093..8741096e4ef0b 100644 --- a/apps/web/src/components/scrawl/ScrawlRuntime.tsx +++ b/apps/web/src/components/scrawl/ScrawlRuntime.tsx @@ -4,7 +4,7 @@ import { useEffect, useSyncExternalStore } from "react"; import { resolveShortcutCommand } from "../../keybindings"; import { getTerminalFocusOwner } from "../../lib/terminalFocus"; -import { isScrawlAvailable, toggleScrawl } from "../../scrawl/scrawlCapture"; +import { getScrawlState, isScrawlAvailable, toggleScrawl } from "../../scrawl/scrawlCapture"; import { getScrawlHistoryState, isScrawlHistoryOpen, @@ -34,7 +34,8 @@ export function ScrawlRuntime() { // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- starts the one rebuild pass of this page load, which runs outside React useEffect(() => { - if (!isScrawlAvailable()) return; + // Asking frees a capture the desktop app still runs, so never while this page has its own. + if (!isScrawlAvailable() || getScrawlState().phase !== "idle") return; void rebuildUnfinishedCaptures().catch((error: unknown) => { console.warn("scrawl: unfinished captures could not be listed", error); }); diff --git a/apps/web/src/scrawl/scrawlCapture.ts b/apps/web/src/scrawl/scrawlCapture.ts index 3d8ce00c5090b..b6cbdf376bc52 100644 --- a/apps/web/src/scrawl/scrawlCapture.ts +++ b/apps/web/src/scrawl/scrawlCapture.ts @@ -33,6 +33,9 @@ import { const SEND_INTERVAL_MS = 500; const SAMPLE_INTERVAL_MS = 750; const VIDEO_TIMESLICE_MS = 2_000; +// The MP4 muxer hands over nothing until the next keyframe, whatever the timeslice, and left alone +// that took over 11 seconds in Chromium: a crash would leave a header and no video. +const VIDEO_KEYFRAME_INTERVAL_MS = 2_000; // Keyframes come out of this video, so small text has to survive it. const VIDEO_BITS_PER_SECOND = 6_000_000; const MAX_FRAME_RATE = 15; @@ -205,7 +208,11 @@ const start = async (bridge: DesktopBridge) => { const format = chooseVideoFormat(); const recorder = new MediaRecorder( new MediaStream([...screen.getVideoTracks(), ...mic.stream.getAudioTracks()]), - { mimeType: format.mimeType, videoBitsPerSecond: VIDEO_BITS_PER_SECOND }, + { + mimeType: format.mimeType, + videoBitsPerSecond: VIDEO_BITS_PER_SECOND, + videoKeyFrameIntervalDuration: VIDEO_KEYFRAME_INTERVAL_MS, + } as MediaRecorderOptions, ); const samples: Array = []; const capture: Running = { diff --git a/docs/README.md b/docs/README.md index af3c45ce567b1..529042ea37e5b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,7 @@ - [Appearance and themes](./user/appearance.md) - [Keyboard shortcuts](./user/keybindings.md) - [SnapShots](./user/snap-shot.md) +- [Screen capture](./user/screen-capture.md) - [Visual replies](./user/html-renders.md) - [Import browser sessions](./user/browser-import.md) - [Use T3 Code as your default browser](./user/default-browser.md) From 302d365b237503196c23b72c3c554d11211a7bb0 Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:34:03 +0100 Subject: [PATCH 8/9] fix(scrawl): names survive the click-through overlays and a second capture Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/scrawl/DesktopScrawl.ts | 12 ++++++---- apps/desktop/src/scrawl/windowAtPoint.ts | 30 ++++++++++++++---------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/apps/desktop/src/scrawl/DesktopScrawl.ts b/apps/desktop/src/scrawl/DesktopScrawl.ts index 4a6c6c132daeb..77730039054ba 100644 --- a/apps/desktop/src/scrawl/DesktopScrawl.ts +++ b/apps/desktop/src/scrawl/DesktopScrawl.ts @@ -273,6 +273,7 @@ const make = Effect.gen(function* () { const nativeId = (window: Electron.BrowserWindow) => Number(window.getNativeWindowHandle().readBigUInt64LE() & 0xffff_ffffn); + const overlayIds = new Set([nativeId(inkWindow), nativeId(pill)]); const relative = (at: number) => at - startedAtMs; const watcher = keys === null @@ -285,7 +286,7 @@ const make = Effect.gen(function* () { Electron.screen.dipToScreenRect(null, display.bounds), Electron.screen.dipToScreenRect(null, display.workArea), ), - ignoredWindowIds: [nativeId(inkWindow), nativeId(pill)], + ignoredWindowIds: [...overlayIds], }, (message) => { if (message.type === "wheel") { @@ -314,11 +315,12 @@ const make = Effect.gen(function* () { if (at.x < x || at.y < y || at.x >= x + width || at.y >= y + height) return; if (keys === null || windowAtPoint === null) return; const point = Electron.screen.dipToScreenPoint(at); - // Asking about our own window would turn on this renderer's accessibility tree. const window = windowAtPoint(point.x, point.y); - if (window === 0n || keys.getWindowThreadAndProcessId(window).processId === process.pid) { - return; - } + if (window === 0n) return; + // Asking about our own window would turn on this renderer's accessibility tree. The overlays + // let clicks through, so what is under them is someone else's. + const overlay = overlayIds.has(Number(window & 0xffff_ffffn)); + if (!overlay && keys.getWindowThreadAndProcessId(window).processId === process.pid) return; current.lookups?.look(key, point.x, point.y); }; diff --git a/apps/desktop/src/scrawl/windowAtPoint.ts b/apps/desktop/src/scrawl/windowAtPoint.ts index 71c8523003a18..36a321822bfc7 100644 --- a/apps/desktop/src/scrawl/windowAtPoint.ts +++ b/apps/desktop/src/scrawl/windowAtPoint.ts @@ -1,15 +1,21 @@ // WindowFromPoint takes its POINT by value, which on x64 travels as one 64-bit integer. -export async function loadWindowAtPoint(): Promise<(x: number, y: number) => bigint> { - const { DataType, load, open } = await import("ffi-rs"); - const library = "scrawl-point-user32"; - open({ library, path: "user32.dll" }); - return (x, y) => - load({ - library, - funcName: "WindowFromPoint", - retType: DataType.BigInt, - paramsType: [DataType.BigInt], - paramsValue: [(BigInt(Math.round(y)) << 32n) | (BigInt(Math.round(x)) & 0xffff_ffffn)], - }) as bigint; +type WindowAtPoint = (x: number, y: number) => bigint; + +let windowAtPoint: Promise | undefined; + +export function loadWindowAtPoint(): Promise { + windowAtPoint ??= import("ffi-rs").then(({ DataType, load, open }) => { + const library = "scrawl-point-user32"; + open({ library, path: "user32.dll" }); + return (x, y) => + load({ + library, + funcName: "WindowFromPoint", + retType: DataType.BigInt, + paramsType: [DataType.BigInt], + paramsValue: [(BigInt(Math.round(y)) << 32n) | (BigInt(Math.round(x)) & 0xffff_ffffn)], + }) as bigint; + }); + return windowAtPoint; } From aee464247b28b7dc5d60898f81c10f07e3d49a05 Mon Sep 17 00:00:00 2001 From: collissimo <148455189+collissimo@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:34:48 +0100 Subject: [PATCH 9/9] docs(scrawl): say --no-ink cannot take ink out of a native capture Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/scrawl/scrawlCli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/desktop/src/scrawl/scrawlCli.ts b/apps/desktop/src/scrawl/scrawlCli.ts index f624bcdf1316f..e004b37186a02 100644 --- a/apps/desktop/src/scrawl/scrawlCli.ts +++ b/apps/desktop/src/scrawl/scrawlCli.ts @@ -58,7 +58,7 @@ const USAGE = `t3-scrawl: look closer at a Scrawl capture and export it. --from 0:04 --to 0:06 [--fps 4] plain PNGs of a range t3-scrawl inspect [--json | --marks [--json]] the text, the manifest, the marks t3-scrawl export [--out demo.mp4] [--zoom] [--crop auto|full|x,y,w,h] [--size 1920] - [--fps N] [--no-clicks] [--no-ink] one mp4 with the audio in sync + [--fps N] [--no-clicks] [--no-ink] one mp4 with the audio in sync (T3 captures have the ink in their video, so --no-ink changes nothing there) zoom and frames write into /zoom/ and print the paths: open the PNGs and look at them. Video and audio are deleted 7 days after a capture; frames/, the contact sheet and scrawl.json stay.`;