diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index 9feb82cb8..b80485b94 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -1,5 +1,7 @@ import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; +import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { PACKAGED_SKILL_INVOCATIONS } from '../../../packaged-skill-invocations'; @@ -130,6 +132,36 @@ describe('feature-demo skill', () => { expect(captureRunner).toContain('pushCursorMove(moveStart, moveEnd, c);'); }); + it.each(['focus', 'reset'])('rejects the unsupported %s action', (action) => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'feature-demo-')); + const scriptPath = path.join(tempDir, 'demo-script.json'); + fs.writeFileSync( + scriptPath, + JSON.stringify({ url: 'https://example.com', beats: [{ a: action }] }), + ); + + try { + const result = spawnSync( + process.execPath, + [path.join(skillDirPath, 'capture/capture.mjs')], + { + encoding: 'utf8', + env: { + ...process.env, + SCRIPT: scriptPath, + AGENT_BROWSER_BIN: 'must-not-run-agent-browser', + }, + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(`unknown beat action: ${action}`); + expect(result.stderr).not.toContain('must-not-run-agent-browser'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('asks the advisor for a flowing narration, not sparse labels', () => { expect(skillContent).toContain('write the NARRATION FIRST'); expect(skillContent).toContain( diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index b4813e33d..f70066690 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -4,13 +4,12 @@ // // Two ideas make the output polished: // 1. The runner performs the real interactions AND logs, for the same clock, -// where the (synthetic) cursor is, when clicks land, and the resolved -// rect of each target — so a zoom can never drift from its element. +// where the (synthetic) cursor is and when clicks land. // 2. The NARRATIVE drives the visuals: each captioned beat holds for as long // as its line takes to speak (the real clip duration when narration was // synthesized before capture; an estimated speaking time for the caption -// text otherwise), and the runner stamps each clip's start at the moment -// its zoom actually lands. Nothing needs retiming afterwards. +// text otherwise), and the runner stamps each clip's start as its visual +// settles. Nothing needs retiming afterwards. // // Usage: SCRIPT=/path/to/demo-script.json OUT_DIR=/tmp/feature-demo/work \ // node capture.mjs @@ -38,6 +37,23 @@ if (!script.url || !Array.isArray(script.beats)) { process.exit(1); } +const SUPPORTED_ACTIONS = new Set([ + 'show', + 'wait', + 'hold', + 'scrollTo', + 'click', + 'type', +]); +const unsupportedBeat = script.beats.find( + (beat) => !SUPPORTED_ACTIONS.has(beat.a), +); + +if (unsupportedBeat) { + console.error(`unknown beat action: ${unsupportedBeat.a}`); + process.exit(1); +} + const VIEWPORT = script.viewport || { w: 1280, h: 800 }; // Narration manifest (pre-capture synthesis). Optional: without it the demo @@ -58,8 +74,6 @@ if (narration && narration.clips.length !== captionedBeatCount) { process.exit(1); } -// The voice starts just before its zoom lands, then speaks over the hold. -const VOICE_LEAD = 0.4; // Breathing room after a line ends before the next beat's motion begins. const LINE_GAP = 0.35; @@ -70,14 +84,6 @@ function estimateSpokenSeconds(text) { return Math.min(10, Math.max(1.8, words / 2.8)); } -// Between focus beats the camera pulls back only partially; pogo-ing to full -// wide between every zoom reads as jumpy. Kept well below the renderer's -// cap on zoom (the largest scale that keeps the whole window on the stage, -// about 1.16 for a wide demo with captions) so a following focus beat still -// reads as a distinct push-in rather than matching the glide. The final -// reset goes fully wide. -const GLIDE_SCALE = 1.06; - const ab = (...args) => execFileSync(AB, args, { encoding: 'utf8', @@ -116,8 +122,6 @@ const timeline = { video: { path: 'recording.mp4', width: VIEWPORT.w, height: VIEWPORT.h }, fps: 30, durationSeconds: 0, - scaleKeys: [{ t: 0, v: 1 }], - focalKeys: [{ t: 0, v: { x: 0.5, y: 0.5 } }], cursorKeys: [{ t: 0, v: { x: 0.5, y: 1.1 } }], clicks: [], captions: [], @@ -126,7 +130,6 @@ const timeline = { ...(script.captionStyle ? { captionStyle: script.captionStyle } : {}), }; -let cur = { scale: 1, focal: { x: 0.5, y: 0.5 } }; // Narrative pacing state: which line is next, and when the previous one // finishes, so consecutive lines never overlap even if beats land early. let lineIndex = 0; @@ -136,8 +139,6 @@ let prevLineEnd = 0; // synthetic cursor would drift toward the next target through every wait, // hold, and scroll in between while the real mouse is stationary. let curCursor = { x: 0.5, y: 1.1 }; -const pushScale = (t, v) => timeline.scaleKeys.push({ t, v }); -const pushFocal = (t, v) => timeline.focalKeys.push({ t, v }); const pushCursorMove = (startT, endT, target) => { timeline.cursorKeys.push({ t: startT, v: curCursor }); timeline.cursorKeys.push({ t: endT, v: target }); @@ -222,59 +223,6 @@ async function run() { continue; } - if (beat.a === 'focus') { - const c = centerNorm(rect(beat.sel)); - const start = now(); - pushScale(start, cur.scale); - pushFocal(start, cur.focal); - // Real hover so the app's hover state shows under the synthetic cursor. - ab( - 'mouse', - 'move', - String((c.x * VIEWPORT.w) | 0), - String((c.y * VIEWPORT.h) | 0), - ); - sleep(beat.moveMs ?? 700); // pace the glide for the eased cursor - const end = now(); - pushScale(end, beat.scale ?? 1.5); - pushFocal(end, c); - pushCursorMove(start, end, c); - cur = { scale: beat.scale ?? 1.5, focal: c }; - - if (beat.caption) { - // The narrative drives the hold: the line starts just before the - // zoom lands and the beat holds until it has been fully spoken, - // plus breathing room. Script holdMs acts as a minimum only. - const lineSeconds = narration - ? narration.clips[lineIndex].durationSeconds - : estimateSpokenSeconds(beat.caption); - const lineStart = - Math.round(Math.max(0.1, prevLineEnd + 0.1, end - VOICE_LEAD) * 1000) / - 1000; - const lineEnd = lineStart + lineSeconds; - prevLineEnd = lineEnd; - - timeline.captions.push({ - start: lineStart, - end: Math.round((lineEnd + 0.25) * 1000) / 1000, - text: beat.caption, - }); - if (narration) { - narration.clips[lineIndex].startSeconds = lineStart; - } - lineIndex += 1; - - const holdSeconds = Math.max( - (beat.holdMs ?? 900) / 1000, - lineEnd + LINE_GAP - now(), - ); - sleep(holdSeconds * 1000); - } else { - sleep(beat.holdMs ?? 900); - } - continue; - } - if (beat.a === 'click') { const c = centerNorm(rect(beat.sel)); const t = now(); @@ -305,35 +253,9 @@ async function run() { continue; } - if (beat.a === 'reset') { - // Partial pull-back keeps the camera alive between beats; `full: true` - // (or the closing reset) returns to wide. - const target = beat.full ? 1 : GLIDE_SCALE; - const start = now(); - pushScale(start, cur.scale); - pushFocal(start, cur.focal); - sleep(beat.ms ?? 600); - const end = now(); - pushScale(end, target); - pushFocal(end, { x: 0.5, y: 0.5 }); - cur = { scale: target, focal: { x: 0.5, y: 0.5 } }; - continue; - } - throw new Error(`unknown beat action: ${beat.a}`); } - // Close on a full wide shot even when the script forgot a final reset. - if (cur.scale !== 1) { - const start = now(); - pushScale(start, cur.scale); - pushFocal(start, cur.focal); - sleep(600); - const end = now(); - pushScale(end, 1); - pushFocal(end, { x: 0.5, y: 0.5 }); - } - timeline.durationSeconds = now(); ab('record', 'stop'); try { @@ -395,7 +317,7 @@ async function run() { writeFileSync(`${OUT_DIR}/timeline.json`, JSON.stringify(timeline, null, 2)); if (narration) { - // Clip start times are now stamped at the moments their zooms landed; + // Clip start times are now stamped at the moments their visuals settled; // this manifest plus the vo/ mp3s next to the script is everything the // renderer needs. writeFileSync( diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json index 5b7bbb061..f1d432b2f 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json @@ -2,8 +2,6 @@ "video": { "path": "recording.mp4", "width": 1280, "height": 800 }, "fps": 30, "durationSeconds": 1, - "scaleKeys": [{ "t": 0, "v": 1 }], - "focalKeys": [{ "t": 0, "v": { "x": 0.5, "y": 0.5 } }], "cursorKeys": [{ "t": 0, "v": { "x": 0.5, "y": 1.1 } }], "clicks": [], "captions": [] diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx index 0df00dbf6..2c5794df0 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx @@ -30,16 +30,6 @@ const smooth = (t: number) => t * t * (3 - 2 * t); const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v)); -function lerpNum(keys: { t: number; v: number }[], t: number): number { - if (t <= keys[0].t) return keys[0].v; - for (let i = 0; i < keys.length - 1; i++) { - const a = keys[i]; - const b = keys[i + 1]; - if (t >= a.t && t <= b.t) - return a.v + (b.v - a.v) * smooth((t - a.t) / Math.max(b.t - a.t, 1e-6)); - } - return keys[keys.length - 1].v; -} function lerpVec(keys: { t: number; v: Vec }[], t: number): Vec { if (t <= keys[0].t) return keys[0].v; for (let i = 0; i < keys.length - 1; i++) { @@ -77,7 +67,7 @@ const Cursor: React.FC<{ invScale: number }> = ({ invScale }) => ( ); // The polished demo "window": recorded video on a rounded panel, with the -// zoom/cursor/ripple/caption effects driven by the captured timeline. Laid +// cursor/ripple/caption effects driven by the captured timeline. Laid // out for whatever canvas size the preset asks for. export const DemoStage: React.FC<{ canvasW: number; @@ -119,8 +109,7 @@ export const DemoStage: React.FC<{ const WIN_X = (canvasW - BASE_W) / 2; const WIN_Y = stageTop + (stageH - BASE_H) / 2; - const sRaw = baseScale + (lerpNum(timeline.scaleKeys, t) - 1); - // Two scales bound how far a zoom can grow cleanly: sCrop is where the + // Two scales bound how far the preset can grow cleanly: sCrop is where the // window starts spilling out of the stage (top and bottom go flush), and // sCover is where it finally reaches the canvas edges. Between them the // window is cropped AND narrower than the canvas, so backdrop shows down @@ -138,8 +127,8 @@ export const DemoStage: React.FC<{ // zoom at a whole window there. Where coverage comes first instead (the // tall vertical preset, where the window covers the width at 1.2x and does // not crop until 2.58x) nothing is capped and full-bleed still works. - const S = sCover > sCrop ? Math.min(sRaw, sCrop) : sRaw; - const focal = lerpVec(timeline.focalKeys, t); + const S = sCover > sCrop ? Math.min(baseScale, sCrop) : baseScale; + const focal = { x: 0.5, y: 0.5 }; const cursor = lerpVec(timeline.cursorKeys, t); const focal0x = WIN_X + focal.x * BASE_W; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs index 9636f1035..cc30a7d15 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs @@ -1,8 +1,8 @@ // Post-capture trim. With narration synthesized BEFORE capture, the runner // paces every beat to its line and stamps clip starts as they land, so no // retiming is needed here — the only remaining job is cutting the dead -// opening hold (page-load settle) so the demo starts immediately. All key, -// caption, and clip times shift together with the video's startFrom. +// opening hold (page-load settle) so the demo starts immediately. Cursor, +// caption, click, and clip times shift together with the video's startFrom. // // Usage: WORK_DIR=/tmp/feature-demo/work node fit-timing.mjs @@ -20,10 +20,10 @@ const narration = existsSync(narrationPath) : null; // The first visible action is the earlier of the first caption/line start -// and the first motion key after t=0. +// and the first cursor motion key after t=0. const firstMotion = Math.min( ...timeline.captions.map((c) => c.start), - ...timeline.scaleKeys.filter((k) => k.t > 0).map((k) => k.t), + ...timeline.cursorKeys.filter((k) => k.t > 0).map((k) => k.t), Number.POSITIVE_INFINITY, ); const trim = @@ -51,8 +51,6 @@ function shiftKeys(keys) { } if (trim > 0) { - timeline.scaleKeys = shiftKeys(timeline.scaleKeys); - timeline.focalKeys = shiftKeys(timeline.focalKeys); timeline.cursorKeys = shiftKeys(timeline.cursorKeys); timeline.clicks = timeline.clicks .filter((c) => c.t >= trim)