diff --git a/.github/workflows/droid-control-tests.yml b/.github/workflows/droid-control-tests.yml new file mode 100644 index 0000000..ff86b35 --- /dev/null +++ b/.github/workflows/droid-control-tests.yml @@ -0,0 +1,73 @@ +name: droid-control tests + +on: + pull_request: + paths: + - 'plugins/droid-control/scripts/render-showcase.sh' + - 'plugins/droid-control/tests/**' + - 'plugins/droid-control/remotion/**' + - '.github/workflows/droid-control-tests.yml' + push: + branches: [master] + paths: + - 'plugins/droid-control/scripts/render-showcase.sh' + - 'plugins/droid-control/tests/**' + - 'plugins/droid-control/remotion/**' + - '.github/workflows/droid-control-tests.yml' + +env: + PYTHONDONTWRITEBYTECODE: 1 + +jobs: + render-helper: + name: render-showcase.sh behavior + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install ffmpeg and agg + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg + curl -fsSL -o agg https://github.com/asciinema/agg/releases/download/v1.9.0/agg-x86_64-unknown-linux-gnu + echo "f111e315cd71056b116302342553dd765b7297579ed511f111d0cedb442aeda6 agg" | sha256sum -c - + sudo install -m 0755 agg /usr/local/bin/agg + + - name: Run helper integration tests + run: python3 -m unittest discover -v -s plugins/droid-control/tests -p 'test_*.py' + + remotion: + name: Remotion typecheck, duration tests, real render + runs-on: ubuntu-latest + defaults: + run: + working-directory: plugins/droid-control/remotion + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + # node --test with built-in type stripping (22.18+); Remotion runtime itself needs >= 18 + node-version: 22 + cache: npm + cache-dependency-path: plugins/droid-control/remotion/package-lock.json + + - run: npm ci + + - run: npx tsc --noEmit -p . + + - run: npm test + + - name: Install ffmpeg and the Remotion browser + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends ffmpeg + npx remotion browser ensure + + - name: Render a synthetic clip through render-showcase.sh and check the encoded file + working-directory: . + env: + RENDER_SHOWCASE_E2E: 1 + run: python3 -m unittest discover -v -s plugins/droid-control/tests -p 'test_*.py' -k RealRender diff --git a/.gitignore b/.gitignore index c2a48c4..13af873 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .factory/ node_modules/ +# Per-render clip staging created by render-showcase.sh; only survives a SIGKILL +plugins/droid-control/remotion/public/render-*/ diff --git a/.skillsrc b/.skillsrc index 4e26c3d..fe6db51 100644 --- a/.skillsrc +++ b/.skillsrc @@ -1,14 +1,14 @@ # Skills listed here are excluded from the top-level skills/ directory. # One skill path per line, relative to the repo root (plugin/skills/skill-name). # Lines starting with # are comments. Blank lines are ignored. -droid-control/skills/agent-browser +droid-control/skills/browser-use droid-control/skills/capture droid-control/skills/compose -droid-control/skills/desktop-control +droid-control/skills/desktop-use droid-control/skills/droid-cli droid-control/skills/droid-control droid-control/skills/pty-capture droid-control/skills/showcase +droid-control/skills/terminal-use droid-control/skills/true-input -droid-control/skills/tuistory droid-control/skills/verify diff --git a/README.md b/README.md index 3ef0559..5119f2a 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Terminal, browser, and computer automation for Droids. Record demos, verify beha **Commands:** `/demo`, `/verify`, `/qa-test` -**Skills:** `droid-control` (orchestrator), `tuistory`, `true-input`, `agent-browser`, `droid-cli`, `pty-capture`, `capture`, `compose`, `verify`, `showcase` +**Skills:** `droid-control` (orchestrator), `terminal-use`, `true-input`, `browser-use`, `desktop-use`, `droid-cli`, `pty-capture`, `capture`, `compose`, `verify`, `showcase` See [plugins/droid-control/README.md](plugins/droid-control/README.md) for details. diff --git a/plugins/droid-control/.factory-plugin/plugin.json b/plugins/droid-control/.factory-plugin/plugin.json index 8f6c249..72a9699 100644 --- a/plugins/droid-control/.factory-plugin/plugin.json +++ b/plugins/droid-control/.factory-plugin/plugin.json @@ -1,5 +1,5 @@ { "name": "droid-control", "description": "Terminal, browser, and native desktop automation for testing, demos, QA, and computer-use tasks", - "version": "1.0.0" + "version": "1.1.0" } diff --git a/plugins/droid-control/ARCHITECTURE.md b/plugins/droid-control/ARCHITECTURE.md index cb0b699..b4f5d5f 100644 --- a/plugins/droid-control/ARCHITECTURE.md +++ b/plugins/droid-control/ARCHITECTURE.md @@ -12,7 +12,7 @@ The plugin is designed to keep a droid focused while it operates real software: - **Low context load:** load the Linux tuistory path without dragging in Windows KVM notes, macOS VM controls, browser automation, and Remotion internals. - **Evidence-first workflows:** every command starts by making commitments, then ends by verifying the artifact against those commitments. -- **Parallel execution:** before/after captures and render jobs can run in worker droids without sharing session names or output paths. +- **Parallel execution:** independent capture environments and render jobs can run in workers. Shared desktop input stays serialized; session names do not isolate focus. - **Clear ownership:** commands decide *what* must be produced; atom skills decide *how* to execute their slice. - **Platform specificity:** OS-specific mechanics live in platform subdocuments, not in global instructions. @@ -24,7 +24,7 @@ The three user-facing commands are deliberately thin: |---|---| | `/demo` | Turn a PR or feature description into a visible proof story and a video deliverable. | | `/verify` | Test a claim as an investigator and report whether the evidence confirms or refutes it. | -| `/qa-test` | Drive a terminal, browser, or Electron flow and report step-level pass/fail evidence. | +| `/qa-test` | Drive a terminal, browser, Electron, or native desktop flow and report step-level `PASS` / `FAIL` / `BLOCKED` evidence. | A command parses arguments into **commitments**: layout, comparison mode, evidence type, video/showcase requirements, keystroke overlays, and any user-specified constraints. Those commitments are not suggestions. The `verify` stage later checks them explicitly. @@ -48,7 +48,7 @@ Each atom skill is a self-contained surface the droid reads at a specific point | Atom type | Skills | Responsibility | |---|---|---| -| Driver atoms | `tuistory`, `true-input`, `agent-browser`, `desktop-control` | How to drive a class of environment. | +| Driver atoms | `terminal-use`, `true-input`, `browser-use`, `desktop-use` | How to drive a class of environment. `terminal-use` is the terminal entrypoint; it runs the tuistory backend and routes real-terminal proof to `true-input`. | | Target atoms | `droid-cli`, `pty-capture` | Target-specific shortcuts, launch rules, and byte-capture patterns. | | Stage atoms | `capture`, `compose`, `verify` | Lifecycle phases with explicit inputs and outputs. | | Polish atom | `showcase` | Visual presets and cinematic layer guidance. | @@ -85,9 +85,10 @@ The parent droid keeps judgment. Workers get exact commands. | Work | Owner | Reason | |---|---|---| +| Short interactive desktop task | Parent | One controller owns observation, input, permission waits, and cleanup. | | Interpret PR / claim / QA goal | Parent | Requires context and judgment. | | Write the interaction script | Parent | Defines the proof story. | -| Capture baseline and candidate branches | Worker droids | Independent, mechanical, parallelizable. | +| Capture baseline and candidate branches | Workers only for independent environments | A shared desktop must be captured serially. | | Render Remotion video | Worker droid | Mechanical once props and clips are fixed. | | Verify commitments | Parent | Requires the original contract and evidence judgment. | @@ -95,9 +96,9 @@ This boundary follows the stage handoffs. Capture workers need resolved `tctl` c ## Runtime artifact pipeline -![droid-control capture compose verify pipeline](diagrams/capture-compose-verify.svg) +![droid-control terminal comparison pipeline](diagrams/capture-compose-verify.svg) -Editable source: [`diagrams/capture-compose-verify.excalidraw`](diagrams/capture-compose-verify.excalidraw) +Editable source: [`diagrams/capture-compose-verify.excalidraw`](diagrams/capture-compose-verify.excalidraw). The diagram shows the terminal comparison flow, where each branch has an isolated tuistory environment and can run in its own worker; a shared desktop is captured serially by the parent (see Delegation boundaries). Every workflow starts by creating a run scope: @@ -125,14 +126,14 @@ Browser/Electron and native-desktop workflows intentionally do **not** go throug The compose stage uses the Remotion project in `remotion/` as a single video engine. The droid writes a `Showcase` props JSON; `scripts/render-showcase.sh` handles the mechanical rendering pipeline: -1. Normalize props and choose fidelity. -2. Convert `.cast` recordings through `agg` and `ffmpeg`. -3. Stage clips into Remotion `public/`. -4. Auto-detect `clipDuration` with `ffprobe` when omitted. -5. Render the `Showcase` composition. -6. Clean staged clips and temporary conversion outputs. +1. Accept `.cast`, `.mp4`, and `.webm` clips only; normalize props and resolve fidelity (omitted: side-by-side `inspect`, single `standard`). +2. Convert `.cast` recordings through `agg` and `ffmpeg` at 1x, keeping the recording's timeline. +3. Stage clips as `clip-` inside a directory created under Remotion `public/` for this render only. +4. Set `clipDuration` to the longest clip with `ffprobe`. +5. Render the `Showcase` composition as limited-range `yuv420p`/`bt709` H.264, or one frame with `--still`. +6. Remove that render's staged directory and conversion outputs on exit. -This keeps droids out of the common failure modes: stale files in `public/`, mismatched `clipDuration`, wrong `agg` theme, invalid pixel formats, and hand-written Remotion commands with missing encode flags. +`remotion/src/lib/duration.ts` owns the timeline: the composition applies `speed` once to every clip, the clips run for `clipDuration / speed`, and the content sequence is padded by one crossfade on each side so the clips start after the title crossfade and the final frame is held through the outro crossfade. Total length is `4s title + clipDuration / speed + 3.5s outro`; a shorter clip holds its final frame. This keeps droids out of the common failure modes: two `recording.mp4` inputs overwriting each other in `public/`, concurrent renders deleting each other's clips, mismatched `clipDuration`, casts sped up twice, wrong `agg` theme, invalid pixel formats, and hand-written Remotion commands with missing encode flags. ### Composition surface @@ -161,13 +162,15 @@ skills/true-input/platforms/macos.md skills/pty-capture/platforms/linux.md skills/pty-capture/platforms/windows.md skills/pty-capture/platforms/macos.md -skills/desktop-control/platforms/linux.md -skills/desktop-control/platforms/windows.md -skills/desktop-control/platforms/macos.md +skills/desktop-use/SKILL.md ``` A Linux droid reads Linux Wayland instructions. A Windows VM byte-capture task reads Windows KVM instructions. The system does not rely on the droid to skim irrelevant sections correctly. +Desktop-use keeps routine setup, target selection, recovery, and recording rules in one compact entrypoint. It includes a host setup table rather than copied platform manuals. No user-home skill dependency or runtime documentation download is required. + +For ordinary desktop tasks, the driver verifies each action and reports directly. Capture/verify stages are loaded for formal evidence deliverables, and compose only when a produced artifact was requested. Explicit GUI-only or cua-only constraints take precedence over Electron's default browser route. + ## Extending the plugin Use the same composition rules when adding capability: diff --git a/plugins/droid-control/CUA-LICENSE.md b/plugins/droid-control/CUA-LICENSE.md new file mode 100644 index 0000000..b8b198c --- /dev/null +++ b/plugins/droid-control/CUA-LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Cua AI, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/droid-control/NOTICES.md b/plugins/droid-control/NOTICES.md index 0abef16..4249560 100644 --- a/plugins/droid-control/NOTICES.md +++ b/plugins/droid-control/NOTICES.md @@ -1,6 +1,10 @@ # Third-Party Notices -This plugin depends on several third-party tools and libraries. They are not bundled -- each is installed separately by the user. Their respective licenses apply at the point of installation and use. +This plugin depends on several third-party tools and libraries. Executables are installed separately by the user. + +## Desktop automation documentation + +- **[Cua Driver](https://github.com/trycua/cua)** — desktop-use guidance is adapted from Cua under the [included MIT license](CUA-LICENSE.md). Driver code and binaries are not bundled. ## Video rendering diff --git a/plugins/droid-control/README.md b/plugins/droid-control/README.md index 590fbb8..4bd8c46 100644 --- a/plugins/droid-control/README.md +++ b/plugins/droid-control/README.md @@ -2,7 +2,7 @@ Terminal, browser, and computer automation plugin for Droids. -Droids can read and write code. This plugin lets them *operate* it: launch apps, type commands, click buttons, record what happens, and produce polished evidence. No human hands required. +Droids can read and write code. This plugin lets them *operate* it: launch apps, type commands, click buttons, record what happens, and produce evidence. OS permission grants and consequential actions still require the appropriate user or host authorization. ## What you get @@ -12,7 +12,7 @@ Droids can read and write code. This plugin lets them *operate* it: launch apps, /demo pr-1847 ``` -Droid reads the PR, scripts the interactions that prove the change works, records both branches in parallel, and renders a side-by-side comparison video. Use Factory presets for cinematic warmth or macos/minimal presets for clean utilitarian demos. +Droid reads the PR, scripts the interactions that prove the change works, records both branches (in parallel when each has its own isolated terminal or browser environment), and renders a side-by-side comparison video. Use Factory presets for cinematic warmth or macos/minimal presets for clean utilitarian demos. **Verify a behavior claim:** @@ -48,6 +48,10 @@ Or use the `/plugins` UI: Browse tab, select droid-control, install. Then open a Droid session and run `/demo`, `/verify`, or `/qa-test`. +For ordinary desktop work, ask directly: **“Using only cua, open Calculator and compute 17 × 23.”** Desktop-use runs the observe/act/verify loop without loading video-production stages. + +The [desktop-use skill](skills/desktop-use/SKILL.md) includes setup and operating guidance. Install the `cua-driver` executable if missing; no separate Cua skill installation is needed. Installed driver versions and Wayland compositors may support different capabilities. + ## Commands ### `/demo` @@ -69,7 +73,7 @@ Runs automated QA against terminal CLIs, web apps, or Electron apps. Accepts a U 1. **Commands** parse user intent into commitments. 2. **The orchestrator** routes by target, stage, and artifact needs. 3. **Atom skills** provide only the mechanics needed right now: drivers, target patterns, capture, compose, verify, and showcase polish. -4. **Workers** handle mechanical capture/render jobs while the parent droid keeps planning and verification context. +4. **Workers** handle independent capture/render jobs. The parent keeps short interactive desktop tasks, including observations, input, permission waits, and cleanup. 5. **Verify** checks the final evidence against the original commitments. For the full rationale and runtime pipeline, see [`ARCHITECTURE.md`](ARCHITECTURE.md). @@ -78,18 +82,18 @@ For the full rationale and runtime pipeline, see [`ARCHITECTURE.md`](ARCHITECTUR The compose stage uses [Remotion](https://www.remotion.dev/) for video compositing. Presets provide window chrome, spacing, palettes, backgrounds, particles, noise, color grading, configurable transitions (`motion-blur`, `flash`, `whip-pan`, `light-leak`, `glitch-lite`), zooms, spotlights, callout annotations, keystroke overlays, section headers, and syntax-highlighted code annotations. -The `render-showcase.sh` helper owns the full pipeline: `.cast` conversion via `agg`, clip staging, duration detection, Remotion rendering, and cleanup. +The `render-showcase.sh` helper owns the full pipeline: `.cast` conversion via `agg`, per-render clip staging, longest-clip duration, Remotion rendering (or a `--still` preview), and cleanup. Playback `speed` is applied once by the composition to every clip. ## Prerequisites | Stage | Platform | Required | |---|---|---| -| tuistory | All | `tuistory`, `asciinema`, `agg` | +| terminal-use (tuistory) | All | `tuistory`, `asciinema`, `agg` | | true-input | Linux/Wayland | `cage`, `wtype`, Wayland terminal | | true-input | Windows (KVM) | `libvirt`, `qemu`, KVM VM with SSH | | true-input | macOS (QEMU) | `qemu`, `socat`, macOS VM with SSH | -| agent-browser | All | `agent-browser` | -| desktop-control | All | `cua-driver` | +| browser-use | All | `agent-browser` | +| desktop-use | All | `cua-driver` | | compose | All | `ffmpeg`, `ffprobe`, `agg` | | showcase | All | Node.js (>= 18), Chrome/Chromium | @@ -103,4 +107,4 @@ curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scr cd plugins/droid-control/remotion && npm install # Remotion video rendering ``` -Only install what you need for your use case. Terminal demos need tuistory, asciinema, agg, and ffmpeg. Web/Electron automation just needs agent-browser. Native desktop GUI automation just needs cua-driver. +Only install what you need, with approval. Terminal demos need tuistory, asciinema, agg, and ffmpeg. Web/Electron automation defaults to browser-use; an explicit cua-only/native-input request uses desktop-use instead. Native desktop automation needs cua-driver plus the graphical session and OS permissions reported by its preflight. Recording and rendering have additional dependencies; they are not required for ordinary desktop tasks. diff --git a/plugins/droid-control/commands/demo.md b/plugins/droid-control/commands/demo.md index b40b311..5d36cf4 100644 --- a/plugins/droid-control/commands/demo.md +++ b/plugins/droid-control/commands/demo.md @@ -88,31 +88,26 @@ Provide the capture stage with: - The interaction script from above - Whether to emit a keystroke TSV -**Delegation:** For before/after comparisons, capture both branches **in parallel** using worker subagents with `run_in_background=true`. Construct the exact `tctl` commands for each worker (see the delegation section in the droid-control skill). Wait for both to complete before proceeding to compose. +Whether capture runs in the parent or in workers is the **droid-control** skill's delegation decision (independent terminal/browser environments may run in workers; a shared desktop stays in the parent). Wait for every clip before composing. ## Compose -Follow the **compose** atom. It owns the full video assembly pipeline. - -**Delegation:** Launch one worker subagent for the mechanical render: -- Worker A: render the final video via `render-showcase.sh` directly from `.cast` / `.mp4` inputs - -`render-showcase.sh` owns `.cast -> agg -> .mp4`, Remotion composition, fidelity profile selection, duration detection, and cleanup. Wait for the worker to finish, then verify the output. +Follow the **compose** atom. It owns the full video assembly pipeline; rendering may be delegated per the droid-control delegation table. Hand compose a hybrid handoff: ### Mechanical (structured) - layout: side-by-side | single -- fidelity: auto | compact | standard | inspect (optional; auto => side-by-side=inspect, single=standard) +- fidelity: compact | standard | inspect (optional; omitted => side-by-side=inspect, single=standard) - labels: ["BEFORE ()", "AFTER ()"] - speed: 3x - title: "PR #11386 — Add --fork flag" - subtitle: "Demo: --fork creates a forked session from current context" -- clips: [/tmp/before.cast, /tmp/after.cast] -- keys: /tmp/keys.tsv (if committed) +- clips: [${RUN_DIR}/before.cast, ${RUN_DIR}/after.cast] +- keys: ${RUN_DIR}/keys.tsv (if committed) - preset: hero | macos | minimal | presentation | factory | factory-hero (if committed) - effects tier: utilitarian | full | none -- output: /tmp/demo-pr-11386.mp4 +- output: ${RUN_DIR}/demo-pr-11386.mp4 ### Creative (natural language) What the viewer should take away. Which moments to hold. How to frame the story. Whether phase cards are warranted. The compose atom uses this -- along with the effects tier -- for editorial decisions: title card phrasing, trim points, emphasis, and choosing specific effects to apply. diff --git a/plugins/droid-control/commands/qa-test.md b/plugins/droid-control/commands/qa-test.md index 482f24f..4893b64 100644 --- a/plugins/droid-control/commands/qa-test.md +++ b/plugins/droid-control/commands/qa-test.md @@ -1,5 +1,5 @@ --- -description: Run an automated QA test flow against a terminal CLI or web/Electron app +description: Run an automated QA test flow against a terminal, browser, or native desktop app argument-hint: '"" or "" or " [-- focus area]" or ""' --- @@ -9,7 +9,7 @@ Load skills: **droid-control**. `$ARGUMENTS` can be: - **URL** (`https://app.factory.ai`, `localhost:3000`) → web app -- **Electron app name** (`Slack`, `VS Code`, `Figma`) → Electron app via CDP +- **Desktop app name** (`Slack`, `VS Code`, `Calculator`) → use the orchestrator's target route; explicit cua/native-input requirements override Electron's CDP default - **CLI command** (`droid-dev`, `htop`, `my-cli --flag`) → terminal TUI - **PR reference** (`11386`) with optional `-- focus area` → infer target from the diff - **Free-text description** ("test the login flow on staging") → infer target and flow @@ -48,6 +48,8 @@ If the user provides specific steps, use them. Otherwise, design a reasonable fl **Terminal**: launch app → wait for ready → snapshot → exercise primary features → verify output → snapshot → close. +**Native desktop / cua-only**: discover or launch the requested app → select an exact target → observe → act → verify each postcondition. Broaden to visible desktop capture/input only with authorization. Leave personal applications open unless closure is requested; end only the automation run. + If the flow is ambiguous or success criteria are unclear, ask the user. ## Capture @@ -62,6 +64,8 @@ If a step fails: - Continue to the next step for maximum coverage - Unless the failure blocks everything downstream (e.g., login failed) +If a step cannot be observed (capture unavailable, permission wait unresolved, missing connection), record it as `BLOCKED` with the blocker named — see the **verify** atom's status vocabulary. Steps that depend on it are `BLOCKED` too, not `FAIL`. + ## Compose (if committed) Follow the **compose** atom if a video deliverable was committed. Hand it: @@ -91,7 +95,7 @@ Follow the **verify** atom. It checks the deliverable and QA report completeness | Step | Status | Notes | |------|--------|-------| -| ... | PASS/FAIL | ... | +| ... | PASS / FAIL / BLOCKED | ... (BLOCKED: what blocked it, what unblocks it) | ### Issues Found diff --git a/plugins/droid-control/commands/verify.md b/plugins/droid-control/commands/verify.md index 34438d9..4e78b61 100644 --- a/plugins/droid-control/commands/verify.md +++ b/plugins/droid-control/commands/verify.md @@ -62,6 +62,8 @@ Follow the **capture** atom. Provide: **If the behavior does not match the claim:** Do not retry the interaction hoping for a different result. Capture a snapshot or screenshot of the actual state. This is evidence. If you suspect your test procedure is wrong (e.g., wrong branch, missing build step), verify the environment first -- but if the environment is correct and the behavior is wrong, that is a finding, not an error on your part. +**If a step cannot be observed** (capture unavailable, permission wait unresolved, missing binary or connection), mark it `BLOCKED` per the **verify** atom's status vocabulary and keep the partial evidence; do not record it as a failed behavior. + ## Compose (if committed) Follow the **compose** atom if a video deliverable was committed. Hand it: @@ -98,6 +100,8 @@ Follow the **verify** atom. It checks the deliverable against your commitments. ``` +The verdict is about evidence sufficiency. Steps carry `PASS` / `FAIL` / `BLOCKED` (verify atom); a `BLOCKED` step the claim depends on forces `INCONCLUSIVE`, and `INCONCLUSIVE` also covers steps that all ran without deciding the claim. + ### When the claim is refuted If the evidence shows the behavior does not match the claim: @@ -111,7 +115,7 @@ This is a valuable finding. The user asked you to test this claim precisely beca ### When the result is inconclusive -If the environment prevented a clean test (e.g., missing dependency, build failure, test infra crash), report what blocked the test and what would be needed to resolve it. Do not guess at the outcome. +If the environment prevented a clean test (e.g., missing dependency, build failure, test infra crash), list the `BLOCKED` steps, what blocked each, and what would be needed to resolve it. Do not guess at the outcome. ## Do NOT diff --git a/plugins/droid-control/diagrams/capture-compose-verify.excalidraw b/plugins/droid-control/diagrams/capture-compose-verify.excalidraw index 479a624..2324427 100644 --- a/plugins/droid-control/diagrams/capture-compose-verify.excalidraw +++ b/plugins/droid-control/diagrams/capture-compose-verify.excalidraw @@ -17,7 +17,7 @@ "frameId": null, "roundness": null, "seed": 1328261054, - "version": 1, + "version": 2, "versionNonce": 1901493144, "isDeleted": false, "boundElements": null, @@ -29,7 +29,7 @@ "textAlign": "left", "verticalAlign": "middle", "containerId": null, - "originalText": "Runtime pipeline: parent judgment, worker execution, verified artifacts", + "originalText": "Terminal comparison pipeline: parent judgment, isolated workers, proof", "autoResize": true, "lineHeight": 1.25, "id": "title_pipeline", @@ -37,7 +37,7 @@ "y": 30, "width": 980, "height": 37, - "text": "Runtime pipeline: parent judgment, worker execution, verified artifacts" + "text": "Terminal comparison pipeline: parent judgment, isolated workers, proof" }, { "type": "text", @@ -53,7 +53,7 @@ "frameId": null, "roundness": null, "seed": 1850501473, - "version": 1, + "version": 2, "versionNonce": 776605305, "isDeleted": false, "boundElements": null, @@ -65,7 +65,7 @@ "textAlign": "left", "verticalAlign": "middle", "containerId": null, - "originalText": "Run-scoped IDs isolate sessions and outputs so multiple droids can capture and render concurrently.", + "originalText": "Run IDs isolate outputs. Workers need isolated environments; a shared desktop stays with the parent.", "autoResize": true, "lineHeight": 1.25, "id": "subtitle_pipeline", @@ -73,7 +73,7 @@ "y": 78, "width": 980, "height": 24, - "text": "Run-scoped IDs isolate sessions and outputs so multiple droids can capture and render concurrently." + "text": "Run IDs isolate outputs. Workers need isolated environments; a shared desktop stays with the parent." }, { "angle": 0, @@ -928,7 +928,7 @@ "frameId": null, "roundness": null, "seed": 1987262418, - "version": 1, + "version": 2, "versionNonce": 1374389013, "isDeleted": false, "boundElements": null, @@ -940,7 +940,7 @@ "textAlign": "center", "verticalAlign": "middle", "containerId": "render", - "originalText": "Render worker\nrender-showcase.sh\n\n.cast -> agg -> mp4\nstage public clips\nauto-detect duration", + "originalText": "Render worker\nrender-showcase.sh\n\n.cast -> agg -> mp4\nstage per-render clips\nlongest-clip duration", "autoResize": true, "lineHeight": 1.25, "id": "t_render", @@ -948,7 +948,7 @@ "y": 383, "width": 284, "height": 159, - "text": "Render worker\nrender-showcase.sh\n\n.cast -> agg -> mp4\nstage public clips\nauto-detect duration" + "text": "Render worker\nrender-showcase.sh\n\n.cast -> agg -> mp4\nstage per-render clips\nlongest-clip duration" }, { "angle": 0, @@ -1217,7 +1217,7 @@ "frameId": null, "roundness": null, "seed": 1728822696, - "version": 1, + "version": 2, "versionNonce": 677430271, "isDeleted": false, "boundElements": null, @@ -1229,7 +1229,7 @@ "textAlign": "center", "verticalAlign": "middle", "containerId": "verify_final", - "originalText": "Verify\n\nffprobe\nfile size / duration\ncommitment checklist", + "originalText": "Verify\n\nffprobe + decode\nproof frames\ncommitment checklist", "autoResize": true, "lineHeight": 1.25, "id": "t_verify_final", @@ -1237,7 +1237,7 @@ "y": 413, "width": 234, "height": 124, - "text": "Verify\n\nffprobe\nfile size / duration\ncommitment checklist" + "text": "Verify\n\nffprobe + decode\nproof frames\ncommitment checklist" }, { "angle": 0, diff --git a/plugins/droid-control/diagrams/capture-compose-verify.svg b/plugins/droid-control/diagrams/capture-compose-verify.svg index c1a7974..0f664f9 100644 --- a/plugins/droid-control/diagrams/capture-compose-verify.svg +++ b/plugins/droid-control/diagrams/capture-compose-verify.svg @@ -16,8 +16,8 @@ } -Runtime pipeline: parent judgment, worker execution, verified artifacts -Run-scoped IDs isolate sessions and outputs so multiple droids can capture and render concurrently. +Terminal comparison pipeline: parent judgment, isolated workers, proof +Run IDs isolate outputs. Workers need isolated environments; a shared desktop stays with the parent. Parent droid @@ -76,8 +76,8 @@ render-showcase.sh .cast -> agg -> mp4 -stage public clips -auto-detect duration +stage per-render clips +longest-clip duration Remotion Showcase @@ -96,8 +96,8 @@ Verify -ffprobe -file size / duration +ffprobe + decode +proof frames commitment checklist diff --git a/plugins/droid-control/remotion/package.json b/plugins/droid-control/remotion/package.json index 30732ec..893c1d3 100644 --- a/plugins/droid-control/remotion/package.json +++ b/plugins/droid-control/remotion/package.json @@ -5,7 +5,8 @@ "scripts": { "dev": "npx remotion studio", "render": "npx remotion render", - "still": "npx remotion still" + "still": "npx remotion still", + "test": "node --test 'tests/**/*.test.mts'" }, "dependencies": { "@remotion/cli": "4.0.445", diff --git a/plugins/droid-control/remotion/src/Root.tsx b/plugins/droid-control/remotion/src/Root.tsx index 1eb49bb..f2a4d71 100644 --- a/plugins/droid-control/remotion/src/Root.tsx +++ b/plugins/droid-control/remotion/src/Root.tsx @@ -25,6 +25,7 @@ export const RemotionRoot: React.FC = () => { title: 'Demo', subtitle: '', preset: 'factory' as const, + fidelity: 'standard' as const, keys: [], effects: [], width: 1920, diff --git a/plugins/droid-control/remotion/src/compositions/Showcase.tsx b/plugins/droid-control/remotion/src/compositions/Showcase.tsx index 45a6a75..b373886 100644 --- a/plugins/droid-control/remotion/src/compositions/Showcase.tsx +++ b/plugins/droid-control/remotion/src/compositions/Showcase.tsx @@ -1,6 +1,6 @@ import React, { useMemo } from 'react'; import { z } from 'zod'; -import { AbsoluteFill, staticFile, useVideoConfig } from 'remotion'; +import { AbsoluteFill, Sequence, staticFile, useVideoConfig } from 'remotion'; import { Video } from '@remotion/media'; import { TransitionSeries, linearTiming } from '@remotion/transitions'; import { getTransitionPresentation } from '../components/ShowcaseTransition'; @@ -16,6 +16,13 @@ import { } from '../lib/schema'; import { getPalette } from '../lib/palettes'; import { getPresetConfig } from '../lib/presets'; +import { + TITLE_DURATION_S, + OUTRO_DURATION_S, + TRANSITION_FRAMES, + contentSequenceFrames, + playbackSpeed, +} from '../lib/duration'; import { Background } from '../components/Background'; import { WindowChrome } from '../components/WindowChrome'; import { TitleCard } from '../components/TitleCard'; @@ -47,9 +54,13 @@ export const showcaseSchema = z.object({ height: z.number().optional(), speedNote: z.string().optional(), windowTitle: z.string().optional(), + // Longest source clip in source seconds; render-showcase.sh probes it. clipDuration: z.number().optional(), + // Playback multiplier applied once, here, to every clip. Overlay times + // (keys, sections, effects, codeAnnotations) are already output seconds. speed: z.number().positive().optional(), - fidelity: fidelitySchema.optional(), + // Resolved by render-showcase.sh (omitted there => chosen by layout). + fidelity: fidelitySchema, // How clip video is sized inside its panel. // - "contain" (default): preserve aspect ratio, letterbox if needed. Safe default. // - "cover": fill the panel, crop overflow. Use when the clip aspect doesn't match the @@ -66,9 +77,6 @@ export const showcaseSchema = z.object({ transitionStyle: transitionStyleSchema.optional(), }); -const TITLE_DURATION_S = 4; -const TRANSITION_FRAMES = 15; - // Effect types this composition actually renders. Anything schema-valid but // absent from this set is a silent no-op — warn instead of dropping quietly. const RENDERED_FX = new Set(['zoom', 'spotlight', 'callout']); @@ -86,24 +94,45 @@ const warnUnrenderedEffects = (effects: z.infer[]) => { } }; -const resolveFidelity = ( - props: z.infer -): 'compact' | 'standard' | 'inspect' => - props.fidelity ?? (props.layout === 'side-by-side' ? 'inspect' : 'standard'); - const visualTreatmentByFidelity = { compact: { noiseOpacity: 0.03, gradeIntensity: 0.04 }, standard: { noiseOpacity: 0.02, gradeIntensity: 0.025 }, inspect: { noiseOpacity: 0.008, gradeIntensity: 0.012 }, } as const; +type ObjectFit = 'contain' | 'cover' | 'fill'; + +// One playback contract for every clip: `speed` is applied here and nowhere +// else, and a clip that ends before the content sequence does not loop -- it +// holds its last decoded frame, so a shorter comparison panel stays on its +// final state. +const ClipVideo: React.FC<{ + clip: string; + speed: number; + objectFit: ObjectFit; + surface: string; +}> = ({ clip, speed, objectFit, surface }) => ( +