From 19e3613c4604cc32ab383c525580d15c49683003 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Fri, 25 Sep 2026 23:14:09 +1000 Subject: [PATCH 1/9] Probe which bundled plugins a CPU can run, one plugin per process Issue #92's first probe reported every plugin as crashing on a Westmere Mac Pro, because R78 autoloads the whole plugin directory and MVTools faults in a static initializer before any filter runs. Loading each plugin by path into a DISABLE_AUTO_LOADING core, in its own process, makes the result per plugin. The probe discovers plugins from the bundle instead of a hard-coded list and runs with the bundle's own Python, so the same script covers macOS, Linux and Windows. A new workflow runs it under Intel SDE, with zsmooth's AVX2-only haswell build as a positive control that must crash. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014GLXdGLfPwgYjW1AkonGqN --- .github/workflows/probe-cpu-compat.yml | 126 +++++++++ Scripts/probe-plugin-compat.py | 353 +++++++++++++++++++++++++ Scripts/probe-plugin-compat.sh | 30 +++ 3 files changed, 509 insertions(+) create mode 100644 .github/workflows/probe-cpu-compat.yml create mode 100755 Scripts/probe-plugin-compat.py create mode 100755 Scripts/probe-plugin-compat.sh diff --git a/.github/workflows/probe-cpu-compat.yml b/.github/workflows/probe-cpu-compat.yml new file mode 100644 index 0000000..203e367 --- /dev/null +++ b/.github/workflows/probe-cpu-compat.yml @@ -0,0 +1,126 @@ +name: Probe CPU compatibility (SDE) + +# Loads and renders every plugin in a deps bundle under Intel SDE emulating an +# older CPU, one plugin per process (Scripts/probe-plugin-compat.py). Every +# hosted runner has AVX2, so this is the only CI-side way to see what a +# pre-AVX2 machine would do with the bundle (issue #92). +# +# Two controls make a result trustworthy rather than merely green: +# - a native run on the runner itself, which must pass everything; an error +# there is a probe bug, not a CPU finding. +# - zsmooth's haswell build (AVX2, no runtime dispatch — issue #82) must +# CRASH under SDE. If it passes, SDE is not trapping and the run proves +# nothing, so the job fails. + +on: + # Also runs when the probe or this workflow changes, which re-validates the + # probe itself (and lets a branch run it before the file exists on main, + # which workflow_dispatch requires). + push: + paths: + - Scripts/probe-plugin-compat.py + - .github/workflows/probe-cpu-compat.yml + workflow_dispatch: + inputs: + deps_run_id: + description: 'Optional build-deps-* run IDs (comma-separated); default is the release named in deps-version.json' + required: false + default: '' + chip: + description: 'SDE chip to emulate (wsm = Westmere, the Mac Pro 5,1 in #92; nhm = Nehalem; snb = Sandy Bridge, AVX without AVX2)' + required: false + default: 'wsm' + +permissions: + contents: read + actions: read + +jobs: + probe: + name: ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x64 + runner: ubuntu-24.04 + sde: sde64 + python: python/bin/python3 + - platform: windows-x64 + runner: windows-latest + sde: sde.exe + python: vapoursynth/python.exe + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + defaults: + run: + shell: bash + env: + D: deps/${{ matrix.platform }} + steps: + - uses: actions/checkout@v5 + + - name: Download dependencies (${{ matrix.platform }}) + env: + GH_TOKEN: ${{ github.token }} + DEPS_RUN_ID: ${{ inputs.deps_run_id }} + run: | + TAG=$(sed -n 's/.*"releaseTag": *"\([^"]*\)".*/\1/p' app/assets/deps-version.json) + mkdir -p "$D" + ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.platform }}" "$TAG" "${TAG#deps-v}") + if [ "$RUNNER_OS" = Windows ]; then + # The Windows zip uses backslash separators; 7-Zip copes, unzip does not. + 7z x "$ZIP" -o"$D" -y >/dev/null + else + unzip -q -o "$ZIP" -d "$D" + fi + rm -rf "$ZIP" .deps-artifact + + - name: Setup Intel SDE + uses: petarpetrovt/setup-sde@v6.0 + with: + environmentVariableName: SDE_PATH + sdeVersion: 10.13.1 + + - name: Native run (control — must pass on this runner) + run: | + "$D/${{ matrix.python }}" Scripts/probe-plugin-compat.py "$D" \ + --report native.txt --json native.json --fail-on-crash + + - name: SDE run (-${{ inputs.chip || 'wsm' }}) + run: | + SDE_DIR=$(cygpath -m "$SDE_PATH" 2>/dev/null || echo "$SDE_PATH") + "$D/${{ matrix.python }}" Scripts/probe-plugin-compat.py "$D" \ + --report sde.txt --json sde.json --timeout 1800 \ + --wrap "\"$SDE_DIR/${{ matrix.sde }}\" -${{ inputs.chip || 'wsm' }} --" + + - name: Check the positive control and summarise + run: | + "$D/${{ matrix.python }}" - <<'EOF' + import json, os, sys + sde = json.load(open("sde.json"))["results"] + rows = ["| file | namespaces | under SDE |", "|---|---|---|"] + rows += [f"| {r['file']} | {r['namespaces']} | {r['status']} {r['detail']} |" for r in sde] + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write("\n".join(rows) + "\n") + control = [r for r in sde if "zsmooth-haswell" in r["file"]] + if not control: + sys.exit("positive control missing: no zsmooth-haswell build in this bundle") + if control[0]["status"] != "CRASHED": + sys.exit("SDE did not trap the AVX2-only zsmooth-haswell build; results are meaningless") + crashed = [r for r in sde if r["status"] == "CRASHED"] + print(f"control OK; {len(crashed)} file(s) crashed under SDE:") + for r in crashed: + print(f" {r['file']} [{r['namespaces']}] {r['detail']}") + EOF + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v5 + with: + name: cpu-compat-${{ matrix.platform }}-${{ inputs.chip || 'wsm' }} + path: | + native.txt + native.json + sde.txt + sde.json diff --git a/Scripts/probe-plugin-compat.py b/Scripts/probe-plugin-compat.py new file mode 100755 index 0000000..28af70b --- /dev/null +++ b/Scripts/probe-plugin-compat.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +"""Load and render every plugin in a VapourBox deps bundle, one per process. + +Answers "which bundled plugins can this CPU run?" (issue #92). Each plugin is +loaded by path into a core created with DISABLE_AUTO_LOADING, so one plugin +that faults while loading cannot take the others down with it. That masking is +exactly what made the first version of this probe report every plugin as +crashed: R78 autoloads the whole plugin directory, and MVTools faults in a +static initializer before any filter runs. + +The parent never imports vapoursynth. Every test is a child process, so a +SIGILL / 0xC000001D in a plugin is a result, not the end of the run. + +Usage: + python probe-plugin-compat.py [DEPS_DIR] [--report FILE] [--json FILE] + [--wrap "sde64 -nhm --"] [--fail-on-crash] + +DEPS_DIR defaults to the installed bundle for this OS. Run it with the +bundle's own Python (see probe-plugin-compat.sh) so no system Python is needed. +""" + +import argparse +import json +import os +import platform +import shlex +import subprocess +import sys +from datetime import datetime + +# Namespace -> render call. `c` is a 640x480 YUV420P8 BlankClip, 12 frames long +# (temporal filters need neighbours). Calls mirror worker/templates where the +# pipeline uses the plugin, so a pass here means the shipped call runs. A +# namespace missing from this table is still load-tested. +RENDER = { + "mv": ("sup = core.mv.Super(c, pel=2, sharp=1)\n" + "bw = core.mv.Analyse(sup, isb=True, delta=1, blksize=8, overlap=4)\n" + "fw = core.mv.Analyse(sup, isb=False, delta=1, blksize=8, overlap=4)\n" + "c = core.mv.Degrain1(clip=c, super=sup, mvbw=bw, mvfw=fw)"), + "znedi3": "c = core.znedi3.nnedi3(c, field=1, dh=False)", + "nnedi3": "c = core.nnedi3.nnedi3(c, field=1, dh=False)", + "eedi3m": "c = core.eedi3m.EEDI3(c, field=1, dh=False)", + "fmtc": "c = core.fmtc.resample(c, w=320, h=240)", + "dfttest": "c = core.dfttest.DFTTest(c)", + "misc": "c = core.misc.SCDetect(c)", + "rgvs": "c = core.rgvs.RemoveGrain(c, mode=2)", + "grain": "c = core.grain.Add(c, var=4)", + "cas": "c = core.cas.CAS(c, sharpness=0.5)", + "dctf": "c = core.dctf.DCTFilter(c, factors=[1.0] * 8)", + "deblock": "c = core.deblock.Deblock(c, quant=25)", + "warp": "c = core.warp.AWarpSharp2(c, thresh=128, blur=3, type=0)", + # opt=2 (SSE2) is what script_generator::ctmf_opt picks on a CPU without + # AVX2, i.e. on every machine this probe exists for. + "ctmf": "c = core.ctmf.CTMF(c, radius=2, memsize=16777216, opt=2)", + "tcanny": "c = core.tcanny.TCanny(c, sigma=1.5, mode=-1)", + "tmedian": "c = core.tmedian.TemporalMedian(c, radius=1)", + "removedirt": "c = core.removedirt.RestoreMotionBlocks(c, restore=c)", + "lghost": "c = core.lghost.LGhost(c, mode=[1], shift=[2], intensity=[10])", + "bwdif": "c = core.bwdif.Bwdif(c, field=1)", + "zsmooth": "c = core.zsmooth.CCD(c, threshold=4, scale=1)", + "neo_f3kdb": "c = core.neo_f3kdb.Deband(c, y=64, cb=64, cr=64)", + "vivtc": "c = core.vivtc.VFM(c, order=1)", + "fb": "c = core.fb.FillBorders(c, left=2, right=2, top=2, bottom=2, mode='fillmargins')", + "descratch": "c = core.descratch.DeScratch(c)", + "fft3dfilter": "c = core.fft3dfilter.FFT3DFilter(c, sigma=2.0, bt=3)", + "ttmpsm": "c = core.ttmpsm.TTempSmooth(c)", + "flux": "c = core.flux.SmoothT(c, temporal_threshold=7)", + "dedot": "c = core.dedot.Dedot(c)", + "bifrost": "c = core.bifrost.Bifrost(c)", + "retinex": "c = core.retinex.MSRCP(core.std.ShufflePlanes(c, 0, vs.GRAY))", + "akarin": "c = core.akarin.Expr(c, 'x 1 +')", +} + +# OpenCL filters need a GPU to render, which says nothing about the CPU. +LOAD_ONLY = ("knlmeanscl", "nnedi3cl") + +# Plugins that call another plugin internally, by file-name substring. The +# dependency is loaded first; it is also tested on its own, so a crash that +# only shows up here is still attributable. +DEPENDS = {"ttempsmooth": ["miscfilters"]} + +# Exercises the parts of VapourSynth itself every job touches: zimg resizing +# and the std.Expr JIT, both of which pick SIMD code paths at runtime. +CORE_RENDER = ("c = core.resize.Bicubic(c, width=320, height=240, format=vs.YUV444P16)\n" + "c = core.std.Expr(c, 'x 1 +')") + +CHILD = r''' +import sys, vapoursynth as vs + +class _NoAutoload(vs.EnvironmentPolicy): + def on_policy_registered(self, api): + self._env = api.create_environment(vs.CoreCreationFlags.DISABLE_AUTO_LOADING) + def get_current_environment(self): + return self._env + def set_environment(self, env): + prev, self._env = self._env, env + return prev + +vs.register_policy(_NoAutoload()) +core = vs.core +path, render = sys.argv[1], sys.argv[2] +for dep in sys.argv[3:]: + core.std.LoadPlugin(dep) +new_ns = [] +if path: + before = {p.namespace for p in core.plugins()} + try: + core.std.LoadPlugin(path) + except vs.Error as e: + print("NOTPLUGIN " + str(e).replace("\n", " "), flush=True) + sys.exit(3) + new_ns = sorted({p.namespace for p in core.plugins()} - before) +print("NAMESPACES " + ",".join(new_ns), flush=True) +if render == "@load": + sys.exit(0) +c = core.std.BlankClip(width=640, height=480, length=12, format=vs.YUV420P8) +if render == "@auto": + snippets = [RENDER[n] for n in new_ns if n in RENDER] + if not snippets: + print("NORENDER", flush=True) + sys.exit(0) + render = "\n".join(snippets) +exec(render) +for i in range(len(c)): + c.get_frame(i) +print("RENDERED", flush=True) +''' + +ILLEGAL_INSTRUCTION = 0xC000001D +WINDOWS_CRASH_NAMES = { + 0xC000001D: "illegal instruction", + 0xC0000005: "access violation", + 0xC00000FD: "stack overflow", + 0xC0000094: "integer divide by zero", + 0xC0000409: "stack buffer overrun", +} + + +def default_deps_dir(): + system = platform.system() + home = os.path.expanduser("~") + if system == "Darwin": + arch = "macos-arm64" if platform.machine() == "arm64" else "macos-x64" + return os.path.join(home, "Library", "Application Support", "VapourBox", "deps", arch) + if system == "Linux": + arch = "linux-arm64" if platform.machine() == "aarch64" else "linux-x64" + base = os.environ.get("XDG_DATA_HOME") or os.path.join(home, ".local", "share") + return os.path.join(base, "VapourBox", "deps", arch) + return os.path.join(os.getcwd(), "deps", "windows-x64") + + +def bundle_layout(deps): + """Python executable, child environment and plugin files for a bundle.""" + env = {k: v for k, v in os.environ.items() + if k not in ("PYTHONHOME", "PYTHONPATH", "VAPOURSYNTH_EXTRA_PLUGIN_PATH")} + env["PYTHONNOUSERSITE"] = "1" + vs_dir = os.path.join(deps, "vapoursynth") + if platform.system() == "Windows": + python = os.path.join(vs_dir, "python.exe") + env["PYTHONHOME"] = vs_dir + env["PYTHONPATH"] = os.pathsep.join([ + os.path.join(deps, "python-packages"), + os.path.join(vs_dir, "Lib", "site-packages"), + ]) + env["PATH"] = os.pathsep.join([vs_dir, env.get("PATH", "")]) + plugin_dir, exts = os.path.join(vs_dir, "vs-plugins"), (".dll",) + else: + py_home = os.path.join(deps, "python") + python = os.path.join(py_home, "bin", "python3") + env["PYTHONHOME"] = py_home + env["PYTHONPATH"] = os.pathsep.join([ + os.path.join(deps, "python-packages"), + deps, + os.path.join(py_home, "lib", "python3.12", "site-packages"), + ]) + libs = [vs_dir, os.path.join(py_home, "lib"), os.path.join(deps, "lib")] + var = "DYLD_LIBRARY_PATH" if platform.system() == "Darwin" else "LD_LIBRARY_PATH" + env[var] = os.pathsep.join(libs + ([env[var]] if env.get(var) else [])) + plugin_dir = os.path.join(vs_dir, "plugins") + exts = (".dylib",) if platform.system() == "Darwin" else (".so",) + + files = [] + for d in (plugin_dir, os.path.join(vs_dir, "zsmooth")): + if os.path.isdir(d): + files += sorted(os.path.join(d, f) for f in os.listdir(d) + if f.lower().endswith(exts)) + return python, env, files + + +def classify(proc, sde_wrapped): + out, err, code = proc.stdout, proc.stderr, proc.returncode + namespaces = "" + for line in out.splitlines(): + if line.startswith("NAMESPACES "): + namespaces = line[len("NAMESPACES "):] + tail = "\n".join(err.strip().splitlines()[-6:]) + + if sde_wrapped and ("SDE-ERROR" in err or "not valid for specified chip" in err): + return "CRASHED", "illegal instruction (SDE)", namespaces, tail + if code < 0: + sig = -code + try: + import signal + name = signal.Signals(sig).name + except (ValueError, ImportError): + name = f"signal {sig}" + return "CRASHED", name, namespaces, tail + if code & 0xFFFFFFFF in WINDOWS_CRASH_NAMES: + return "CRASHED", WINDOWS_CRASH_NAMES[code & 0xFFFFFFFF], namespaces, tail + if code == 3 and "NOTPLUGIN" in out: + return "SKIPPED", "not a VapourSynth plugin", namespaces, "" + if code == 0 and "NORENDER" in out: + return "LOADED", "no render test for this namespace", namespaces, "" + if code == 0 and "RENDERED" in out: + return "PASS", "", namespaces, "" + if code == 0: + return "LOADED", "load-only test", namespaces, "" + return "ERROR", f"exit {code}", namespaces, tail + + +def cpu_summary(): + lines = [f"Machine: {platform.system()} {platform.release()} ({platform.machine()})"] + system = platform.system() + try: + if system == "Darwin": + def sysctl(key): + r = subprocess.run(["sysctl", "-n", key], capture_output=True, text=True) + return r.stdout.strip() + feats = sysctl("machdep.cpu.features").split() + leaf7 = sysctl("machdep.cpu.leaf7_features").split() + lines.append(f"CPU: {sysctl('machdep.cpu.brand_string')}") + # macOS spells plain AVX as "AVX1.0". + lines.append(f"AVX: {int('AVX1.0' in feats)} AVX2: {int('AVX2' in leaf7)}") + elif system == "Linux": + with open("/proc/cpuinfo") as f: + info = f.read() + model = next((l.split(":", 1)[1].strip() for l in info.splitlines() + if l.startswith("model name")), "unknown") + flags = next((l.split(":", 1)[1].split() for l in info.splitlines() + if l.startswith("flags")), []) + lines.append(f"CPU: {model}") + lines.append(f"AVX: {int('avx' in flags)} AVX2: {int('avx2' in flags)}") + elif system == "Windows": + import ctypes + present = ctypes.windll.kernel32.IsProcessorFeaturePresent + lines.append(f"CPU: {platform.processor()}") + # PF_AVX_INSTRUCTIONS_AVAILABLE = 39, PF_AVX2_INSTRUCTIONS_AVAILABLE = 40 + lines.append(f"AVX: {int(bool(present(39)))} AVX2: {int(bool(present(40)))}") + except Exception as e: # the report is still useful without CPU details + lines.append(f"CPU: unavailable ({e})") + return lines + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("deps", nargs="?", default=None) + ap.add_argument("--report", default=None, help="text report path") + ap.add_argument("--json", default=None, help="machine-readable results path") + ap.add_argument("--wrap", default="", help='prefix for each test, e.g. "sde64 -nhm --"') + ap.add_argument("--timeout", type=int, default=600) + ap.add_argument("--fail-on-crash", action="store_true", + help="exit 1 if anything crashed (for CI gates)") + args = ap.parse_args() + + deps = os.path.abspath(args.deps or default_deps_dir()) + if not os.path.isdir(os.path.join(deps, "vapoursynth")): + sys.exit(f"No VapourBox deps bundle found at {deps}\n" + "Pass the path to your deps folder as the first argument.") + python, env, files = bundle_layout(deps) + wrap = shlex.split(args.wrap) + child_src = f"RENDER = {RENDER!r}\n" + CHILD + + if args.report is None: + desktop = os.path.join(os.path.expanduser("~"), "Desktop") + where = desktop if os.path.isdir(desktop) else os.getcwd() + args.report = os.path.join(where, "vapourbox-plugin-compat-report.txt") + + version = "" + try: + with open(os.path.join(deps, "version.json")) as f: + version = json.dumps(json.load(f)) + except (OSError, ValueError): + pass + + header = ["VapourBox plugin compatibility report", + f"Generated: {datetime.now():%Y-%m-%d %H:%M:%S}", + f"Deps: {deps}", + f"Bundle: {version or 'unknown'}"] + header += cpu_summary() + if wrap: + header.append(f"Wrapped with: {' '.join(wrap)}") + header.append("=" * 72) + + report = open(args.report, "w") + + def emit(line=""): + print(line, flush=True) + report.write(line + "\n") + + for line in header: + emit(line) + + tests = [("(core: VapourSynth, zimg, Expr)", "", CORE_RENDER)] + for f in files: + stem = os.path.basename(f) + tests.append((stem, f, None)) + + def deps_for(path): + low = os.path.basename(path).lower() + wanted = next((v for k, v in DEPENDS.items() if k in low), []) + return [f for f in files + if any(w in os.path.basename(f).lower() for w in wanted)] + + results = [] + for label, path, render in tests: + extra = [] + if render is None: + # Unknown until the child reports its namespaces; OpenCL is load-only. + render = "@auto" + if any(k in os.path.basename(path).lower() for k in LOAD_ONLY): + render = "@load" + extra = deps_for(path) + cmd = wrap + [python, "-c", child_src, path, render] + extra + try: + proc = subprocess.run(cmd, env=env, capture_output=True, text=True, + timeout=args.timeout) + status, detail, ns, tail = classify(proc, bool(wrap)) + except subprocess.TimeoutExpired: + status, detail, ns, tail = "ERROR", f"timed out after {args.timeout}s", "", "" + results.append({"file": label, "namespaces": ns, "status": status, + "detail": detail}) + shown = f"{label} [{ns}]" if ns else label + emit(f"{shown:<46} {status}{' (' + detail + ')' if detail else ''}") + if tail and status in ("CRASHED", "ERROR"): + for t in tail.splitlines(): + emit(f" {t}") + + counts = {s: sum(r["status"] == s for r in results) + for s in ("PASS", "LOADED", "CRASHED", "ERROR", "SKIPPED")} + emit("=" * 72) + emit("Summary: " + ", ".join(f"{n} {s.lower()}" for s, n in counts.items())) + report.close() + print(f"\nReport written to: {args.report}") + + if args.json: + with open(args.json, "w") as f: + json.dump({"deps": deps, "bundle": version, "results": results}, f, indent=2) + if args.fail_on_crash and (counts["CRASHED"] or counts["ERROR"]): + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/Scripts/probe-plugin-compat.sh b/Scripts/probe-plugin-compat.sh new file mode 100755 index 0000000..e8d0469 --- /dev/null +++ b/Scripts/probe-plugin-compat.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Test which bundled VapourSynth plugins this machine's CPU can run (issue #92). +# +# bash probe-plugin-compat.sh [path-to-deps-folder] +# +# Runs probe-plugin-compat.py (which must sit next to this script) with the deps +# bundle's own Python, so no system Python is needed. Writes the report to +# ~/Desktop/vapourbox-plugin-compat-report.txt. + +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" + +if [ $# -ge 1 ]; then + DEPS="$1" +elif [ "$(uname)" = "Darwin" ]; then + [ "$(uname -m)" = "arm64" ] && ID=macos-arm64 || ID=macos-x64 + DEPS="$HOME/Library/Application Support/VapourBox/deps/$ID" +else + [ "$(uname -m)" = "aarch64" ] && ID=linux-arm64 || ID=linux-x64 + DEPS="${XDG_DATA_HOME:-$HOME/.local/share}/VapourBox/deps/$ID" +fi + +PY="$DEPS/python/bin/python3" +if [ ! -x "$PY" ]; then + echo "Could not find VapourBox's bundled Python at: $PY" + echo "Open VapourBox once so it can download its components, then try again." + exit 1 +fi + +exec "$PY" "$HERE/probe-plugin-compat.py" "$DEPS" "${@:2}" From d114e43a728c1e7b41d940042cd22b4f8f25ed1d Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Fri, 25 Sep 2026 23:20:38 +1000 Subject: [PATCH 2/9] Make an SDE probe run prove it is valid before it counts The first survey showed why the positive control alone is not enough: on Windows at a pre-AVX chip, every test "crashed" inside VCRUNTIME140's memcpy, because Microsoft's runtime chooses its AVX path from what the host kernel reports, which SDE cannot emulate. The control plugin crashed too, so the run looked valid while proving nothing. A run now needs the plugin-free core test to pass under SDE, and the control to fault inside its own image. The report names the faulting image, and a fault in an OS runtime DLL is classified as inconclusive rather than blamed on a plugin. Windows moves to Sandy Bridge (AVX, no AVX2), which is the tier boundary and where the runtime's AVX path is legal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014GLXdGLfPwgYjW1AkonGqN --- .github/workflows/probe-cpu-compat.yml | 54 ++++++++++++++++---------- Scripts/probe-plugin-compat.py | 24 +++++++++--- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/.github/workflows/probe-cpu-compat.yml b/.github/workflows/probe-cpu-compat.yml index 203e367..5892739 100644 --- a/.github/workflows/probe-cpu-compat.yml +++ b/.github/workflows/probe-cpu-compat.yml @@ -5,12 +5,15 @@ name: Probe CPU compatibility (SDE) # hosted runner has AVX2, so this is the only CI-side way to see what a # pre-AVX2 machine would do with the bundle (issue #92). # -# Two controls make a result trustworthy rather than merely green: -# - a native run on the runner itself, which must pass everything; an error -# there is a probe bug, not a CPU finding. -# - zsmooth's haswell build (AVX2, no runtime dispatch — issue #82) must -# CRASH under SDE. If it passes, SDE is not trapping and the run proves -# nothing, so the job fails. +# A run only counts if its controls hold: +# - native: the runner itself must pass everything; an error there is a +# probe bug, not a CPU finding. +# - core under SDE: VapourSynth with no plugins must pass. If it doesn't, the +# emulation itself is broken for this OS/chip and every plugin result is +# meaningless. (Windows at pre-AVX chips fails this: Microsoft's runtime +# picks AVX from what the host kernel reports, which SDE cannot emulate.) +# - positive: zsmooth's haswell build (AVX2, no runtime dispatch — issue #82) +# must crash inside its own image under a pre-AVX2 chip, proving SDE traps. on: # Also runs when the probe or this workflow changes, which re-validates the @@ -26,10 +29,6 @@ on: description: 'Optional build-deps-* run IDs (comma-separated); default is the release named in deps-version.json' required: false default: '' - chip: - description: 'SDE chip to emulate (wsm = Westmere, the Mac Pro 5,1 in #92; nhm = Nehalem; snb = Sandy Bridge, AVX without AVX2)' - required: false - default: 'wsm' permissions: contents: read @@ -37,19 +36,24 @@ permissions: jobs: probe: - name: ${{ matrix.platform }} + name: ${{ matrix.platform }} (-${{ matrix.chip }}) strategy: fail-fast: false matrix: include: + # wsm = Westmere, the Mac Pro 5,1 in #92: no AVX at all. - platform: linux-x64 runner: ubuntu-24.04 sde: sde64 python: python/bin/python3 + chip: wsm + # snb = Sandy Bridge: AVX without AVX2 — the v3/v2 tier boundary. + # Windows can only be probed here; see the header. - platform: windows-x64 runner: windows-latest sde: sde.exe python: vapoursynth/python.exe + chip: snb runs-on: ${{ matrix.runner }} timeout-minutes: 90 defaults: @@ -87,14 +91,14 @@ jobs: "$D/${{ matrix.python }}" Scripts/probe-plugin-compat.py "$D" \ --report native.txt --json native.json --fail-on-crash - - name: SDE run (-${{ inputs.chip || 'wsm' }}) + - name: SDE run (-${{ matrix.chip }}) run: | SDE_DIR=$(cygpath -m "$SDE_PATH" 2>/dev/null || echo "$SDE_PATH") "$D/${{ matrix.python }}" Scripts/probe-plugin-compat.py "$D" \ --report sde.txt --json sde.json --timeout 1800 \ - --wrap "\"$SDE_DIR/${{ matrix.sde }}\" -${{ inputs.chip || 'wsm' }} --" + --wrap "\"$SDE_DIR/${{ matrix.sde }}\" -${{ matrix.chip }} --" - - name: Check the positive control and summarise + - name: Check the controls and summarise run: | "$D/${{ matrix.python }}" - <<'EOF' import json, os, sys @@ -103,22 +107,30 @@ jobs: rows += [f"| {r['file']} | {r['namespaces']} | {r['status']} {r['detail']} |" for r in sde] with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: f.write("\n".join(rows) + "\n") + + core = next(r for r in sde if r["file"].startswith("(core")) + if core["status"] != "PASS": + sys.exit(f"core control failed under SDE ({core['status']}: {core['detail']}); " + "the emulation is not valid for this OS/chip, so no plugin result means anything") control = [r for r in sde if "zsmooth-haswell" in r["file"]] if not control: sys.exit("positive control missing: no zsmooth-haswell build in this bundle") - if control[0]["status"] != "CRASHED": - sys.exit("SDE did not trap the AVX2-only zsmooth-haswell build; results are meaningless") - crashed = [r for r in sde if r["status"] == "CRASHED"] - print(f"control OK; {len(crashed)} file(s) crashed under SDE:") - for r in crashed: - print(f" {r['file']} [{r['namespaces']}] {r['detail']}") + c = control[0] + if c["status"] != "CRASHED" or "zsmooth-haswell" not in c["detail"]: + sys.exit(f"SDE did not trap the AVX2-only zsmooth-haswell build in its own code " + f"({c['status']}: {c['detail']}); results are meaningless") + + bad = [r for r in sde if r["status"] in ("CRASHED", "INCONCLUSIVE") and r is not c] + print(f"controls OK; {len(bad)} other file(s) crashed or were inconclusive:") + for r in bad: + print(f" {r['file']} [{r['namespaces']}] {r['status']}: {r['detail']}") EOF - name: Upload reports if: always() uses: actions/upload-artifact@v5 with: - name: cpu-compat-${{ matrix.platform }}-${{ inputs.chip || 'wsm' }} + name: cpu-compat-${{ matrix.platform }}-${{ matrix.chip }} path: | native.txt native.json diff --git a/Scripts/probe-plugin-compat.py b/Scripts/probe-plugin-compat.py index 28af70b..2c31f33 100755 --- a/Scripts/probe-plugin-compat.py +++ b/Scripts/probe-plugin-compat.py @@ -126,7 +126,14 @@ def set_environment(self, env): print("RENDERED", flush=True) ''' -ILLEGAL_INSTRUCTION = 0xC000001D +# Under SDE on Windows, Microsoft's runtime DLLs pick SIMD paths from what the +# *host* kernel reports (IsProcessorFeaturePresent reads shared kernel memory, +# which SDE cannot virtualise), so they execute AVX that SDE then flags — on +# code that is fine on real pre-AVX hardware. A fault in one of these says +# nothing about the plugin under test. +EMULATION_ARTIFACT_IMAGES = ("vcruntime140", "ucrtbase", "msvcp140", "ntdll", + "kernelbase", "kernel32") + WINDOWS_CRASH_NAMES = { 0xC000001D: "illegal instruction", 0xC0000005: "access violation", @@ -196,7 +203,14 @@ def classify(proc, sde_wrapped): tail = "\n".join(err.strip().splitlines()[-6:]) if sde_wrapped and ("SDE-ERROR" in err or "not valid for specified chip" in err): - return "CRASHED", "illegal instruction (SDE)", namespaces, tail + image = next((l.split("Image:", 1)[1].strip() for l in err.splitlines() + if "Image:" in l), "") + where = os.path.basename(image.replace("\\", "/")) or "unknown image" + if any(a in where.lower() for a in EMULATION_ARTIFACT_IMAGES): + return ("INCONCLUSIVE", + f"SDE flagged the OS runtime ({where}), not this plugin", + namespaces, tail) + return "CRASHED", f"illegal instruction in {where}", namespaces, tail if code < 0: sig = -code try: @@ -259,7 +273,7 @@ def main(): ap.add_argument("--wrap", default="", help='prefix for each test, e.g. "sde64 -nhm --"') ap.add_argument("--timeout", type=int, default=600) ap.add_argument("--fail-on-crash", action="store_true", - help="exit 1 if anything crashed (for CI gates)") + help="exit 1 if anything crashed, errored or was inconclusive (for CI gates)") args = ap.parse_args() deps = os.path.abspath(args.deps or default_deps_dir()) @@ -336,7 +350,7 @@ def deps_for(path): emit(f" {t}") counts = {s: sum(r["status"] == s for r in results) - for s in ("PASS", "LOADED", "CRASHED", "ERROR", "SKIPPED")} + for s in ("PASS", "LOADED", "CRASHED", "ERROR", "INCONCLUSIVE", "SKIPPED")} emit("=" * 72) emit("Summary: " + ", ".join(f"{n} {s.lower()}" for s, n in counts.items())) report.close() @@ -345,7 +359,7 @@ def deps_for(path): if args.json: with open(args.json, "w") as f: json.dump({"deps": deps, "bundle": version, "results": results}, f, indent=2) - if args.fail_on_crash and (counts["CRASHED"] or counts["ERROR"]): + if args.fail_on_crash and (counts["CRASHED"] or counts["ERROR"] or counts["INCONCLUSIVE"]): sys.exit(1) From 8f8188a4c147df49d3e8e735ae58ef834e5a7021 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Fri, 25 Sep 2026 23:27:27 +1000 Subject: [PATCH 3/9] Probe: don't pass an empty argument for "no plugin" sde.exe drops empty arguments when it relaunches the child, so the core control's empty plugin path vanished and shifted every argument after it. Natively the same test passed, which is what made it look like a CPU finding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014GLXdGLfPwgYjW1AkonGqN --- Scripts/probe-plugin-compat.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Scripts/probe-plugin-compat.py b/Scripts/probe-plugin-compat.py index 2c31f33..50aafd9 100755 --- a/Scripts/probe-plugin-compat.py +++ b/Scripts/probe-plugin-compat.py @@ -102,7 +102,7 @@ def set_environment(self, env): for dep in sys.argv[3:]: core.std.LoadPlugin(dep) new_ns = [] -if path: +if path != "-": before = {p.namespace for p in core.plugins()} try: core.std.LoadPlugin(path) @@ -314,7 +314,9 @@ def emit(line=""): for line in header: emit(line) - tests = [("(core: VapourSynth, zimg, Expr)", "", CORE_RENDER)] + # "-" rather than "" for "no plugin": sde.exe drops empty arguments when + # it relaunches the child, which shifted every later argument. + tests = [("(core: VapourSynth, zimg, Expr)", "-", CORE_RENDER)] for f in files: stem = os.path.basename(f) tests.append((stem, f, None)) From d2137173ac315806a371e916dbf1b4343665bc43 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Fri, 25 Sep 2026 23:45:08 +1000 Subject: [PATCH 4/9] Worker: one CPU-tier decision, and zsmooth back to plain autoload The x86 deps bundles are splitting into CPU tiers (issue #92): v3 for x86-64-v3 CPUs and v2 for anything older, where the prebuilt macOS MVTools faults with SIGILL inside dlopen because its AVX2 files build their lookup tables at load time. cpu::cpu_tier() is the single decision: v3 needs the whole x86-64-v3 set, not just AVX2, because the v3 plugins are compiled for the whole level. --probe-cpu reports it for the app to choose a bundle, and ctmf_opt derives from it, so the worker has one answer to "what is this machine". With one zsmooth build per bundle, the explicit-path machinery that picked between two builds (issue #82) goes: the bundle's zsmooth autoloads like every other plugin. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014GLXdGLfPwgYjW1AkonGqN --- worker/src/cpu.rs | 120 +++++++++++++++++++++ worker/src/dependency_locator.rs | 133 ------------------------ worker/src/lib.rs | 1 + worker/src/main.rs | 69 +++++------- worker/src/pipeline_executor.rs | 3 +- worker/src/script_generator.rs | 68 ++---------- worker/templates/pipeline_template.vpy | 10 -- worker/templates/preview_template.vpy | 7 -- worker/tests/filter_integration_test.rs | 116 ++++----------------- 9 files changed, 178 insertions(+), 349 deletions(-) create mode 100644 worker/src/cpu.rs diff --git a/worker/src/cpu.rs b/worker/src/cpu.rs new file mode 100644 index 0000000..50d3f55 --- /dev/null +++ b/worker/src/cpu.rs @@ -0,0 +1,120 @@ +//! Which x86 deps bundle this CPU can run (issue #92). +//! +//! The x86 bundles ship in two capability tiers. `v3` is built for the +//! x86-64-v3 psABI level (Haswell, 2013, and later); `v2` runs on anything +//! older. This is the single place that decides between them: the app asks +//! `vapourbox-worker --probe-cpu` which bundle to download, and the worker's own +//! CPU-dependent choices (`script_generator::ctmf_opt`) derive from the same +//! answer, so the two can never disagree about what this machine is. +//! +//! Detection is a runtime CPUID query that also checks the OS has enabled the +//! AVX register state, so it reports what the process can really execute — +//! including under Rosetta, which exposes AVX2 on macOS 15+ and nothing earlier. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CpuTier { + V2, + V3, +} + +impl CpuTier { + pub fn as_str(self) -> &'static str { + match self { + CpuTier::V2 => "v2", + CpuTier::V3 => "v3", + } + } +} + +/// The x86-64-v3 feature set, as compilers define `-march=x86-64-v3`. +/// +/// All of it, not just AVX2: zsmooth's haswell build and the other v3 plugins +/// are compiled for the whole level, so a CPU with AVX2 but without (say) MOVBE +/// can still fault in them. Such CPUs are rare; the cost of sending one to v2 +/// is throughput, while the cost of the reverse is a crash. +pub const V3_FEATURES: [&str; 8] = ["avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "lzcnt", "movbe"]; + +/// The tier for a CPU, given a feature test. Pure, so it is testable on any host. +pub fn tier_from_features(has: impl Fn(&str) -> bool) -> CpuTier { + if V3_FEATURES.iter().all(|f| has(f)) { + CpuTier::V3 + } else { + CpuTier::V2 + } +} + +/// This machine's tier, or `None` off x86 (ARM bundles are not tiered). +pub fn cpu_tier() -> Option { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + Some(tier_from_features(has_feature)) + } + #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] + { + None + } +} + +/// Runtime feature test by name. `is_x86_feature_detected!` only takes +/// literals, hence the match; an unknown name is reported as absent. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub fn has_feature(name: &str) -> bool { + match name { + "sse2" => std::is_x86_feature_detected!("sse2"), + "sse4.1" => std::is_x86_feature_detected!("sse4.1"), + "sse4.2" => std::is_x86_feature_detected!("sse4.2"), + "avx" => std::is_x86_feature_detected!("avx"), + "avx2" => std::is_x86_feature_detected!("avx2"), + "bmi1" => std::is_x86_feature_detected!("bmi1"), + "bmi2" => std::is_x86_feature_detected!("bmi2"), + "f16c" => std::is_x86_feature_detected!("f16c"), + "fma" => std::is_x86_feature_detected!("fma"), + "lzcnt" => std::is_x86_feature_detected!("lzcnt"), + "movbe" => std::is_x86_feature_detected!("movbe"), + "avx512f" => std::is_x86_feature_detected!("avx512f"), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_v3_feature_is_required() { + assert_eq!(tier_from_features(|_| true), CpuTier::V3); + for missing in V3_FEATURES { + assert_eq!( + tier_from_features(|f| f != missing), + CpuTier::V2, + "a CPU without {missing} must not be offered the v3 bundle" + ); + } + } + + #[test] + fn a_pre_avx_cpu_is_v2() { + // Westmere, the Mac Pro 5,1 from issue #92: SSE4.2, no AVX at all. + let westmere = ["sse2", "sse4.1", "sse4.2"]; + assert_eq!(tier_from_features(|f| westmere.contains(&f)), CpuTier::V2); + } + + #[test] + fn tier_is_reported_exactly_where_bundles_are_tiered() { + let tier = cpu_tier(); + if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { + assert!(tier.is_some(), "x86 must always resolve to a tier"); + } else { + assert_eq!(tier, None, "ARM bundles are not tiered"); + } + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[test] + fn the_runtime_answer_agrees_with_the_feature_test() { + assert_eq!(cpu_tier(), Some(tier_from_features(has_feature))); + // Guard the literal match: a misspelt arm would silently read "absent" + // and push every machine to v2. + assert!(has_feature("sse2"), "sse2 is architectural on x86_64"); + } +} diff --git a/worker/src/dependency_locator.rs b/worker/src/dependency_locator.rs index 4ad660d..23de2eb 100644 --- a/worker/src/dependency_locator.rs +++ b/worker/src/dependency_locator.rs @@ -574,83 +574,6 @@ impl DependencyLocator { } } - /// Directory holding the zsmooth builds, which are deliberately NOT in the - /// autoload directory. - /// - /// Upstream publishes zsmooth only for `haswell` (an AVX2 baseline, no - /// runtime dispatch) and `znver4`, so the bundled binary hard-crashes with - /// an illegal instruction on any pre-2013 x86 CPU the moment a zsmooth - /// filter runs — issue #82, on a Celeron J4105 and a Core i7 870. A second - /// `x86_64_v2` build covers those machines, and it cannot simply sit beside - /// the first: both register the namespace `zsmooth`, so whichever autoloads - /// second is rejected. Exactly one is therefore loaded explicitly, by - /// [`Self::zsmooth_plugin`], from here. - pub fn zsmooth_dir(&self) -> PathBuf { - self.platform_dir().join("vapoursynth").join("zsmooth") - } - - /// The zsmooth build this machine can actually execute, or `None` when the - /// bundle predates the split. - /// - /// `None` is not a failure: deps bundles up to 1.9.0 ship a single zsmooth - /// inside the autoload directory, and on those the generated script must - /// emit no `LoadPlugin` at all and let autoload do what it has always done. - /// That keeps a newer worker working against an older bundle, which matters - /// because the app can be upgraded before the deps download completes. - /// - /// The choice is made here, in the worker, rather than in the script for the - /// same reason as `script_generator::ctmf_opt`: it is a property of the - /// machine, and no preceding pass can change the answer. - pub fn zsmooth_plugin(&self) -> Option { - let dir = self.zsmooth_dir(); - for name in Self::zsmooth_candidates() { - let path = dir.join(name); - if path.exists() { - return Some(path); - } - } - None - } - - /// Candidate filenames in preference order: the fastest build this CPU can - /// run first, then the one that runs anywhere. - /// - /// Never fall back the other way. Choosing haswell where AVX2 is absent is - /// not a slow job, it is a dead one. - fn zsmooth_candidates() -> Vec { - let ext = if cfg!(target_os = "windows") { - "dll" - } else if cfg!(target_os = "macos") { - "dylib" - } else { - "so" - }; - let prefix = if cfg!(target_os = "windows") { "" } else { "lib" }; - - let mut variants: Vec<&str> = Vec::new(); - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - if std::is_x86_feature_detected!("avx2") { - variants.push("haswell"); - } - variants.push("x86_64_v2"); - } - // A single build on non-x86: aarch64 has one NEON baseline and upstream - // publishes no variants for it. - variants.push(""); - - variants - .into_iter() - .map(|v| { - if v.is_empty() { - format!("{}zsmooth.{}", prefix, ext) - } else { - format!("{}zsmooth-{}.{}", prefix, v, ext) - } - }) - .collect() - } - /// Get the NNEDI3CL weights path. pub fn nnedi3cl_weights_path(&self) -> PathBuf { #[cfg(target_os = "windows")] @@ -927,62 +850,6 @@ impl DependencyLocator { mod tests { use super::*; - #[test] - fn zsmooth_never_offers_a_build_this_cpu_cannot_run() { - // The whole point of the split: naming the AVX2 build on a machine - // without AVX2 is not a slow job, it is `0xC000001D` and a dead one - // (issue #82, on a Celeron J4105 and a Core i7 870). This assertion - // runs on every platform whatever hardware CI draws, which is the - // durable half of the guard — the end-to-end test can only confirm - // opportunistically, and GitHub's fleet is a mixed draw. - let candidates = DependencyLocator::zsmooth_candidates(); - assert!(!candidates.is_empty(), "there must always be a candidate"); - - let haswell = candidates.iter().any(|c| c.contains("haswell")); - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - assert_eq!( - haswell, - std::is_x86_feature_detected!("avx2"), - "the haswell build may be offered only where AVX2 exists" - ); - // The fallback has to be present on x86 regardless, or a non-AVX2 - // machine has nothing to load. - assert!( - candidates.iter().any(|c| c.contains("x86_64_v2")), - "x86 must always offer the x86_64_v2 fallback: {candidates:?}" - ); - if haswell { - assert!( - candidates[0].contains("haswell"), - "where AVX2 exists the fastest build must be preferred: {candidates:?}" - ); - } - } - #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] - { - assert!(!haswell, "no x86 build may be offered off x86: {candidates:?}"); - } - } - - #[test] - fn zsmooth_candidates_use_this_platforms_library_naming() { - // A wrong prefix or extension makes every candidate miss, which - // degrades silently to the autoload path — i.e. to the bug. - let candidates = DependencyLocator::zsmooth_candidates(); - let (prefix, ext) = if cfg!(target_os = "windows") { - ("", ".dll") - } else if cfg!(target_os = "macos") { - ("lib", ".dylib") - } else { - ("lib", ".so") - }; - for c in &candidates { - assert!(c.starts_with(&format!("{prefix}zsmooth")), "bad prefix: {c}"); - assert!(c.ends_with(ext), "bad extension: {c}"); - } - } - #[test] fn test_platform_suffix() { let locator = DependencyLocator { diff --git a/worker/src/lib.rs b/worker/src/lib.rs index edff205..b97214d 100644 --- a/worker/src/lib.rs +++ b/worker/src/lib.rs @@ -3,6 +3,7 @@ //! Provides video processing functionality using VapourSynth. pub mod models; +pub mod cpu; pub mod dependency_locator; pub mod dvd_reader; pub mod filter_registry; diff --git a/worker/src/main.rs b/worker/src/main.rs index 5a73020..d3430f0 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -19,6 +19,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; mod models; +mod cpu; mod dependency_locator; mod dvd_reader; mod pipeline_executor; @@ -203,55 +204,38 @@ fn run_probe_opencl() -> ExitCode { } -/// Report the CPU architecture and the dispatch-relevant instruction set -/// extensions as JSON. +/// Report the CPU architecture, the dispatch-relevant instruction set +/// extensions and, on x86, the deps bundle tier, as JSON. /// -/// This is diagnostic, not a decision: nothing in the pipeline reads it. It -/// exists so a test run can *state* which hardware it tested, because several -/// bundled plugins select a code path from these bits and a green run is -/// otherwise silent about which path it took. +/// `tier` is a decision: the app reads it to choose which x86 deps bundle to +/// download (see `cpu::cpu_tier`). It is absent off x86, where bundles are not +/// tiered, and the app treats a missing or unreadable answer on x86 as `v2`. /// -/// The motivating case: `ctmf.CTMF`'s AVX-512 kernel for 8-bit input crashes -/// the process, and GitHub's hosted Windows runners are a mixed fleet — so the -/// nightly passed for days on non-AVX-512 machines, went red the night it drew -/// an AVX-512 one, and looked like a spontaneous failure against an unchanged -/// tree. Printing this next to the result turns "it passed" into "it passed on -/// this hardware". +/// `features` is diagnostic. It lets a test run *state* which hardware it +/// tested, because several bundled plugins select a code path from these bits +/// and a green run is otherwise silent about which path it took — the CTMF +/// AVX-512 crash passed for days on non-AVX-512 runners before drawing one. fn run_probe_cpu() -> ExitCode { - let features = detected_cpu_features(); - println!( - "{}", - serde_json::json!({ - "arch": std::env::consts::ARCH, - "features": features, - }) - ); + let mut out = serde_json::json!({ + "arch": std::env::consts::ARCH, + "features": detected_cpu_features(), + }); + if let Some(tier) = cpu::cpu_tier() { + out["tier"] = serde_json::json!(tier.as_str()); + } + println!("{out}"); ExitCode::SUCCESS } -/// The instruction set extensions that bundled plugins actually dispatch on. -/// -/// Deliberately a short list rather than everything detectable: these are the -/// ones that change which kernel a plugin runs here. Detection is a runtime -/// CPUID query, so it reports what the process can really execute — including -/// under emulation, where an x86_64 worker on Apple Silicon correctly reports -/// whatever Rosetta exposes rather than what the binary was compiled for. +/// The instruction set extensions that bundled plugins dispatch on, plus the +/// whole x86-64-v3 set that decides the bundle tier. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn detected_cpu_features() -> Vec<&'static str> { - let mut features = Vec::new(); - for (name, present) in [ - ("sse2", std::is_x86_feature_detected!("sse2")), - ("sse4.1", std::is_x86_feature_detected!("sse4.1")), - ("avx", std::is_x86_feature_detected!("avx")), - ("avx2", std::is_x86_feature_detected!("avx2")), - ("fma", std::is_x86_feature_detected!("fma")), - ("avx512f", std::is_x86_feature_detected!("avx512f")), - ] { - if present { - features.push(name); - } - } - features + let names = ["sse2", "sse4.1", "sse4.2"] + .into_iter() + .chain(cpu::V3_FEATURES) + .chain(["avx512f"]); + names.filter(|n| cpu::has_feature(n)).collect() } #[cfg(target_arch = "aarch64")] @@ -523,8 +507,7 @@ fn run_worker( let knlm_available = deps.as_ref().map(|d| d.knlm_available()).unwrap_or(true); let script_generator = ScriptGenerator::new()? .with_opencl_available(opencl_available) - .with_knlm_available(knlm_available) - .with_zsmooth_plugin(deps.as_ref().and_then(|d| d.zsmooth_plugin())); + .with_knlm_available(knlm_available); let script_path = script_generator .generate(&job) .with_context(|| "Failed to generate VapourSynth script")?; diff --git a/worker/src/pipeline_executor.rs b/worker/src/pipeline_executor.rs index 505b6f3..4c74104 100644 --- a/worker/src/pipeline_executor.rs +++ b/worker/src/pipeline_executor.rs @@ -1252,8 +1252,7 @@ impl PipelineExecutor { // FPS as rational let script_generator = ScriptGenerator::new()? .with_opencl_available(self.deps.opencl_available()) - .with_knlm_available(self.deps.knlm_available()) - .with_zsmooth_plugin(self.deps.zsmooth_plugin()); + .with_knlm_available(self.deps.knlm_available()); let (fps_num, fps_den) = script_generator.frame_rate_to_rational(frame_rate); let preview_params = PreviewParams { diff --git a/worker/src/script_generator.rs b/worker/src/script_generator.rs index 3bcbcea..ab54825 100644 --- a/worker/src/script_generator.rs +++ b/worker/src/script_generator.rs @@ -28,32 +28,23 @@ use crate::models::{ /// /// The plugin does **not** verify that the CPU can run the level it is handed, /// so this has to be a real capability query rather than a constant: 3 (AVX2) -/// where the CPU has it, otherwise 2 (SSE2, which every x86-64 CPU has by -/// definition). The three non-AVX-512 levels are bit-identical, so this costs -/// throughput and nothing else. Non-x86 builds of the plugin compile the -/// dispatch out and ignore the value. +/// on a v3-tier CPU, otherwise 2 (SSE2, which every x86-64 CPU has by +/// definition). It follows `cpu::cpu_tier`, the same answer that chose the deps +/// bundle, so the worker has one notion of what this machine is. The three +/// non-AVX-512 levels are bit-identical, so a v2 machine that happens to have +/// AVX2 loses throughput and nothing else. Non-x86 builds of the plugin compile +/// the dispatch out and ignore the value. /// /// This belongs in the worker rather than in the script because it is a /// property of the machine, not of the clip — unlike the depth scalings, no /// preceding pass can change the answer. pub fn ctmf_opt() -> u8 { - if cpu_has_avx2() { - 3 - } else { - 2 + match crate::cpu::cpu_tier() { + Some(crate::cpu::CpuTier::V3) => 3, + _ => 2, } } -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -fn cpu_has_avx2() -> bool { - std::is_x86_feature_detected!("avx2") -} - -#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] -fn cpu_has_avx2() -> bool { - false -} - /// Generates VapourSynth scripts from templates. pub struct ScriptGenerator { template: String, @@ -69,12 +60,6 @@ pub struct ScriptGenerator { /// `knlm.KNLMeansCL: CL_INVALID_VALUE` / a missing-namespace error. Defaults /// to true; callers set it from `DependencyLocator::knlm_available()`. knlm_available: bool, - /// Absolute path to the zsmooth build this machine can execute, from - /// `DependencyLocator::zsmooth_plugin()`. `None` means the bundle predates - /// the per-CPU split and still autoloads a single zsmooth, so no - /// `LoadPlugin` is emitted — see that method for why that has to keep - /// working. - zsmooth_plugin: Option, } /// Parameters for preview script generation. @@ -106,7 +91,6 @@ impl ScriptGenerator { preview_template, opencl_available: true, knlm_available: true, - zsmooth_plugin: None, }) } @@ -125,18 +109,6 @@ impl ScriptGenerator { self } - /// Set the zsmooth build to load explicitly (from - /// `DependencyLocator::zsmooth_plugin()`). - /// - /// Both the encode and the preview path must be given the same value: they - /// are separate scripts, and a preview that loaded a different build than - /// the render would show a different picture than it produced — the same - /// class of split the field-order derivation exists to prevent. - pub fn with_zsmooth_plugin(mut self, plugin: Option) -> Self { - self.zsmooth_plugin = plugin; - self - } - /// Generate a .vpy script file for the given job. pub fn generate(&self, job: &VideoJob) -> Result { let pipeline = job.effective_pipeline(); @@ -160,7 +132,6 @@ impl ScriptGenerator { // Start with preview template and substitute preview-specific params let mut script = self.preview_template.clone(); - script = self.substitute_zsmooth(script); // Pipe source directory (same as main pipeline) let pipe_source_dir = Self::pipe_source_dir().unwrap_or_else(|_| env::temp_dir()); @@ -281,30 +252,9 @@ impl ScriptGenerator { } } - /// Emit (or elide) the explicit `LoadPlugin` for zsmooth. - /// - /// One function for both scripts on purpose: the encode and the preview must - /// load the same build, and doing this twice is how they would drift. - fn substitute_zsmooth(&self, script: String) -> String { - match self.zsmooth_plugin.as_ref() { - Some(path) => { - // The template uses r"..." so backslashes are literal, exactly - // as {{PIPE_SOURCE_DIR}} relies on. - let script = script.replace("{{ZSMOOTH_PLUGIN}}", &path.to_string_lossy()); - script - .replace("{{#LOAD_ZSMOOTH}}\n", "") - .replace("{{/LOAD_ZSMOOTH}}\n", "") - .replace("{{#LOAD_ZSMOOTH}}", "") - .replace("{{/LOAD_ZSMOOTH}}", "") - } - None => remove_block("{{#LOAD_ZSMOOTH}}", "{{/LOAD_ZSMOOTH}}", script), - } - } - /// Substitute parameters in a script string. fn substitute_parameters(&self, template: &str, job: &VideoJob, pipeline: &ProcessingPipeline, _input_path: &str) -> String { let mut script = template.to_string(); - script = self.substitute_zsmooth(script); // Pipe source parameters — FFmpeg decodes, pipes raw frames to VapourSynth via stdin let pipe_source_dir = Self::pipe_source_dir().unwrap_or_else(|_| env::temp_dir()); diff --git a/worker/templates/pipeline_template.vpy b/worker/templates/pipeline_template.vpy index 17a2b0d..eb1fd4b 100644 --- a/worker/templates/pipeline_template.vpy +++ b/worker/templates/pipeline_template.vpy @@ -15,16 +15,6 @@ core = vs.core # 1GB default, can be adjusted based on system memory core.max_cache_size = 1024 -{{#LOAD_ZSMOOTH}} -# zsmooth is loaded here rather than autoloaded, because the bundle ships more -# than one build of it and only one may register the namespace. Upstream -# compiles it for an AVX2 baseline with no runtime dispatch, so that build dies -# with an illegal instruction on a pre-2013 CPU (issue #82); the worker picks the -# fastest build this machine can execute and substitutes its path. The block is -# absent on a deps bundle old enough to autoload a single zsmooth. -core.std.LoadPlugin(r"{{ZSMOOTH_PLUGIN}}") -{{/LOAD_ZSMOOTH}} - # Load input video from stdin pipe (FFmpeg decodes → raw frames → VapourSynth) # This eliminates FFMS2 indexing which blocks on large/NAS files. import os diff --git a/worker/templates/preview_template.vpy b/worker/templates/preview_template.vpy index 2642ab7..f85d56d 100644 --- a/worker/templates/preview_template.vpy +++ b/worker/templates/preview_template.vpy @@ -14,13 +14,6 @@ core = vs.core # Configure cache size for optimal performance with temporal filters core.max_cache_size = 1024 -{{#LOAD_ZSMOOTH}} -# Same explicit zsmooth load as the encode template, from the same worker-side -# choice — a preview that loaded a different build than the render would show a -# different picture than it produced. See pipeline_template.vpy for why. -core.std.LoadPlugin(r"{{ZSMOOTH_PLUGIN}}") -{{/LOAD_ZSMOOTH}} - # Load raw frames piped from FFmpeg via stdin sys.path.insert(0, r"{{PIPE_SOURCE_DIR}}") from pipe_source import create_pipe_clip diff --git a/worker/tests/filter_integration_test.rs b/worker/tests/filter_integration_test.rs index cec5d92..5715a58 100644 --- a/worker/tests/filter_integration_test.rs +++ b/worker/tests/filter_integration_test.rs @@ -5655,53 +5655,14 @@ fn test_153_ctmf_opt_is_a_level_the_cpu_can_actually_run() { // The plugin does NOT verify that the CPU supports the level it is handed — // opt=3 on a pre-AVX2 machine installs the AVX2 kernels and crashes exactly // as opt=4 does on this one. So this has to stay a real capability query, - // not a constant: 3 only where AVX2 was detected, otherwise 2 (SSE2, which - // every x86-64 CPU has by definition). Both are bit-identical to the C path. + // not a constant: 3 only on a v3-tier CPU, otherwise 2 (SSE2, which every + // x86-64 CPU has by definition). Both are bit-identical to the C path. It + // follows the same tier that chose the deps bundle (issue #92), so the + // worker has one answer to "what is this machine". + use vapourbox_worker::cpu::{cpu_tier, CpuTier}; let opt = vapourbox_worker::script_generator::ctmf_opt(); - assert!( - opt == 2 || opt == 3, - "opt must be SSE2 or AVX2, got {opt}" - ); - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let expected = if std::is_x86_feature_detected!("avx2") { 3 } else { 2 }; - assert_eq!(opt, expected, "opt must follow what the CPU actually has"); - } -} - -/// Generate both scripts with an explicit zsmooth build selected, the way the -/// worker does once a bundle carries the per-CPU split. -fn generate_both_scripts_with_zsmooth( - job: &VideoJob, - plugin: Option, -) -> (String, String) { - let generator = ScriptGenerator::new() - .expect("create generator") - .with_zsmooth_plugin(plugin); - let encode_path = generator.generate(job).expect("generate encode script"); - let encode = std::fs::read_to_string(&encode_path).expect("read encode script"); - - let params = PreviewParams { - width: job.input_width.unwrap_or(720), - height: job.input_height.unwrap_or(480), - pix_fmt: job - .input_pixel_format - .clone() - .unwrap_or_else(|| "yuv420p".to_string()), - num_frames: 11, - fps_num: 30000, - fps_den: 1001, - output_index: 5, - }; - let preview_path = generator - .generate_preview(job, ¶ms) - .expect("generate preview script"); - let preview = std::fs::read_to_string(&preview_path).expect("read preview script"); - - let _ = std::fs::remove_file(&encode_path); - let _ = std::fs::remove_file(&preview_path); - (encode, preview) + let expected = if cpu_tier() == Some(CpuTier::V3) { 3 } else { 2 }; + assert_eq!(opt, expected, "opt must follow the CPU tier"); } fn chroma_denoise_job(id: &str) -> VideoJob { @@ -5719,69 +5680,34 @@ fn chroma_denoise_job(id: &str) -> VideoJob { } #[test] -fn test_154_zsmooth_is_loaded_explicitly_from_the_chosen_build() { - // zsmooth is bundled twice — upstream builds it for an AVX2 baseline with no - // runtime dispatch, so that binary dies with an illegal instruction - // (0xC000001D) on a pre-2013 CPU the instant a filter runs (issue #82). Both - // builds register the namespace `zsmooth`, so neither may autoload and - // exactly one is loaded by path. +fn test_154_zsmooth_is_autoloaded_from_the_tiered_bundle() { + // zsmooth used to ship twice per x86 bundle (an AVX2-only haswell build and + // an x86_64_v2 fallback, issue #82) and was loaded by explicit path. Since + // the bundles are split by CPU tier (issue #92), each carries exactly one + // build in the autoload directory, so the scripts must load nothing + // themselves. A LoadPlugin here would fail every job: the zsmooth/ + // directory it pointed into no longer exists. // // Both scripts, because the reporter in #82 hit the preview first. create_output_dir(); - let job = chroma_denoise_job("test_154_zsmooth_load"); - let chosen = std::path::PathBuf::from("/deps/vapoursynth/zsmooth/zsmooth-x86_64_v2.dll"); - let (encode, preview) = generate_both_scripts_with_zsmooth(&job, Some(chosen.clone())); + let job = chroma_denoise_job("test_154_zsmooth_autoload"); + let (encode, preview) = generate_both_scripts(&job); for (name, script) in [("encode", &encode), ("preview", &preview)] { assert!( - script.contains(&format!("core.std.LoadPlugin(r\"{}\")", chosen.display())), - "{name} script must load the chosen zsmooth build explicitly" + !script.contains("LoadPlugin"), + "{name} script must leave zsmooth to autoload" ); assert!( script.contains("core.zsmooth.CCD("), "{name} script should still call the filter" ); - // An unsubstituted marker is valid Python nowhere and would fail the job + // A leftover marker is valid Python nowhere and would fail the job // with a SyntaxError that reads like a template bug. - for leftover in ["{{#LOAD_ZSMOOTH}}", "{{/LOAD_ZSMOOTH}}", "{{ZSMOOTH_PLUGIN}}"] { - assert!( - !script.contains(leftover), - "{name} script left {leftover} unsubstituted" - ); - } - // The load has to precede the first use, or the namespace is missing - // when the filter is constructed. - let load = script.find("core.std.LoadPlugin(r\"").expect("load present"); - let use_ = script.find("core.zsmooth.CCD(").expect("call present"); - assert!(load < use_, "{name} script loads zsmooth after using it"); - } -} - -#[test] -fn test_155_a_bundle_without_the_split_still_autoloads_zsmooth() { - // Deps bundles up to 1.9.0 ship one zsmooth inside the autoload directory. - // A worker that emitted a LoadPlugin for a path those bundles do not have - // would fail every job on them — and the app can be upgraded before the - // deps download finishes, so that window is real. No path selected must - // therefore mean no LoadPlugin, leaving the script as it was before the - // split existed. - create_output_dir(); - let job = chroma_denoise_job("test_155_zsmooth_autoload"); - let (encode, preview) = generate_both_scripts_with_zsmooth(&job, None); - - for (name, script) in [("encode", &encode), ("preview", &preview)] { - assert!( - !script.contains("LoadPlugin"), - "{name} script must not load a plugin the bundle may not have" - ); - assert!( - script.contains("core.zsmooth.CCD("), - "{name} script should still call the filter, via autoload" - ); - for leftover in ["{{#LOAD_ZSMOOTH}}", "{{/LOAD_ZSMOOTH}}", "{{ZSMOOTH_PLUGIN}}"] { + for leftover in ["LOAD_ZSMOOTH", "ZSMOOTH_PLUGIN"] { assert!( !script.contains(leftover), - "{name} script left {leftover} unsubstituted" + "{name} script still carries {leftover}" ); } } From 5f2fee14d1177d0b52c670b88d9ebd44b01ece80 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Fri, 25 Sep 2026 23:45:08 +1000 Subject: [PATCH 5/9] App: download the deps bundle for this CPU's tier, and re-check it at startup The tier lives only in which release asset is downloaded (macos-x64-v2 etc.) and in version.json; the install directory stays deps/, so the worker, dev paths and tests need no tier awareness. The app asks the bundled worker (--probe-cpu) rather than re-deriving the tier, so both share one definition. VAPOURBOX_DEPS_TIER overrides it, and any unclear answer on x86 means v2, which runs everywhere and only costs speed. Startup compares the installed tier with this CPU's and re-downloads on a mismatch, before the version check, so an install carried over to (or from) a different machine is replaced even when it is newer than expected. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014GLXdGLfPwgYjW1AkonGqN --- app/lib/services/dependency_manager.dart | 140 +++++++++++++++++- app/lib/services/tool_locator.dart | 10 +- app/lib/views/dependency_download_dialog.dart | 4 + app/test/attribution_test.dart | 30 ++-- app/test/deps_tier_test.dart | 131 ++++++++++++++++ app/test/support/worker_harness.dart | 41 +++-- app/test/vapoursynth_integration_test.dart | 74 ++------- 7 files changed, 322 insertions(+), 108 deletions(-) create mode 100644 app/test/deps_tier_test.dart diff --git a/app/lib/services/dependency_manager.dart b/app/lib/services/dependency_manager.dart index de68427..cad0dd9 100644 --- a/app/lib/services/dependency_manager.dart +++ b/app/lib/services/dependency_manager.dart @@ -10,6 +10,7 @@ import 'package:rhttp/rhttp.dart'; import 'package:path/path.dart' as path; import 'temp_directory_service.dart'; +import 'tool_locator.dart'; /// Status of the dependency installation. enum DependencyStatus { @@ -22,6 +23,13 @@ enum DependencyStatus { /// Dependencies are installed but older than this app expects outdated, + /// Installed deps were built for a different CPU tier than this machine's + /// (see [DependencyManager.depsTier]) — typically the install was carried + /// over from, or to, another computer. Replaced even when the installed + /// version is newer than expected: a bundle this CPU cannot run is not + /// "newer", it is unusable. + wrongTier, + /// Installed deps are NEWER than the version this app was built against. /// /// Treated as usable, not as something to "fix": replacing them would be a @@ -179,9 +187,14 @@ class InstalledDepsInfo { final String version; final DateTime? installedAt; + /// CPU tier the bundle was built for (`v2`/`v3`), or null. Bundles from + /// before tiering carry no tier; on x86 those are the v3 build. + final String? tier; + InstalledDepsInfo({ required this.version, this.installedAt, + this.tier, }); factory InstalledDepsInfo.fromJson(Map json) { @@ -190,6 +203,7 @@ class InstalledDepsInfo { installedAt: json['installedAt'] != null ? DateTime.tryParse(json['installedAt'] as String) : null, + tier: json['tier'] as String?, ); } @@ -197,6 +211,7 @@ class InstalledDepsInfo { return { 'version': version, 'installedAt': installedAt?.toIso8601String(), + if (tier != null) 'tier': tier, }; } } @@ -233,6 +248,98 @@ class DependencyManager { throw UnsupportedError('Unsupported platform'); } + /// Whether a platform's deps ship in CPU tiers. Only x86 does: the ARM + /// bundles have a single NEON baseline. + static bool isTiered(String platformId) => platformId.endsWith('-x64'); + + /// The release asset id for a platform and tier: the v3 bundle keeps the + /// plain platform name (so every pre-tiering release and install stays + /// valid), the v2 bundle gets a `-v2` suffix. The *install* directory is + /// always [platformId] — the tier lives only in which zip is downloaded and in + /// version.json — so the worker, dev paths and tests need no tier awareness. + static String assetIdFor(String platformId, String? tier) => + tier == 'v2' ? '$platformId-v2' : platformId; + + String? _cachedTier; + bool _tierResolved = false; + + /// Test seam: replaces the worker probe. Returns the probe's raw `tier` + /// value, or null to simulate a failed probe. + @visibleForTesting + Future Function()? tierProbeOverride; + + /// Forget the cached tier (tests only). + @visibleForTesting + void resetTierForTesting() { + _cachedTier = null; + _tierResolved = false; + } + + /// Which deps tier this machine needs: `v3` or `v2` on x86, null on ARM. + /// + /// The worker decides (`vapourbox-worker --probe-cpu`, see + /// worker/src/cpu.rs) so the app and the worker share one definition of the + /// tiers — re-deriving it here from sysctl or /proc would be a second + /// implementation, and sysctl is wrong under Rosetta anyway. + /// `VAPOURBOX_DEPS_TIER=v2|v3` overrides the probe: an escape hatch for a VM + /// that misreports its CPU, and for dev builds with no worker built yet. + /// + /// Anything short of a clear answer on x86 means `v2`. The v2 bundle runs on + /// every x86-64 CPU and only costs speed; guessing `v3` wrongly costs a crash + /// on every job (issue #92). + Future depsTier() async { + if (_tierResolved) return _cachedTier; + final id = platformId; + final override = Platform.environment['VAPOURBOX_DEPS_TIER']; + String? probed; + if (isTiered(id) && override != 'v2' && override != 'v3') { + probed = await (tierProbeOverride ?? _probeWorkerTier)(); + if (probed != 'v3' && probed != 'v2') { + print('DependencyManager: CPU tier probe gave no answer ' + '(${probed ?? 'failed'}) - using the v2 bundle, which runs anywhere'); + } + } + _cachedTier = resolveTier(platformId: id, override: override, probed: probed); + _tierResolved = true; + return _cachedTier; + } + + /// The tier decision itself, pure so it is testable on any host. + static String? resolveTier( + {required String platformId, String? override, String? probed}) { + if (!isTiered(platformId)) return null; + if (override == 'v2' || override == 'v3') return override; + return probed == 'v3' ? 'v3' : 'v2'; + } + + /// Whether an installed bundle's tier suits this machine. A bundle from + /// before tiering records no tier; on x86 that was always the v3 build. + static bool tierMatches({String? installed, String? machine}) => + (installed ?? (machine == null ? null : 'v3')) == machine; + + /// This machine's release asset id (see [assetIdFor]). + Future assetPlatformId() async => + assetIdFor(platformId, await depsTier()); + + Future _probeWorkerTier() async { + final worker = ToolLocator.findWorkerExecutable(); + if (worker == null) return null; + try { + final result = await Process.run(worker, ['--probe-cpu']) + .timeout(const Duration(seconds: 15)); + if (result.exitCode != 0) return null; + for (final line in result.stdout.toString().split('\n')) { + final trimmed = line.trim(); + if (!trimmed.startsWith('{')) continue; + final json = jsonDecode(trimmed); + if (json is Map && json['tier'] is String) return json['tier'] as String; + } + } catch (e) { + print('DependencyManager: CPU tier probe failed: $e'); + } + return null; + } + /// Get the dependencies directory path. Future getDepsDirectory() async { // Explicit override (used by `flutter test` in CI, where the executable is @@ -384,12 +491,25 @@ class DependencyManager { return DependencyStatus.missing; } + // A bundle for the wrong CPU tier is replaced before the version is even + // considered: the v3 bundle faults on every job on a CPU below x86-64-v3 + // (issue #92), and an install can outlive the machine it was chosen for + // (a migrated account, a restored backup, a CPU upgrade). This deliberately + // wins over newerThanExpected, so a newer wrong-tier install is replaced + // with the expected version — a downgrade, but one that runs. + final tier = await depsTier(); + if (!tierMatches(installed: installed.tier, machine: tier)) { + print('DependencyManager: Installed deps are the ' + '${installed.tier ?? 'v3'} bundle but this CPU needs $tier'); + return DependencyStatus.wrongTier; + } + // Check version match (per-platform: a platform may pin its own version). // Direction matters. Older than expected is an upgrade; newer is not a // fault at all, and treating it as one downgraded a deliberately newer // bundle back to the released one — destructively, since installing wipes // and replaces. - final expectedVersion = expected.versionFor(platformId); + final expectedVersion = expected.versionFor(assetIdFor(platformId, tier)); if (installed.version != expectedVersion) { final order = compareVersions(installed.version, expectedVersion); if (order > 0) { @@ -535,10 +655,13 @@ class DependencyManager { /// installation is done or fails. Future downloadAndInstall() async { final expected = await getExpectedVersion(); + final tier = await depsTier(); + final assetId = assetIdFor(platformId, tier); - // Construct download URL from release tag (filename is derived). - final downloadUrl = expected.getDownloadUrl(platformId); - final filename = expected.filenameFor(platformId); + // Construct download URL from release tag (filename is derived). The asset + // carries the tier; the install directory does not. + final downloadUrl = expected.getDownloadUrl(assetId); + final filename = expected.filenameFor(assetId); print('DependencyManager: Downloading from $downloadUrl'); @@ -552,7 +675,7 @@ class DependencyManager { // is best-effort: if the sidecar is missing/unreadable we still install (the // download is over HTTPS), matching prior behaviour when no hash was set. final expectedSha256 = - await _fetchExpectedSha256(expected.getManifestUrl(platformId)); + await _fetchExpectedSha256(expected.getManifestUrl(assetId)); // The zip is downloaded into a stable cache path rather than a throwaway // temp directory, and kept if anything after the download fails. Everything @@ -671,8 +794,8 @@ class DependencyManager { // Write version file (per-platform version, so the next check matches). // Still the last thing written into the tree, so a staged directory that // never gets swapped in can never look complete. - await _writeInstalledVersion(expected.versionFor(platformId), - depsDirOverride: target); + await _writeInstalledVersion(expected.versionFor(assetId), + tier: tier, depsDirOverride: target); // Swap. If the second rename fails we have already moved the old install // aside, so put it back rather than leaving the user with no deps at all. @@ -1133,12 +1256,13 @@ class DependencyManager { /// Write the installed version file. Future _writeInstalledVersion(String version, - {Directory? depsDirOverride}) async { + {String? tier, Directory? depsDirOverride}) async { final versionFile = await _getInstalledVersionFile(depsDirOverride: depsDirOverride); final info = InstalledDepsInfo( version: version, installedAt: DateTime.now(), + tier: tier, ); await versionFile.writeAsString( const JsonEncoder.withIndent(' ').convert(info.toJson()), diff --git a/app/lib/services/tool_locator.dart b/app/lib/services/tool_locator.dart index 3e0a4c2..2f75cc4 100644 --- a/app/lib/services/tool_locator.dart +++ b/app/lib/services/tool_locator.dart @@ -109,8 +109,14 @@ class ToolLocator { return File(p).existsSync() ? p : null; } - /// Resolve the vapourbox-worker executable path. - String? _resolveWorker() { + String? _resolveWorker() => findWorkerExecutable(); + + /// Locate the vapourbox-worker executable. + /// + /// Static and independent of the deps directory, because the worker is also + /// needed *before* any deps exist: `DependencyManager.depsTier()` asks it + /// which x86 bundle this CPU can run, and that decides what gets downloaded. + static String? findWorkerExecutable() { // Explicit override, mirroring VAPOURBOX_DEPS_DIR above. Under `flutter // test` the resolved executable is the test runner, not the app bundle, so // neither the production nor the dev path below can find the worker — which diff --git a/app/lib/views/dependency_download_dialog.dart b/app/lib/views/dependency_download_dialog.dart index 7eaab99..31c14e0 100644 --- a/app/lib/views/dependency_download_dialog.dart +++ b/app/lib/views/dependency_download_dialog.dart @@ -104,6 +104,10 @@ class _DependencyDownloadDialogState extends State { case DependencyStatus.outdated: return 'A new version of the processing components is available.\n\n' 'Updating to ensure compatibility.'; + case DependencyStatus.wrongTier: + return 'The installed processing components were built for a ' + 'different processor.\n\n' + 'Downloading the version for this computer.'; case DependencyStatus.corrupted: return 'Some processing components are damaged or incomplete.\n\n' 'Re-downloading to fix the issue.'; diff --git a/app/test/attribution_test.dart b/app/test/attribution_test.dart index 738d61b..983b3df 100644 --- a/app/test/attribution_test.dart +++ b/app/test/attribution_test.dart @@ -92,20 +92,10 @@ const _pluginToNotice = { const _mustNotAppear = ['ffms2', 'BestSource']; String _stem(String filename) { - // A manifest entry may be a bundle-relative path rather than a bare filename - // (zsmooth ships outside the autoload directory), and may carry a CPU-target - // suffix because it ships once per baseline. Credit is owed to the project, - // not to each build of it, so both are normalised away. - var s = filename.toLowerCase().split('/').last; + var s = filename.toLowerCase(); final dot = s.lastIndexOf('.'); if (dot > 0) s = s.substring(0, dot); if (s.startsWith('lib') && s.length > 3) s = s.substring(3); - for (final variant in const ['-haswell', '-x86_64_v2']) { - if (s.endsWith(variant)) { - s = s.substring(0, s.length - variant.length); - break; - } - } return s; } @@ -149,6 +139,24 @@ void main() { ); }); + test('both CPU tiers of an x64 bundle ship the same plugins', () { + // The v2 bundle (issue #92) differs from v3 in how some plugins are + // BUILT, never in which ones ship: a plugin missing from v2 would just + // be a pass that fails on older machines. The packaging guard checks + // each list separately, so only this keeps the two lists in step. + final manifest = jsonDecode( + File(p.join(root, 'Scripts', 'deps-expected-plugins.json')).readAsStringSync(), + ) as Map; + for (final platform in ['macos-x64', 'windows-x64', 'linux-x64']) { + expect(manifest['$platform-v2'], isNotNull, reason: 'no $platform-v2 list'); + expect( + (manifest['$platform-v2'] as List).toSet(), + (manifest[platform] as List).toSet(), + reason: '$platform-v2 must list exactly the plugins $platform does', + ); + } + }); + test('does not credit components that are no longer shipped', () { for (final gone in _mustNotAppear) { expect( diff --git a/app/test/deps_tier_test.dart b/app/test/deps_tier_test.dart new file mode 100644 index 0000000..ba0aca4 --- /dev/null +++ b/app/test/deps_tier_test.dart @@ -0,0 +1,131 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:vapourbox/services/dependency_manager.dart'; + +/// The x86 deps bundles ship in two CPU tiers (issue #92): v3 for x86-64-v3 +/// CPUs, v2 for anything older. These pin the rules that pick one, name its +/// release asset, and decide when an existing install is for the wrong CPU. +void main() { + group('resolveTier', () { + test('ARM platforms are not tiered, whatever the probe says', () { + for (final id in ['macos-arm64', 'linux-arm64']) { + expect(DependencyManager.resolveTier(platformId: id, probed: 'v3'), isNull); + expect(DependencyManager.resolveTier(platformId: id, override: 'v2'), isNull); + } + }); + + test('x86 follows the worker probe', () { + for (final id in ['macos-x64', 'windows-x64', 'linux-x64']) { + expect(DependencyManager.resolveTier(platformId: id, probed: 'v3'), 'v3'); + expect(DependencyManager.resolveTier(platformId: id, probed: 'v2'), 'v2'); + } + }); + + test('no clear answer on x86 means v2, which runs everywhere', () { + // Guessing v3 wrongly is a crash on every job; v2 only costs speed. + for (final probed in [null, '', 'v4', 'V3']) { + expect( + DependencyManager.resolveTier(platformId: 'macos-x64', probed: probed), + 'v2', + reason: 'probe "$probed"'); + } + }); + + test('VAPOURBOX_DEPS_TIER overrides the probe in both directions', () { + expect( + DependencyManager.resolveTier( + platformId: 'linux-x64', override: 'v2', probed: 'v3'), + 'v2'); + expect( + DependencyManager.resolveTier( + platformId: 'linux-x64', override: 'v3', probed: 'v2'), + 'v3'); + }); + + test('an invalid override is ignored rather than trusted', () { + expect( + DependencyManager.resolveTier( + platformId: 'linux-x64', override: 'fast', probed: 'v3'), + 'v3'); + }); + }); + + group('release assets', () { + test('v3 keeps the pre-tiering asset name; v2 is suffixed', () { + expect(DependencyManager.assetIdFor('macos-x64', 'v3'), 'macos-x64'); + expect(DependencyManager.assetIdFor('macos-x64', 'v2'), 'macos-x64-v2'); + expect(DependencyManager.assetIdFor('macos-arm64', null), 'macos-arm64'); + }); + + test('the v2 asset resolves to its own zip and sidecar', () { + // Must agree with what the package-deps-* scripts upload. + final info = DepsVersionInfo.fromJson({ + 'version': '1.11.0', + 'releaseTag': 'deps-v1.11.0', + 'githubRepo': 'StuartCameronCode/VapourBox', + }); + final id = DependencyManager.assetIdFor('windows-x64', 'v2'); + expect(info.filenameFor(id), 'VapourBox-deps-1.11.0-windows-x64-v2.zip'); + expect( + info.getManifestUrl(id), + 'https://github.com/StuartCameronCode/VapourBox/releases/download/' + 'deps-v1.11.0/VapourBox-deps-1.11.0-windows-x64-v2.zip.sha256.json'); + }); + }); + + group('tierMatches', () { + test('a pre-tiering install is the v3 bundle', () { + // Every x86 install before tiering is the v3 build — which is exactly + // what faults on the #92 Mac Pro, so it must read as the wrong tier there. + expect(DependencyManager.tierMatches(installed: null, machine: 'v3'), isTrue); + expect(DependencyManager.tierMatches(installed: null, machine: 'v2'), isFalse); + }); + + test('a recorded tier must equal the machine tier', () { + expect(DependencyManager.tierMatches(installed: 'v2', machine: 'v2'), isTrue); + expect(DependencyManager.tierMatches(installed: 'v3', machine: 'v3'), isTrue); + expect(DependencyManager.tierMatches(installed: 'v2', machine: 'v3'), isFalse); + expect(DependencyManager.tierMatches(installed: 'v3', machine: 'v2'), isFalse); + }); + + test('ARM installs never mismatch', () { + expect(DependencyManager.tierMatches(installed: null, machine: null), isTrue); + }); + }); + + group('version.json', () { + test('round-trips the tier', () { + final info = InstalledDepsInfo(version: '1.11.0', tier: 'v2'); + final back = InstalledDepsInfo.fromJson(info.toJson()); + expect(back.tier, 'v2'); + expect(back.version, '1.11.0'); + }); + + test('omits the tier where there is none, and reads old files', () { + expect(InstalledDepsInfo(version: '1.11.0').toJson().containsKey('tier'), + isFalse); + expect(InstalledDepsInfo.fromJson({'version': '1.10.0'}).tier, isNull); + }); + }); + + group('depsTier', () { + final manager = DependencyManager.instance; + tearDown(() { + manager.tierProbeOverride = null; + manager.resetTierForTesting(); + }); + + test('asks the probe only where bundles are tiered, and caches it', () async { + var calls = 0; + manager.tierProbeOverride = () async { + calls++; + return 'v3'; + }; + final tiered = DependencyManager.isTiered(manager.platformId); + final first = await manager.depsTier(); + final second = await manager.depsTier(); + expect(first, tiered ? 'v3' : isNull); + expect(second, first); + expect(calls, tiered ? 1 : 0); + }); + }); +} diff --git a/app/test/support/worker_harness.dart b/app/test/support/worker_harness.dart index 88f5d28..53aa932 100644 --- a/app/test/support/worker_harness.dart +++ b/app/test/support/worker_harness.dart @@ -244,27 +244,17 @@ class WorkerHarness { return '${probe['arch']} [${features.isEmpty ? 'none detected' : features.join(' ')}]'; } - /// The instruction set extensions the worker process can actually execute, - /// as a set. Empty when the probe could not run at all. - /// - /// A test that needs to know whether a CPU-specific binary is safe to load - /// must ask this rather than parse `/proc/cpuinfo` or `sysctl` itself: a - /// second implementation of the same decision is how the interface and the - /// pipeline come to disagree, and the worker's answer is the one that governs - /// what the pipeline loads. Used by the zsmooth build check in - /// `vapoursynth_integration_test`. - static Future> cpuFeatures() async { + /// The deps tier to fetch when downloading, decided exactly as the app does + /// (`DependencyManager.resolveTier`): `VAPOURBOX_DEPS_TIER` if set, else the + /// worker's `--probe-cpu` answer. The override is how CI tests the v2 bundle + /// on runners that all probe as v3. + static Future depsTier() async { final probe = await _probeCpu(); - if (probe == null) return {}; - return (probe['features'] as List).cast().toSet(); - } - - /// True when the worker reports an x86 architecture (so the x86-specific - /// zsmooth builds are the relevant ones). - static Future isX86() async { - final probe = await _probeCpu(); - final arch = probe?['arch'] as String?; - return arch != null && (arch.startsWith('x86') || arch == 'amd64'); + return DependencyManager.resolveTier( + platformId: platform, + override: Platform.environment['VAPOURBOX_DEPS_TIER'], + probed: probe?['tier'] as String?, + ); } /// Never throws: this is diagnostic, and it is asked for before @@ -773,11 +763,13 @@ class WorkerHarness { /// Download + extract the pinned deps zip into [destDir]. static Future _downloadDeps(String destDir) async { final info = _loadDepsVersion(); - final url = info.getDownloadUrl(platform); - final expectedSha = await _fetchSidecarSha(info.getManifestUrl(platform)); + final tier = await depsTier(); + final assetId = DependencyManager.assetIdFor(platform, tier); + final url = info.getDownloadUrl(assetId); + final expectedSha = await _fetchSidecarSha(info.getManifestUrl(assetId)); final tmp = await Directory.systemTemp.createTemp('vb_deps_'); - final zip = File(p.join(tmp.path, info.filenameFor(platform))); + final zip = File(p.join(tmp.path, info.filenameFor(assetId))); try { // ignore: avoid_print print('WorkerHarness: downloading $url'); @@ -830,8 +822,9 @@ class WorkerHarness { // Stamp installed version so a normal app run treats it as up-to-date. File(p.join(destDir, 'version.json')).writeAsStringSync( const JsonEncoder.withIndent(' ').convert({ - 'version': info.versionFor(platform), + 'version': info.versionFor(assetId), 'installedAt': DateTime.now().toIso8601String(), + if (tier != null) 'tier': tier, }), ); } finally { diff --git a/app/test/vapoursynth_integration_test.dart b/app/test/vapoursynth_integration_test.dart index a452920..722b4da 100644 --- a/app/test/vapoursynth_integration_test.dart +++ b/app/test/vapoursynth_integration_test.dart @@ -10,7 +10,6 @@ import 'dart:io'; import 'package:flutter_test/flutter_test.dart'; import 'package:path/path.dart' as path; -import 'support/worker_harness.dart'; void main() { late String depsDir; @@ -117,17 +116,15 @@ core = vs.core # for on the paths the app exposes. A plugin missing here is a FILTER that fails # at job time with "No attribute with the name exists" — which is what an # incomplete or stale deps install looks like from the user's side. `zsmooth` -# (Chroma Denoise / CCD) was added to the bundle after this list was written and -# went uncovered, so a bundle without it passed the suite and failed the filter. -# It is no longer in THIS list because it is deliberately not autoloaded — it -# ships once per CPU baseline and the worker loads one by path. The test below -# covers it. +# (Chroma Denoise / CCD) was once missed here, so a bundle without it passed the +# suite and failed the filter. Since the x86 bundles split by CPU tier (#92) +# each ships exactly one zsmooth build, autoloaded like the rest. required = ['std', 'resize', 'mv', 'znedi3', 'eedi3m', 'fmtc', 'dfttest', 'neo_f3kdb', 'cas', 'dctf', 'deblock', 'rgvs', 'ctmf', 'warp', 'misc', 'grain', 'tcanny', 'descratch', 'vivtc', 'ttmpsm', 'tmedian', 'fft3dfilter', 'flux', 'bifrost', 'retinex', - 'bwdif', 'fb', 'removedirt', 'dedot', 'lghost'] + 'bwdif', 'fb', 'removedirt', 'dedot', 'lghost', 'zsmooth'] # On ARM, `nnedi3` is load-bearing and `znedi3` is only the fallback: znedi3's # SIMD is x86-only, so the ARM bundles build it scalar and both the templates' @@ -173,66 +170,17 @@ else: expect(result.stdout.toString(), contains('All plugins loaded')); }); - test('the zsmooth build for this CPU loads and runs', () async { - // zsmooth is deliberately OUTSIDE the autoload directory: upstream builds - // it for an AVX2 baseline with no runtime dispatch, so that binary dies - // with an illegal instruction on a pre-2013 CPU the instant a filter runs - // (issue #82). The bundle ships one build per CPU baseline and the worker - // loads exactly one by path. - // - // The Rust side (test_154) proves the generated script asks for the right - // file on any hardware; only running it proves the file is there and - // executes — and only ever for the CPU that ran it, which is why the - // choice itself is asserted in Rust and not here. - final zsmoothDir = Directory(path.join(depsDir, 'vapoursynth', 'zsmooth')); - expect( - zsmoothDir.existsSync(), - isTrue, - reason: 'deps bundle has no vapoursynth/zsmooth directory: ' - '${zsmoothDir.path} — a bundle older than deps 1.10.0, or a failed ' - 'zsmooth build', - ); - - // Asked of the worker, never derived here: loading the AVX2 build on a - // CPU without AVX2 is the crash this whole split exists to prevent, so a - // second-guessed answer is worse than none. - final features = await WorkerHarness.cpuFeatures(); - final isX86 = await WorkerHarness.isX86(); - final ext = Platform.isWindows - ? 'dll' - : Platform.isMacOS - ? 'dylib' - : 'so'; - final prefix = Platform.isWindows ? '' : 'lib'; - // Same preference order as DependencyLocator::zsmooth_candidates. With no - // probe (worker not built), only the portable build is considered — it - // runs everywhere, so the check degrades rather than risking the crash. - final candidates = [ - if (isX86 && features.contains('avx2')) '${prefix}zsmooth-haswell.$ext', - if (isX86) '${prefix}zsmooth-x86_64_v2.$ext', - '${prefix}zsmooth.$ext', - ]; - final chosen = candidates - .map((f) => File(path.join(zsmoothDir.path, f))) - .where((f) => f.existsSync()) - .firstOrNull; - expect( - chosen, - isNotNull, - reason: 'no zsmooth build this CPU can run in ${zsmoothDir.path}: ' - 'looked for $candidates, found ' - '${zsmoothDir.listSync().map((e) => path.basename(e.path)).toList()} ' - '(CPU features: $features)', - ); - + test('the bundled zsmooth build runs on this CPU', () async { + // zsmooth has no runtime dispatch: a build above this CPU's baseline + // loads fine and then dies with an illegal instruction the instant a + // filter runs (issue #82). Each x86 bundle tier ships the one build its + // CPUs can run, so the autoloaded namespace must render here. + // Constructing the node is not enough — the fault is in the kernel, so a + // frame has to be rendered. final script = ''' import vapoursynth as vs core = vs.core -core.std.LoadPlugin(r"${chosen!.path}") clip = core.std.BlankClip(width=160, height=120, format=vs.YUV420P8, length=2) -# CCD is what the Chroma Denoise pass is made of and what #82 was reported -# against. Constructing the node is not enough: the fault is in the kernel, so a -# frame has to be rendered. clip = core.zsmooth.CCD(clip, threshold=4, scale=1) clip.get_frame(0) print("zsmooth OK") From 4e834244bbfcabaf4ef9bff2ba6b5d7c0261e81d Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Fri, 25 Sep 2026 23:45:08 +1000 Subject: [PATCH 6/9] Build every x86 deps bundle in two CPU tiers, and gate the v2 one Each download-deps script takes --tier v3|v2 (-Tier on Windows) and maps it in one block at the top. The only differences, from running every bundled plugin under Intel SDE and on a real Westmere Mac Pro: - zsmooth: the haswell build for v3, an x86_64_v2 build for v2; each bundle ships exactly one, in the autoload directory. - macOS MVTools: Stefan-Olt's prebuilt for v3; for v2, v24 from source with patches/mvtools-v24-no-avx2.patch, which builds none of the AVX2 files and masks the AVX2 flag, removing the load-time VEX that crashed dlopen. The package scripts read the tier from version.json and name the asset from it. A v2 bundle is gated before it can be published: under SDE on Linux (Westmere) and Windows (Sandy Bridge, as SDE cannot emulate pre-AVX Windows), and on macOS by check-load-time-simd.py, which proves statically that no plugin runs AVX while it loads. Release upload moves after the gate. release.sh no longer packages deps locally: one checkout holds only one tier per platform, so it would publish an incomplete release. Deps 1.11.0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014GLXdGLfPwgYjW1AkonGqN --- .github/workflows/build-deps-linux.yml | 49 +++- .github/workflows/build-deps-macos.yml | 25 +- .github/workflows/build-deps-windows.yml | 52 +++- .github/workflows/nightly.yml | 32 ++- .github/workflows/probe-cpu-compat.yml | 122 ++++++--- Scripts/check-deps-changed.sh | 8 + Scripts/check-load-time-simd.py | 198 ++++++++++++++ Scripts/deps-expected-plugins.json | 123 ++++++++- Scripts/download-deps-linux.sh | 176 ++++++------ Scripts/download-deps-macos.sh | 312 +++++++++++++--------- Scripts/download-deps-windows.ps1 | 127 +++++---- Scripts/package-deps-linux.sh | 39 +-- Scripts/package-deps-macos.sh | 53 ++-- Scripts/package-deps-windows.ps1 | 32 ++- Scripts/patches/mvtools-v24-no-avx2.patch | 120 +++++++++ Scripts/probe-plugin-compat.py | 6 +- Scripts/release.sh | 103 ++----- app/assets/deps-version.json | 6 +- 18 files changed, 1116 insertions(+), 467 deletions(-) create mode 100644 Scripts/check-load-time-simd.py create mode 100644 Scripts/patches/mvtools-v24-no-avx2.patch diff --git a/.github/workflows/build-deps-linux.yml b/.github/workflows/build-deps-linux.yml index 6e08a22..7047b60 100644 --- a/.github/workflows/build-deps-linux.yml +++ b/.github/workflows/build-deps-linux.yml @@ -23,11 +23,20 @@ on: permissions: contents: write + actions: read jobs: + # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for + # anything older. Same script, one --tier switch. build-x64: if: ${{ inputs.arch == 'x64' || inputs.arch == 'both' }} runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + tier: [v3, v2] + env: + ASSET: VapourBox-deps-${{ inputs.version }}-linux-x64${{ matrix.tier == 'v2' && '-v2' || '' }} steps: - uses: actions/checkout@v5 @@ -47,27 +56,51 @@ jobs: gcc --version | head -1 pip3 install meson - - name: Build dependencies - run: ./Scripts/download-deps-linux.sh --force + - name: Build dependencies (${{ matrix.tier }}) + run: ./Scripts/download-deps-linux.sh --force --tier ${{ matrix.tier }} - name: Package dependencies run: ./Scripts/package-deps-linux.sh --version "${{ inputs.version }}" --arch x64 - uses: actions/upload-artifact@v5 with: - name: VapourBox-deps-${{ inputs.version }}-linux-x64 + name: ${{ env.ASSET }} path: | - dist/VapourBox-deps-${{ inputs.version }}-linux-x64.zip - dist/VapourBox-deps-${{ inputs.version }}-linux-x64.zip.sha256.json + dist/${{ env.ASSET }}.zip + dist/${{ env.ASSET }}.zip.sha256.json + # The v2 bundle's promise is that it runs on CPUs without AVX. Every hosted + # runner has AVX2, so prove it by running every plugin under Intel SDE + # emulating Westmere (the Mac Pro 5,1 from issue #92). + gate-x64-v2: + needs: build-x64 + uses: ./.github/workflows/probe-cpu-compat.yml + with: + artifact_prefix: VapourBox-deps-${{ inputs.version }}- + tier: v2 + gate: true + matrix_json: '[{"platform":"linux-x64","runner":"ubuntu-24.04","sde":"sde64","python":"python/bin/python3","chip":"wsm"}]' + + # Published only once the v2 gate has passed, so a bundle that would crash on + # an older CPU never reaches a release. + upload-x64: + needs: [build-x64, gate-x64-v2] + if: inputs.release_tag != '' + runs-on: ubuntu-24.04 + steps: + - uses: actions/download-artifact@v5 + with: + pattern: VapourBox-deps-${{ inputs.version }}-linux-x64* + path: dist + merge-multiple: true - name: Upload to release - if: inputs.release_tag != '' env: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ - "dist/VapourBox-deps-${{ inputs.version }}-linux-x64.zip" \ - "dist/VapourBox-deps-${{ inputs.version }}-linux-x64.zip.sha256.json" --clobber + --repo "${{ github.repository }}" \ + dist/VapourBox-deps-${{ inputs.version }}-linux-x64*.zip \ + dist/VapourBox-deps-${{ inputs.version }}-linux-x64*.zip.sha256.json --clobber build-arm64: if: ${{ inputs.arch == 'arm64' || inputs.arch == 'both' }} diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml index 811e884..5b900ea 100644 --- a/.github/workflows/build-deps-macos.yml +++ b/.github/workflows/build-deps-macos.yml @@ -60,28 +60,39 @@ jobs: "dist/VapourBox-deps-${{ inputs.version }}-macos-arm64.zip" \ "dist/VapourBox-deps-${{ inputs.version }}-macos-arm64.zip.sha256.json" --clobber + # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for + # anything older. Same script, one --tier switch. The v2 bundle is gated by + # package-deps-macos.sh itself (Scripts/check-load-time-simd.py): macOS has + # no Intel SDE, so it proves statically that no plugin runs AVX while it + # loads — and fails before anything below can upload it. build-x64: if: ${{ inputs.arch == 'x64' || inputs.arch == 'both' }} runs-on: macos-15-intel # only hosted Intel image (macos-13 retired; last one until ~Aug 2027) + strategy: + fail-fast: false + matrix: + tier: [v3, v2] + env: + ASSET: VapourBox-deps-${{ inputs.version }}-macos-x64${{ matrix.tier == 'v2' && '-v2' || '' }} steps: - uses: actions/checkout@v5 - - name: Build dependencies (x64, native) + - name: Build dependencies (x64 ${{ matrix.tier }}, native) # STRICT_MIN_OS=1: fail the build if any bundled Mach-O targets newer than # the macOS 12 floor (issue #39) instead of silently shipping it. env: STRICT_MIN_OS: "1" - run: ./Scripts/download-deps-macos.sh --force + run: ./Scripts/download-deps-macos.sh --force --tier ${{ matrix.tier }} - name: Package dependencies run: ./Scripts/package-deps-macos.sh --version "${{ inputs.version }}" --arch x64 - uses: actions/upload-artifact@v5 with: - name: VapourBox-deps-${{ inputs.version }}-macos-x64 + name: ${{ env.ASSET }} path: | - dist/VapourBox-deps-${{ inputs.version }}-macos-x64.zip - dist/VapourBox-deps-${{ inputs.version }}-macos-x64.zip.sha256.json + dist/${{ env.ASSET }}.zip + dist/${{ env.ASSET }}.zip.sha256.json - name: Upload to release if: inputs.release_tag != '' @@ -89,5 +100,5 @@ jobs: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ - "dist/VapourBox-deps-${{ inputs.version }}-macos-x64.zip" \ - "dist/VapourBox-deps-${{ inputs.version }}-macos-x64.zip.sha256.json" --clobber + "dist/${{ env.ASSET }}.zip" \ + "dist/${{ env.ASSET }}.zip.sha256.json" --clobber diff --git a/.github/workflows/build-deps-windows.yml b/.github/workflows/build-deps-windows.yml index 8418723..95f4bf4 100644 --- a/.github/workflows/build-deps-windows.yml +++ b/.github/workflows/build-deps-windows.yml @@ -20,16 +20,25 @@ on: permissions: contents: write + actions: read jobs: + # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for + # anything older. Same script, one -Tier switch. build-x64: runs-on: windows-latest + strategy: + fail-fast: false + matrix: + tier: [v3, v2] + env: + ASSET: VapourBox-deps-${{ inputs.version }}-windows-x64${{ matrix.tier == 'v2' && '-v2' || '' }} steps: - uses: actions/checkout@v5 - - name: Build dependencies + - name: Build dependencies (${{ matrix.tier }}) shell: pwsh - run: ./Scripts/download-deps-windows.ps1 + run: ./Scripts/download-deps-windows.ps1 -Tier ${{ matrix.tier }} - name: Package dependencies shell: pwsh @@ -37,17 +46,40 @@ jobs: - uses: actions/upload-artifact@v5 with: - name: VapourBox-deps-${{ inputs.version }}-windows-x64 + name: ${{ env.ASSET }} path: | - dist/VapourBox-deps-${{ inputs.version }}-windows-x64.zip - dist/VapourBox-deps-${{ inputs.version }}-windows-x64.zip.sha256.json + dist/${{ env.ASSET }}.zip + dist/${{ env.ASSET }}.zip.sha256.json + # The v2 bundle must run below x86-64-v3, and every hosted runner has AVX2. + # Windows is probed at Sandy Bridge (AVX without AVX2): at pre-AVX chips SDE + # cannot emulate what Microsoft's runtime reads from the kernel, so the core + # control fails and nothing could be proven (see probe-cpu-compat.yml). + gate-x64-v2: + needs: build-x64 + uses: ./.github/workflows/probe-cpu-compat.yml + with: + artifact_prefix: VapourBox-deps-${{ inputs.version }}- + tier: v2 + gate: true + matrix_json: '[{"platform":"windows-x64","runner":"windows-latest","sde":"sde.exe","python":"vapoursynth/python.exe","chip":"snb"}]' + + # Published only once the v2 gate has passed. + upload-x64: + needs: [build-x64, gate-x64-v2] + if: inputs.release_tag != '' + runs-on: ubuntu-24.04 + steps: + - uses: actions/download-artifact@v5 + with: + pattern: VapourBox-deps-${{ inputs.version }}-windows-x64* + path: dist + merge-multiple: true - name: Upload to release - if: inputs.release_tag != '' - shell: pwsh env: GH_TOKEN: ${{ github.token }} run: | - gh release upload "${{ inputs.release_tag }}" ` - "dist/VapourBox-deps-${{ inputs.version }}-windows-x64.zip" ` - "dist/VapourBox-deps-${{ inputs.version }}-windows-x64.zip.sha256.json" --clobber + gh release upload "${{ inputs.release_tag }}" \ + --repo "${{ github.repository }}" \ + dist/VapourBox-deps-${{ inputs.version }}-windows-x64*.zip \ + dist/VapourBox-deps-${{ inputs.version }}-windows-x64*.zip.sha256.json --clobber diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a62332c..7400e4b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -44,11 +44,20 @@ jobs: strategy: fail-fast: false matrix: + # x64 runs both CPU-tier bundles (issue #92). The runner is a v3 CPU, so + # the v2 run proves the v2 bundle is functionally equivalent — not that + # it runs on an older CPU; that is build-deps-macos.yml's static gate. include: - arch: arm64 runner: macos-15 + asset: macos-arm64 - arch: x64 runner: macos-15-intel # last native Intel image, available until ~Aug 2027 + asset: macos-x64 + - arch: x64 + runner: macos-15-intel + asset: macos-x64-v2 + name: macos (${{ matrix.asset }}) runs-on: ${{ matrix.runner }} env: VAPOURBOX_DEPS_DIR: ${{ github.workspace }}/deps/macos-${{ matrix.arch }} @@ -71,14 +80,15 @@ jobs: - name: Setup Rust uses: dtolnay/rust-toolchain@stable - - name: Download dependencies (${{ matrix.arch }}) + - name: Download dependencies (${{ matrix.asset }}) env: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | + # The tier is in the asset name only; either installs to deps/. PLATFORM="macos-${{ matrix.arch }}" mkdir -p "deps/$PLATFORM" - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "$PLATFORM" \ + ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") unzip -q -o "$ZIP" -d "deps/$PLATFORM" rm -rf "$ZIP" .deps-artifact @@ -123,6 +133,12 @@ jobs: windows: if: ${{ github.event_name == 'schedule' || inputs.platform == 'all' || inputs.platform == 'windows' }} + # Both CPU-tier bundles (issue #92); see the macOS job for what the v2 run proves. + strategy: + fail-fast: false + matrix: + asset: [windows-x64, windows-x64-v2] + name: windows (${{ matrix.asset }}) runs-on: windows-latest env: VAPOURBOX_DEPS_DIR: ${{ github.workspace }}/deps/windows-x64 @@ -153,7 +169,7 @@ jobs: DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | mkdir -p deps/windows-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh windows-x64 \ + ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") 7z x "$ZIP" -o"deps/windows-x64" -y >/dev/null rm -rf "$ZIP" .deps-artifact @@ -197,6 +213,12 @@ jobs: linux: if: ${{ github.event_name == 'schedule' || inputs.platform == 'all' || inputs.platform == 'linux' }} + # Both CPU-tier bundles (issue #92); see the macOS job for what the v2 run proves. + strategy: + fail-fast: false + matrix: + asset: [linux-x64, linux-x64-v2] + name: linux (${{ matrix.asset }}) runs-on: ubuntu-24.04 # must match the deps-build runner (glibc) env: VAPOURBOX_DEPS_DIR: ${{ github.workspace }}/deps/linux-x64 @@ -225,13 +247,13 @@ jobs: - name: Setup Rust uses: dtolnay/rust-toolchain@stable - - name: Download dependencies (linux-x64) + - name: Download dependencies (${{ matrix.asset }}) env: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | mkdir -p deps/linux-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh linux-x64 \ + ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") unzip -q -o "$ZIP" -d deps/linux-x64 rm -rf "$ZIP" .deps-artifact diff --git a/.github/workflows/probe-cpu-compat.yml b/.github/workflows/probe-cpu-compat.yml index 5892739..57a4354 100644 --- a/.github/workflows/probe-cpu-compat.yml +++ b/.github/workflows/probe-cpu-compat.yml @@ -2,8 +2,14 @@ name: Probe CPU compatibility (SDE) # Loads and renders every plugin in a deps bundle under Intel SDE emulating an # older CPU, one plugin per process (Scripts/probe-plugin-compat.py). Every -# hosted runner has AVX2, so this is the only CI-side way to see what a -# pre-AVX2 machine would do with the bundle (issue #92). +# hosted runner has AVX2, so this is the only CI-side way to see what an older +# machine would do with a bundle (issue #92). +# +# Two modes: +# - survey (push / workflow_dispatch): reports what the bundle does; crashes +# are findings, not failures. +# - gate (workflow_call from build-deps-*.yml with gate: true): the v2 bundle +# promises to run on these CPUs, so any crash fails the build. # # A run only counts if its controls hold: # - native: the runner itself must pass everything; an error there is a @@ -11,9 +17,11 @@ name: Probe CPU compatibility (SDE) # - core under SDE: VapourSynth with no plugins must pass. If it doesn't, the # emulation itself is broken for this OS/chip and every plugin result is # meaningless. (Windows at pre-AVX chips fails this: Microsoft's runtime -# picks AVX from what the host kernel reports, which SDE cannot emulate.) -# - positive: zsmooth's haswell build (AVX2, no runtime dispatch — issue #82) -# must crash inside its own image under a pre-AVX2 chip, proving SDE traps. +# picks AVX from what the host kernel reports, which SDE cannot emulate — +# so Windows is probed at Sandy Bridge, AVX without AVX2.) +# - positive: upstream's zsmooth haswell build (AVX2, no runtime dispatch, +# issue #82), fetched here independently of the bundle, must crash inside +# its own image — proving SDE actually traps. on: # Also runs when the probe or this workflow changes, which re-validates the @@ -29,31 +37,44 @@ on: description: 'Optional build-deps-* run IDs (comma-separated); default is the release named in deps-version.json' required: false default: '' + tier: + description: 'x64 bundle tier to probe' + type: choice + options: [v3, v2] + default: v3 + workflow_call: + inputs: + artifact_prefix: + description: 'Take the bundle from THIS run''s artifacts: ' + type: string + required: true + tier: + type: string + default: v2 + matrix_json: + description: 'JSON list of {platform, runner, sde, python, chip} entries' + type: string + required: true + gate: + type: boolean + default: true permissions: contents: read actions: read +env: + ZSMOOTH_VERSION: '0.19.0' + jobs: probe: - name: ${{ matrix.platform }} (-${{ matrix.chip }}) + name: ${{ matrix.platform }}${{ (inputs.tier || 'v3') == 'v2' && '-v2' || '' }} (-${{ matrix.chip }}) strategy: fail-fast: false matrix: - include: - # wsm = Westmere, the Mac Pro 5,1 in #92: no AVX at all. - - platform: linux-x64 - runner: ubuntu-24.04 - sde: sde64 - python: python/bin/python3 - chip: wsm - # snb = Sandy Bridge: AVX without AVX2 — the v3/v2 tier boundary. - # Windows can only be probed here; see the header. - - platform: windows-x64 - runner: windows-latest - sde: sde.exe - python: vapoursynth/python.exe - chip: snb + # wsm = Westmere (the Mac Pro 5,1 in #92): no AVX at all. + # snb = Sandy Bridge: AVX without AVX2 — the v3/v2 tier boundary. + include: ${{ fromJSON(inputs.matrix_json || '[{"platform":"linux-x64","runner":"ubuntu-24.04","sde":"sde64","python":"python/bin/python3","chip":"wsm"},{"platform":"windows-x64","runner":"windows-latest","sde":"sde.exe","python":"vapoursynth/python.exe","chip":"snb"}]') }} runs-on: ${{ matrix.runner }} timeout-minutes: 90 defaults: @@ -61,17 +82,31 @@ jobs: shell: bash env: D: deps/${{ matrix.platform }} + ASSET_ID: ${{ matrix.platform }}${{ (inputs.tier || 'v3') == 'v2' && '-v2' || '' }} steps: - uses: actions/checkout@v5 - - name: Download dependencies (${{ matrix.platform }}) + # Gate mode: the bundle was built earlier in this same run. + - name: Take the bundle from this run + if: inputs.artifact_prefix != '' + uses: actions/download-artifact@v5 + with: + name: ${{ inputs.artifact_prefix }}${{ env.ASSET_ID }} + path: .deps-artifact + + - name: Download dependencies (${{ env.ASSET_ID }}) env: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | - TAG=$(sed -n 's/.*"releaseTag": *"\([^"]*\)".*/\1/p' app/assets/deps-version.json) mkdir -p "$D" - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.platform }}" "$TAG" "${TAG#deps-v}") + if [ -d .deps-artifact ]; then + ZIP=$(find .deps-artifact -name "*-$ASSET_ID.zip" | head -1) + else + TAG=$(sed -n 's/.*"releaseTag": *"\([^"]*\)".*/\1/p' app/assets/deps-version.json) + ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "$ASSET_ID" "$TAG" "${TAG#deps-v}") + fi + [ -f "$ZIP" ] || { echo "no bundle zip for $ASSET_ID"; exit 1; } if [ "$RUNNER_OS" = Windows ]; then # The Windows zip uses backslash separators; 7-Zip copes, unzip does not. 7z x "$ZIP" -o"$D" -y >/dev/null @@ -80,6 +115,24 @@ jobs: fi rm -rf "$ZIP" .deps-artifact + - name: Fetch the positive control (upstream zsmooth, AVX2-only) + run: | + if [ "$RUNNER_OS" = Windows ]; then + ASSET=zsmooth-x86_64-windows.zip; EXT=dll + else + ASSET=zsmooth-x86_64-linux-gnu.zip; EXT=so + fi + curl -fsSL -o control.zip \ + "https://github.com/adworacz/zsmooth/releases/download/${ZSMOOTH_VERSION}/${ASSET}" + if [ "$RUNNER_OS" = Windows ]; then + 7z x control.zip -ocontrol -y >/dev/null + else + unzip -q -o control.zip -d control + fi + FOUND=$(find control -name "*zsmooth*.$EXT" -type f | head -1) + cp "$FOUND" "control-zsmooth-haswell.$EXT" + echo "CONTROL=$PWD/control-zsmooth-haswell.$EXT" >> "$GITHUB_ENV" + - name: Setup Intel SDE uses: petarpetrovt/setup-sde@v6.0 with: @@ -95,10 +148,12 @@ jobs: run: | SDE_DIR=$(cygpath -m "$SDE_PATH" 2>/dev/null || echo "$SDE_PATH") "$D/${{ matrix.python }}" Scripts/probe-plugin-compat.py "$D" \ - --report sde.txt --json sde.json --timeout 1800 \ + --report sde.txt --json sde.json --timeout 1800 --extra "$CONTROL" \ --wrap "\"$SDE_DIR/${{ matrix.sde }}\" -${{ matrix.chip }} --" - - name: Check the controls and summarise + - name: Check the controls${{ inputs.gate && ' and gate the bundle' || '' }} + env: + GATE: ${{ inputs.gate && 'true' || 'false' }} run: | "$D/${{ matrix.python }}" - <<'EOF' import json, os, sys @@ -112,25 +167,24 @@ jobs: if core["status"] != "PASS": sys.exit(f"core control failed under SDE ({core['status']}: {core['detail']}); " "the emulation is not valid for this OS/chip, so no plugin result means anything") - control = [r for r in sde if "zsmooth-haswell" in r["file"]] - if not control: - sys.exit("positive control missing: no zsmooth-haswell build in this bundle") - c = control[0] - if c["status"] != "CRASHED" or "zsmooth-haswell" not in c["detail"]: - sys.exit(f"SDE did not trap the AVX2-only zsmooth-haswell build in its own code " + c = next(r for r in sde if r["file"].startswith("control-zsmooth-haswell")) + if c["status"] != "CRASHED" or "control-zsmooth-haswell" not in c["detail"]: + sys.exit(f"SDE did not trap the AVX2-only control build in its own code " f"({c['status']}: {c['detail']}); results are meaningless") - bad = [r for r in sde if r["status"] in ("CRASHED", "INCONCLUSIVE") and r is not c] - print(f"controls OK; {len(bad)} other file(s) crashed or were inconclusive:") + bad = [r for r in sde if r["status"] in ("CRASHED", "INCONCLUSIVE", "ERROR") and r is not c] + print(f"controls OK; {len(bad)} bundle file(s) crashed, errored or were inconclusive:") for r in bad: print(f" {r['file']} [{r['namespaces']}] {r['status']}: {r['detail']}") + if bad and os.environ["GATE"] == "true": + sys.exit("the bundle does not run on this CPU") EOF - name: Upload reports if: always() uses: actions/upload-artifact@v5 with: - name: cpu-compat-${{ matrix.platform }}-${{ matrix.chip }} + name: cpu-compat-${{ env.ASSET_ID }}-${{ matrix.chip }} path: | native.txt native.json diff --git a/Scripts/check-deps-changed.sh b/Scripts/check-deps-changed.sh index f73fe31..253e942 100755 --- a/Scripts/check-deps-changed.sh +++ b/Scripts/check-deps-changed.sh @@ -18,10 +18,18 @@ fi # Files to check for changes. # Deps binaries are no longer committed (reproduced by the download scripts, which # are the source of truth), so "deps changed" == "a download script changed". +# Everything that changes what goes into a bundle, not just the download +# scripts: a patch (e.g. the MVTools no-AVX2 patch) or a packaging change +# alters the bundle just as surely. DEPS_PATHS=( "Scripts/download-deps-windows.ps1" "Scripts/download-deps-macos.sh" "Scripts/download-deps-linux.sh" + "Scripts/patches" + "Scripts/package-deps-windows.ps1" + "Scripts/package-deps-macos.sh" + "Scripts/package-deps-linux.sh" + "Scripts/deps-expected-plugins.json" ) # Get the last deps release tag diff --git a/Scripts/check-load-time-simd.py b/Scripts/check-load-time-simd.py new file mode 100644 index 0000000..98d665e --- /dev/null +++ b/Scripts/check-load-time-simd.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Fail if an x86_64 Mach-O plugin can run AVX instructions while it loads. + +Issue #92: prebuilt MVTools compiles six files with -mavx2, and their static +initializers — run by dyld inside dlopen, before any plugin code can check the +CPU — contain VEX instructions. VapourSynth autoloads every plugin when a core +starts, so on a CPU without AVX every job died with SIGILL whatever it asked +for. Runtime SIMD dispatch in the plugin's filters cannot help: this runs +first. + +macOS has no Intel SDE, so the v2 bundle is checked statically here instead +(Linux and Windows are checked by running under SDE; see +.github/workflows/probe-cpu-compat.yml). Initializers are found structurally, +from __init_offsets / __mod_init_func, never by symbol name — LTO and +toolchains rename them freely, and a name-based scan once came back clean on a +binary it simply could not see into. Direct calls and tail calls are followed, +because the VEX can sit in a helper the initializer calls rather than in the +initializer itself. + +Usage: check-load-time-simd.py [--depth N] BINARY... +Exit 1 if any binary can reach a VEX instruction from an initializer. +""" + +import argparse +import bisect +import re +import shutil +import struct +import subprocess +import sys + +LC_SEGMENT_64 = 0x19 +MH_MAGIC_64 = 0xFEEDFACF +FAT_MAGIC = 0xCAFEBABE +CPU_TYPE_X86_64 = 0x01000007 + +# VEX-encoded (AVX and later) mnemonics start with "v" in LLVM's AT&T syntax; +# these few legacy instructions also do and are not VEX. +NOT_VEX = {"verr", "verw"} + + +def x86_64_slice(data): + """Return (bytes, offset) of the x86_64 Mach-O image in a thin or fat file.""" + magic = struct.unpack(">I", data[:4])[0] + if magic == FAT_MAGIC: + nfat = struct.unpack(">I", data[4:8])[0] + for i in range(nfat): + cputype, _, offset, size, _ = struct.unpack(">iiIII", data[8 + i * 20:28 + i * 20]) + if cputype == CPU_TYPE_X86_64: + return data[offset:offset + size] + return None + if struct.unpack(":$") + insn = re.compile(r"^\s*([0-9a-f]+):\s+(\S+)\s*(.*)$") + # Direct targets only ("0x1234 "). An indirect `jmpq *0x..(%rip)` + # names a GOT slot, not code, and following it links unrelated functions. + target = re.compile(r"^0x([0-9a-f]+)") + for line in out.splitlines(): + m = label.match(line) + if m: + cur = int(m.group(1), 16) + funcs[cur] = {"name": m.group(2), "vex": [], "calls": set()} + continue + m = insn.match(line) + if not m or cur is None: + continue + addr, mnem, ops = int(m.group(1), 16), m.group(2), m.group(3) + if mnem.startswith("v") and mnem not in NOT_VEX: + funcs[cur]["vex"].append(f"{addr:#x}: {mnem} {ops}".strip()) + if mnem.startswith("call") or mnem.startswith("jmp"): + t = target.match(ops) + if t: + funcs[cur]["calls"].add(int(t.group(1), 16)) + # Stub tables are trampolines into OTHER images (libc++, libSystem), which + # this check cannot see into; never walk through them. + return {a: f for a, f in funcs.items() + if f["name"] not in ("__stubs", "__stub_helper", "__auth_stubs")} + + +def check(path, depth): + with open(path, "rb") as f: + image = x86_64_slice(f.read()) + if image is None: + return f"{path}: no x86_64 image, skipped", [] + roots = initializer_addresses(image) + if not roots: + # Nothing runs at load time. Checked before disassembling: some valid + # images (Zig-linked zsmooth) have a header layout llvm-objdump rejects. + return f"{path}: 0 initializers", [] + try: + funcs = disassemble(path) + except subprocess.CalledProcessError as e: + # Initializers we cannot inspect are a failure, not a pass. + return (f"{path}: {len(roots)} initializers", [ + f"could not disassemble: {e.stderr.strip().splitlines()[-1] if e.stderr else e}"]) + starts = sorted(funcs) + + def owner(addr): + i = bisect.bisect_right(starts, addr) - 1 + return starts[i] if i >= 0 else None + + findings = [] + seen = set() + frontier = [(owner(r), [owner(r)]) for r in roots if owner(r) is not None] + for _ in range(depth + 1): + nxt = [] + for fn, chain in frontier: + if fn in seen: + continue + seen.add(fn) + info = funcs[fn] + if info["vex"]: + names = " -> ".join(funcs[c]["name"] for c in chain) + findings.append(f"{names}\n first: {info['vex'][0]} " + f"({len(info['vex'])} VEX instructions)") + for t in info["calls"]: + o = owner(t) + if o is not None and o != fn: + nxt.append((o, chain + [o])) + frontier = nxt + return f"{path}: {len(roots)} initializers, {len(seen)} functions reachable", findings + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("binaries", nargs="+") + ap.add_argument("--depth", type=int, default=6, + help="call depth to follow from each initializer") + args = ap.parse_args() + bad = 0 + for b in args.binaries: + summary, findings = check(b, args.depth) + print(("FAIL " if findings else "ok ") + summary) + for f in findings: + print(" " + f) + bad += bool(findings) + sys.exit(1 if bad else 0) + + +if __name__ == "__main__": + main() diff --git a/Scripts/deps-expected-plugins.json b/Scripts/deps-expected-plugins.json index e2f398f..a0f1784 100644 --- a/Scripts/deps-expected-plugins.json +++ b/Scripts/deps-expected-plugins.json @@ -1,17 +1,30 @@ { - "_comment": "Required VapourSynth plugin filenames per platform — the contract for a COMPLETE deps bundle. The package-deps-* scripts assert every file listed here exists in the staged bundle before zipping and FAIL the build if any are missing, so a silently-failed download (e.g. a dead upstream URL) becomes a red build instead of an incomplete bundle shipping. Plugin directory: windows-x64 = vapoursynth/vs-plugins, macos/linux = vapoursynth/plugins. Lists exclude data files (nnedi3 weights) and runtime libs (fftw); they cover the VapourSynth plugin binaries only. Update this when adding or removing a plugin. An entry containing a '/' is a path relative to the BUNDLE ROOT rather than a filename in the plugin directory: zsmooth ships one build per CPU baseline in vapoursynth/zsmooth/, deliberately outside the autoload directory, because both builds register the same namespace and the worker loads exactly one by path (DependencyLocator::zsmooth_plugin).", + "_comment": "Required VapourSynth plugin filenames per platform — the contract for a COMPLETE deps bundle. The package-deps-* scripts assert every file listed here exists in the staged bundle's plugin directory before zipping and FAIL the build if any are missing, so a silently-failed download (e.g. a dead upstream URL) becomes a red build instead of an incomplete bundle shipping. Plugin directory: windows-x64 = vapoursynth/vs-plugins, macos/linux = vapoursynth/plugins. Lists exclude data files (nnedi3 weights) and runtime libs (fftw); they cover the VapourSynth plugin binaries only. Update this when adding or removing a plugin. The x64 platforms ship in two CPU tiers (issue #92): the plain key is the v3 bundle, the -v2 key the bundle for CPUs below x86-64-v3. Both tiers carry the same set of plugins, built differently, so their lists must stay identical (asserted by app/test/attribution_test.dart).", "windows-x64": [ "AddGrain.dll", + "bifrost.dll", + "bwdif.dll", "CAS.dll", "CTMF.dll", "DCTFilter.dll", - "DFTTest.dll", - "DeScratch.dll", "Deblock.dll", + "dedot.dll", + "DeScratch.dll", + "DFTTest.dll", "EEDI3m.dll", + "fft3dfilter.dll", + "fmtconv.dll", "KNLMeansCL.dll", "LGhost.dll", + "libakarin.dll", + "libawarpsharp2.dll", + "libfillborders.dll", + "libfluxsmooth.dll", + "libmvtools.dll", + "libtemporalmedian.dll", + "libzstd.dll", "MiscFilters.dll", + "neo-f3kdb.dll", "NNEDI3CL.dll", "RemoveDirt.dll", "RemoveGrainVS.dll", @@ -19,11 +32,25 @@ "TCanny.dll", "TTempSmooth.dll", "VIVTC.dll", + "vsznedi3.dll", + "zsmooth.dll" + ], + "windows-x64-v2": [ + "AddGrain.dll", "bifrost.dll", "bwdif.dll", + "CAS.dll", + "CTMF.dll", + "DCTFilter.dll", + "Deblock.dll", "dedot.dll", + "DeScratch.dll", + "DFTTest.dll", + "EEDI3m.dll", "fft3dfilter.dll", "fmtconv.dll", + "KNLMeansCL.dll", + "LGhost.dll", "libakarin.dll", "libawarpsharp2.dll", "libfillborders.dll", @@ -31,10 +58,17 @@ "libmvtools.dll", "libtemporalmedian.dll", "libzstd.dll", + "MiscFilters.dll", "neo-f3kdb.dll", - "vapoursynth/zsmooth/zsmooth-haswell.dll", - "vapoursynth/zsmooth/zsmooth-x86_64_v2.dll", - "vsznedi3.dll" + "NNEDI3CL.dll", + "RemoveDirt.dll", + "RemoveGrainVS.dll", + "Retinex.dll", + "TCanny.dll", + "TTempSmooth.dll", + "VIVTC.dll", + "vsznedi3.dll", + "zsmooth.dll" ], "macos-arm64": [ "libaddgrain.dylib", @@ -70,7 +104,7 @@ "libttempsmooth.dylib", "libvivtc.dylib", "libznedi3.dylib", - "vapoursynth/zsmooth/libzsmooth.dylib" + "libzsmooth.dylib" ], "macos-x64": [ "libaddgrain.dylib", @@ -103,8 +137,40 @@ "libttempsmooth.dylib", "libvivtc.dylib", "libznedi3.dylib", - "vapoursynth/zsmooth/libzsmooth-haswell.dylib", - "vapoursynth/zsmooth/libzsmooth-x86_64_v2.dylib" + "libzsmooth.dylib" + ], + "macos-x64-v2": [ + "libaddgrain.dylib", + "libawarpsharp2.dylib", + "libbifrost.dylib", + "libbwdif.dylib", + "libcas.dylib", + "libctmf.dylib", + "libdctfilter.dylib", + "libdeblock.dylib", + "libdedot.dylib", + "libdescratch.dylib", + "libdfttest.dylib", + "libeedi3m.dylib", + "libfft3dfilter.dylib", + "libfillborders.dylib", + "libfluxsmooth.dylib", + "libfmtconv.dylib", + "libknlmeanscl.dylib", + "liblghost.dylib", + "libmiscfilters.dylib", + "libmvtools.dylib", + "libneo-f3kdb.dylib", + "libnnedi3cl.dylib", + "libremovedirt.dylib", + "libremovegrain.dylib", + "libretinex.dylib", + "libtcanny.dylib", + "libtmedian.dylib", + "libttempsmooth.dylib", + "libvivtc.dylib", + "libznedi3.dylib", + "libzsmooth.dylib" ], "linux-x64": [ "libaddgrain.so", @@ -138,8 +204,41 @@ "libttempsmooth.so", "libvivtc.so", "libznedi3.so", - "vapoursynth/zsmooth/libzsmooth-haswell.so", - "vapoursynth/zsmooth/libzsmooth-x86_64_v2.so" + "libzsmooth.so" + ], + "linux-x64-v2": [ + "libaddgrain.so", + "libakarin.so", + "libawarpsharp2.so", + "libbifrost.so", + "libbwdif.so", + "libcas.so", + "libctmf.so", + "libdctfilter.so", + "libdeblock.so", + "libdedot.so", + "libdescratch.so", + "libdfttest.so", + "libeedi3m.so", + "libfft3dfilter.so", + "libfillborders.so", + "libfluxsmooth.so", + "libfmtconv.so", + "libknlmeanscl.so", + "liblghost.so", + "libmiscfilters.so", + "libmvtools.so", + "libneo-f3kdb.so", + "libnnedi3cl.so", + "libremovedirt.so", + "libremovegrain.so", + "libretinex.so", + "libtcanny.so", + "libtmedian.so", + "libttempsmooth.so", + "libvivtc.so", + "libznedi3.so", + "libzsmooth.so" ], "linux-arm64": [ "libaddgrain.so", @@ -174,6 +273,6 @@ "libttempsmooth.so", "libvivtc.so", "libznedi3.so", - "vapoursynth/zsmooth/libzsmooth.so" + "libzsmooth.so" ] } diff --git a/Scripts/download-deps-linux.sh b/Scripts/download-deps-linux.sh index 10db031..81cca58 100755 --- a/Scripts/download-deps-linux.sh +++ b/Scripts/download-deps-linux.sh @@ -9,20 +9,25 @@ # libfftw3-dev libboost-filesystem-dev libboost-atomic-dev \ # ocl-icd-opencl-dev libdvdread-dev # -# Usage: ./Scripts/download-deps-linux.sh [--force] +# Usage: ./Scripts/download-deps-linux.sh [--force] [--tier v3|v2] set -e FORCE=false +TIER=v3 while [[ $# -gt 0 ]]; do case $1 in --force) FORCE=true shift ;; + --tier) + TIER="$2" + shift 2 + ;; *) echo "Unknown option: $1" - echo "Usage: $0 [--force]" + echo "Usage: $0 [--force] [--tier v3|v2]" exit 1 ;; esac @@ -39,10 +44,47 @@ else exit 1 fi +# CPU tier (issue #92). The x86 bundle ships twice: v3 for x86-64-v3 CPUs +# (Haswell, 2013, and later) and v2 for anything older, chosen by the app from +# `vapourbox-worker --probe-cpu`. This block is the ONLY place the tier is +# interpreted; everything below reads these variables and never $TIER itself. +# ZSMOOTH_CPU zsmooth's baseline (upstream has no runtime dispatch) +# That is the whole difference on Linux: every other plugin in this bundle +# renders under Intel SDE emulating Westmere (no AVX at all) — see +# .github/workflows/probe-cpu-compat.yml — including MVTools, whose macOS +# prebuilt is the one that does not. +if [ "$ARCH" = "x86_64" ]; then + case "$TIER" in + v3) ZSMOOTH_CPU=haswell ;; + v2) ZSMOOTH_CPU=x86_64_v2 ;; + *) echo "Unknown tier: $TIER (expected v3 or v2)"; exit 1 ;; + esac +else + if [ "$TIER" != "v3" ]; then + echo "--tier is x86-only; the arm64 bundle is not tiered." + exit 1 + fi + TIER="" +fi + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" DEPS_DIR="$PROJECT_ROOT/deps/$PLATFORM_DIR" PLUGINS_DIR="$DEPS_DIR/vapoursynth/plugins" + +# Both tiers build into the same deps/ directory (the tier lives in +# version.json, not the path), and most steps skip what already exists. So a +# tier switch without --force would keep the previous tier's zsmooth and +# silently produce a mixed bundle. +if [ -n "$TIER" ] && [ "$FORCE" = false ] && [ -f "$DEPS_DIR/version.json" ]; then + EXISTING_TIER=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('tier','v3'))" \ + "$DEPS_DIR/version.json" 2>/dev/null || echo "v3") + if [ "$EXISTING_TIER" != "$TIER" ]; then + echo "ERROR: $DEPS_DIR holds the $EXISTING_TIER bundle; building $TIER over it" + echo "would mix the two. Re-run with --force." + exit 1 + fi +fi PYTHON_DIR="$DEPS_DIR/python" PYTHON_PACKAGES_DIR="$DEPS_DIR/python-packages" BUILD_DIR="/tmp/vapourbox-build-$$" @@ -1024,24 +1066,22 @@ build_plugin "tmedian" \ "libtmedian.so" \ "$PLUGIN_BUILD_ENV meson setup build --buildtype=release && ninja -C build" -# zsmooth — one build per CPU baseline +# zsmooth — exactly one build, in the autoload directory # # core.zsmooth.CCD (also Cnr4 and a set of RemoveGrain/TemporalMedian-family -# filters). Upstream publishes only `haswell` (an AVX2 baseline) and `znver4` -# x86 builds, compiled throughout with NO runtime dispatch — so on a pre-2013 -# CPU the library loads fine and then dies with an illegal instruction the -# instant a filter runs. That is issue #82 (reported on Windows, but this -# bundle took the same haswell asset), and it is silent: vspipe prints nothing. -# -# So x86 ships both builds outside the autoload directory and the worker loads -# exactly one by path (DependencyLocator::zsmooth_plugin). They cannot share a -# directory: each registers the namespace `zsmooth`, so whichever autoloads -# second is rejected. aarch64 has a single NEON baseline and needs no split. +# filters). zsmooth has NO runtime dispatch: each build is compiled for one CPU +# baseline throughout, so a build above the machine's baseline loads fine and +# then dies with an illegal instruction the instant a filter runs — issue #82, +# silently, because vspipe prints nothing. # -# Measured at 720x576: `x86_64` is 2.0x slower than haswell on CCD and 3.0x on -# Cnr4, `x86_64_v2` 1.4x on both — which is why the portable build is v2 -# (SSE4.2/POPCNT, everything from Nehalem 2009 on) and why the fast build is -# still shipped rather than dropped for one portable binary. +# aarch64 has a single NEON baseline and takes the author's build. x86 takes +# the $ZSMOOTH_CPU chosen by the tier block at the top of this script: the +# author's haswell asset for v3, or an x86_64_v2 build compiled here for v2 +# (SSE4.2/POPCNT, everything from Nehalem 2009 on; measured 1.4x slower than +# haswell on CCD and Cnr4, against 2-3x for plain x86_64). One build per bundle +# means it autoloads like any other plugin; bundles up to 1.10.0 shipped both +# x86 builds in a separate zsmooth/ directory for the worker to load by path, +# which the tiers replace. # # Keep ZSMOOTH_VERSION in step across download-deps-{macos,linux}.sh and # download-deps-windows.ps1 — a version skew would make the same job produce @@ -1049,25 +1089,51 @@ build_plugin "tmedian" \ ZSMOOTH_VERSION="0.19.0" # Must satisfy zsmooth's build.zig.zon `minimum_zig_version` (0.15.2 for 0.19.0). ZIG_VERSION="0.15.2" -ZSMOOTH_DIR="$DEPS_DIR/vapoursynth/zsmooth" -mkdir -p "$ZSMOOTH_DIR" +ZSMOOTH_OUT="$PLUGINS_DIR/libzsmooth.so" +rm -rf "$DEPS_DIR/vapoursynth/zsmooth" echo "" echo "=== Installing zsmooth ===" -# The pre-built asset: haswell on x86, the only build on aarch64. case "$ARCH" in - aarch64|arm64) - ZSMOOTH_ASSET="zsmooth-aarch64-linux-gnu.zip" - ZSMOOTH_PREBUILT="$ZSMOOTH_DIR/libzsmooth.so" - ;; - *) - ZSMOOTH_ASSET="zsmooth-x86_64-linux-gnu.zip" - ZSMOOTH_PREBUILT="$ZSMOOTH_DIR/libzsmooth-haswell.so" - ;; + aarch64|arm64) ZSMOOTH_ASSET="zsmooth-aarch64-linux-gnu.zip" ;; + *) ZSMOOTH_ASSET="zsmooth-x86_64-linux-gnu.zip" ;; # the haswell build esac -if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_PREBUILT" ]; then +if [ "$FORCE" = false ] && [ -f "$ZSMOOTH_OUT" ]; then + echo " libzsmooth.so already exists, skipping" +elif [ "${ZSMOOTH_CPU:-}" = "x86_64_v2" ]; then + # No upstream asset for this baseline. Zig brings its own libc and builds + # zsmooth's fftw dependency itself, so this adds no apt package — only + # network access, since `zig build` fetches zsmooth's own Zig dependencies. + echo " Building zsmooth $ZSMOOTH_VERSION (x86_64_v2, runs without AVX2)..." + # Subshell so a failure cannot abort the script under `set -e`; the file + # check below decides whether it worked. + ( + set -e + cd "$BUILD_DIR" + rm -rf zig-toolchain zsmooth-src zig.tar.xz + curl -fsSL -o zig.tar.xz \ + "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" + mkdir -p zig-toolchain + tar -xf zig.tar.xz -C zig-toolchain --strip-components=1 + git clone --depth 1 --branch "$ZSMOOTH_VERSION" \ + https://github.com/adworacz/zsmooth.git zsmooth-src + cd zsmooth-src + "$BUILD_DIR/zig-toolchain/zig" build \ + -Doptimize=ReleaseFast -Dcpu=x86_64_v2 + cp zig-out/lib/libzsmooth.so "$ZSMOOTH_OUT" + ) || true + if [ -f "$ZSMOOTH_OUT" ]; then + patchelf --set-rpath '$ORIGIN:$ORIGIN/../../lib' "$ZSMOOTH_OUT" 2>/dev/null || true + echo " Built zsmooth (x86_64_v2)" + BUILT_PLUGINS+=("zsmooth") + else + echo " Warning: failed to build zsmooth (x86_64_v2)" + FAILED_PLUGINS+=("zsmooth") + fi + rm -rf "$BUILD_DIR/zig-toolchain" "$BUILD_DIR/zsmooth-src" "$BUILD_DIR/zig.tar.xz" +else rm -rf "$BUILD_DIR/zsmooth" mkdir -p "$BUILD_DIR/zsmooth" if curl -sL -o "$BUILD_DIR/zsmooth/zsmooth.zip" \ @@ -1075,9 +1141,9 @@ if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_PREBUILT" ]; then && unzip -q -o "$BUILD_DIR/zsmooth/zsmooth.zip" -d "$BUILD_DIR/zsmooth"; then so_path=$(find "$BUILD_DIR/zsmooth" -name "*.so" -type f 2>/dev/null | head -1) if [ -n "$so_path" ]; then - cp "$so_path" "$ZSMOOTH_PREBUILT" - patchelf --set-rpath '$ORIGIN:$ORIGIN/../../lib' "$ZSMOOTH_PREBUILT" 2>/dev/null || true - echo " Downloaded pre-built zsmooth -> $(basename "$ZSMOOTH_PREBUILT")" + cp "$so_path" "$ZSMOOTH_OUT" + patchelf --set-rpath '$ORIGIN:$ORIGIN/../../lib' "$ZSMOOTH_OUT" 2>/dev/null || true + echo " Downloaded pre-built zsmooth" BUILT_PLUGINS+=("zsmooth") else echo " Failed: no .so in the zsmooth archive" @@ -1088,52 +1154,8 @@ if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_PREBUILT" ]; then FAILED_PLUGINS+=("zsmooth") fi rm -rf "$BUILD_DIR/zsmooth" -else - echo " $(basename "$ZSMOOTH_PREBUILT") already exists, skipping" fi -# The portable x86 build has no upstream asset and must be compiled. Zig brings -# its own libc and builds zsmooth's fftw dependency itself, so this adds no apt -# package — only network access, since `zig build` fetches zsmooth's own Zig -# dependencies. -case "$ARCH" in - aarch64|arm64) : ;; - *) - ZSMOOTH_V2="$ZSMOOTH_DIR/libzsmooth-x86_64_v2.so" - if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_V2" ]; then - echo " Building zsmooth $ZSMOOTH_VERSION (x86_64_v2, runs without AVX2)..." - # Subshell so a failure cannot abort the script under `set -e`; the - # file check below decides whether it worked. - ( - set -e - cd "$BUILD_DIR" - rm -rf zig-toolchain zsmooth-src zig.tar.xz - curl -fsSL -o zig.tar.xz \ - "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" - mkdir -p zig-toolchain - tar -xf zig.tar.xz -C zig-toolchain --strip-components=1 - git clone --depth 1 --branch "$ZSMOOTH_VERSION" \ - https://github.com/adworacz/zsmooth.git zsmooth-src - cd zsmooth-src - "$BUILD_DIR/zig-toolchain/zig" build \ - -Doptimize=ReleaseFast -Dcpu=x86_64_v2 - cp zig-out/lib/libzsmooth.so "$ZSMOOTH_V2" - ) || true - - if [ -f "$ZSMOOTH_V2" ]; then - patchelf --set-rpath '$ORIGIN:$ORIGIN/../../lib' "$ZSMOOTH_V2" 2>/dev/null || true - echo " Built zsmooth -> $(basename "$ZSMOOTH_V2")" - else - echo " Warning: failed to build the portable zsmooth" - FAILED_PLUGINS+=("zsmooth-x86_64_v2") - fi - rm -rf "$BUILD_DIR/zig-toolchain" "$BUILD_DIR/zsmooth-src" "$BUILD_DIR/zig.tar.xz" - else - echo " $(basename "$ZSMOOTH_V2") already exists, skipping" - fi - ;; -esac - # DeScratch (core.descratch.DeScratch - vertical scratch removal) # Built from source: the repo carries the VapourSynth + AviSynthPlus headers as # submodules, so a recursive clone is required (build_plugin can't fetch those). @@ -1794,7 +1816,7 @@ cat > "$DEPS_DIR/version.json" << EOF "version": "$EXPECTED_DEPS_VERSION", "installedAt": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")", "platform": "$PLATFORM_DIR", - "architecture": "$ARCH", + "architecture": "$ARCH",$([ -n "$TIER" ] && printf '\n "tier": "%s",' "$TIER") "buildType": "source" } EOF diff --git a/Scripts/download-deps-macos.sh b/Scripts/download-deps-macos.sh index 5abe82f..b7a8fcd 100755 --- a/Scripts/download-deps-macos.sh +++ b/Scripts/download-deps-macos.sh @@ -13,20 +13,25 @@ # - Homebrew (for build tools only, not runtime) # - Xcode Command Line Tools # -# Usage: ./scripts/download-deps-macos.sh [--force] +# Usage: ./scripts/download-deps-macos.sh [--force] [--tier v3|v2] set -e FORCE=false +TIER=v3 while [[ $# -gt 0 ]]; do case $1 in --force) FORCE=true shift ;; + --tier) + TIER="$2" + shift 2 + ;; *) echo "Unknown option: $1" - echo "Usage: $0 [--force]" + echo "Usage: $0 [--force] [--tier v3|v2]" exit 1 ;; esac @@ -43,6 +48,29 @@ else exit 1 fi +# CPU tier (issue #92). The x86 bundle ships twice: v3 for x86-64-v3 CPUs +# (Haswell, 2013, and later) and v2 for anything older, chosen by the app from +# `vapourbox-worker --probe-cpu`. This block is the ONLY place the tier is +# interpreted; everything below reads these variables and never $TIER itself. +# ZSMOOTH_CPU zsmooth's Zig -Dcpu target (upstream has no dispatch) +# MVTOOLS_SOURCE prebuilt: Stefan-Olt's build, whose AVX2 static +# initializers SIGILL at load on a CPU without AVX; +# source-no-avx2: v24 from source with those files stubbed +# out (patches/mvtools-v24-no-avx2.patch) +if [ "$ARCH" = "x86_64" ]; then + case "$TIER" in + v3) ZSMOOTH_CPU=haswell; MVTOOLS_SOURCE=prebuilt ;; + v2) ZSMOOTH_CPU=x86_64_v2; MVTOOLS_SOURCE=source-no-avx2 ;; + *) echo "Unknown tier: $TIER (expected v3 or v2)"; exit 1 ;; + esac +else + if [ "$TIER" != "v3" ]; then + echo "--tier is x86-only; the arm64 bundle is not tiered." + exit 1 + fi + TIER="" +fi + # NOTE: x64 deps are built natively on an Intel Mac / the macos-15-intel CI # runner (uname -m reports x86_64 -> macos-x64). To build x64 deps on an Apple # Silicon Mac instead, run this script translated through Rosetta 2 with an @@ -84,6 +112,20 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" DEPS_DIR="$PROJECT_ROOT/deps/$PLATFORM_DIR" PLUGINS_DIR="$DEPS_DIR/vapoursynth/plugins" + +# Both tiers build into the same deps/ directory (the tier lives in +# version.json, not the path), and most steps skip what already exists. So a +# tier switch without --force would keep the previous tier's MVTools and +# zsmooth and silently produce a mixed bundle. +if [ -n "$TIER" ] && [ "$FORCE" = false ] && [ -f "$DEPS_DIR/version.json" ]; then + EXISTING_TIER=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('tier','v3'))" \ + "$DEPS_DIR/version.json" 2>/dev/null || echo "v3") + if [ "$EXISTING_TIER" != "$TIER" ]; then + echo "ERROR: $DEPS_DIR holds the $EXISTING_TIER bundle; building $TIER over it" + echo "would mix the two. Re-run with --force." + exit 1 + fi +fi PYTHON_DIR="$DEPS_DIR/python" PYTHON_PACKAGES_DIR="$DEPS_DIR/python-packages" BUILD_DIR="/tmp/vapourbox-build-$$" @@ -852,6 +894,48 @@ download_prebuilt_plugin() { return 1 } +# MVTools for the v2 tier: v24 — the same version the v3 tier's prebuilt uses — +# from source, with patches/mvtools-v24-no-avx2.patch. Stefan-Olt's build +# compiles six files with -mavx2, and three of them construct lookup tables at +# load time with VEX instructions, so dlopen itself SIGILLs on a CPU without +# AVX; autoload makes that every job (issue #92). The patch builds none of the +# AVX2 code and masks the AVX2 flag so nothing dispatches to it; the runtime +# dispatch to SSE2/AVX assembly is unchanged. +build_mvtools_no_avx2() { + if [ "$FORCE" = false ] && [ -f "$PLUGINS_DIR/libmvtools.dylib" ]; then + echo " MVTools already exists, skipping" + return 0 + fi + echo ""; echo "=== Building MVTools v24 without AVX2 (x86_64, v2 tier) ===" + cd "$BUILD_DIR" + rm -rf mvtools + git clone --depth 1 --branch v24 https://github.com/dubhater/vapoursynth-mvtools.git mvtools + # Hard failure: an unapplied patch would quietly ship the crashing build. + (cd mvtools && git apply "$SCRIPT_DIR/patches/mvtools-v24-no-avx2.patch") || { + echo " ERROR: patches/mvtools-v24-no-avx2.patch did not apply to MVTools v24." >&2 + exit 1 + } + if PKG_CONFIG_PATH="$VS_PC_DIR:$BREW_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" \ + meson setup mvtools/build mvtools --buildtype=release \ + && ninja -C mvtools/build; then + cp mvtools/build/libmvtools.dylib "$PLUGINS_DIR/libmvtools.dylib" + install_name_tool -id "@loader_path/libmvtools.dylib" "$PLUGINS_DIR/libmvtools.dylib" + # Link against Homebrew's fftw at build time, load the bundled copy. + for dep in $(otool -L "$PLUGINS_DIR/libmvtools.dylib" | awk '/libfftw3f/ {print $1}'); do + install_name_tool -change "$dep" "@loader_path/../../lib/$(basename "$dep")" \ + "$PLUGINS_DIR/libmvtools.dylib" + done + codesign -s - -f "$PLUGINS_DIR/libmvtools.dylib" 2>/dev/null || true + echo " Built MVTools (no AVX2)" + else + echo " Failed to build MVTools" + FAILED_PLUGINS+=("MVTools") + cd "$BUILD_DIR" + return 1 + fi + cd "$BUILD_DIR" +} + STEFANOLT="https://github.com/Stefan-Olt/vs-plugin-build/releases/download/vsplugin" # pkg-config dir + header dir of the from-source VapourSynth install. Used by the @@ -924,7 +1008,11 @@ if [ "$ARCH" = "x86_64" ]; then # ======================================================================== echo "" echo "=== Downloading pre-built x86_64 plugins (Stefan-Olt/vs-plugin-build) ===" - download_prebuilt_plugin "MVTools" "libmvtools.dylib" "$STEFANOLT/com.nodame.mvtools/v24/darwin-x86_64/2024-09-30T17.08.24%2B00.00Z/MVTools-v24-darwin-x86_64.zip" + if [ "$MVTOOLS_SOURCE" = prebuilt ]; then + download_prebuilt_plugin "MVTools" "libmvtools.dylib" "$STEFANOLT/com.nodame.mvtools/v24/darwin-x86_64/2024-09-30T17.08.24%2B00.00Z/MVTools-v24-darwin-x86_64.zip" + else + build_mvtools_no_avx2 + fi download_prebuilt_plugin "ZNEDI3" "libznedi3.dylib" "$STEFANOLT/xxx.abc.znedi3/3bd542a/darwin-x86_64/2026-01-10T23.47.38%2B00.00Z/ZNEDI3-3bd542a-darwin-x86_64.zip" download_prebuilt_plugin "EEDI3m" "libeedi3m.dylib" "$STEFANOLT/com.holywu.eedi3/r8/darwin-x86_64/2026-01-15T20.48.25%2B00.00Z/EEDI3m-r8-darwin-x86_64.zip" download_prebuilt_plugin "fmtconv" "libfmtconv.dylib" "$STEFANOLT/fmtconv/git-18a9cecb/darwin-x86_64/2024-10-10T14.48.08%2B00.00Z/fmtconv-git-18a9cecb-darwin-x86_64.zip" @@ -1489,60 +1577,50 @@ build_plugin "retinex" \ "libretinex.dylib" \ "meson setup build --buildtype=release && ninja -C build" -# zsmooth — one build per CPU baseline +# zsmooth — exactly one build, in the autoload directory # # core.zsmooth.CCD (also Cnr4 and a set of RemoveGrain/TemporalMedian-family -# filters). Upstream publishes only `haswell` (an AVX2 baseline) and `znver4` -# x86 builds, compiled throughout with NO runtime dispatch, so on a pre-2013 -# CPU they die with an illegal instruction the instant a filter runs — issue -# #82, silently, because vspipe prints nothing on a native crash. -# -# x86_64 therefore ships TWO builds outside the autoload directory and the -# worker loads exactly one by path (DependencyLocator::zsmooth_plugin); they -# cannot share a directory, because each registers the namespace `zsmooth` and -# whichever autoloads second is rejected. arm64 has one NEON baseline and needs -# no split. +# filters). zsmooth has NO runtime dispatch: each build is compiled for one CPU +# baseline throughout, so a build above the machine's baseline dies with an +# illegal instruction the instant a filter runs (issue #82). # -# Note this arch was ALREADY affected in the other direction: the x64 build -# below has always been compiled at Zig's default baseline (SSE2), which -# measures 2.0x slower than haswell on CCD and 3.0x on Cnr4. Every Intel Mac -# that can run macOS 12 is at least Nehalem and most are Haswell or newer, so -# building both here makes the common case fast for the first time as well as -# keeping the oldest ones working. +# arm64 has a single NEON baseline and takes the author's build. x64 builds the +# $ZSMOOTH_CPU chosen by the tier block at the top of this script: haswell (the +# fast path) for v3, x86_64_v2 (SSE4.2/POPCNT — every Intel Mac that can run +# macOS 12) for v2. One build per bundle means it autoloads like any other +# plugin; bundles up to 1.10.0 shipped both x64 builds in a separate zsmooth/ +# directory and had the worker load one by path, which the tiers replace. # # Keep ZSMOOTH_VERSION in step across download-deps-{macos,linux}.sh and # download-deps-windows.ps1 — a version skew would make the same job produce # different chroma per OS. ZSMOOTH_VERSION="0.19.0" -ZSMOOTH_DIR="$DEPS_DIR/vapoursynth/zsmooth" -mkdir -p "$ZSMOOTH_DIR" - -if [ "$ARCH" = "arm64" ]; then - # arm64 takes the author's build: it is minos 13, comfortably under this - # arch's 15.0 target. One build, no variants. - if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_DIR/libzsmooth.dylib" ]; then - tmp="$BUILD_DIR/prebuilt-zsmooth" - rm -rf "$tmp"; mkdir -p "$tmp" - zs_url="https://github.com/adworacz/zsmooth/releases/download/${ZSMOOTH_VERSION}/zsmooth-aarch64-macos.zip" - if curl -sL "$zs_url" -o "$tmp/plugin.zip" && unzip -q -o "$tmp/plugin.zip" -d "$tmp"; then - found=$(find "$tmp" -name "*.dylib" -type f 2>/dev/null | head -1) - if [ -n "$found" ]; then - cp "$found" "$ZSMOOTH_DIR/libzsmooth.dylib" - install_name_tool -id "@loader_path/libzsmooth.dylib" "$ZSMOOTH_DIR/libzsmooth.dylib" 2>/dev/null || true - codesign -s - -f "$ZSMOOTH_DIR/libzsmooth.dylib" 2>/dev/null || true - echo " Downloaded pre-built zsmooth" - else - echo " Warning: no dylib in the zsmooth archive" - FAILED_PLUGINS+=("zsmooth") - fi +ZSMOOTH_OUT="$PLUGINS_DIR/libzsmooth.dylib" +rm -rf "$DEPS_DIR/vapoursynth/zsmooth" + +if [ "$FORCE" = false ] && [ -f "$ZSMOOTH_OUT" ]; then + echo " zsmooth already exists, skipping" +elif [ "$ARCH" = "arm64" ]; then + # The author's build: minos 13, comfortably under this arch's 15.0 target. + tmp="$BUILD_DIR/prebuilt-zsmooth" + rm -rf "$tmp"; mkdir -p "$tmp" + zs_url="https://github.com/adworacz/zsmooth/releases/download/${ZSMOOTH_VERSION}/zsmooth-aarch64-macos.zip" + if curl -sL "$zs_url" -o "$tmp/plugin.zip" && unzip -q -o "$tmp/plugin.zip" -d "$tmp"; then + found=$(find "$tmp" -name "*.dylib" -type f 2>/dev/null | head -1) + if [ -n "$found" ]; then + cp "$found" "$ZSMOOTH_OUT" + install_name_tool -id "@loader_path/libzsmooth.dylib" "$ZSMOOTH_OUT" 2>/dev/null || true + codesign -s - -f "$ZSMOOTH_OUT" 2>/dev/null || true + echo " Downloaded pre-built zsmooth" else - echo " Warning: failed to fetch pre-built zsmooth" + echo " Warning: no dylib in the zsmooth archive" FAILED_PLUGINS+=("zsmooth") fi - rm -rf "$tmp" else - echo " zsmooth already exists, skipping" + echo " Warning: failed to fetch pre-built zsmooth" + FAILED_PLUGINS+=("zsmooth") fi + rm -rf "$tmp" else # x64 builds from source, for two reasons: the author's x86_64 build is # minos 13.0 and this bundle targets 12.0, so the minos guard at the end of @@ -1568,66 +1646,54 @@ else *) ZIG_MACOS_MIN="${MACOS_MIN_VERSION}.0" ;; esac - ZIG_BIN="" - # x86_64_v2 is SSE4.2/POPCNT — every Intel Mac that can run macOS 12. - # haswell is the fast path for 2013-and-later machines. Order matters only - # for the log; the worker picks by CPUID at job time. - for zs_target in haswell x86_64_v2; do - out="$ZSMOOTH_DIR/libzsmooth-${zs_target}.dylib" - if [ "$FORCE" = false ] && [ -f "$out" ]; then - echo " zsmooth ($zs_target) already exists, skipping" - continue + echo "" + echo "=== Building zsmooth $ZSMOOTH_CPU (x64, targeting macOS $MACOS_MIN_VERSION) ===" + # Subshell so a failure here can't abort the whole script under `set -e`; + # the file check below decides whether it worked. + ( + set -e + cd "$BUILD_DIR" + rm -rf zig-toolchain zig.tar.xz zsmooth fftw-patched + curl -fsSL -o zig.tar.xz \ + "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-macos-${ZIG_VERSION}.tar.xz" + mkdir -p zig-toolchain + tar -xf zig.tar.xz -C zig-toolchain --strip-components=1 + git clone --depth 1 --branch "$ZSMOOTH_VERSION" \ + https://github.com/adworacz/zsmooth.git zsmooth + + # zsmooth's Zig fftw port declares HAVE_MEMALIGN on every non-Windows + # target, but macOS has no memalign() — it is declared in , + # which the SAME file already knows macOS lacks (HAVE_MALLOC_H is + # gated on !is_mac). fftw's kalloc.c only reaches that branch when + # MIN_ALIGNMENT is 32, i.e. when AVX is enabled, so the bug is + # invisible at the SSE-level baselines and kills ONLY the haswell + # build, with a clang implicit-declaration error inside a dependency. + # HAVE_POSIX_MEMALIGN is already true, so clearing this falls through + # to posix_memalign, which macOS does have. Applied for both tiers: it + # is a correct fix on macOS whichever branch fftw takes. + # + # Patched via a local path dependency rather than by editing Zig's + # global package cache: path deps take no hash, so this is + # deterministic and cannot be invalidated by a cache wipe. + git clone --depth 1 --branch "$FFTW_FORK_TAG" \ + https://github.com/adworacz/fftw.git fftw-patched + if ! grep -q '.HAVE_MEMALIGN = if (!is_windows) true else null,' \ + fftw-patched/build.zig; then + echo " ERROR: the fftw HAVE_MEMALIGN line is not what the patch expects." >&2 + echo " Upstream may have fixed it — re-check before removing this patch." >&2 + exit 1 fi - echo "" - echo "=== Building zsmooth $zs_target (x64, targeting macOS $MACOS_MIN_VERSION) ===" - # Subshell so a failure here can't abort the whole script under `set -e`; - # the file check below decides whether it worked. - ( - set -e - cd "$BUILD_DIR" - if [ -z "$ZIG_BIN" ]; then - rm -rf zig-toolchain zig.tar.xz - curl -fsSL -o zig.tar.xz \ - "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-macos-${ZIG_VERSION}.tar.xz" - mkdir -p zig-toolchain - tar -xf zig.tar.xz -C zig-toolchain --strip-components=1 - fi - rm -rf zsmooth fftw-patched - git clone --depth 1 --branch "$ZSMOOTH_VERSION" \ - https://github.com/adworacz/zsmooth.git zsmooth - - # zsmooth's Zig fftw port declares HAVE_MEMALIGN on every non-Windows - # target, but macOS has no memalign() — it is declared in , - # which the SAME file already knows macOS lacks (HAVE_MALLOC_H is - # gated on !is_mac). fftw's kalloc.c only reaches that branch when - # MIN_ALIGNMENT is 32, i.e. when AVX is enabled, so the bug is - # invisible at the SSE-level baselines and kills ONLY the haswell - # build, with a clang implicit-declaration error inside a dependency. - # HAVE_POSIX_MEMALIGN is already true, so clearing this falls through - # to posix_memalign, which macOS does have. - # - # Patched via a local path dependency rather than by editing Zig's - # global package cache: path deps take no hash, so this is - # deterministic and cannot be invalidated by a cache wipe. - git clone --depth 1 --branch "$FFTW_FORK_TAG" \ - https://github.com/adworacz/fftw.git fftw-patched - if ! grep -q '.HAVE_MEMALIGN = if (!is_windows) true else null,' \ - fftw-patched/build.zig; then - echo " ERROR: the fftw HAVE_MEMALIGN line is not what the patch expects." >&2 - echo " Upstream may have fixed it — re-check before removing this patch." >&2 - exit 1 - fi - # `is_mac` is already defined in that file. - sed -i.bak \ - 's/\.HAVE_MEMALIGN = if (!is_windows) true else null,/.HAVE_MEMALIGN = if (!is_windows and !is_mac) true else null,/' \ - fftw-patched/build.zig - grep -q '.HAVE_MEMALIGN = if (!is_windows and !is_mac) true else null,' \ - fftw-patched/build.zig || { echo " ERROR: fftw memalign patch did not apply" >&2; exit 1; } - - cd zsmooth - # Repoint the fftw dependency at the patched clone. A path dependency - # carries no hash field, so the url+hash pair is replaced wholesale. - "$PYTHON_BIN" - <<'ZONEOF' + # `is_mac` is already defined in that file. + sed -i.bak \ + 's/\.HAVE_MEMALIGN = if (!is_windows) true else null,/.HAVE_MEMALIGN = if (!is_windows and !is_mac) true else null,/' \ + fftw-patched/build.zig + grep -q '.HAVE_MEMALIGN = if (!is_windows and !is_mac) true else null,' \ + fftw-patched/build.zig || { echo " ERROR: fftw memalign patch did not apply" >&2; exit 1; } + + cd zsmooth + # Repoint the fftw dependency at the patched clone. A path dependency + # carries no hash field, so the url+hash pair is replaced wholesale. + "$PYTHON_BIN" - <<'ZONEOF' import io, re p = "build.zig.zon" s = io.open(p, encoding="utf-8").read() @@ -1639,25 +1705,23 @@ io.open(p, "w", encoding="utf-8").write(s) print(" fftw repointed to the patched local clone") ZONEOF - "$BUILD_DIR/zig-toolchain/zig" build \ - -Doptimize=ReleaseFast \ - -Dtarget="x86_64-macos.${ZIG_MACOS_MIN}" \ - -Dcpu="$zs_target" - cp zig-out/lib/libzsmooth.dylib "$out" - ) || true - - if [ -f "$out" ]; then - ZIG_BIN="$BUILD_DIR/zig-toolchain/zig" - install_name_tool -id "@loader_path/$(basename "$out")" "$out" 2>/dev/null || true - codesign -s - -f "$out" 2>/dev/null || true - echo " Built zsmooth -> $(basename "$out")" - else - echo " Warning: failed to build zsmooth ($zs_target)" - FAILED_PLUGINS+=("zsmooth-$zs_target") - fi - rm -rf "$BUILD_DIR/zsmooth" "$BUILD_DIR/fftw-patched" - done - rm -rf "$BUILD_DIR/zig-toolchain" "$BUILD_DIR/zig.tar.xz" + "$BUILD_DIR/zig-toolchain/zig" build \ + -Doptimize=ReleaseFast \ + -Dtarget="x86_64-macos.${ZIG_MACOS_MIN}" \ + -Dcpu="$ZSMOOTH_CPU" + cp zig-out/lib/libzsmooth.dylib "$ZSMOOTH_OUT" + ) || true + + if [ -f "$ZSMOOTH_OUT" ]; then + install_name_tool -id "@loader_path/libzsmooth.dylib" "$ZSMOOTH_OUT" 2>/dev/null || true + codesign -s - -f "$ZSMOOTH_OUT" 2>/dev/null || true + echo " Built zsmooth ($ZSMOOTH_CPU)" + else + echo " Warning: failed to build zsmooth ($ZSMOOTH_CPU)" + FAILED_PLUGINS+=("zsmooth") + fi + rm -rf "$BUILD_DIR/zsmooth" "$BUILD_DIR/fftw-patched" \ + "$BUILD_DIR/zig-toolchain" "$BUILD_DIR/zig.tar.xz" fi # ============================================================================ @@ -1771,7 +1835,7 @@ download_prebuilt_plugin "RemoveDirt" "libremovedirt.dylib" "$REMOVEDIRT_URL" # 15.0, and this bundle's Intel floor is 12.0 with STRICT_MIN_OS=1 — the wheel # would fail the guard and, shipped anyway, would refuse to load on Monterey # (issue #39). It is one C++ file with no SIMD and no dependencies, so the x64 -# branch compiles it directly, exactly as zsmooth splits for the same reason. +# branch compiles it directly, as zsmooth's x64 build does for the same reason. # Wheel 3.0 and git tag v3 are the same release; keep the two in step. DEDOT_VERSION="3.0" DEDOT_TAG="v3" @@ -2311,7 +2375,7 @@ cat > "$DEPS_DIR/version.json" << EOF "version": "$EXPECTED_DEPS_VERSION", "installedAt": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")", "platform": "$PLATFORM_DIR", - "architecture": "$ARCH", + "architecture": "$ARCH",$([ -n "$TIER" ] && printf '\n "tier": "%s",' "$TIER") "buildType": "source" } EOF diff --git a/Scripts/download-deps-windows.ps1 b/Scripts/download-deps-windows.ps1 index 8141295..049321b 100644 --- a/Scripts/download-deps-windows.ps1 +++ b/Scripts/download-deps-windows.ps1 @@ -16,13 +16,20 @@ .PARAMETER TargetDir The target directory for dependencies. Default: deps/windows-x64 +.PARAMETER Tier + CPU tier to build: v3 (x86-64-v3, Haswell and later; the default) or v2 + (anything older). See the tier block below. + .EXAMPLE .\download-deps-windows.ps1 + .\download-deps-windows.ps1 -Tier v2 .\download-deps-windows.ps1 -TargetDir "C:\vapourbox\deps\windows-x64" #> param( - [string]$TargetDir = "deps\windows-x64" + [string]$TargetDir = "deps\windows-x64", + [ValidateSet("v3", "v2")] + [string]$Tier = "v3" ) $ErrorActionPreference = "Stop" @@ -33,10 +40,34 @@ $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $ProjectRoot = Split-Path -Parent $ScriptDir $FullTargetDir = Join-Path $ProjectRoot $TargetDir +# CPU tier (issue #92). The x86 bundle ships twice: v3 for x86-64-v3 CPUs +# (Haswell, 2013, and later) and v2 for anything older, chosen by the app from +# `vapourbox-worker --probe-cpu`. This block is the ONLY place the tier is +# interpreted; everything below reads these variables and never $Tier itself. +# $ZsmoothCpu zsmooth's baseline (upstream has no runtime dispatch) +# That is the whole difference on Windows: every other plugin in this bundle +# renders under Intel SDE emulating Sandy Bridge (AVX without AVX2) — see +# .github/workflows/probe-cpu-compat.yml. CPUs without AVX at all cannot be +# checked there (Microsoft's runtime trips SDE first), so they are unverified. +$ZsmoothCpu = if ($Tier -eq "v3") { "haswell" } else { "x86_64_v2" } + Write-Host "=== VapourBox Windows Dependency Downloader ===" -ForegroundColor Cyan Write-Host "Target directory: $FullTargetDir" +Write-Host "CPU tier: $Tier" Write-Host "" +# Both tiers build into the same directory (the tier lives in version.json, not +# the path), and every step skips what already exists. So building one tier +# over the other would keep the previous zsmooth and silently mix the bundles. +$ExistingVersionFile = Join-Path $FullTargetDir "version.json" +if (Test-Path $ExistingVersionFile) { + $ExistingTier = (Get-Content $ExistingVersionFile -Raw | ConvertFrom-Json).tier + if (-not $ExistingTier) { $ExistingTier = "v3" } + if ($ExistingTier -ne $Tier) { + throw "$FullTargetDir holds the $ExistingTier bundle; building $Tier over it would mix the two. Delete it first." + } +} + # Create directory structure $Directories = @( "$FullTargetDir\vapoursynth\vs-plugins", @@ -396,8 +427,8 @@ $PluginsZip = @( Url = "https://github.com/Khanattila/KNLMeansCL/releases/download/v1.1.1/KNLMeansCL-v1.1.1.zip" Check = "KNLMeansCL.dll" } - # zsmooth is NOT here: it ships as two CPU-specific builds outside the - # autoload directory. See section 4a below. + # zsmooth is NOT here: which build ships depends on the CPU tier. See + # section 4a below. ) foreach ($Plugin in $Plugins7z) { @@ -510,43 +541,37 @@ if ($BadArch.Count -gt 0) { Write-Host " All plugin DLLs are x64" -ForegroundColor Green # ============================================================================= -# 4a. zsmooth — one build per CPU baseline +# 4a. zsmooth — exactly one build, in vs-plugins # ============================================================================= # core.zsmooth.CCD (also Cnr4 and a set of RemoveGrain/TemporalMedian-family -# filters). Upstream publishes only `haswell` (an AVX2 baseline) and `znver4` -# builds, compiled throughout with NO runtime dispatch — so on a pre-2013 x86 -# CPU the DLL loads fine and then dies with an illegal instruction -# (0xC000001D) the instant a filter runs. That is issue #82, reported on a -# Celeron J4105 and a Core i7 870, and it is silent: vspipe prints nothing, so -# the encode surfaces as ffmpeg reading an empty pipe. +# filters). zsmooth has NO runtime dispatch: each build is compiled for one CPU +# baseline throughout, so a build above the machine's baseline loads fine and +# then dies with an illegal instruction (0xC000001D) the instant a filter runs. +# That is issue #82, reported on a Celeron J4105 and a Core i7 870, and it is +# silent: vspipe prints nothing, so the encode surfaces as ffmpeg reading an +# empty pipe. # -# Both builds are therefore shipped and the worker loads exactly one by path -# (DependencyLocator::zsmooth_plugin). They cannot both sit in vs-plugins — -# each registers the namespace `zsmooth`, so whichever autoloads second is -# rejected — hence the separate directory, which is deliberately not on the -# plugin path. -# -# Why not just ship the portable build for everyone: measured on this plugin at -# 720x576, `x86_64` is 2.0x slower than haswell on CCD and 3.0x on Cnr4 -# (`x86_64_v2` 1.4x and 1.4x). Paying that on every modern machine to serve the -# rare old one is the wrong trade; picking at runtime costs ~4 MB of zip. +# The tier block at the top picks $ZsmoothCpu: the author's haswell build for +# v3, or an x86_64_v2 build compiled here for v2 (SSE4.2/POPCNT, everything +# from Nehalem 2009 on; measured 1.4x slower than haswell on CCD and Cnr4, +# against 2-3x for plain x86_64). One build per bundle means it loads from +# vs-plugins like any other plugin; bundles up to 1.10.0 shipped both builds in +# a separate zsmooth\ directory for the worker to load by path. # # Keep ZSMOOTH_VERSION in step with download-deps-{macos,linux}.sh: a skew # would make the same job produce different chroma per OS. Write-Host "" -Write-Host "[4a/8] Installing zsmooth (per-CPU builds)..." -ForegroundColor Yellow +Write-Host "[4a/8] Installing zsmooth ($ZsmoothCpu)..." -ForegroundColor Yellow $ZsmoothVersion = "0.19.0" # Must satisfy zsmooth's build.zig.zon `minimum_zig_version`; 0.15.2 for 0.19.0. $ZigVersion = "0.15.2" -$ZsmoothDir = "$FullTargetDir\vapoursynth\zsmooth" -if (-not (Test-Path $ZsmoothDir)) { - New-Item -ItemType Directory -Force -Path $ZsmoothDir | Out-Null -} +$ZsmoothOut = "$PluginsDir\zsmooth.dll" +Remove-Item "$FullTargetDir\vapoursynth\zsmooth" -Recurse -Force -ErrorAction SilentlyContinue -# The AVX2 build comes pre-built from upstream. -$HaswellPath = "$ZsmoothDir\zsmooth-haswell.dll" -if (-not (Test-Path $HaswellPath)) { +if (Test-Path $ZsmoothOut) { + Write-Host " zsmooth.dll already installed" -ForegroundColor Gray +} elseif ($ZsmoothCpu -eq "haswell") { Write-Host " Downloading zsmooth $ZsmoothVersion (haswell/AVX2)..." -ForegroundColor Gray try { $ZsZip = Join-Path $TempDir "zsmooth-haswell.zip" @@ -555,24 +580,19 @@ if (-not (Test-Path $HaswellPath)) { Expand-Archive -Path $ZsZip -DestinationPath $ZsExtract -Force $Dll = Get-ChildItem -Path $ZsExtract -Recurse -Filter "zsmooth.dll" | Select-Object -First 1 if (-not $Dll) { throw "no zsmooth.dll in the upstream archive" } - Copy-Item $Dll.FullName $HaswellPath -Force + Copy-Item $Dll.FullName $ZsmoothOut -Force Remove-Item $ZsZip -Force -ErrorAction SilentlyContinue Remove-Item $ZsExtract -Recurse -Force -ErrorAction SilentlyContinue - Write-Host " Installed: zsmooth-haswell.dll" -ForegroundColor Gray + Write-Host " Installed: zsmooth.dll (haswell)" -ForegroundColor Gray } catch { Write-Host " Failed: $_" -ForegroundColor Red } } else { - Write-Host " zsmooth-haswell.dll already installed" -ForegroundColor Gray -} - -# The portable build has no upstream asset and must be compiled. This is the -# first from-source build in this script; Zig cross-compiles with its own libc -# and builds zsmooth's fftw dependency itself, so it needs no MSVC — only git -# and network access (zig build fetches zsmooth's own Zig dependencies). -$V2Path = "$ZsmoothDir\zsmooth-x86_64_v2.dll" -if (-not (Test-Path $V2Path)) { - Write-Host " Building zsmooth $ZsmoothVersion (x86_64_v2, runs without AVX2)..." -ForegroundColor Gray + # The portable build has no upstream asset and must be compiled. Zig + # cross-compiles with its own libc and builds zsmooth's fftw dependency + # itself, so it needs no MSVC — only git and network access (zig build + # fetches zsmooth's own Zig dependencies). + Write-Host " Building zsmooth $ZsmoothVersion ($ZsmoothCpu, runs without AVX2)..." -ForegroundColor Gray try { $ZigDir = Join-Path $TempDir "zig-toolchain" $ZigZip = Join-Path $TempDir "zig.zip" @@ -595,10 +615,7 @@ if (-not (Test-Path $V2Path)) { Push-Location $ZsSrc try { - # -Dcpu=x86_64_v2 is SSE4.2/POPCNT: everything from Nehalem (2009) - # on, which covers both CPUs in issue #82. Plain `x86_64` would add - # pre-2009 chips at roughly half the CCD/Cnr4 throughput again. - & $ZigExe build -Doptimize=ReleaseFast -Dtarget=x86_64-windows-gnu -Dcpu=x86_64_v2 + & $ZigExe build -Doptimize=ReleaseFast -Dtarget=x86_64-windows-gnu -Dcpu=$ZsmoothCpu if ($LASTEXITCODE -ne 0) { throw "zig build failed (exit $LASTEXITCODE)" } } finally { Pop-Location @@ -606,26 +623,21 @@ if (-not (Test-Path $V2Path)) { $Built = Get-ChildItem -Path (Join-Path $ZsSrc "zig-out") -Recurse -Filter "zsmooth.dll" | Select-Object -First 1 if (-not $Built) { throw "zig build produced no zsmooth.dll" } - Copy-Item $Built.FullName $V2Path -Force + Copy-Item $Built.FullName $ZsmoothOut -Force Remove-Item $ZigZip -Force -ErrorAction SilentlyContinue Remove-Item $ZigDir, $ZsSrc -Recurse -Force -ErrorAction SilentlyContinue - Write-Host " Built: zsmooth-x86_64_v2.dll" -ForegroundColor Gray + Write-Host " Built: zsmooth.dll ($ZsmoothCpu)" -ForegroundColor Gray } catch { Write-Host " Failed: $_" -ForegroundColor Red } -} else { - Write-Host " zsmooth-x86_64_v2.dll already installed" -ForegroundColor Gray } -# A missing build here is not a warning to scroll past: without the AVX2 one -# every modern machine loses the pass, and without the portable one issue #82 -# comes straight back. deps-expected-plugins.json also covers both, so the -# packaging step would fail — this just fails nearer the cause. -$MissingZsmooth = @(@($HaswellPath, $V2Path) | Where-Object { -not (Test-Path $_) }) -if ($MissingZsmooth.Count -gt 0) { - throw "zsmooth build(s) missing: $(($MissingZsmooth | Split-Path -Leaf) -join ', ')" +# Fail near the cause rather than at packaging: without zsmooth the Chroma +# Denoise pass is gone on every machine. +if (-not (Test-Path $ZsmoothOut)) { + throw "zsmooth ($ZsmoothCpu) is missing" } -Write-Host " zsmooth: both CPU builds present" -ForegroundColor Green +Write-Host " zsmooth: $ZsmoothCpu build present" -ForegroundColor Green # ============================================================================= # 4b. FFTW Library (required by DFTTest) @@ -1135,9 +1147,10 @@ $ExpectedVersion = (Get-Content $DepsVersionJson -Raw | ConvertFrom-Json).versio version = $ExpectedVersion installedAt = (Get-Date).ToUniversalTime().ToString("o") platform = "windows-x64" + tier = $Tier buildType = "source" } | ConvertTo-Json | Set-Content -Path "$FullTargetDir\version.json" -Encoding utf8 -Write-Host " version.json written ($ExpectedVersion)" -ForegroundColor Green +Write-Host " version.json written ($ExpectedVersion, $Tier)" -ForegroundColor Green # ============================================================================= # 8. Cleanup diff --git a/Scripts/package-deps-linux.sh b/Scripts/package-deps-linux.sh index 2151e84..05bfd12 100755 --- a/Scripts/package-deps-linux.sh +++ b/Scripts/package-deps-linux.sh @@ -54,7 +54,17 @@ PACKAGE_INCOMPLETE="" package_arch() { local ARCH_NAME=$1 local DEPS_DIR="$PROJECT_ROOT/deps/linux-$ARCH_NAME" - local PACKAGE_NAME="VapourBox-deps-$VERSION-linux-$ARCH_NAME" + # The CPU tier (issue #92) is whatever download-deps-linux.sh stamped into + # the build's version.json; only x64 is tiered. v3 keeps the plain asset + # name, v2 is suffixed — the same rule as DependencyManager.assetIdFor. + local TIER="" + if [ "$ARCH_NAME" = "x64" ]; then + TIER=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('tier','v3'))" \ + "$DEPS_DIR/version.json" 2>/dev/null || echo "v3") + fi + local PLATFORM_KEY="linux-$ARCH_NAME" + [ "$TIER" = "v2" ] && PLATFORM_KEY="linux-$ARCH_NAME-v2" + local PACKAGE_NAME="VapourBox-deps-$VERSION-$PLATFORM_KEY" local PACKAGE_DIR="$DIST_DIR/$PACKAGE_NAME" echo "[1/4] Checking prerequisites for $ARCH_NAME..." @@ -121,7 +131,7 @@ package_arch() { echo " Creating version file..." cat > "$PACKAGE_DIR/version.json" << EOF { - "version": "$VERSION", + "version": "$VERSION",$([ -n "$TIER" ] && printf '\n "tier": "%s",' "$TIER") "installedAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" } EOF @@ -135,30 +145,27 @@ EOF # Completeness guard: every required plugin must be present before we zip, # so a silently-failed build/download can't ship an incomplete bundle. # Contract: Scripts/deps-expected-plugins.json. - echo " Verifying required plugins for $ARCH_NAME..." + echo " Verifying required plugins for $PLATFORM_KEY..." local MANIFEST="$PROJECT_ROOT/Scripts/deps-expected-plugins.json" local MISSING - MISSING=$(python3 - "$MANIFEST" "linux-$ARCH_NAME" "$PACKAGE_DIR/vapoursynth/plugins" "$PACKAGE_DIR" <<'PY' + MISSING=$(python3 - "$MANIFEST" "$PLATFORM_KEY" "$PACKAGE_DIR/vapoursynth/plugins" <<'PY' import json, os, sys -manifest, key, plugin_dir, bundle_root = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] -expected = json.load(open(manifest)).get(key, []) -# An entry with a "/" is bundle-root-relative, not a plugin-directory filename: -# zsmooth ships one build per CPU baseline OUTSIDE the autoload directory (both -# register the same namespace, so the worker loads exactly one by path), and a -# guard that only looked in plugins/ would stop covering it. -def target(f): - return os.path.join(bundle_root, *f.split("/")) if "/" in f else os.path.join(plugin_dir, f) -print("\n".join(f for f in expected if not os.path.isfile(target(f)))) +manifest, key, plugin_dir = sys.argv[1], sys.argv[2], sys.argv[3] +expected = json.load(open(manifest)).get(key) +if expected is None: + print(f"(no {key} entry in deps-expected-plugins.json)") + sys.exit() +print("\n".join(f for f in expected if not os.path.isfile(os.path.join(plugin_dir, f)))) PY ) if [ -n "$MISSING" ]; then - echo "ERROR: linux-$ARCH_NAME bundle is missing required plugins:" >&2 + echo "ERROR: $PLATFORM_KEY bundle is missing required plugins:" >&2 echo "$MISSING" | sed 's/^/ - /' >&2 echo "A plugin build/download likely failed - check the download-deps-linux.sh output." >&2 - PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE linux-$ARCH_NAME" + PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE $PLATFORM_KEY" return 1 fi - echo " All required plugins present for $ARCH_NAME" + echo " All required plugins present for $PLATFORM_KEY" echo "[4/4] Creating zip archive for $ARCH_NAME..." local ZIP_FILE="$DIST_DIR/$PACKAGE_NAME.zip" diff --git a/Scripts/package-deps-macos.sh b/Scripts/package-deps-macos.sh index 6ed9c78..f8851b9 100755 --- a/Scripts/package-deps-macos.sh +++ b/Scripts/package-deps-macos.sh @@ -54,7 +54,17 @@ PACKAGE_INCOMPLETE="" package_arch() { local ARCH_NAME=$1 local DEPS_DIR="$PROJECT_ROOT/deps/macos-$ARCH_NAME" - local PACKAGE_NAME="VapourBox-deps-$VERSION-macos-$ARCH_NAME" + # The CPU tier (issue #92) is whatever download-deps-macos.sh stamped into + # the build's version.json; only x64 is tiered. v3 keeps the plain asset + # name, v2 is suffixed — the same rule as DependencyManager.assetIdFor. + local TIER="" + if [ "$ARCH_NAME" = "x64" ]; then + TIER=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('tier','v3'))" \ + "$DEPS_DIR/version.json" 2>/dev/null || echo "v3") + fi + local PLATFORM_KEY="macos-$ARCH_NAME" + [ "$TIER" = "v2" ] && PLATFORM_KEY="macos-$ARCH_NAME-v2" + local PACKAGE_NAME="VapourBox-deps-$VERSION-$PLATFORM_KEY" local PACKAGE_DIR="$DIST_DIR/$PACKAGE_NAME" echo "[1/4] Checking prerequisites for $ARCH_NAME..." @@ -126,7 +136,7 @@ package_arch() { echo " Creating version file..." cat > "$PACKAGE_DIR/version.json" << EOF { - "version": "$VERSION", + "version": "$VERSION",$([ -n "$TIER" ] && printf '\n "tier": "%s",' "$TIER") "installedAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" } EOF @@ -142,30 +152,41 @@ EOF # Completeness guard: every required plugin must be present before we zip, # so a silently-failed build/download can't ship an incomplete bundle. # Contract: Scripts/deps-expected-plugins.json. - echo " Verifying required plugins for $ARCH_NAME..." + echo " Verifying required plugins for $PLATFORM_KEY..." local MANIFEST="$PROJECT_ROOT/Scripts/deps-expected-plugins.json" local MISSING - MISSING=$(python3 - "$MANIFEST" "macos-$ARCH_NAME" "$PACKAGE_DIR/vapoursynth/plugins" "$PACKAGE_DIR" <<'PY' + MISSING=$(python3 - "$MANIFEST" "$PLATFORM_KEY" "$PACKAGE_DIR/vapoursynth/plugins" <<'PY' import json, os, sys -manifest, key, plugin_dir, bundle_root = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] -expected = json.load(open(manifest)).get(key, []) -# An entry with a "/" is bundle-root-relative, not a plugin-directory filename: -# zsmooth ships one build per CPU baseline OUTSIDE the autoload directory (both -# register the same namespace, so the worker loads exactly one by path), and a -# guard that only looked in plugins/ would stop covering it. -def target(f): - return os.path.join(bundle_root, *f.split("/")) if "/" in f else os.path.join(plugin_dir, f) -print("\n".join(f for f in expected if not os.path.isfile(target(f)))) +manifest, key, plugin_dir = sys.argv[1], sys.argv[2], sys.argv[3] +expected = json.load(open(manifest)).get(key) +if expected is None: + print(f"(no {key} entry in deps-expected-plugins.json)") + sys.exit() +print("\n".join(f for f in expected if not os.path.isfile(os.path.join(plugin_dir, f)))) PY ) if [ -n "$MISSING" ]; then - echo "ERROR: macos-$ARCH_NAME bundle is missing required plugins:" >&2 + echo "ERROR: $PLATFORM_KEY bundle is missing required plugins:" >&2 echo "$MISSING" | sed 's/^/ - /' >&2 echo "A plugin build/download likely failed - check the download-deps-macos.sh output." >&2 - PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE macos-$ARCH_NAME" + PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE $PLATFORM_KEY" return 1 fi - echo " All required plugins present for $ARCH_NAME" + echo " All required plugins present for $PLATFORM_KEY" + + # The v2 bundle's whole promise is that it loads on a CPU without AVX, and + # macOS has no Intel SDE to prove that by running it. So prove statically + # that no plugin can execute a VEX instruction while it loads — the fault + # issue #92 was: MVTools' AVX2 static initializers SIGILL inside dlopen. + if [ "$TIER" = "v2" ]; then + echo " Checking no plugin runs AVX while loading (v2 tier)..." + if ! python3 "$PROJECT_ROOT/Scripts/check-load-time-simd.py" \ + "$PACKAGE_DIR"/vapoursynth/plugins/*.dylib; then + echo "ERROR: a plugin in the $PLATFORM_KEY bundle runs AVX at load time" >&2 + PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE $PLATFORM_KEY" + return 1 + fi + fi echo "[4/4] Creating zip archive for $ARCH_NAME..." local ZIP_FILE="$DIST_DIR/$PACKAGE_NAME.zip" diff --git a/Scripts/package-deps-windows.ps1 b/Scripts/package-deps-windows.ps1 index c91ec44..3918a43 100644 --- a/Scripts/package-deps-windows.ps1 +++ b/Scripts/package-deps-windows.ps1 @@ -17,17 +17,29 @@ $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $ProjectRoot = Split-Path -Parent $ScriptDir $DistDir = Join-Path $ProjectRoot "dist" $AppName = "VapourBox" -$PackageName = "$AppName-deps-$Version-windows-x64" +$DepsDir = Join-Path $ProjectRoot "deps\windows-x64" + +# The CPU tier (issue #92) is whatever download-deps-windows.ps1 stamped into the +# build's version.json. v3 keeps the plain asset name, v2 is suffixed — the same +# rule as DependencyManager.assetIdFor. A build from before tiering is v3. +$Tier = "v3" +$BuiltVersionFile = Join-Path $DepsDir "version.json" +if (Test-Path $BuiltVersionFile) { + $Stamped = (Get-Content $BuiltVersionFile -Raw | ConvertFrom-Json).tier + if ($Stamped) { $Tier = $Stamped } +} +$PlatformKey = if ($Tier -eq "v2") { "windows-x64-v2" } else { "windows-x64" } +$PackageName = "$AppName-deps-$Version-$PlatformKey" $PackageDir = Join-Path $DistDir $PackageName Write-Host "=== Packaging VapourBox Dependencies for Windows ===" -ForegroundColor Cyan Write-Host "Version: $Version" +Write-Host "CPU tier: $Tier ($PlatformKey)" Write-Host "" # Check prerequisites Write-Host "[1/5] Checking prerequisites..." -ForegroundColor Yellow -$DepsDir = Join-Path $ProjectRoot "deps\windows-x64" if (-not (Test-Path $DepsDir)) { Write-Host "ERROR: Dependencies not found at $DepsDir" -ForegroundColor Red Write-Host "Run '.\Scripts\download-deps-windows.ps1' first" -ForegroundColor Red @@ -141,6 +153,7 @@ Copy-Item (Join-Path $DepsDir "ffmpeg\ffprobe.exe") "$PackageDir\ffmpeg\" -Error Write-Host " Creating version file..." $VersionInfo = @{ version = $Version + tier = $Tier installedAt = (Get-Date).ToString("o") } | ConvertTo-Json -Depth 10 Set-Content -Path "$PackageDir\version.json" -Value $VersionInfo @@ -150,16 +163,13 @@ Set-Content -Path "$PackageDir\version.json" -Value $VersionInfo # otherwise ship an incomplete bundle. Contract: Scripts/deps-expected-plugins.json. Write-Host "[4b/5] Verifying required plugins..." -ForegroundColor Yellow $ManifestPath = Join-Path $ProjectRoot "Scripts\deps-expected-plugins.json" -$ExpectedPlugins = (Get-Content $ManifestPath -Raw | ConvertFrom-Json)."windows-x64" +$ExpectedPlugins = (Get-Content $ManifestPath -Raw | ConvertFrom-Json).$PlatformKey +if (-not $ExpectedPlugins) { + Write-Host "ERROR: no $PlatformKey entry in deps-expected-plugins.json" -ForegroundColor Red + exit 1 +} $StagedPluginDir = Join-Path "$PackageDir\vapoursynth" "vs-plugins" -# An entry with a '/' is bundle-root-relative, not a plugin-directory filename. -# zsmooth ships one build per CPU baseline outside the autoload directory (both -# register the same namespace, so the worker loads exactly one by path), and a -# guard that only ever looked in vs-plugins would stop covering it. -$MissingPlugins = @($ExpectedPlugins | Where-Object { - $Target = if ($_ -match '/') { Join-Path $PackageDir ($_ -replace '/', '\') } else { Join-Path $StagedPluginDir $_ } - -not (Test-Path $Target) -}) +$MissingPlugins = @($ExpectedPlugins | Where-Object { -not (Test-Path (Join-Path $StagedPluginDir $_)) }) if ($MissingPlugins.Count -gt 0) { Write-Host "ERROR: bundle is missing $($MissingPlugins.Count) required plugin(s):" -ForegroundColor Red $MissingPlugins | ForEach-Object { Write-Host " - $_" -ForegroundColor Red } diff --git a/Scripts/patches/mvtools-v24-no-avx2.patch b/Scripts/patches/mvtools-v24-no-avx2.patch new file mode 100644 index 0000000..e19b26c --- /dev/null +++ b/Scripts/patches/mvtools-v24-no-avx2.patch @@ -0,0 +1,120 @@ +diff --git a/meson.build b/meson.build +index f24a406..63b0298 100644 +--- a/meson.build ++++ b/meson.build +@@ -150,18 +150,13 @@ if host_cpu_family.startswith('x86') + + + libavx2_sources = [ +- 'src/MaskFun_AVX2.cpp', +- 'src/MVDegrains_AVX2.cpp', +- 'src/MVFrame_AVX2.cpp', +- 'src/Overlap_AVX2.cpp', +- 'src/SADFunctions_AVX2.cpp', +- 'src/SimpleResize_AVX2.cpp', ++ 'src/AVX2_stubs.cpp', + ] + + helper_libs += static_library('avx2', + libavx2_sources, + dependencies: vapoursynth_dep, +- cpp_args: [cflags, '-mavx2', '-mtune=haswell'], ++ cpp_args: cflags, + install: false) + endif + +diff --git a/src/AVX2_stubs.cpp b/src/AVX2_stubs.cpp +new file mode 100644 +index 0000000..96a5503 +--- /dev/null ++++ b/src/AVX2_stubs.cpp +@@ -0,0 +1,77 @@ ++// VapourBox v2-tier build: the AVX2 translation units are replaced by these ++// stubs, compiled at the baseline like everything else (issue #92). ++// ++// Upstream builds six files with -mavx2. Their static initializers (the ++// kernel lookup tables) then contain VEX instructions, which run inside dlopen, ++// before any CPU check, and fault with SIGILL on every CPU without AVX. The ++// shared template code those files instantiate can also win the linker's ++// choice of copy and leak AVX2 into callers that are not guarded at all. ++// ++// So the v2 build contains no AVX2-compiled code, and EntryPoint.c masks ++// X264_CPU_AVX2 out of g_cpuinfo so no dispatch site ever selects these. They ++// abort rather than do nothing: reaching one means the mask was lost, and a ++// silent no-op would render blank frames instead of failing. ++ ++#include ++#include ++#include ++ ++#include "MaskFun.h" ++#include "MVDegrains.h" ++#include "Overlap.h" ++#include "SADFunctions.h" ++#include "SimpleResize.h" ++ ++[[noreturn]] static void mvtools_avx2_unreachable(const char *name) { ++ std::fprintf(stderr, "MVTools (VapourBox v2 build): AVX2 path %s reached; " ++ "the X264_CPU_AVX2 mask in EntryPoint.c is missing\n", name); ++ std::abort(); ++} ++ ++void selectFlowInterFunctions_AVX2(FlowInterSimpleFunction *, FlowInterFunction *, FlowInterExtraFunction *, int) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++DenoiseFunction selectDegrainFunctionAVX2(unsigned, unsigned, unsigned, unsigned) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++OverlapsFunction selectOverlapsFunctionAVX2(unsigned, unsigned, unsigned) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++SADFunction selectSADFunctionAVX2(unsigned, unsigned, unsigned) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void Average2_avx2(uint8_t *, const uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void VerticalBilinear_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void HorizontalBilinear_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void DiagonalBilinear_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void VerticalWiener_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void HorizontalWiener_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void simpleResize_uint8_t_avx2(const SimpleResize *, uint8_t *, int, const uint8_t *, int, int) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void simpleResize_int16_t_avx2(const SimpleResize *, int16_t *, int, const int16_t *, int, int) { ++ mvtools_avx2_unreachable(__func__); ++} +diff --git a/src/EntryPoint.c b/src/EntryPoint.c +index 6b34eab..ca30663 100644 +--- a/src/EntryPoint.c ++++ b/src/EntryPoint.c +@@ -44,5 +44,6 @@ VapourSynthPluginInit(VSConfigPlugin configFunc, VSRegisterFunction registerFunc + mvscdetectionRegister(registerFunc, plugin); + mvdepanRegister(registerFunc, plugin); + +- g_cpuinfo = cpu_detect(); ++ // VapourBox v2 tier: no AVX2 code is built (see AVX2_stubs.cpp). ++ g_cpuinfo = cpu_detect() & ~X264_CPU_AVX2; + } diff --git a/Scripts/probe-plugin-compat.py b/Scripts/probe-plugin-compat.py index 50aafd9..643d82c 100755 --- a/Scripts/probe-plugin-compat.py +++ b/Scripts/probe-plugin-compat.py @@ -13,7 +13,8 @@ Usage: python probe-plugin-compat.py [DEPS_DIR] [--report FILE] [--json FILE] - [--wrap "sde64 -nhm --"] [--fail-on-crash] + [--wrap "sde64 -nhm --"] [--extra FILE] + [--fail-on-crash] DEPS_DIR defaults to the installed bundle for this OS. Run it with the bundle's own Python (see probe-plugin-compat.sh) so no system Python is needed. @@ -271,6 +272,8 @@ def main(): ap.add_argument("--report", default=None, help="text report path") ap.add_argument("--json", default=None, help="machine-readable results path") ap.add_argument("--wrap", default="", help='prefix for each test, e.g. "sde64 -nhm --"') + ap.add_argument("--extra", action="append", default=[], metavar="FILE", + help="also test this plugin file (e.g. a known-bad control build)") ap.add_argument("--timeout", type=int, default=600) ap.add_argument("--fail-on-crash", action="store_true", help="exit 1 if anything crashed, errored or was inconclusive (for CI gates)") @@ -281,6 +284,7 @@ def main(): sys.exit(f"No VapourBox deps bundle found at {deps}\n" "Pass the path to your deps folder as the first argument.") python, env, files = bundle_layout(deps) + files += [os.path.abspath(f) for f in args.extra] wrap = shlex.split(args.wrap) child_src = f"RENDER = {RENDER!r}\n" + CHILD diff --git a/Scripts/release.sh b/Scripts/release.sh index 51b7c0a..12971ad 100755 --- a/Scripts/release.sh +++ b/Scripts/release.sh @@ -216,94 +216,25 @@ fi echo "" -# Step 1: Package and release dependencies if changed +# Step 1: Dependencies, if changed. +# +# Deps are built and published by CI only. The x64 bundles ship in two CPU +# tiers (issue #92), each built into the same deps/ directory, so a +# local tree only ever holds ONE tier of each — packaging it here would publish +# a release missing the other, and every machine on that tier would fail to +# download. (Windows deps cannot be built on macOS at all.) The build-deps-* +# workflows build every tier, gate the v2 bundles, and upload to the release. if $DEPS_CHANGED; then - echo -e "${BLUE}[1/6] Packaging dependencies...${NC}" - - # Package macOS deps - if [ -d "$PROJECT_ROOT/deps/macos-arm64" ] || [ -d "$PROJECT_ROOT/deps/macos-x64" ]; then - "$SCRIPT_DIR/package-deps-macos.sh" --version "$DEPS_VERSION" --arch both || true - fi - - # Package Windows deps (if on Windows or deps exist) - if [ -d "$PROJECT_ROOT/deps/windows-x64" ]; then - echo -e "${YELLOW}Windows deps found. Package manually on Windows or copy existing.${NC}" - # On macOS we can still create the zip if deps directory exists - WINDOWS_PACKAGE_DIR="$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-windows-x64" - rm -rf "$WINDOWS_PACKAGE_DIR" - mkdir -p "$WINDOWS_PACKAGE_DIR" - cp -r "$PROJECT_ROOT/deps/windows-x64/"* "$WINDOWS_PACKAGE_DIR/" - - # Create version file - cat > "$WINDOWS_PACKAGE_DIR/version.json" << EOF -{ - "version": "$DEPS_VERSION", - "installedAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" -} -EOF - - # Create zip - cd "$PROJECT_ROOT/dist" - zip -r -q "VapourBox-deps-$DEPS_VERSION-windows-x64.zip" "VapourBox-deps-$DEPS_VERSION-windows-x64" - rm -rf "$WINDOWS_PACKAGE_DIR" - echo "Created: dist/VapourBox-deps-$DEPS_VERSION-windows-x64.zip" - - # Integrity sidecar (matches package-deps-windows.ps1 output) so the app - # can verify the download. This manual path bypasses the PowerShell - # packager, so write the sidecar here too. - WIN_ZIP="$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-windows-x64.zip" - WIN_SHA=$(shasum -a 256 "$WIN_ZIP" | cut -d' ' -f1) - WIN_SIZE=$(stat -f%z "$WIN_ZIP" 2>/dev/null || stat -c%s "$WIN_ZIP") - cat > "$WIN_ZIP.sha256.json" << EOF -{ - "filename": "VapourBox-deps-$DEPS_VERSION-windows-x64.zip", - "sha256": "$WIN_SHA", - "size": $WIN_SIZE, - "version": "$DEPS_VERSION" -} -EOF - fi - - # Package Linux deps (if deps exist) - if [ -d "$PROJECT_ROOT/deps/linux-x64" ] || [ -d "$PROJECT_ROOT/deps/linux-arm64" ]; then - "$SCRIPT_DIR/package-deps-linux.sh" --version "$DEPS_VERSION" --arch both || true - fi - + echo -e "${RED}Dependencies changed: build and publish them with CI first.${NC}" echo "" - echo -e "${BLUE}[2/6] Creating deps release on GitHub...${NC}" - - # Create deps release - DEPS_NOTES="## VapourBox Dependencies $DEPS_VERSION - -This release contains pre-built dependencies for VapourBox. - -### Contents -- VapourSynth portable with plugins -- FFmpeg -- Python packages (havsfunc, mvsfunc, etc.) - -### Downloads -- \`VapourBox-deps-$DEPS_VERSION-windows-x64.zip\` - Windows x64 -- \`VapourBox-deps-$DEPS_VERSION-macos-arm64.zip\` - macOS Apple Silicon -- \`VapourBox-deps-$DEPS_VERSION-macos-x64.zip\` - macOS Intel - -These dependencies are automatically downloaded by the app on first launch." - - gh release create "$DEPS_TAG" \ - --repo "$GITHUB_REPO" \ - --title "Dependencies $DEPS_VERSION" \ - --notes "$DEPS_NOTES" \ - --latest=false \ - "$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-"*.zip \ - "$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-"*.zip.sha256.json 2>/dev/null || { - echo -e "${YELLOW}Uploading assets to existing release...${NC}" - for f in "$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-"*.zip \ - "$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-"*.zip.sha256.json; do - [ -f "$f" ] && gh release upload "$DEPS_TAG" "$f" --repo "$GITHUB_REPO" --clobber - done - } - - echo -e "${GREEN}Deps release created: $DEPS_TAG${NC}" + echo " gh release create $DEPS_TAG --repo $GITHUB_REPO --title \"VapourBox deps $DEPS_VERSION\" \\" + echo " --notes \"Dependencies $DEPS_VERSION\" --latest=false" + echo " gh workflow run build-deps-macos.yml -f version=$DEPS_VERSION -f release_tag=$DEPS_TAG -f arch=both" + echo " gh workflow run build-deps-windows.yml -f version=$DEPS_VERSION -f release_tag=$DEPS_TAG" + echo " gh workflow run build-deps-linux.yml -f version=$DEPS_VERSION -f release_tag=$DEPS_TAG -f arch=both" + echo "" + echo "Then re-run this script with --skip-deps-check." + exit 1 else echo -e "${BLUE}[1/6] Skipping deps packaging (unchanged)${NC}" echo -e "${BLUE}[2/6] Skipping deps release (unchanged)${NC}" diff --git a/app/assets/deps-version.json b/app/assets/deps-version.json index 4477365..ebe26f8 100644 --- a/app/assets/deps-version.json +++ b/app/assets/deps-version.json @@ -1,6 +1,6 @@ { - "version": "1.10.0", - "releaseTag": "deps-v1.10.0", - "releaseDate": "2026-08-28", + "version": "1.11.0", + "releaseTag": "deps-v1.11.0", + "releaseDate": "2026-09-25", "githubRepo": "StuartCameronCode/VapourBox" } From 2243ffc2264795e8fbe72cfb87eb7c72f58ed18e Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sat, 26 Sep 2026 10:40:09 +1000 Subject: [PATCH 7/9] Windows deps: quote zig's -Dcpu so the tier's CPU is actually passed PowerShell handed zig the literal text "$ZsmoothCpu": a bare native-command argument that starts with a dash is not expanded, so the v2 build failed with "unknown CPU". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014GLXdGLfPwgYjW1AkonGqN --- Scripts/download-deps-windows.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Scripts/download-deps-windows.ps1 b/Scripts/download-deps-windows.ps1 index 049321b..a6077f7 100644 --- a/Scripts/download-deps-windows.ps1 +++ b/Scripts/download-deps-windows.ps1 @@ -615,7 +615,9 @@ if (Test-Path $ZsmoothOut) { Push-Location $ZsSrc try { - & $ZigExe build -Doptimize=ReleaseFast -Dtarget=x86_64-windows-gnu -Dcpu=$ZsmoothCpu + # Quoted: PowerShell passes a bare -Dcpu=$ZsmoothCpu to a native command + # literally, unexpanded. + & $ZigExe build -Doptimize=ReleaseFast -Dtarget=x86_64-windows-gnu "-Dcpu=$ZsmoothCpu" if ($LASTEXITCODE -ne 0) { throw "zig build failed (exit $LASTEXITCODE)" } } finally { Pop-Location From b4efbf82adcf6f20136f027f6df17d56f28ff5a3 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sat, 26 Sep 2026 10:47:13 +1000 Subject: [PATCH 8/9] Docs: CPU-tiered deps bundles (issue #92) CLAUDE.md states the tier rules (one decision in the worker, the tier only in the asset name and version.json, one --tier block per script, v2 gated before publishing) in place of the zsmooth-by-path rule they replace, plus the 1.11.0 history row, the load-time SIGILL debugging tip, and the Rosetta caveat. ENGINEERING_NOTES records the investigation, including the traps that produced misleading results on the way. README tells users older CPUs are supported. The Apple Silicon x64-deps instructions are removed: Homebrew's installer no longer creates the Intel prefix they depend on. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014GLXdGLfPwgYjW1AkonGqN --- CLAUDE.md | 94 ++++++++++++++++++++++++++++--------- README.md | 2 + docs/BUILDING.md | 19 ++++---- docs/ENGINEERING_NOTES.md | 97 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 181 insertions(+), 31 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ad22d58..cfef9ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,6 +100,7 @@ VapourBox/ | `worker/src/script_generator.rs` | Template substitution for .vpy | | `worker/src/pipeline_executor.rs` | vspipe \| ffmpeg execution | | `worker/src/pixel_format.rs` | Source `pix_fmt` → pipe format (see "Source Pixel Formats") | +| `worker/src/cpu.rs` | The x86 deps CPU tier (`v3`/`v2`) — the single decision; see "CPU tiers" | | `worker/templates/pipeline_template.vpy` | VapourSynth script template | | `worker/tests/filter_integration_test.rs` | Filter integration tests | @@ -143,19 +144,24 @@ VapourBox/ # it targets deps/macos-x64. FFmpeg comes pre-built from evermeet.cx, most # plugins from Stefan-Olt; descratch/neo-f3kdb/nnedi3cl/vivtc build from source. ./scripts/download-deps-macos.sh -# To build x64 deps on Apple Silicon instead, run it under Rosetta 2 with an -# Intel Homebrew prefix first in PATH: -# arch -x86_64 /bin/bash -lc 'PATH=/usr/local/bin:$PATH ./Scripts/download-deps-macos.sh --force' +# x64 deps can NOT be built on Apple Silicon any more: Homebrew's installer +# refuses to create the Intel prefix under Rosetta. Use CI, or unzip a +# published/artifact macos-x64 bundle into deps/macos-x64. # Windows (PowerShell) .\Scripts\download-deps-windows.ps1 # Linux ./Scripts/download-deps-linux.sh + +# Any x64 platform, older-CPU tier (see "CPU tiers"); switching tier in an +# existing deps/ needs --force (Windows: delete it first) +./Scripts/download-deps-linux.sh --force --tier v2 ``` Both macOS architectures are produced in CI by `build-deps-macos.yml` (arm64 on -`macos-15`, x64 natively on `macos-15-intel`). +`macos-15`, x64 natively on `macos-15-intel`), and every x64 platform's +workflow builds both CPU tiers. ### Build Rust Worker @@ -360,9 +366,13 @@ Adding a filter touches many files. Missing any step causes silent failures (fil platform that ships it. The `package-deps-*` scripts assert this list is present before zipping and fail the build if any are missing, so a dead download URL becomes a red build instead of a silently-incomplete bundle. - A bare filename is looked up in the plugin directory; an entry containing a - `/` is resolved from the **bundle root** instead, which is how zsmooth's - per-CPU builds outside the autoload directory stay covered. + x64 platforms have a second key per CPU tier (`macos-x64-v2`, …) that must + list the same plugins — `attribution_test.dart` asserts it. +- **Check it for load-time SIMD on x64** (see "CPU tiers" below): a prebuilt + x86 binary built with `-mavx2`-style flags can SIGILL inside `dlopen` on an + older CPU, and autoload makes that every job. Run + `Scripts/check-load-time-simd.py` on a macOS dylib; the v2 gates in the + `build-deps-*` workflows cover all three platforms. - For local testing, run the download script to populate `deps/` (e.g. `deps/windows-x64/vapoursynth/vs-plugins/`, `deps/macos-arm64/vapoursynth/plugins/`). - **Credit it**: add an entry to `licenses/NOTICES.txt` and a `_ComponentTile` @@ -656,19 +666,39 @@ disagree. Pin the **series** against a rolling host tag (e.g. BtbN's `n9.0-latest`), never an exact old build — a fixed old tag gets garbage-collected off a rolling `latest` alias and 404s the deps build outright. -**zsmooth ships one build per x86 CPU baseline (`haswell`, `x86_64_v2`), loaded -by explicit path, not autoloaded.** Upstream's single AVX2-baseline build -crashes with an illegal instruction on any pre-2013 CPU the instant a zsmooth -filter runs (`CCD`, `Cnr4`, `SpotLess`→`RemoveDirt`, `mClean`, -`TemporalDegrain2`, hybrid_mv all reach it). `DependencyLocator::zsmooth_plugin()` -returns `None` on an older bundle lacking the split, and the generated script -must stay byte-identical to the pre-split form in that case. +**CPU tiers (issue #92): every x86 deps bundle ships twice.** `v3` is built +for x86-64-v3 CPUs (Haswell, 2013, and later); `v2` runs on anything older. +The rules: + +- **One decision, in the worker.** `cpu::cpu_tier()` (`worker/src/cpu.rs`) + requires the *whole* x86-64-v3 set, not just AVX2. `--probe-cpu` reports it, + the app downloads by it, and `ctmf_opt` derives from it. Never re-derive the + tier elsewhere (sysctl is wrong under Rosetta); on x86 an unclear answer means + `v2`, which only costs speed. `VAPOURBOX_DEPS_TIER=v2|v3` overrides it. +- **The tier lives only in the asset name and version.json.** `v3` keeps the + plain names (`VapourBox-deps-X-macos-x64.zip`), `v2` is suffixed + (`…-macos-x64-v2.zip`); both install to `deps/`. The app re-checks + the installed tier at startup (`DependencyStatus.wrongTier`) and replaces a + mismatch even when it is newer than expected. +- **One `--tier` block per `download-deps-*` script** maps the tier to build + variables; nothing else in the script branches on it. The two tiers differ + only where running the bundle on an old CPU proved they must: zsmooth + (`haswell` / `x86_64_v2`, one build per bundle, autoloaded — upstream has no + runtime dispatch) everywhere, and on macOS MVTools (v2 builds v24 from source + with `patches/mvtools-v24-no-avx2.patch`, because Stefan-Olt's build SIGILLs + in its AVX2 static initializers inside `dlopen`). +- **A v2 bundle is gated before it can be published**: under Intel SDE by + `probe-cpu-compat.yml` (Linux at Westmere; Windows at Sandy Bridge, since SDE + cannot emulate pre-AVX Windows — Microsoft's runtime reads the host's CPU + features from the kernel), and on macOS statically by + `Scripts/check-load-time-simd.py` inside `package-deps-macos.sh`. **CTMF's `opt` (SIMD level) must be chosen by the worker from the CPU, never left at the plugin's own auto-detect (`opt=0`).** Its AVX-512 kernel for 8-bit input crashes with an access violation on real hardware — auto-detect is exactly what selects that broken kernel. `script_generator::ctmf_opt` picks 3 -(AVX2) or 2 (SSE2) based on `is_x86_feature_detected!`, never 0. +(AVX2) on a v3-tier CPU, else 2 (SSE2), never 0. This is independent of the +tiers: a v3 machine can have AVX-512. See docs/ENGINEERING_NOTES.md for the specific plugin-by-plugin decisions, probe-round methodology, and dated write-ups behind all of the above. @@ -1046,15 +1076,15 @@ can render fine and still fail the preview. > `vapoursynth_integration_test`'s "all required plugins load" list is the > runtime contract for a **complete deps install** and must name every -> namespace a filter can reach — except `zsmooth`, which is deliberately not -> autoloaded and has its own load-and-render test. Add the namespace whenever +> namespace a filter can reach. Add the namespace whenever > you add a plugin, or a bundle missing it passes CI and fails at job time. > OpenCL-only plugins (`nnedi3cl`, `knlm`) stay **out** of the list — the app > degrades to a CPU path without them. The harness honors `$VAPOURBOX_DEPS_DIR`, else uses repo-root `deps/`, else **downloads the deps release pinned in `app/assets/deps-version.json`** -(opt out with `$VAPOURBOX_SKIP_DEPS_DOWNLOAD=1`). The worker binary is found under +(opt out with `$VAPOURBOX_SKIP_DEPS_DOWNLOAD=1`), in the tier the app would +pick (`$VAPOURBOX_DEPS_TIER` overrides it). The worker binary is found under `worker/target/{release,debug}` (CI's `cargo test`/`cargo build` produces debug). Subtitle heavy tests skip when the whisper add-on is absent. @@ -1081,9 +1111,19 @@ integration tests. Matrix: macOS **arm64** (`macos-15`), macOS **x64** (`macos-15-intel`), **Windows x64**, **Linux x64** (`ubuntu-24.04`). The heavy full-encode integration tests run separately in `.github/workflows/nightly.yml` (cron + `workflow_dispatch`) via `flutter test --tags heavy` on the same -4-platform matrix. Fixtures (`small_clip.mp4`, telecine/interlaced clips) are +4-platform matrix, running each x64 platform against **both** CPU-tier bundles. +Fixtures (`small_clip.mp4`, telecine/interlaced clips) are committed under `Tests/TestResources/`. +> **Old-CPU behaviour cannot be tested on the runners, or on Apple Silicon.** +> Every hosted runner is a v3 CPU, and Rosetta 2 translates AVX/AVX2 on macOS +> 15+ — so a plugin that SIGILLs on a pre-AVX2 Intel Mac runs fine under +> Rosetta. The tools that do answer it: `probe-cpu-compat.yml` (every plugin +> under Intel SDE, with controls that prove the emulation is valid), and +> `Scripts/probe-plugin-compat.sh` for a user to run on the real machine +> (one plugin per process, into a `DISABLE_AUTO_LOADING` core — under autoload +> one faulting plugin masks every other result). + > **A green CI run on hosted hardware is not proof about CPU-dispatched code.** > The runner fleet is mixed for features like AVX-512, so a filter that only > crashes on that instruction set can pass for days and then fail on no code @@ -1301,6 +1341,12 @@ placement and a version skew would change chroma per-OS. under vspipe with passes commented out; on Windows, `$LASTEXITCODE` `0xC0000005` = access violation, `0xC000001D` = illegal instruction (CPU feature the machine lacks) — both are decoded by `format_exit_status`. + **If it happens with every pass disabled, it is load-time, not a filter**: + VapourSynth autoloads every plugin when the core starts, so one plugin whose + static initializers use a missing instruction set kills every job (issue #92, + MVTools' AVX2 tables, `SIGILL` = signal 4 on macOS/Linux). Commenting out + passes cannot find it; the OS crash report (`.ips` on macOS) names the image, + and `Scripts/probe-plugin-compat.sh` tests each plugin in isolation. ## Platform-Specific Notes @@ -1376,7 +1422,7 @@ Plugin lists for all platforms: see `deps/` directories or download scripts. ## Dependency Versioning and Auto-Download -Dependencies are versioned separately from the app via `app/assets/deps-version.json` and distributed as separate GitHub releases (tag: `deps-vX.Y.Z`). The app auto-downloads deps on launch if missing or outdated. +Dependencies are versioned separately from the app via `app/assets/deps-version.json` and distributed as separate GitHub releases (tag: `deps-vX.Y.Z`). The app auto-downloads deps on launch if missing, outdated, or built for a different CPU tier (x64 only — see "CPU tiers"; the asset is `DependencyManager.assetIdFor(platformId, tier)`). `deps-version.json` is a **slim pointer** — `{version, releaseTag, githubRepo}`. Integrity metadata is **not** stored here: each `package-deps-*` script writes a **sidecar** `.sha256.json` uploaded next to the zip, which the app fetches and verifies at download time (best-effort if the sidecar is missing). **Net effect: a new deps release only needs a `version`/`releaseTag` bump.** @@ -1402,6 +1448,9 @@ App and deps use **separate release tags** so unchanged deps aren't re-uploaded ``` This prompts for version, checks deps changes, builds, packages, and creates draft GitHub releases. +It does **not** package deps: if they changed, it stops and prints the +`build-deps-*` workflow commands. A local checkout holds only one CPU tier per +x64 platform, so a locally packaged deps release would be missing the other. ### CI Build and Release @@ -1479,8 +1528,8 @@ Notes: 1. **Confirm version** — ask user, update `pubspec.yaml` 2. **Check deps** — run `check-deps-changed.sh`; if changed, bump `version`/`releaseTag` in `deps-version.json`. -3. **Build & package** — use packaging scripts (or `release.sh`). Each `package-deps-*` writes the zip **and** its `.sha256.json` sidecar. -4. **Upload deps assets** — upload each platform's zip **and its `.sha256.json` sidecar** to the deps release. +3. **Build & package deps in CI** — dispatch `build-deps-{macos,windows,linux}.yml` with `release_tag`. They build both x64 tiers, gate the v2 bundles, and only then upload each zip **and** its `.sha256.json` sidecar. +4. **Check the deps release** — every platform, both tiers of each x64: `macos-arm64`, `macos-x64`, `macos-x64-v2`, `windows-x64`, `windows-x64-v2`, `linux-x64`, `linux-x64-v2`, `linux-arm64`. 5. **Test** — fresh install + upgrade test 6. **Create GitHub releases** — deps release first (if changed, tag `deps-vX.Y.Z`), then app release (tag `vX.Y.Z`) @@ -1581,3 +1630,4 @@ Full write-ups (root causes, measurements) for each entry are in | 1.8.0 | 2026-08-07 | VapourSynth **R73 → R78** everywhere (Windows Python-wheel layout; `deps//vapoursynth/` is the Python package on macOS/Linux). Adds **akarin** (LLVM JIT for `std.Expr`, ~4x on arm64 QTGMC; not on macos-x64). Fixes nnedi3 on linux-arm64. Removes BestSource. Linux now needs glibc 2.39 | | 1.9.0 | 2026-08-15 | Adds **fluxsmooth** (unlocks havsfunc's STPresso), **bifrost** (temporal rainbow/dot-crawl removal), **retinex** (shadow-detail lift) — all pinned to the newest release with a published Windows binary | | 1.10.0 | 2026-08-31 | **Issue #82**: zsmooth now ships one build per x86 CPU baseline (haswell, x86_64_v2), fixing an illegal-instruction crash on pre-2013 CPUs. **FFmpeg pinned to 9.0 on all four platforms** (they had silently diverged: Windows on an unpinned post-9.0 master, macOS on floating 9.0.1, Linux stuck at 7.1 after BtbN garbage-collected the pinned tag) | +| 1.11.0 | 2026-09 | **Issue #92**: every x86 bundle ships in two **CPU tiers** — `v3` (x86-64-v3; the plain asset names) and `v2` (`…-x64-v2`, anything older), chosen by the app from `vapourbox-worker --probe-cpu`. Each bundle carries one autoloaded zsmooth (replacing 1.10.0's two-builds-loaded-by-path). macOS v2 builds MVTools v24 from source without its AVX2 files, whose static initializers SIGILLed inside `dlopen` on pre-AVX Macs. v2 bundles are gated (SDE on Linux/Windows, static check on macOS) before publishing | diff --git a/README.md b/README.md index 15a9dfc..1dd5891 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,8 @@ GPU-accelerated deinterlacing (NNEDI3CL) needs your GPU's OpenCL driver installe +**Older Intel and AMD processors** (before about 2013 — no AVX2) are supported on all three platforms. On first launch VapourBox checks the processor and downloads the matching set of processing components; a machine moved to a different CPU gets the right set on its next launch. The older-CPU set runs some filters more slowly, never differently. Every build of that set is checked for pre-AVX processors (e.g. a 2010 Mac Pro) on macOS and Linux; on Windows it is checked for processors with AVX but no AVX2, and older ones are expected to work but can't be tested automatically. + ## Output formats | | | diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 36edc90..cce5a0f 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -116,14 +116,17 @@ come pre-built from ./Scripts/download-deps-macos.sh --force ``` -To build the x64 deps on Apple Silicon instead, run it under Rosetta 2 with an -Intel Homebrew prefix (`macos-13` Intel runners were retired Dec 2025): - -```bash -softwareupdate --install-rosetta --agree-to-license -arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" -arch -x86_64 /bin/bash -lc 'PATH=/usr/local/bin:$PATH ./Scripts/download-deps-macos.sh --force' -``` +The x64 deps can no longer be built on Apple Silicon: the script needs an Intel +Homebrew prefix, and Homebrew's installer now refuses to create one under +Rosetta ("Homebrew on macOS is only supported on Apple Silicon processors!"). +Build them in CI (`build-deps-macos.yml`, natively on `macos-15-intel`), or +unzip a published or artifact `macos-x64` bundle into `deps/macos-x64`. + +**CPU tiers (x64 only).** Every x64 bundle exists in two tiers: `v3` (the +default; x86-64-v3 CPUs) and `v2` (older CPUs). Choose with `--tier v2` +(`-Tier v2` on Windows). Both write to the same `deps/`, so switching +tier needs `--force` (Windows: delete the directory first) — the scripts refuse +to build one tier over the other rather than produce a mixture. In CI, deps are produced by **Build macOS Deps** (`build-deps-macos.yml`, arm64 on `macos-15` and x64 natively on `macos-15-intel`), **Build Linux Deps** diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index f11740c..8d1ad12 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -384,7 +384,102 @@ Note the version parser accepts `n9.0.1` and `9.0.1` and deliberately **rejects a `master` build** (`N-125978-...`), so reverting any platform to an unpinned master URL is a red build rather than a silent regression. -### zsmooth ships once per CPU baseline, and is loaded by path (issue #82, 2026-08-28) +### x86 bundles split into CPU tiers (issue #92, 2026-09-25) + +**The report.** A Mac Pro 5,1 (Xeon X5690, Westmere — SSE4.2, no AVX at all) +failed every preview at once with `vspipe exited with signal 4` (SIGILL), even +with every pass disabled. The reporter's `.ips` put the fault in +`__GLOBAL__sub_I_MVDegrains_AVX2.cpp` inside `libmvtools.dylib`, called from +`dlopen` ← `VSCore::loadAllPluginsInPath` ← `createCore`. + +**Why every job.** Stefan-Olt's macOS MVTools compiles six files with +`-mavx2`. Three of them (`MVDegrains_AVX2`, `Overlap_AVX2`, `SADFunctions_AVX2`) +build a `static const std::unordered_map` of kernels at load time, and the +compiler used VEX instructions for it. Static initializers run inside `dlopen`, +before any plugin code can check the CPU, and R78 autoloads every plugin when a +core is created — so one plugin faulting at load kills every job, whatever it +asks for. Runtime dispatch in the filters is irrelevant; they never get to run. + +**How it was pinned down, and the traps on the way.** +- A disassembly scan of initializers *by name* (`_GLOBAL__sub_I_*`) came back + clean on the Linux bundle only because GCC's LTO renames them + (`__static_initialization_and_destruction_0v.lto_priv.N`). Name-based scans + are not evidence. `Scripts/check-load-time-simd.py` finds initializers from + `__init_offsets` / `__mod_init_func` instead, follows direct calls (never + into `__stubs`), and flags the stock 1.10.0 MVTools and nothing else in the + bundle. +- The first real-hardware probe reported all 18 prebuilt plugins as crashing, + because each test autoloaded the whole directory, MVTools included. + `Scripts/probe-plugin-compat.py` loads one plugin per process into a + `CoreCreationFlags.DISABLE_AUTO_LOADING` core instead. +- Rosetta 2 translates AVX/AVX2 on macOS 15+ (not AVX-512), so the crashing + MVTools ran perfectly on Apple Silicon. Nothing about pre-AVX2 behaviour can + be learned there. +- The reporter's Ivy Bridge MacBook (AVX, no AVX2) loaded everything: the + initializer VEX is plain AVX, so it is the *pre-AVX* machines that fail. + (macOS spells the feature `AVX1.0` in `machdep.cpu.features`.) + +**Surveyed under Intel SDE** (`probe-cpu-compat.yml`), with three controls: the +runner itself must pass everything; a plugin-free core must pass under SDE; and +upstream's AVX2-only zsmooth must crash *in its own image*. +- **Linux at Westmere:** every plugin in the 1.10.0 bundle renders, MVTools + included (GCC's build keeps VEX out of that load path), except the zsmooth + haswell build — which the worker never loaded there anyway. +- **Windows at Westmere is unmeasurable:** every test, including the core + control, faults in `VCRUNTIME140.dll`'s `memcpy` on a `vmovdqu ymm`. Microsoft's + runtime takes its AVX path from what the *host kernel* reports + (`IsProcessorFeaturePresent` reads shared kernel memory), which SDE cannot + virtualise, and python.org's runtime runs fine on real pre-AVX Windows. Without + the core control this run looked valid, because the positive control also + "crashed" — in vcruntime. **Windows at Sandy Bridge** (AVX, no AVX2) is valid, + and there everything but zsmooth-haswell renders. +- `sde.exe` drops empty arguments when relaunching the child, which once made a + probe bug look like a CPU finding. + +**The design.** Rather than more per-plugin special cases (zsmooth's two +builds loaded by path was the first), each x86 platform ships two bundles: +`v3` (x86-64-v3; the pre-existing asset names) and `v2` (`-v2`). One gateway +chooses — the app downloads by `vapourbox-worker --probe-cpu`'s `tier`, and the +tier lives only in the asset name and `version.json`, never the install path, +so the worker, dev paths and tests are untouched. v3 needs the whole +x86-64-v3 feature set (a v3 build may use FMA/BMI2/MOVBE as well as AVX2). +Startup re-checks the tier, so an install that moved machines is replaced. + +What actually differs between the tiers is only what the measurements above +proved must: zsmooth's baseline on all three platforms, and on macOS MVTools. +The v2 MVTools is v24 from source with `patches/mvtools-v24-no-avx2.patch`: + +- It replaces the six AVX2 files with stubs compiled at the baseline. With no + `-mavx2` code in the binary, neither the initializer VEX nor the other + hazard remains: an `-mavx2` file's copy of shared template code can win the + linker's choice and put AVX2 into unguarded callers. +- It masks `X264_CPU_AVX2` out of `g_cpuinfo`, the single point where the CPU + flags are set. Every call into the AVX2 code is gated on that flag. +- The stubs `abort()` rather than return nothing. They are unreachable, and a + silent no-op would render blank frames on an AVX2 machine running the v2 + bundle if the mask were ever lost. +- Every AVX2 selector call site null-checks (`if (tmp) degrain = tmp;`). +- The nasm assembly stays, and is still runtime-dispatched. +- The patched binary has no compiler-generated VEX outside the asm kernels. + +**Gates.** A v2 bundle cannot be published without passing: SDE in +`build-deps-{linux,windows}.yml` (the upload job waits for it), the static +check inside `package-deps-macos.sh`. `release.sh` stopped packaging deps +locally — a checkout holds one tier per platform, so it would have published +half a release. + +**Open.** Pre-AVX *Windows* remains unverified (no SDE run is possible, and +`check-load-time-simd.py` reads Mach-O only). MVTools' Windows build is the +official MSVC v24 — if it turns out to have the same initializer problem, the +Windows v2 tier would need an MVTools source build, which the Windows deps +script has no toolchain for today. + +### zsmooth ships once per CPU baseline, and is loaded by path (issue #82, 2026-08-28) — superseded by the CPU tiers above + +Since deps 1.11.0 each tier's bundle carries exactly one zsmooth build, in the +autoload directory; the two-builds-loaded-by-path mechanism below was removed. +The measurements still hold and are why v2 uses `x86_64_v2` rather than +`x86_64`. A plugin can also have **no** dispatch at all. zsmooth is compiled for a whole CPU baseline — upstream publishes only `haswell` (AVX2) and `znver4` for x86, From b66a96fea2014fb2696bf62baa3ef5c545bdc7e3 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sat, 26 Sep 2026 10:54:53 +1000 Subject: [PATCH 9/9] vapoursynth_integration_test: honour VAPOURBOX_DEPS_DIR Its own deps resolver ignored the override that the worker harness and the app honour, so a run pointed at an unreleased bundle silently tested whatever was in the repo's deps/ instead. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014GLXdGLfPwgYjW1AkonGqN --- app/test/vapoursynth_integration_test.dart | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/test/vapoursynth_integration_test.dart b/app/test/vapoursynth_integration_test.dart index 722b4da..a66fb71 100644 --- a/app/test/vapoursynth_integration_test.dart +++ b/app/test/vapoursynth_integration_test.dart @@ -48,8 +48,12 @@ void main() { 'Unsupported platform: ${Platform.operatingSystem}'); } - // Try different possible locations for deps + // $VAPOURBOX_DEPS_DIR first, as the worker harness and the app honour it — + // without that, a run pointed at an unreleased bundle silently tested + // whatever happened to be in the repo's deps/ instead. + final override = Platform.environment['VAPOURBOX_DEPS_DIR']; final possibleDepsPaths = [ + if (override != null && override.isNotEmpty) override, path.join(scriptDir, '..', 'deps', depsPlatform), path.join(scriptDir, 'deps', depsPlatform), ];