diff --git a/.github/workflows/build-deps-linux.yml b/.github/workflows/build-deps-linux.yml index 6e08a22..7047b60 100644 --- a/.github/workflows/build-deps-linux.yml +++ b/.github/workflows/build-deps-linux.yml @@ -23,11 +23,20 @@ on: permissions: contents: write + actions: read jobs: + # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for + # anything older. Same script, one --tier switch. build-x64: if: ${{ inputs.arch == 'x64' || inputs.arch == 'both' }} runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + tier: [v3, v2] + env: + ASSET: VapourBox-deps-${{ inputs.version }}-linux-x64${{ matrix.tier == 'v2' && '-v2' || '' }} steps: - uses: actions/checkout@v5 @@ -47,27 +56,51 @@ jobs: gcc --version | head -1 pip3 install meson - - name: Build dependencies - run: ./Scripts/download-deps-linux.sh --force + - name: Build dependencies (${{ matrix.tier }}) + run: ./Scripts/download-deps-linux.sh --force --tier ${{ matrix.tier }} - name: Package dependencies run: ./Scripts/package-deps-linux.sh --version "${{ inputs.version }}" --arch x64 - uses: actions/upload-artifact@v5 with: - name: VapourBox-deps-${{ inputs.version }}-linux-x64 + name: ${{ env.ASSET }} path: | - dist/VapourBox-deps-${{ inputs.version }}-linux-x64.zip - dist/VapourBox-deps-${{ inputs.version }}-linux-x64.zip.sha256.json + dist/${{ env.ASSET }}.zip + dist/${{ env.ASSET }}.zip.sha256.json + # The v2 bundle's promise is that it runs on CPUs without AVX. Every hosted + # runner has AVX2, so prove it by running every plugin under Intel SDE + # emulating Westmere (the Mac Pro 5,1 from issue #92). + gate-x64-v2: + needs: build-x64 + uses: ./.github/workflows/probe-cpu-compat.yml + with: + artifact_prefix: VapourBox-deps-${{ inputs.version }}- + tier: v2 + gate: true + matrix_json: '[{"platform":"linux-x64","runner":"ubuntu-24.04","sde":"sde64","python":"python/bin/python3","chip":"wsm"}]' + + # Published only once the v2 gate has passed, so a bundle that would crash on + # an older CPU never reaches a release. + upload-x64: + needs: [build-x64, gate-x64-v2] + if: inputs.release_tag != '' + runs-on: ubuntu-24.04 + steps: + - uses: actions/download-artifact@v5 + with: + pattern: VapourBox-deps-${{ inputs.version }}-linux-x64* + path: dist + merge-multiple: true - name: Upload to release - if: inputs.release_tag != '' env: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ - "dist/VapourBox-deps-${{ inputs.version }}-linux-x64.zip" \ - "dist/VapourBox-deps-${{ inputs.version }}-linux-x64.zip.sha256.json" --clobber + --repo "${{ github.repository }}" \ + dist/VapourBox-deps-${{ inputs.version }}-linux-x64*.zip \ + dist/VapourBox-deps-${{ inputs.version }}-linux-x64*.zip.sha256.json --clobber build-arm64: if: ${{ inputs.arch == 'arm64' || inputs.arch == 'both' }} diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml index 811e884..5b900ea 100644 --- a/.github/workflows/build-deps-macos.yml +++ b/.github/workflows/build-deps-macos.yml @@ -60,28 +60,39 @@ jobs: "dist/VapourBox-deps-${{ inputs.version }}-macos-arm64.zip" \ "dist/VapourBox-deps-${{ inputs.version }}-macos-arm64.zip.sha256.json" --clobber + # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for + # anything older. Same script, one --tier switch. The v2 bundle is gated by + # package-deps-macos.sh itself (Scripts/check-load-time-simd.py): macOS has + # no Intel SDE, so it proves statically that no plugin runs AVX while it + # loads — and fails before anything below can upload it. build-x64: if: ${{ inputs.arch == 'x64' || inputs.arch == 'both' }} runs-on: macos-15-intel # only hosted Intel image (macos-13 retired; last one until ~Aug 2027) + strategy: + fail-fast: false + matrix: + tier: [v3, v2] + env: + ASSET: VapourBox-deps-${{ inputs.version }}-macos-x64${{ matrix.tier == 'v2' && '-v2' || '' }} steps: - uses: actions/checkout@v5 - - name: Build dependencies (x64, native) + - name: Build dependencies (x64 ${{ matrix.tier }}, native) # STRICT_MIN_OS=1: fail the build if any bundled Mach-O targets newer than # the macOS 12 floor (issue #39) instead of silently shipping it. env: STRICT_MIN_OS: "1" - run: ./Scripts/download-deps-macos.sh --force + run: ./Scripts/download-deps-macos.sh --force --tier ${{ matrix.tier }} - name: Package dependencies run: ./Scripts/package-deps-macos.sh --version "${{ inputs.version }}" --arch x64 - uses: actions/upload-artifact@v5 with: - name: VapourBox-deps-${{ inputs.version }}-macos-x64 + name: ${{ env.ASSET }} path: | - dist/VapourBox-deps-${{ inputs.version }}-macos-x64.zip - dist/VapourBox-deps-${{ inputs.version }}-macos-x64.zip.sha256.json + dist/${{ env.ASSET }}.zip + dist/${{ env.ASSET }}.zip.sha256.json - name: Upload to release if: inputs.release_tag != '' @@ -89,5 +100,5 @@ jobs: GH_TOKEN: ${{ github.token }} run: | gh release upload "${{ inputs.release_tag }}" \ - "dist/VapourBox-deps-${{ inputs.version }}-macos-x64.zip" \ - "dist/VapourBox-deps-${{ inputs.version }}-macos-x64.zip.sha256.json" --clobber + "dist/${{ env.ASSET }}.zip" \ + "dist/${{ env.ASSET }}.zip.sha256.json" --clobber diff --git a/.github/workflows/build-deps-windows.yml b/.github/workflows/build-deps-windows.yml index 8418723..95f4bf4 100644 --- a/.github/workflows/build-deps-windows.yml +++ b/.github/workflows/build-deps-windows.yml @@ -20,16 +20,25 @@ on: permissions: contents: write + actions: read jobs: + # x64 ships in two CPU tiers (issue #92): v3 for x86-64-v3 CPUs, v2 for + # anything older. Same script, one -Tier switch. build-x64: runs-on: windows-latest + strategy: + fail-fast: false + matrix: + tier: [v3, v2] + env: + ASSET: VapourBox-deps-${{ inputs.version }}-windows-x64${{ matrix.tier == 'v2' && '-v2' || '' }} steps: - uses: actions/checkout@v5 - - name: Build dependencies + - name: Build dependencies (${{ matrix.tier }}) shell: pwsh - run: ./Scripts/download-deps-windows.ps1 + run: ./Scripts/download-deps-windows.ps1 -Tier ${{ matrix.tier }} - name: Package dependencies shell: pwsh @@ -37,17 +46,40 @@ jobs: - uses: actions/upload-artifact@v5 with: - name: VapourBox-deps-${{ inputs.version }}-windows-x64 + name: ${{ env.ASSET }} path: | - dist/VapourBox-deps-${{ inputs.version }}-windows-x64.zip - dist/VapourBox-deps-${{ inputs.version }}-windows-x64.zip.sha256.json + dist/${{ env.ASSET }}.zip + dist/${{ env.ASSET }}.zip.sha256.json + # The v2 bundle must run below x86-64-v3, and every hosted runner has AVX2. + # Windows is probed at Sandy Bridge (AVX without AVX2): at pre-AVX chips SDE + # cannot emulate what Microsoft's runtime reads from the kernel, so the core + # control fails and nothing could be proven (see probe-cpu-compat.yml). + gate-x64-v2: + needs: build-x64 + uses: ./.github/workflows/probe-cpu-compat.yml + with: + artifact_prefix: VapourBox-deps-${{ inputs.version }}- + tier: v2 + gate: true + matrix_json: '[{"platform":"windows-x64","runner":"windows-latest","sde":"sde.exe","python":"vapoursynth/python.exe","chip":"snb"}]' + + # Published only once the v2 gate has passed. + upload-x64: + needs: [build-x64, gate-x64-v2] + if: inputs.release_tag != '' + runs-on: ubuntu-24.04 + steps: + - uses: actions/download-artifact@v5 + with: + pattern: VapourBox-deps-${{ inputs.version }}-windows-x64* + path: dist + merge-multiple: true - name: Upload to release - if: inputs.release_tag != '' - shell: pwsh env: GH_TOKEN: ${{ github.token }} run: | - gh release upload "${{ inputs.release_tag }}" ` - "dist/VapourBox-deps-${{ inputs.version }}-windows-x64.zip" ` - "dist/VapourBox-deps-${{ inputs.version }}-windows-x64.zip.sha256.json" --clobber + gh release upload "${{ inputs.release_tag }}" \ + --repo "${{ github.repository }}" \ + dist/VapourBox-deps-${{ inputs.version }}-windows-x64*.zip \ + dist/VapourBox-deps-${{ inputs.version }}-windows-x64*.zip.sha256.json --clobber diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a62332c..7400e4b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -44,11 +44,20 @@ jobs: strategy: fail-fast: false matrix: + # x64 runs both CPU-tier bundles (issue #92). The runner is a v3 CPU, so + # the v2 run proves the v2 bundle is functionally equivalent — not that + # it runs on an older CPU; that is build-deps-macos.yml's static gate. include: - arch: arm64 runner: macos-15 + asset: macos-arm64 - arch: x64 runner: macos-15-intel # last native Intel image, available until ~Aug 2027 + asset: macos-x64 + - arch: x64 + runner: macos-15-intel + asset: macos-x64-v2 + name: macos (${{ matrix.asset }}) runs-on: ${{ matrix.runner }} env: VAPOURBOX_DEPS_DIR: ${{ github.workspace }}/deps/macos-${{ matrix.arch }} @@ -71,14 +80,15 @@ jobs: - name: Setup Rust uses: dtolnay/rust-toolchain@stable - - name: Download dependencies (${{ matrix.arch }}) + - name: Download dependencies (${{ matrix.asset }}) env: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | + # The tier is in the asset name only; either installs to deps/. PLATFORM="macos-${{ matrix.arch }}" mkdir -p "deps/$PLATFORM" - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "$PLATFORM" \ + ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") unzip -q -o "$ZIP" -d "deps/$PLATFORM" rm -rf "$ZIP" .deps-artifact @@ -123,6 +133,12 @@ jobs: windows: if: ${{ github.event_name == 'schedule' || inputs.platform == 'all' || inputs.platform == 'windows' }} + # Both CPU-tier bundles (issue #92); see the macOS job for what the v2 run proves. + strategy: + fail-fast: false + matrix: + asset: [windows-x64, windows-x64-v2] + name: windows (${{ matrix.asset }}) runs-on: windows-latest env: VAPOURBOX_DEPS_DIR: ${{ github.workspace }}/deps/windows-x64 @@ -153,7 +169,7 @@ jobs: DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | mkdir -p deps/windows-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh windows-x64 \ + ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") 7z x "$ZIP" -o"deps/windows-x64" -y >/dev/null rm -rf "$ZIP" .deps-artifact @@ -197,6 +213,12 @@ jobs: linux: if: ${{ github.event_name == 'schedule' || inputs.platform == 'all' || inputs.platform == 'linux' }} + # Both CPU-tier bundles (issue #92); see the macOS job for what the v2 run proves. + strategy: + fail-fast: false + matrix: + asset: [linux-x64, linux-x64-v2] + name: linux (${{ matrix.asset }}) runs-on: ubuntu-24.04 # must match the deps-build runner (glibc) env: VAPOURBOX_DEPS_DIR: ${{ github.workspace }}/deps/linux-x64 @@ -225,13 +247,13 @@ jobs: - name: Setup Rust uses: dtolnay/rust-toolchain@stable - - name: Download dependencies (linux-x64) + - name: Download dependencies (${{ matrix.asset }}) env: GH_TOKEN: ${{ github.token }} DEPS_RUN_ID: ${{ inputs.deps_run_id }} run: | mkdir -p deps/linux-x64 - ZIP=$(bash .github/scripts/fetch-deps-bundle.sh linux-x64 \ + ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "${{ matrix.asset }}" \ "${{ steps.deps.outputs.tag }}" "${{ steps.deps.outputs.ver }}") unzip -q -o "$ZIP" -d deps/linux-x64 rm -rf "$ZIP" .deps-artifact diff --git a/.github/workflows/probe-cpu-compat.yml b/.github/workflows/probe-cpu-compat.yml new file mode 100644 index 0000000..57a4354 --- /dev/null +++ b/.github/workflows/probe-cpu-compat.yml @@ -0,0 +1,192 @@ +name: Probe CPU compatibility (SDE) + +# Loads and renders every plugin in a deps bundle under Intel SDE emulating an +# older CPU, one plugin per process (Scripts/probe-plugin-compat.py). Every +# hosted runner has AVX2, so this is the only CI-side way to see what an older +# machine would do with a bundle (issue #92). +# +# Two modes: +# - survey (push / workflow_dispatch): reports what the bundle does; crashes +# are findings, not failures. +# - gate (workflow_call from build-deps-*.yml with gate: true): the v2 bundle +# promises to run on these CPUs, so any crash fails the build. +# +# A run only counts if its controls hold: +# - native: the runner itself must pass everything; an error there is a +# probe bug, not a CPU finding. +# - core under SDE: VapourSynth with no plugins must pass. If it doesn't, the +# emulation itself is broken for this OS/chip and every plugin result is +# meaningless. (Windows at pre-AVX chips fails this: Microsoft's runtime +# picks AVX from what the host kernel reports, which SDE cannot emulate — +# so Windows is probed at Sandy Bridge, AVX without AVX2.) +# - positive: upstream's zsmooth haswell build (AVX2, no runtime dispatch, +# issue #82), fetched here independently of the bundle, must crash inside +# its own image — proving SDE actually traps. + +on: + # Also runs when the probe or this workflow changes, which re-validates the + # probe itself (and lets a branch run it before the file exists on main, + # which workflow_dispatch requires). + push: + paths: + - Scripts/probe-plugin-compat.py + - .github/workflows/probe-cpu-compat.yml + workflow_dispatch: + inputs: + deps_run_id: + description: 'Optional build-deps-* run IDs (comma-separated); default is the release named in deps-version.json' + required: false + default: '' + tier: + description: 'x64 bundle tier to probe' + type: choice + options: [v3, v2] + default: v3 + workflow_call: + inputs: + artifact_prefix: + description: 'Take the bundle from THIS run''s artifacts: ' + type: string + required: true + tier: + type: string + default: v2 + matrix_json: + description: 'JSON list of {platform, runner, sde, python, chip} entries' + type: string + required: true + gate: + type: boolean + default: true + +permissions: + contents: read + actions: read + +env: + ZSMOOTH_VERSION: '0.19.0' + +jobs: + probe: + name: ${{ matrix.platform }}${{ (inputs.tier || 'v3') == 'v2' && '-v2' || '' }} (-${{ matrix.chip }}) + strategy: + fail-fast: false + matrix: + # wsm = Westmere (the Mac Pro 5,1 in #92): no AVX at all. + # snb = Sandy Bridge: AVX without AVX2 — the v3/v2 tier boundary. + include: ${{ fromJSON(inputs.matrix_json || '[{"platform":"linux-x64","runner":"ubuntu-24.04","sde":"sde64","python":"python/bin/python3","chip":"wsm"},{"platform":"windows-x64","runner":"windows-latest","sde":"sde.exe","python":"vapoursynth/python.exe","chip":"snb"}]') }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + defaults: + run: + shell: bash + env: + D: deps/${{ matrix.platform }} + ASSET_ID: ${{ matrix.platform }}${{ (inputs.tier || 'v3') == 'v2' && '-v2' || '' }} + steps: + - uses: actions/checkout@v5 + + # Gate mode: the bundle was built earlier in this same run. + - name: Take the bundle from this run + if: inputs.artifact_prefix != '' + uses: actions/download-artifact@v5 + with: + name: ${{ inputs.artifact_prefix }}${{ env.ASSET_ID }} + path: .deps-artifact + + - name: Download dependencies (${{ env.ASSET_ID }}) + env: + GH_TOKEN: ${{ github.token }} + DEPS_RUN_ID: ${{ inputs.deps_run_id }} + run: | + mkdir -p "$D" + if [ -d .deps-artifact ]; then + ZIP=$(find .deps-artifact -name "*-$ASSET_ID.zip" | head -1) + else + TAG=$(sed -n 's/.*"releaseTag": *"\([^"]*\)".*/\1/p' app/assets/deps-version.json) + ZIP=$(bash .github/scripts/fetch-deps-bundle.sh "$ASSET_ID" "$TAG" "${TAG#deps-v}") + fi + [ -f "$ZIP" ] || { echo "no bundle zip for $ASSET_ID"; exit 1; } + if [ "$RUNNER_OS" = Windows ]; then + # The Windows zip uses backslash separators; 7-Zip copes, unzip does not. + 7z x "$ZIP" -o"$D" -y >/dev/null + else + unzip -q -o "$ZIP" -d "$D" + fi + rm -rf "$ZIP" .deps-artifact + + - name: Fetch the positive control (upstream zsmooth, AVX2-only) + run: | + if [ "$RUNNER_OS" = Windows ]; then + ASSET=zsmooth-x86_64-windows.zip; EXT=dll + else + ASSET=zsmooth-x86_64-linux-gnu.zip; EXT=so + fi + curl -fsSL -o control.zip \ + "https://github.com/adworacz/zsmooth/releases/download/${ZSMOOTH_VERSION}/${ASSET}" + if [ "$RUNNER_OS" = Windows ]; then + 7z x control.zip -ocontrol -y >/dev/null + else + unzip -q -o control.zip -d control + fi + FOUND=$(find control -name "*zsmooth*.$EXT" -type f | head -1) + cp "$FOUND" "control-zsmooth-haswell.$EXT" + echo "CONTROL=$PWD/control-zsmooth-haswell.$EXT" >> "$GITHUB_ENV" + + - name: Setup Intel SDE + uses: petarpetrovt/setup-sde@v6.0 + with: + environmentVariableName: SDE_PATH + sdeVersion: 10.13.1 + + - name: Native run (control — must pass on this runner) + run: | + "$D/${{ matrix.python }}" Scripts/probe-plugin-compat.py "$D" \ + --report native.txt --json native.json --fail-on-crash + + - name: SDE run (-${{ matrix.chip }}) + run: | + SDE_DIR=$(cygpath -m "$SDE_PATH" 2>/dev/null || echo "$SDE_PATH") + "$D/${{ matrix.python }}" Scripts/probe-plugin-compat.py "$D" \ + --report sde.txt --json sde.json --timeout 1800 --extra "$CONTROL" \ + --wrap "\"$SDE_DIR/${{ matrix.sde }}\" -${{ matrix.chip }} --" + + - name: Check the controls${{ inputs.gate && ' and gate the bundle' || '' }} + env: + GATE: ${{ inputs.gate && 'true' || 'false' }} + run: | + "$D/${{ matrix.python }}" - <<'EOF' + import json, os, sys + sde = json.load(open("sde.json"))["results"] + rows = ["| file | namespaces | under SDE |", "|---|---|---|"] + rows += [f"| {r['file']} | {r['namespaces']} | {r['status']} {r['detail']} |" for r in sde] + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write("\n".join(rows) + "\n") + + core = next(r for r in sde if r["file"].startswith("(core")) + if core["status"] != "PASS": + sys.exit(f"core control failed under SDE ({core['status']}: {core['detail']}); " + "the emulation is not valid for this OS/chip, so no plugin result means anything") + c = next(r for r in sde if r["file"].startswith("control-zsmooth-haswell")) + if c["status"] != "CRASHED" or "control-zsmooth-haswell" not in c["detail"]: + sys.exit(f"SDE did not trap the AVX2-only control build in its own code " + f"({c['status']}: {c['detail']}); results are meaningless") + + bad = [r for r in sde if r["status"] in ("CRASHED", "INCONCLUSIVE", "ERROR") and r is not c] + print(f"controls OK; {len(bad)} bundle file(s) crashed, errored or were inconclusive:") + for r in bad: + print(f" {r['file']} [{r['namespaces']}] {r['status']}: {r['detail']}") + if bad and os.environ["GATE"] == "true": + sys.exit("the bundle does not run on this CPU") + EOF + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v5 + with: + name: cpu-compat-${{ env.ASSET_ID }}-${{ matrix.chip }} + path: | + native.txt + native.json + sde.txt + sde.json diff --git a/CLAUDE.md b/CLAUDE.md index ad22d58..cfef9ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,6 +100,7 @@ VapourBox/ | `worker/src/script_generator.rs` | Template substitution for .vpy | | `worker/src/pipeline_executor.rs` | vspipe \| ffmpeg execution | | `worker/src/pixel_format.rs` | Source `pix_fmt` → pipe format (see "Source Pixel Formats") | +| `worker/src/cpu.rs` | The x86 deps CPU tier (`v3`/`v2`) — the single decision; see "CPU tiers" | | `worker/templates/pipeline_template.vpy` | VapourSynth script template | | `worker/tests/filter_integration_test.rs` | Filter integration tests | @@ -143,19 +144,24 @@ VapourBox/ # it targets deps/macos-x64. FFmpeg comes pre-built from evermeet.cx, most # plugins from Stefan-Olt; descratch/neo-f3kdb/nnedi3cl/vivtc build from source. ./scripts/download-deps-macos.sh -# To build x64 deps on Apple Silicon instead, run it under Rosetta 2 with an -# Intel Homebrew prefix first in PATH: -# arch -x86_64 /bin/bash -lc 'PATH=/usr/local/bin:$PATH ./Scripts/download-deps-macos.sh --force' +# x64 deps can NOT be built on Apple Silicon any more: Homebrew's installer +# refuses to create the Intel prefix under Rosetta. Use CI, or unzip a +# published/artifact macos-x64 bundle into deps/macos-x64. # Windows (PowerShell) .\Scripts\download-deps-windows.ps1 # Linux ./Scripts/download-deps-linux.sh + +# Any x64 platform, older-CPU tier (see "CPU tiers"); switching tier in an +# existing deps/ needs --force (Windows: delete it first) +./Scripts/download-deps-linux.sh --force --tier v2 ``` Both macOS architectures are produced in CI by `build-deps-macos.yml` (arm64 on -`macos-15`, x64 natively on `macos-15-intel`). +`macos-15`, x64 natively on `macos-15-intel`), and every x64 platform's +workflow builds both CPU tiers. ### Build Rust Worker @@ -360,9 +366,13 @@ Adding a filter touches many files. Missing any step causes silent failures (fil platform that ships it. The `package-deps-*` scripts assert this list is present before zipping and fail the build if any are missing, so a dead download URL becomes a red build instead of a silently-incomplete bundle. - A bare filename is looked up in the plugin directory; an entry containing a - `/` is resolved from the **bundle root** instead, which is how zsmooth's - per-CPU builds outside the autoload directory stay covered. + x64 platforms have a second key per CPU tier (`macos-x64-v2`, …) that must + list the same plugins — `attribution_test.dart` asserts it. +- **Check it for load-time SIMD on x64** (see "CPU tiers" below): a prebuilt + x86 binary built with `-mavx2`-style flags can SIGILL inside `dlopen` on an + older CPU, and autoload makes that every job. Run + `Scripts/check-load-time-simd.py` on a macOS dylib; the v2 gates in the + `build-deps-*` workflows cover all three platforms. - For local testing, run the download script to populate `deps/` (e.g. `deps/windows-x64/vapoursynth/vs-plugins/`, `deps/macos-arm64/vapoursynth/plugins/`). - **Credit it**: add an entry to `licenses/NOTICES.txt` and a `_ComponentTile` @@ -656,19 +666,39 @@ disagree. Pin the **series** against a rolling host tag (e.g. BtbN's `n9.0-latest`), never an exact old build — a fixed old tag gets garbage-collected off a rolling `latest` alias and 404s the deps build outright. -**zsmooth ships one build per x86 CPU baseline (`haswell`, `x86_64_v2`), loaded -by explicit path, not autoloaded.** Upstream's single AVX2-baseline build -crashes with an illegal instruction on any pre-2013 CPU the instant a zsmooth -filter runs (`CCD`, `Cnr4`, `SpotLess`→`RemoveDirt`, `mClean`, -`TemporalDegrain2`, hybrid_mv all reach it). `DependencyLocator::zsmooth_plugin()` -returns `None` on an older bundle lacking the split, and the generated script -must stay byte-identical to the pre-split form in that case. +**CPU tiers (issue #92): every x86 deps bundle ships twice.** `v3` is built +for x86-64-v3 CPUs (Haswell, 2013, and later); `v2` runs on anything older. +The rules: + +- **One decision, in the worker.** `cpu::cpu_tier()` (`worker/src/cpu.rs`) + requires the *whole* x86-64-v3 set, not just AVX2. `--probe-cpu` reports it, + the app downloads by it, and `ctmf_opt` derives from it. Never re-derive the + tier elsewhere (sysctl is wrong under Rosetta); on x86 an unclear answer means + `v2`, which only costs speed. `VAPOURBOX_DEPS_TIER=v2|v3` overrides it. +- **The tier lives only in the asset name and version.json.** `v3` keeps the + plain names (`VapourBox-deps-X-macos-x64.zip`), `v2` is suffixed + (`…-macos-x64-v2.zip`); both install to `deps/`. The app re-checks + the installed tier at startup (`DependencyStatus.wrongTier`) and replaces a + mismatch even when it is newer than expected. +- **One `--tier` block per `download-deps-*` script** maps the tier to build + variables; nothing else in the script branches on it. The two tiers differ + only where running the bundle on an old CPU proved they must: zsmooth + (`haswell` / `x86_64_v2`, one build per bundle, autoloaded — upstream has no + runtime dispatch) everywhere, and on macOS MVTools (v2 builds v24 from source + with `patches/mvtools-v24-no-avx2.patch`, because Stefan-Olt's build SIGILLs + in its AVX2 static initializers inside `dlopen`). +- **A v2 bundle is gated before it can be published**: under Intel SDE by + `probe-cpu-compat.yml` (Linux at Westmere; Windows at Sandy Bridge, since SDE + cannot emulate pre-AVX Windows — Microsoft's runtime reads the host's CPU + features from the kernel), and on macOS statically by + `Scripts/check-load-time-simd.py` inside `package-deps-macos.sh`. **CTMF's `opt` (SIMD level) must be chosen by the worker from the CPU, never left at the plugin's own auto-detect (`opt=0`).** Its AVX-512 kernel for 8-bit input crashes with an access violation on real hardware — auto-detect is exactly what selects that broken kernel. `script_generator::ctmf_opt` picks 3 -(AVX2) or 2 (SSE2) based on `is_x86_feature_detected!`, never 0. +(AVX2) on a v3-tier CPU, else 2 (SSE2), never 0. This is independent of the +tiers: a v3 machine can have AVX-512. See docs/ENGINEERING_NOTES.md for the specific plugin-by-plugin decisions, probe-round methodology, and dated write-ups behind all of the above. @@ -1046,15 +1076,15 @@ can render fine and still fail the preview. > `vapoursynth_integration_test`'s "all required plugins load" list is the > runtime contract for a **complete deps install** and must name every -> namespace a filter can reach — except `zsmooth`, which is deliberately not -> autoloaded and has its own load-and-render test. Add the namespace whenever +> namespace a filter can reach. Add the namespace whenever > you add a plugin, or a bundle missing it passes CI and fails at job time. > OpenCL-only plugins (`nnedi3cl`, `knlm`) stay **out** of the list — the app > degrades to a CPU path without them. The harness honors `$VAPOURBOX_DEPS_DIR`, else uses repo-root `deps/`, else **downloads the deps release pinned in `app/assets/deps-version.json`** -(opt out with `$VAPOURBOX_SKIP_DEPS_DOWNLOAD=1`). The worker binary is found under +(opt out with `$VAPOURBOX_SKIP_DEPS_DOWNLOAD=1`), in the tier the app would +pick (`$VAPOURBOX_DEPS_TIER` overrides it). The worker binary is found under `worker/target/{release,debug}` (CI's `cargo test`/`cargo build` produces debug). Subtitle heavy tests skip when the whisper add-on is absent. @@ -1081,9 +1111,19 @@ integration tests. Matrix: macOS **arm64** (`macos-15`), macOS **x64** (`macos-15-intel`), **Windows x64**, **Linux x64** (`ubuntu-24.04`). The heavy full-encode integration tests run separately in `.github/workflows/nightly.yml` (cron + `workflow_dispatch`) via `flutter test --tags heavy` on the same -4-platform matrix. Fixtures (`small_clip.mp4`, telecine/interlaced clips) are +4-platform matrix, running each x64 platform against **both** CPU-tier bundles. +Fixtures (`small_clip.mp4`, telecine/interlaced clips) are committed under `Tests/TestResources/`. +> **Old-CPU behaviour cannot be tested on the runners, or on Apple Silicon.** +> Every hosted runner is a v3 CPU, and Rosetta 2 translates AVX/AVX2 on macOS +> 15+ — so a plugin that SIGILLs on a pre-AVX2 Intel Mac runs fine under +> Rosetta. The tools that do answer it: `probe-cpu-compat.yml` (every plugin +> under Intel SDE, with controls that prove the emulation is valid), and +> `Scripts/probe-plugin-compat.sh` for a user to run on the real machine +> (one plugin per process, into a `DISABLE_AUTO_LOADING` core — under autoload +> one faulting plugin masks every other result). + > **A green CI run on hosted hardware is not proof about CPU-dispatched code.** > The runner fleet is mixed for features like AVX-512, so a filter that only > crashes on that instruction set can pass for days and then fail on no code @@ -1301,6 +1341,12 @@ placement and a version skew would change chroma per-OS. under vspipe with passes commented out; on Windows, `$LASTEXITCODE` `0xC0000005` = access violation, `0xC000001D` = illegal instruction (CPU feature the machine lacks) — both are decoded by `format_exit_status`. + **If it happens with every pass disabled, it is load-time, not a filter**: + VapourSynth autoloads every plugin when the core starts, so one plugin whose + static initializers use a missing instruction set kills every job (issue #92, + MVTools' AVX2 tables, `SIGILL` = signal 4 on macOS/Linux). Commenting out + passes cannot find it; the OS crash report (`.ips` on macOS) names the image, + and `Scripts/probe-plugin-compat.sh` tests each plugin in isolation. ## Platform-Specific Notes @@ -1376,7 +1422,7 @@ Plugin lists for all platforms: see `deps/` directories or download scripts. ## Dependency Versioning and Auto-Download -Dependencies are versioned separately from the app via `app/assets/deps-version.json` and distributed as separate GitHub releases (tag: `deps-vX.Y.Z`). The app auto-downloads deps on launch if missing or outdated. +Dependencies are versioned separately from the app via `app/assets/deps-version.json` and distributed as separate GitHub releases (tag: `deps-vX.Y.Z`). The app auto-downloads deps on launch if missing, outdated, or built for a different CPU tier (x64 only — see "CPU tiers"; the asset is `DependencyManager.assetIdFor(platformId, tier)`). `deps-version.json` is a **slim pointer** — `{version, releaseTag, githubRepo}`. Integrity metadata is **not** stored here: each `package-deps-*` script writes a **sidecar** `.sha256.json` uploaded next to the zip, which the app fetches and verifies at download time (best-effort if the sidecar is missing). **Net effect: a new deps release only needs a `version`/`releaseTag` bump.** @@ -1402,6 +1448,9 @@ App and deps use **separate release tags** so unchanged deps aren't re-uploaded ``` This prompts for version, checks deps changes, builds, packages, and creates draft GitHub releases. +It does **not** package deps: if they changed, it stops and prints the +`build-deps-*` workflow commands. A local checkout holds only one CPU tier per +x64 platform, so a locally packaged deps release would be missing the other. ### CI Build and Release @@ -1479,8 +1528,8 @@ Notes: 1. **Confirm version** — ask user, update `pubspec.yaml` 2. **Check deps** — run `check-deps-changed.sh`; if changed, bump `version`/`releaseTag` in `deps-version.json`. -3. **Build & package** — use packaging scripts (or `release.sh`). Each `package-deps-*` writes the zip **and** its `.sha256.json` sidecar. -4. **Upload deps assets** — upload each platform's zip **and its `.sha256.json` sidecar** to the deps release. +3. **Build & package deps in CI** — dispatch `build-deps-{macos,windows,linux}.yml` with `release_tag`. They build both x64 tiers, gate the v2 bundles, and only then upload each zip **and** its `.sha256.json` sidecar. +4. **Check the deps release** — every platform, both tiers of each x64: `macos-arm64`, `macos-x64`, `macos-x64-v2`, `windows-x64`, `windows-x64-v2`, `linux-x64`, `linux-x64-v2`, `linux-arm64`. 5. **Test** — fresh install + upgrade test 6. **Create GitHub releases** — deps release first (if changed, tag `deps-vX.Y.Z`), then app release (tag `vX.Y.Z`) @@ -1581,3 +1630,4 @@ Full write-ups (root causes, measurements) for each entry are in | 1.8.0 | 2026-08-07 | VapourSynth **R73 → R78** everywhere (Windows Python-wheel layout; `deps//vapoursynth/` is the Python package on macOS/Linux). Adds **akarin** (LLVM JIT for `std.Expr`, ~4x on arm64 QTGMC; not on macos-x64). Fixes nnedi3 on linux-arm64. Removes BestSource. Linux now needs glibc 2.39 | | 1.9.0 | 2026-08-15 | Adds **fluxsmooth** (unlocks havsfunc's STPresso), **bifrost** (temporal rainbow/dot-crawl removal), **retinex** (shadow-detail lift) — all pinned to the newest release with a published Windows binary | | 1.10.0 | 2026-08-31 | **Issue #82**: zsmooth now ships one build per x86 CPU baseline (haswell, x86_64_v2), fixing an illegal-instruction crash on pre-2013 CPUs. **FFmpeg pinned to 9.0 on all four platforms** (they had silently diverged: Windows on an unpinned post-9.0 master, macOS on floating 9.0.1, Linux stuck at 7.1 after BtbN garbage-collected the pinned tag) | +| 1.11.0 | 2026-09 | **Issue #92**: every x86 bundle ships in two **CPU tiers** — `v3` (x86-64-v3; the plain asset names) and `v2` (`…-x64-v2`, anything older), chosen by the app from `vapourbox-worker --probe-cpu`. Each bundle carries one autoloaded zsmooth (replacing 1.10.0's two-builds-loaded-by-path). macOS v2 builds MVTools v24 from source without its AVX2 files, whose static initializers SIGILLed inside `dlopen` on pre-AVX Macs. v2 bundles are gated (SDE on Linux/Windows, static check on macOS) before publishing | diff --git a/README.md b/README.md index 15a9dfc..1dd5891 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,8 @@ GPU-accelerated deinterlacing (NNEDI3CL) needs your GPU's OpenCL driver installe +**Older Intel and AMD processors** (before about 2013 — no AVX2) are supported on all three platforms. On first launch VapourBox checks the processor and downloads the matching set of processing components; a machine moved to a different CPU gets the right set on its next launch. The older-CPU set runs some filters more slowly, never differently. Every build of that set is checked for pre-AVX processors (e.g. a 2010 Mac Pro) on macOS and Linux; on Windows it is checked for processors with AVX but no AVX2, and older ones are expected to work but can't be tested automatically. + ## Output formats | | | diff --git a/Scripts/check-deps-changed.sh b/Scripts/check-deps-changed.sh index f73fe31..253e942 100755 --- a/Scripts/check-deps-changed.sh +++ b/Scripts/check-deps-changed.sh @@ -18,10 +18,18 @@ fi # Files to check for changes. # Deps binaries are no longer committed (reproduced by the download scripts, which # are the source of truth), so "deps changed" == "a download script changed". +# Everything that changes what goes into a bundle, not just the download +# scripts: a patch (e.g. the MVTools no-AVX2 patch) or a packaging change +# alters the bundle just as surely. DEPS_PATHS=( "Scripts/download-deps-windows.ps1" "Scripts/download-deps-macos.sh" "Scripts/download-deps-linux.sh" + "Scripts/patches" + "Scripts/package-deps-windows.ps1" + "Scripts/package-deps-macos.sh" + "Scripts/package-deps-linux.sh" + "Scripts/deps-expected-plugins.json" ) # Get the last deps release tag diff --git a/Scripts/check-load-time-simd.py b/Scripts/check-load-time-simd.py new file mode 100644 index 0000000..98d665e --- /dev/null +++ b/Scripts/check-load-time-simd.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Fail if an x86_64 Mach-O plugin can run AVX instructions while it loads. + +Issue #92: prebuilt MVTools compiles six files with -mavx2, and their static +initializers — run by dyld inside dlopen, before any plugin code can check the +CPU — contain VEX instructions. VapourSynth autoloads every plugin when a core +starts, so on a CPU without AVX every job died with SIGILL whatever it asked +for. Runtime SIMD dispatch in the plugin's filters cannot help: this runs +first. + +macOS has no Intel SDE, so the v2 bundle is checked statically here instead +(Linux and Windows are checked by running under SDE; see +.github/workflows/probe-cpu-compat.yml). Initializers are found structurally, +from __init_offsets / __mod_init_func, never by symbol name — LTO and +toolchains rename them freely, and a name-based scan once came back clean on a +binary it simply could not see into. Direct calls and tail calls are followed, +because the VEX can sit in a helper the initializer calls rather than in the +initializer itself. + +Usage: check-load-time-simd.py [--depth N] BINARY... +Exit 1 if any binary can reach a VEX instruction from an initializer. +""" + +import argparse +import bisect +import re +import shutil +import struct +import subprocess +import sys + +LC_SEGMENT_64 = 0x19 +MH_MAGIC_64 = 0xFEEDFACF +FAT_MAGIC = 0xCAFEBABE +CPU_TYPE_X86_64 = 0x01000007 + +# VEX-encoded (AVX and later) mnemonics start with "v" in LLVM's AT&T syntax; +# these few legacy instructions also do and are not VEX. +NOT_VEX = {"verr", "verw"} + + +def x86_64_slice(data): + """Return (bytes, offset) of the x86_64 Mach-O image in a thin or fat file.""" + magic = struct.unpack(">I", data[:4])[0] + if magic == FAT_MAGIC: + nfat = struct.unpack(">I", data[4:8])[0] + for i in range(nfat): + cputype, _, offset, size, _ = struct.unpack(">iiIII", data[8 + i * 20:28 + i * 20]) + if cputype == CPU_TYPE_X86_64: + return data[offset:offset + size] + return None + if struct.unpack(":$") + insn = re.compile(r"^\s*([0-9a-f]+):\s+(\S+)\s*(.*)$") + # Direct targets only ("0x1234 "). An indirect `jmpq *0x..(%rip)` + # names a GOT slot, not code, and following it links unrelated functions. + target = re.compile(r"^0x([0-9a-f]+)") + for line in out.splitlines(): + m = label.match(line) + if m: + cur = int(m.group(1), 16) + funcs[cur] = {"name": m.group(2), "vex": [], "calls": set()} + continue + m = insn.match(line) + if not m or cur is None: + continue + addr, mnem, ops = int(m.group(1), 16), m.group(2), m.group(3) + if mnem.startswith("v") and mnem not in NOT_VEX: + funcs[cur]["vex"].append(f"{addr:#x}: {mnem} {ops}".strip()) + if mnem.startswith("call") or mnem.startswith("jmp"): + t = target.match(ops) + if t: + funcs[cur]["calls"].add(int(t.group(1), 16)) + # Stub tables are trampolines into OTHER images (libc++, libSystem), which + # this check cannot see into; never walk through them. + return {a: f for a, f in funcs.items() + if f["name"] not in ("__stubs", "__stub_helper", "__auth_stubs")} + + +def check(path, depth): + with open(path, "rb") as f: + image = x86_64_slice(f.read()) + if image is None: + return f"{path}: no x86_64 image, skipped", [] + roots = initializer_addresses(image) + if not roots: + # Nothing runs at load time. Checked before disassembling: some valid + # images (Zig-linked zsmooth) have a header layout llvm-objdump rejects. + return f"{path}: 0 initializers", [] + try: + funcs = disassemble(path) + except subprocess.CalledProcessError as e: + # Initializers we cannot inspect are a failure, not a pass. + return (f"{path}: {len(roots)} initializers", [ + f"could not disassemble: {e.stderr.strip().splitlines()[-1] if e.stderr else e}"]) + starts = sorted(funcs) + + def owner(addr): + i = bisect.bisect_right(starts, addr) - 1 + return starts[i] if i >= 0 else None + + findings = [] + seen = set() + frontier = [(owner(r), [owner(r)]) for r in roots if owner(r) is not None] + for _ in range(depth + 1): + nxt = [] + for fn, chain in frontier: + if fn in seen: + continue + seen.add(fn) + info = funcs[fn] + if info["vex"]: + names = " -> ".join(funcs[c]["name"] for c in chain) + findings.append(f"{names}\n first: {info['vex'][0]} " + f"({len(info['vex'])} VEX instructions)") + for t in info["calls"]: + o = owner(t) + if o is not None and o != fn: + nxt.append((o, chain + [o])) + frontier = nxt + return f"{path}: {len(roots)} initializers, {len(seen)} functions reachable", findings + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("binaries", nargs="+") + ap.add_argument("--depth", type=int, default=6, + help="call depth to follow from each initializer") + args = ap.parse_args() + bad = 0 + for b in args.binaries: + summary, findings = check(b, args.depth) + print(("FAIL " if findings else "ok ") + summary) + for f in findings: + print(" " + f) + bad += bool(findings) + sys.exit(1 if bad else 0) + + +if __name__ == "__main__": + main() diff --git a/Scripts/deps-expected-plugins.json b/Scripts/deps-expected-plugins.json index e2f398f..a0f1784 100644 --- a/Scripts/deps-expected-plugins.json +++ b/Scripts/deps-expected-plugins.json @@ -1,17 +1,30 @@ { - "_comment": "Required VapourSynth plugin filenames per platform — the contract for a COMPLETE deps bundle. The package-deps-* scripts assert every file listed here exists in the staged bundle before zipping and FAIL the build if any are missing, so a silently-failed download (e.g. a dead upstream URL) becomes a red build instead of an incomplete bundle shipping. Plugin directory: windows-x64 = vapoursynth/vs-plugins, macos/linux = vapoursynth/plugins. Lists exclude data files (nnedi3 weights) and runtime libs (fftw); they cover the VapourSynth plugin binaries only. Update this when adding or removing a plugin. An entry containing a '/' is a path relative to the BUNDLE ROOT rather than a filename in the plugin directory: zsmooth ships one build per CPU baseline in vapoursynth/zsmooth/, deliberately outside the autoload directory, because both builds register the same namespace and the worker loads exactly one by path (DependencyLocator::zsmooth_plugin).", + "_comment": "Required VapourSynth plugin filenames per platform — the contract for a COMPLETE deps bundle. The package-deps-* scripts assert every file listed here exists in the staged bundle's plugin directory before zipping and FAIL the build if any are missing, so a silently-failed download (e.g. a dead upstream URL) becomes a red build instead of an incomplete bundle shipping. Plugin directory: windows-x64 = vapoursynth/vs-plugins, macos/linux = vapoursynth/plugins. Lists exclude data files (nnedi3 weights) and runtime libs (fftw); they cover the VapourSynth plugin binaries only. Update this when adding or removing a plugin. The x64 platforms ship in two CPU tiers (issue #92): the plain key is the v3 bundle, the -v2 key the bundle for CPUs below x86-64-v3. Both tiers carry the same set of plugins, built differently, so their lists must stay identical (asserted by app/test/attribution_test.dart).", "windows-x64": [ "AddGrain.dll", + "bifrost.dll", + "bwdif.dll", "CAS.dll", "CTMF.dll", "DCTFilter.dll", - "DFTTest.dll", - "DeScratch.dll", "Deblock.dll", + "dedot.dll", + "DeScratch.dll", + "DFTTest.dll", "EEDI3m.dll", + "fft3dfilter.dll", + "fmtconv.dll", "KNLMeansCL.dll", "LGhost.dll", + "libakarin.dll", + "libawarpsharp2.dll", + "libfillborders.dll", + "libfluxsmooth.dll", + "libmvtools.dll", + "libtemporalmedian.dll", + "libzstd.dll", "MiscFilters.dll", + "neo-f3kdb.dll", "NNEDI3CL.dll", "RemoveDirt.dll", "RemoveGrainVS.dll", @@ -19,11 +32,25 @@ "TCanny.dll", "TTempSmooth.dll", "VIVTC.dll", + "vsznedi3.dll", + "zsmooth.dll" + ], + "windows-x64-v2": [ + "AddGrain.dll", "bifrost.dll", "bwdif.dll", + "CAS.dll", + "CTMF.dll", + "DCTFilter.dll", + "Deblock.dll", "dedot.dll", + "DeScratch.dll", + "DFTTest.dll", + "EEDI3m.dll", "fft3dfilter.dll", "fmtconv.dll", + "KNLMeansCL.dll", + "LGhost.dll", "libakarin.dll", "libawarpsharp2.dll", "libfillborders.dll", @@ -31,10 +58,17 @@ "libmvtools.dll", "libtemporalmedian.dll", "libzstd.dll", + "MiscFilters.dll", "neo-f3kdb.dll", - "vapoursynth/zsmooth/zsmooth-haswell.dll", - "vapoursynth/zsmooth/zsmooth-x86_64_v2.dll", - "vsznedi3.dll" + "NNEDI3CL.dll", + "RemoveDirt.dll", + "RemoveGrainVS.dll", + "Retinex.dll", + "TCanny.dll", + "TTempSmooth.dll", + "VIVTC.dll", + "vsznedi3.dll", + "zsmooth.dll" ], "macos-arm64": [ "libaddgrain.dylib", @@ -70,7 +104,7 @@ "libttempsmooth.dylib", "libvivtc.dylib", "libznedi3.dylib", - "vapoursynth/zsmooth/libzsmooth.dylib" + "libzsmooth.dylib" ], "macos-x64": [ "libaddgrain.dylib", @@ -103,8 +137,40 @@ "libttempsmooth.dylib", "libvivtc.dylib", "libznedi3.dylib", - "vapoursynth/zsmooth/libzsmooth-haswell.dylib", - "vapoursynth/zsmooth/libzsmooth-x86_64_v2.dylib" + "libzsmooth.dylib" + ], + "macos-x64-v2": [ + "libaddgrain.dylib", + "libawarpsharp2.dylib", + "libbifrost.dylib", + "libbwdif.dylib", + "libcas.dylib", + "libctmf.dylib", + "libdctfilter.dylib", + "libdeblock.dylib", + "libdedot.dylib", + "libdescratch.dylib", + "libdfttest.dylib", + "libeedi3m.dylib", + "libfft3dfilter.dylib", + "libfillborders.dylib", + "libfluxsmooth.dylib", + "libfmtconv.dylib", + "libknlmeanscl.dylib", + "liblghost.dylib", + "libmiscfilters.dylib", + "libmvtools.dylib", + "libneo-f3kdb.dylib", + "libnnedi3cl.dylib", + "libremovedirt.dylib", + "libremovegrain.dylib", + "libretinex.dylib", + "libtcanny.dylib", + "libtmedian.dylib", + "libttempsmooth.dylib", + "libvivtc.dylib", + "libznedi3.dylib", + "libzsmooth.dylib" ], "linux-x64": [ "libaddgrain.so", @@ -138,8 +204,41 @@ "libttempsmooth.so", "libvivtc.so", "libznedi3.so", - "vapoursynth/zsmooth/libzsmooth-haswell.so", - "vapoursynth/zsmooth/libzsmooth-x86_64_v2.so" + "libzsmooth.so" + ], + "linux-x64-v2": [ + "libaddgrain.so", + "libakarin.so", + "libawarpsharp2.so", + "libbifrost.so", + "libbwdif.so", + "libcas.so", + "libctmf.so", + "libdctfilter.so", + "libdeblock.so", + "libdedot.so", + "libdescratch.so", + "libdfttest.so", + "libeedi3m.so", + "libfft3dfilter.so", + "libfillborders.so", + "libfluxsmooth.so", + "libfmtconv.so", + "libknlmeanscl.so", + "liblghost.so", + "libmiscfilters.so", + "libmvtools.so", + "libneo-f3kdb.so", + "libnnedi3cl.so", + "libremovedirt.so", + "libremovegrain.so", + "libretinex.so", + "libtcanny.so", + "libtmedian.so", + "libttempsmooth.so", + "libvivtc.so", + "libznedi3.so", + "libzsmooth.so" ], "linux-arm64": [ "libaddgrain.so", @@ -174,6 +273,6 @@ "libttempsmooth.so", "libvivtc.so", "libznedi3.so", - "vapoursynth/zsmooth/libzsmooth.so" + "libzsmooth.so" ] } diff --git a/Scripts/download-deps-linux.sh b/Scripts/download-deps-linux.sh index 10db031..81cca58 100755 --- a/Scripts/download-deps-linux.sh +++ b/Scripts/download-deps-linux.sh @@ -9,20 +9,25 @@ # libfftw3-dev libboost-filesystem-dev libboost-atomic-dev \ # ocl-icd-opencl-dev libdvdread-dev # -# Usage: ./Scripts/download-deps-linux.sh [--force] +# Usage: ./Scripts/download-deps-linux.sh [--force] [--tier v3|v2] set -e FORCE=false +TIER=v3 while [[ $# -gt 0 ]]; do case $1 in --force) FORCE=true shift ;; + --tier) + TIER="$2" + shift 2 + ;; *) echo "Unknown option: $1" - echo "Usage: $0 [--force]" + echo "Usage: $0 [--force] [--tier v3|v2]" exit 1 ;; esac @@ -39,10 +44,47 @@ else exit 1 fi +# CPU tier (issue #92). The x86 bundle ships twice: v3 for x86-64-v3 CPUs +# (Haswell, 2013, and later) and v2 for anything older, chosen by the app from +# `vapourbox-worker --probe-cpu`. This block is the ONLY place the tier is +# interpreted; everything below reads these variables and never $TIER itself. +# ZSMOOTH_CPU zsmooth's baseline (upstream has no runtime dispatch) +# That is the whole difference on Linux: every other plugin in this bundle +# renders under Intel SDE emulating Westmere (no AVX at all) — see +# .github/workflows/probe-cpu-compat.yml — including MVTools, whose macOS +# prebuilt is the one that does not. +if [ "$ARCH" = "x86_64" ]; then + case "$TIER" in + v3) ZSMOOTH_CPU=haswell ;; + v2) ZSMOOTH_CPU=x86_64_v2 ;; + *) echo "Unknown tier: $TIER (expected v3 or v2)"; exit 1 ;; + esac +else + if [ "$TIER" != "v3" ]; then + echo "--tier is x86-only; the arm64 bundle is not tiered." + exit 1 + fi + TIER="" +fi + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" DEPS_DIR="$PROJECT_ROOT/deps/$PLATFORM_DIR" PLUGINS_DIR="$DEPS_DIR/vapoursynth/plugins" + +# Both tiers build into the same deps/ directory (the tier lives in +# version.json, not the path), and most steps skip what already exists. So a +# tier switch without --force would keep the previous tier's zsmooth and +# silently produce a mixed bundle. +if [ -n "$TIER" ] && [ "$FORCE" = false ] && [ -f "$DEPS_DIR/version.json" ]; then + EXISTING_TIER=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('tier','v3'))" \ + "$DEPS_DIR/version.json" 2>/dev/null || echo "v3") + if [ "$EXISTING_TIER" != "$TIER" ]; then + echo "ERROR: $DEPS_DIR holds the $EXISTING_TIER bundle; building $TIER over it" + echo "would mix the two. Re-run with --force." + exit 1 + fi +fi PYTHON_DIR="$DEPS_DIR/python" PYTHON_PACKAGES_DIR="$DEPS_DIR/python-packages" BUILD_DIR="/tmp/vapourbox-build-$$" @@ -1024,24 +1066,22 @@ build_plugin "tmedian" \ "libtmedian.so" \ "$PLUGIN_BUILD_ENV meson setup build --buildtype=release && ninja -C build" -# zsmooth — one build per CPU baseline +# zsmooth — exactly one build, in the autoload directory # # core.zsmooth.CCD (also Cnr4 and a set of RemoveGrain/TemporalMedian-family -# filters). Upstream publishes only `haswell` (an AVX2 baseline) and `znver4` -# x86 builds, compiled throughout with NO runtime dispatch — so on a pre-2013 -# CPU the library loads fine and then dies with an illegal instruction the -# instant a filter runs. That is issue #82 (reported on Windows, but this -# bundle took the same haswell asset), and it is silent: vspipe prints nothing. -# -# So x86 ships both builds outside the autoload directory and the worker loads -# exactly one by path (DependencyLocator::zsmooth_plugin). They cannot share a -# directory: each registers the namespace `zsmooth`, so whichever autoloads -# second is rejected. aarch64 has a single NEON baseline and needs no split. +# filters). zsmooth has NO runtime dispatch: each build is compiled for one CPU +# baseline throughout, so a build above the machine's baseline loads fine and +# then dies with an illegal instruction the instant a filter runs — issue #82, +# silently, because vspipe prints nothing. # -# Measured at 720x576: `x86_64` is 2.0x slower than haswell on CCD and 3.0x on -# Cnr4, `x86_64_v2` 1.4x on both — which is why the portable build is v2 -# (SSE4.2/POPCNT, everything from Nehalem 2009 on) and why the fast build is -# still shipped rather than dropped for one portable binary. +# aarch64 has a single NEON baseline and takes the author's build. x86 takes +# the $ZSMOOTH_CPU chosen by the tier block at the top of this script: the +# author's haswell asset for v3, or an x86_64_v2 build compiled here for v2 +# (SSE4.2/POPCNT, everything from Nehalem 2009 on; measured 1.4x slower than +# haswell on CCD and Cnr4, against 2-3x for plain x86_64). One build per bundle +# means it autoloads like any other plugin; bundles up to 1.10.0 shipped both +# x86 builds in a separate zsmooth/ directory for the worker to load by path, +# which the tiers replace. # # Keep ZSMOOTH_VERSION in step across download-deps-{macos,linux}.sh and # download-deps-windows.ps1 — a version skew would make the same job produce @@ -1049,25 +1089,51 @@ build_plugin "tmedian" \ ZSMOOTH_VERSION="0.19.0" # Must satisfy zsmooth's build.zig.zon `minimum_zig_version` (0.15.2 for 0.19.0). ZIG_VERSION="0.15.2" -ZSMOOTH_DIR="$DEPS_DIR/vapoursynth/zsmooth" -mkdir -p "$ZSMOOTH_DIR" +ZSMOOTH_OUT="$PLUGINS_DIR/libzsmooth.so" +rm -rf "$DEPS_DIR/vapoursynth/zsmooth" echo "" echo "=== Installing zsmooth ===" -# The pre-built asset: haswell on x86, the only build on aarch64. case "$ARCH" in - aarch64|arm64) - ZSMOOTH_ASSET="zsmooth-aarch64-linux-gnu.zip" - ZSMOOTH_PREBUILT="$ZSMOOTH_DIR/libzsmooth.so" - ;; - *) - ZSMOOTH_ASSET="zsmooth-x86_64-linux-gnu.zip" - ZSMOOTH_PREBUILT="$ZSMOOTH_DIR/libzsmooth-haswell.so" - ;; + aarch64|arm64) ZSMOOTH_ASSET="zsmooth-aarch64-linux-gnu.zip" ;; + *) ZSMOOTH_ASSET="zsmooth-x86_64-linux-gnu.zip" ;; # the haswell build esac -if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_PREBUILT" ]; then +if [ "$FORCE" = false ] && [ -f "$ZSMOOTH_OUT" ]; then + echo " libzsmooth.so already exists, skipping" +elif [ "${ZSMOOTH_CPU:-}" = "x86_64_v2" ]; then + # No upstream asset for this baseline. Zig brings its own libc and builds + # zsmooth's fftw dependency itself, so this adds no apt package — only + # network access, since `zig build` fetches zsmooth's own Zig dependencies. + echo " Building zsmooth $ZSMOOTH_VERSION (x86_64_v2, runs without AVX2)..." + # Subshell so a failure cannot abort the script under `set -e`; the file + # check below decides whether it worked. + ( + set -e + cd "$BUILD_DIR" + rm -rf zig-toolchain zsmooth-src zig.tar.xz + curl -fsSL -o zig.tar.xz \ + "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" + mkdir -p zig-toolchain + tar -xf zig.tar.xz -C zig-toolchain --strip-components=1 + git clone --depth 1 --branch "$ZSMOOTH_VERSION" \ + https://github.com/adworacz/zsmooth.git zsmooth-src + cd zsmooth-src + "$BUILD_DIR/zig-toolchain/zig" build \ + -Doptimize=ReleaseFast -Dcpu=x86_64_v2 + cp zig-out/lib/libzsmooth.so "$ZSMOOTH_OUT" + ) || true + if [ -f "$ZSMOOTH_OUT" ]; then + patchelf --set-rpath '$ORIGIN:$ORIGIN/../../lib' "$ZSMOOTH_OUT" 2>/dev/null || true + echo " Built zsmooth (x86_64_v2)" + BUILT_PLUGINS+=("zsmooth") + else + echo " Warning: failed to build zsmooth (x86_64_v2)" + FAILED_PLUGINS+=("zsmooth") + fi + rm -rf "$BUILD_DIR/zig-toolchain" "$BUILD_DIR/zsmooth-src" "$BUILD_DIR/zig.tar.xz" +else rm -rf "$BUILD_DIR/zsmooth" mkdir -p "$BUILD_DIR/zsmooth" if curl -sL -o "$BUILD_DIR/zsmooth/zsmooth.zip" \ @@ -1075,9 +1141,9 @@ if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_PREBUILT" ]; then && unzip -q -o "$BUILD_DIR/zsmooth/zsmooth.zip" -d "$BUILD_DIR/zsmooth"; then so_path=$(find "$BUILD_DIR/zsmooth" -name "*.so" -type f 2>/dev/null | head -1) if [ -n "$so_path" ]; then - cp "$so_path" "$ZSMOOTH_PREBUILT" - patchelf --set-rpath '$ORIGIN:$ORIGIN/../../lib' "$ZSMOOTH_PREBUILT" 2>/dev/null || true - echo " Downloaded pre-built zsmooth -> $(basename "$ZSMOOTH_PREBUILT")" + cp "$so_path" "$ZSMOOTH_OUT" + patchelf --set-rpath '$ORIGIN:$ORIGIN/../../lib' "$ZSMOOTH_OUT" 2>/dev/null || true + echo " Downloaded pre-built zsmooth" BUILT_PLUGINS+=("zsmooth") else echo " Failed: no .so in the zsmooth archive" @@ -1088,52 +1154,8 @@ if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_PREBUILT" ]; then FAILED_PLUGINS+=("zsmooth") fi rm -rf "$BUILD_DIR/zsmooth" -else - echo " $(basename "$ZSMOOTH_PREBUILT") already exists, skipping" fi -# The portable x86 build has no upstream asset and must be compiled. Zig brings -# its own libc and builds zsmooth's fftw dependency itself, so this adds no apt -# package — only network access, since `zig build` fetches zsmooth's own Zig -# dependencies. -case "$ARCH" in - aarch64|arm64) : ;; - *) - ZSMOOTH_V2="$ZSMOOTH_DIR/libzsmooth-x86_64_v2.so" - if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_V2" ]; then - echo " Building zsmooth $ZSMOOTH_VERSION (x86_64_v2, runs without AVX2)..." - # Subshell so a failure cannot abort the script under `set -e`; the - # file check below decides whether it worked. - ( - set -e - cd "$BUILD_DIR" - rm -rf zig-toolchain zsmooth-src zig.tar.xz - curl -fsSL -o zig.tar.xz \ - "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-linux-${ZIG_VERSION}.tar.xz" - mkdir -p zig-toolchain - tar -xf zig.tar.xz -C zig-toolchain --strip-components=1 - git clone --depth 1 --branch "$ZSMOOTH_VERSION" \ - https://github.com/adworacz/zsmooth.git zsmooth-src - cd zsmooth-src - "$BUILD_DIR/zig-toolchain/zig" build \ - -Doptimize=ReleaseFast -Dcpu=x86_64_v2 - cp zig-out/lib/libzsmooth.so "$ZSMOOTH_V2" - ) || true - - if [ -f "$ZSMOOTH_V2" ]; then - patchelf --set-rpath '$ORIGIN:$ORIGIN/../../lib' "$ZSMOOTH_V2" 2>/dev/null || true - echo " Built zsmooth -> $(basename "$ZSMOOTH_V2")" - else - echo " Warning: failed to build the portable zsmooth" - FAILED_PLUGINS+=("zsmooth-x86_64_v2") - fi - rm -rf "$BUILD_DIR/zig-toolchain" "$BUILD_DIR/zsmooth-src" "$BUILD_DIR/zig.tar.xz" - else - echo " $(basename "$ZSMOOTH_V2") already exists, skipping" - fi - ;; -esac - # DeScratch (core.descratch.DeScratch - vertical scratch removal) # Built from source: the repo carries the VapourSynth + AviSynthPlus headers as # submodules, so a recursive clone is required (build_plugin can't fetch those). @@ -1794,7 +1816,7 @@ cat > "$DEPS_DIR/version.json" << EOF "version": "$EXPECTED_DEPS_VERSION", "installedAt": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")", "platform": "$PLATFORM_DIR", - "architecture": "$ARCH", + "architecture": "$ARCH",$([ -n "$TIER" ] && printf '\n "tier": "%s",' "$TIER") "buildType": "source" } EOF diff --git a/Scripts/download-deps-macos.sh b/Scripts/download-deps-macos.sh index 5abe82f..b7a8fcd 100755 --- a/Scripts/download-deps-macos.sh +++ b/Scripts/download-deps-macos.sh @@ -13,20 +13,25 @@ # - Homebrew (for build tools only, not runtime) # - Xcode Command Line Tools # -# Usage: ./scripts/download-deps-macos.sh [--force] +# Usage: ./scripts/download-deps-macos.sh [--force] [--tier v3|v2] set -e FORCE=false +TIER=v3 while [[ $# -gt 0 ]]; do case $1 in --force) FORCE=true shift ;; + --tier) + TIER="$2" + shift 2 + ;; *) echo "Unknown option: $1" - echo "Usage: $0 [--force]" + echo "Usage: $0 [--force] [--tier v3|v2]" exit 1 ;; esac @@ -43,6 +48,29 @@ else exit 1 fi +# CPU tier (issue #92). The x86 bundle ships twice: v3 for x86-64-v3 CPUs +# (Haswell, 2013, and later) and v2 for anything older, chosen by the app from +# `vapourbox-worker --probe-cpu`. This block is the ONLY place the tier is +# interpreted; everything below reads these variables and never $TIER itself. +# ZSMOOTH_CPU zsmooth's Zig -Dcpu target (upstream has no dispatch) +# MVTOOLS_SOURCE prebuilt: Stefan-Olt's build, whose AVX2 static +# initializers SIGILL at load on a CPU without AVX; +# source-no-avx2: v24 from source with those files stubbed +# out (patches/mvtools-v24-no-avx2.patch) +if [ "$ARCH" = "x86_64" ]; then + case "$TIER" in + v3) ZSMOOTH_CPU=haswell; MVTOOLS_SOURCE=prebuilt ;; + v2) ZSMOOTH_CPU=x86_64_v2; MVTOOLS_SOURCE=source-no-avx2 ;; + *) echo "Unknown tier: $TIER (expected v3 or v2)"; exit 1 ;; + esac +else + if [ "$TIER" != "v3" ]; then + echo "--tier is x86-only; the arm64 bundle is not tiered." + exit 1 + fi + TIER="" +fi + # NOTE: x64 deps are built natively on an Intel Mac / the macos-15-intel CI # runner (uname -m reports x86_64 -> macos-x64). To build x64 deps on an Apple # Silicon Mac instead, run this script translated through Rosetta 2 with an @@ -84,6 +112,20 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" DEPS_DIR="$PROJECT_ROOT/deps/$PLATFORM_DIR" PLUGINS_DIR="$DEPS_DIR/vapoursynth/plugins" + +# Both tiers build into the same deps/ directory (the tier lives in +# version.json, not the path), and most steps skip what already exists. So a +# tier switch without --force would keep the previous tier's MVTools and +# zsmooth and silently produce a mixed bundle. +if [ -n "$TIER" ] && [ "$FORCE" = false ] && [ -f "$DEPS_DIR/version.json" ]; then + EXISTING_TIER=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('tier','v3'))" \ + "$DEPS_DIR/version.json" 2>/dev/null || echo "v3") + if [ "$EXISTING_TIER" != "$TIER" ]; then + echo "ERROR: $DEPS_DIR holds the $EXISTING_TIER bundle; building $TIER over it" + echo "would mix the two. Re-run with --force." + exit 1 + fi +fi PYTHON_DIR="$DEPS_DIR/python" PYTHON_PACKAGES_DIR="$DEPS_DIR/python-packages" BUILD_DIR="/tmp/vapourbox-build-$$" @@ -852,6 +894,48 @@ download_prebuilt_plugin() { return 1 } +# MVTools for the v2 tier: v24 — the same version the v3 tier's prebuilt uses — +# from source, with patches/mvtools-v24-no-avx2.patch. Stefan-Olt's build +# compiles six files with -mavx2, and three of them construct lookup tables at +# load time with VEX instructions, so dlopen itself SIGILLs on a CPU without +# AVX; autoload makes that every job (issue #92). The patch builds none of the +# AVX2 code and masks the AVX2 flag so nothing dispatches to it; the runtime +# dispatch to SSE2/AVX assembly is unchanged. +build_mvtools_no_avx2() { + if [ "$FORCE" = false ] && [ -f "$PLUGINS_DIR/libmvtools.dylib" ]; then + echo " MVTools already exists, skipping" + return 0 + fi + echo ""; echo "=== Building MVTools v24 without AVX2 (x86_64, v2 tier) ===" + cd "$BUILD_DIR" + rm -rf mvtools + git clone --depth 1 --branch v24 https://github.com/dubhater/vapoursynth-mvtools.git mvtools + # Hard failure: an unapplied patch would quietly ship the crashing build. + (cd mvtools && git apply "$SCRIPT_DIR/patches/mvtools-v24-no-avx2.patch") || { + echo " ERROR: patches/mvtools-v24-no-avx2.patch did not apply to MVTools v24." >&2 + exit 1 + } + if PKG_CONFIG_PATH="$VS_PC_DIR:$BREW_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" \ + meson setup mvtools/build mvtools --buildtype=release \ + && ninja -C mvtools/build; then + cp mvtools/build/libmvtools.dylib "$PLUGINS_DIR/libmvtools.dylib" + install_name_tool -id "@loader_path/libmvtools.dylib" "$PLUGINS_DIR/libmvtools.dylib" + # Link against Homebrew's fftw at build time, load the bundled copy. + for dep in $(otool -L "$PLUGINS_DIR/libmvtools.dylib" | awk '/libfftw3f/ {print $1}'); do + install_name_tool -change "$dep" "@loader_path/../../lib/$(basename "$dep")" \ + "$PLUGINS_DIR/libmvtools.dylib" + done + codesign -s - -f "$PLUGINS_DIR/libmvtools.dylib" 2>/dev/null || true + echo " Built MVTools (no AVX2)" + else + echo " Failed to build MVTools" + FAILED_PLUGINS+=("MVTools") + cd "$BUILD_DIR" + return 1 + fi + cd "$BUILD_DIR" +} + STEFANOLT="https://github.com/Stefan-Olt/vs-plugin-build/releases/download/vsplugin" # pkg-config dir + header dir of the from-source VapourSynth install. Used by the @@ -924,7 +1008,11 @@ if [ "$ARCH" = "x86_64" ]; then # ======================================================================== echo "" echo "=== Downloading pre-built x86_64 plugins (Stefan-Olt/vs-plugin-build) ===" - download_prebuilt_plugin "MVTools" "libmvtools.dylib" "$STEFANOLT/com.nodame.mvtools/v24/darwin-x86_64/2024-09-30T17.08.24%2B00.00Z/MVTools-v24-darwin-x86_64.zip" + if [ "$MVTOOLS_SOURCE" = prebuilt ]; then + download_prebuilt_plugin "MVTools" "libmvtools.dylib" "$STEFANOLT/com.nodame.mvtools/v24/darwin-x86_64/2024-09-30T17.08.24%2B00.00Z/MVTools-v24-darwin-x86_64.zip" + else + build_mvtools_no_avx2 + fi download_prebuilt_plugin "ZNEDI3" "libznedi3.dylib" "$STEFANOLT/xxx.abc.znedi3/3bd542a/darwin-x86_64/2026-01-10T23.47.38%2B00.00Z/ZNEDI3-3bd542a-darwin-x86_64.zip" download_prebuilt_plugin "EEDI3m" "libeedi3m.dylib" "$STEFANOLT/com.holywu.eedi3/r8/darwin-x86_64/2026-01-15T20.48.25%2B00.00Z/EEDI3m-r8-darwin-x86_64.zip" download_prebuilt_plugin "fmtconv" "libfmtconv.dylib" "$STEFANOLT/fmtconv/git-18a9cecb/darwin-x86_64/2024-10-10T14.48.08%2B00.00Z/fmtconv-git-18a9cecb-darwin-x86_64.zip" @@ -1489,60 +1577,50 @@ build_plugin "retinex" \ "libretinex.dylib" \ "meson setup build --buildtype=release && ninja -C build" -# zsmooth — one build per CPU baseline +# zsmooth — exactly one build, in the autoload directory # # core.zsmooth.CCD (also Cnr4 and a set of RemoveGrain/TemporalMedian-family -# filters). Upstream publishes only `haswell` (an AVX2 baseline) and `znver4` -# x86 builds, compiled throughout with NO runtime dispatch, so on a pre-2013 -# CPU they die with an illegal instruction the instant a filter runs — issue -# #82, silently, because vspipe prints nothing on a native crash. -# -# x86_64 therefore ships TWO builds outside the autoload directory and the -# worker loads exactly one by path (DependencyLocator::zsmooth_plugin); they -# cannot share a directory, because each registers the namespace `zsmooth` and -# whichever autoloads second is rejected. arm64 has one NEON baseline and needs -# no split. +# filters). zsmooth has NO runtime dispatch: each build is compiled for one CPU +# baseline throughout, so a build above the machine's baseline dies with an +# illegal instruction the instant a filter runs (issue #82). # -# Note this arch was ALREADY affected in the other direction: the x64 build -# below has always been compiled at Zig's default baseline (SSE2), which -# measures 2.0x slower than haswell on CCD and 3.0x on Cnr4. Every Intel Mac -# that can run macOS 12 is at least Nehalem and most are Haswell or newer, so -# building both here makes the common case fast for the first time as well as -# keeping the oldest ones working. +# arm64 has a single NEON baseline and takes the author's build. x64 builds the +# $ZSMOOTH_CPU chosen by the tier block at the top of this script: haswell (the +# fast path) for v3, x86_64_v2 (SSE4.2/POPCNT — every Intel Mac that can run +# macOS 12) for v2. One build per bundle means it autoloads like any other +# plugin; bundles up to 1.10.0 shipped both x64 builds in a separate zsmooth/ +# directory and had the worker load one by path, which the tiers replace. # # Keep ZSMOOTH_VERSION in step across download-deps-{macos,linux}.sh and # download-deps-windows.ps1 — a version skew would make the same job produce # different chroma per OS. ZSMOOTH_VERSION="0.19.0" -ZSMOOTH_DIR="$DEPS_DIR/vapoursynth/zsmooth" -mkdir -p "$ZSMOOTH_DIR" - -if [ "$ARCH" = "arm64" ]; then - # arm64 takes the author's build: it is minos 13, comfortably under this - # arch's 15.0 target. One build, no variants. - if [ "$FORCE" = true ] || [ ! -f "$ZSMOOTH_DIR/libzsmooth.dylib" ]; then - tmp="$BUILD_DIR/prebuilt-zsmooth" - rm -rf "$tmp"; mkdir -p "$tmp" - zs_url="https://github.com/adworacz/zsmooth/releases/download/${ZSMOOTH_VERSION}/zsmooth-aarch64-macos.zip" - if curl -sL "$zs_url" -o "$tmp/plugin.zip" && unzip -q -o "$tmp/plugin.zip" -d "$tmp"; then - found=$(find "$tmp" -name "*.dylib" -type f 2>/dev/null | head -1) - if [ -n "$found" ]; then - cp "$found" "$ZSMOOTH_DIR/libzsmooth.dylib" - install_name_tool -id "@loader_path/libzsmooth.dylib" "$ZSMOOTH_DIR/libzsmooth.dylib" 2>/dev/null || true - codesign -s - -f "$ZSMOOTH_DIR/libzsmooth.dylib" 2>/dev/null || true - echo " Downloaded pre-built zsmooth" - else - echo " Warning: no dylib in the zsmooth archive" - FAILED_PLUGINS+=("zsmooth") - fi +ZSMOOTH_OUT="$PLUGINS_DIR/libzsmooth.dylib" +rm -rf "$DEPS_DIR/vapoursynth/zsmooth" + +if [ "$FORCE" = false ] && [ -f "$ZSMOOTH_OUT" ]; then + echo " zsmooth already exists, skipping" +elif [ "$ARCH" = "arm64" ]; then + # The author's build: minos 13, comfortably under this arch's 15.0 target. + tmp="$BUILD_DIR/prebuilt-zsmooth" + rm -rf "$tmp"; mkdir -p "$tmp" + zs_url="https://github.com/adworacz/zsmooth/releases/download/${ZSMOOTH_VERSION}/zsmooth-aarch64-macos.zip" + if curl -sL "$zs_url" -o "$tmp/plugin.zip" && unzip -q -o "$tmp/plugin.zip" -d "$tmp"; then + found=$(find "$tmp" -name "*.dylib" -type f 2>/dev/null | head -1) + if [ -n "$found" ]; then + cp "$found" "$ZSMOOTH_OUT" + install_name_tool -id "@loader_path/libzsmooth.dylib" "$ZSMOOTH_OUT" 2>/dev/null || true + codesign -s - -f "$ZSMOOTH_OUT" 2>/dev/null || true + echo " Downloaded pre-built zsmooth" else - echo " Warning: failed to fetch pre-built zsmooth" + echo " Warning: no dylib in the zsmooth archive" FAILED_PLUGINS+=("zsmooth") fi - rm -rf "$tmp" else - echo " zsmooth already exists, skipping" + echo " Warning: failed to fetch pre-built zsmooth" + FAILED_PLUGINS+=("zsmooth") fi + rm -rf "$tmp" else # x64 builds from source, for two reasons: the author's x86_64 build is # minos 13.0 and this bundle targets 12.0, so the minos guard at the end of @@ -1568,66 +1646,54 @@ else *) ZIG_MACOS_MIN="${MACOS_MIN_VERSION}.0" ;; esac - ZIG_BIN="" - # x86_64_v2 is SSE4.2/POPCNT — every Intel Mac that can run macOS 12. - # haswell is the fast path for 2013-and-later machines. Order matters only - # for the log; the worker picks by CPUID at job time. - for zs_target in haswell x86_64_v2; do - out="$ZSMOOTH_DIR/libzsmooth-${zs_target}.dylib" - if [ "$FORCE" = false ] && [ -f "$out" ]; then - echo " zsmooth ($zs_target) already exists, skipping" - continue + echo "" + echo "=== Building zsmooth $ZSMOOTH_CPU (x64, targeting macOS $MACOS_MIN_VERSION) ===" + # Subshell so a failure here can't abort the whole script under `set -e`; + # the file check below decides whether it worked. + ( + set -e + cd "$BUILD_DIR" + rm -rf zig-toolchain zig.tar.xz zsmooth fftw-patched + curl -fsSL -o zig.tar.xz \ + "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-macos-${ZIG_VERSION}.tar.xz" + mkdir -p zig-toolchain + tar -xf zig.tar.xz -C zig-toolchain --strip-components=1 + git clone --depth 1 --branch "$ZSMOOTH_VERSION" \ + https://github.com/adworacz/zsmooth.git zsmooth + + # zsmooth's Zig fftw port declares HAVE_MEMALIGN on every non-Windows + # target, but macOS has no memalign() — it is declared in , + # which the SAME file already knows macOS lacks (HAVE_MALLOC_H is + # gated on !is_mac). fftw's kalloc.c only reaches that branch when + # MIN_ALIGNMENT is 32, i.e. when AVX is enabled, so the bug is + # invisible at the SSE-level baselines and kills ONLY the haswell + # build, with a clang implicit-declaration error inside a dependency. + # HAVE_POSIX_MEMALIGN is already true, so clearing this falls through + # to posix_memalign, which macOS does have. Applied for both tiers: it + # is a correct fix on macOS whichever branch fftw takes. + # + # Patched via a local path dependency rather than by editing Zig's + # global package cache: path deps take no hash, so this is + # deterministic and cannot be invalidated by a cache wipe. + git clone --depth 1 --branch "$FFTW_FORK_TAG" \ + https://github.com/adworacz/fftw.git fftw-patched + if ! grep -q '.HAVE_MEMALIGN = if (!is_windows) true else null,' \ + fftw-patched/build.zig; then + echo " ERROR: the fftw HAVE_MEMALIGN line is not what the patch expects." >&2 + echo " Upstream may have fixed it — re-check before removing this patch." >&2 + exit 1 fi - echo "" - echo "=== Building zsmooth $zs_target (x64, targeting macOS $MACOS_MIN_VERSION) ===" - # Subshell so a failure here can't abort the whole script under `set -e`; - # the file check below decides whether it worked. - ( - set -e - cd "$BUILD_DIR" - if [ -z "$ZIG_BIN" ]; then - rm -rf zig-toolchain zig.tar.xz - curl -fsSL -o zig.tar.xz \ - "https://ziglang.org/download/${ZIG_VERSION}/zig-x86_64-macos-${ZIG_VERSION}.tar.xz" - mkdir -p zig-toolchain - tar -xf zig.tar.xz -C zig-toolchain --strip-components=1 - fi - rm -rf zsmooth fftw-patched - git clone --depth 1 --branch "$ZSMOOTH_VERSION" \ - https://github.com/adworacz/zsmooth.git zsmooth - - # zsmooth's Zig fftw port declares HAVE_MEMALIGN on every non-Windows - # target, but macOS has no memalign() — it is declared in , - # which the SAME file already knows macOS lacks (HAVE_MALLOC_H is - # gated on !is_mac). fftw's kalloc.c only reaches that branch when - # MIN_ALIGNMENT is 32, i.e. when AVX is enabled, so the bug is - # invisible at the SSE-level baselines and kills ONLY the haswell - # build, with a clang implicit-declaration error inside a dependency. - # HAVE_POSIX_MEMALIGN is already true, so clearing this falls through - # to posix_memalign, which macOS does have. - # - # Patched via a local path dependency rather than by editing Zig's - # global package cache: path deps take no hash, so this is - # deterministic and cannot be invalidated by a cache wipe. - git clone --depth 1 --branch "$FFTW_FORK_TAG" \ - https://github.com/adworacz/fftw.git fftw-patched - if ! grep -q '.HAVE_MEMALIGN = if (!is_windows) true else null,' \ - fftw-patched/build.zig; then - echo " ERROR: the fftw HAVE_MEMALIGN line is not what the patch expects." >&2 - echo " Upstream may have fixed it — re-check before removing this patch." >&2 - exit 1 - fi - # `is_mac` is already defined in that file. - sed -i.bak \ - 's/\.HAVE_MEMALIGN = if (!is_windows) true else null,/.HAVE_MEMALIGN = if (!is_windows and !is_mac) true else null,/' \ - fftw-patched/build.zig - grep -q '.HAVE_MEMALIGN = if (!is_windows and !is_mac) true else null,' \ - fftw-patched/build.zig || { echo " ERROR: fftw memalign patch did not apply" >&2; exit 1; } - - cd zsmooth - # Repoint the fftw dependency at the patched clone. A path dependency - # carries no hash field, so the url+hash pair is replaced wholesale. - "$PYTHON_BIN" - <<'ZONEOF' + # `is_mac` is already defined in that file. + sed -i.bak \ + 's/\.HAVE_MEMALIGN = if (!is_windows) true else null,/.HAVE_MEMALIGN = if (!is_windows and !is_mac) true else null,/' \ + fftw-patched/build.zig + grep -q '.HAVE_MEMALIGN = if (!is_windows and !is_mac) true else null,' \ + fftw-patched/build.zig || { echo " ERROR: fftw memalign patch did not apply" >&2; exit 1; } + + cd zsmooth + # Repoint the fftw dependency at the patched clone. A path dependency + # carries no hash field, so the url+hash pair is replaced wholesale. + "$PYTHON_BIN" - <<'ZONEOF' import io, re p = "build.zig.zon" s = io.open(p, encoding="utf-8").read() @@ -1639,25 +1705,23 @@ io.open(p, "w", encoding="utf-8").write(s) print(" fftw repointed to the patched local clone") ZONEOF - "$BUILD_DIR/zig-toolchain/zig" build \ - -Doptimize=ReleaseFast \ - -Dtarget="x86_64-macos.${ZIG_MACOS_MIN}" \ - -Dcpu="$zs_target" - cp zig-out/lib/libzsmooth.dylib "$out" - ) || true - - if [ -f "$out" ]; then - ZIG_BIN="$BUILD_DIR/zig-toolchain/zig" - install_name_tool -id "@loader_path/$(basename "$out")" "$out" 2>/dev/null || true - codesign -s - -f "$out" 2>/dev/null || true - echo " Built zsmooth -> $(basename "$out")" - else - echo " Warning: failed to build zsmooth ($zs_target)" - FAILED_PLUGINS+=("zsmooth-$zs_target") - fi - rm -rf "$BUILD_DIR/zsmooth" "$BUILD_DIR/fftw-patched" - done - rm -rf "$BUILD_DIR/zig-toolchain" "$BUILD_DIR/zig.tar.xz" + "$BUILD_DIR/zig-toolchain/zig" build \ + -Doptimize=ReleaseFast \ + -Dtarget="x86_64-macos.${ZIG_MACOS_MIN}" \ + -Dcpu="$ZSMOOTH_CPU" + cp zig-out/lib/libzsmooth.dylib "$ZSMOOTH_OUT" + ) || true + + if [ -f "$ZSMOOTH_OUT" ]; then + install_name_tool -id "@loader_path/libzsmooth.dylib" "$ZSMOOTH_OUT" 2>/dev/null || true + codesign -s - -f "$ZSMOOTH_OUT" 2>/dev/null || true + echo " Built zsmooth ($ZSMOOTH_CPU)" + else + echo " Warning: failed to build zsmooth ($ZSMOOTH_CPU)" + FAILED_PLUGINS+=("zsmooth") + fi + rm -rf "$BUILD_DIR/zsmooth" "$BUILD_DIR/fftw-patched" \ + "$BUILD_DIR/zig-toolchain" "$BUILD_DIR/zig.tar.xz" fi # ============================================================================ @@ -1771,7 +1835,7 @@ download_prebuilt_plugin "RemoveDirt" "libremovedirt.dylib" "$REMOVEDIRT_URL" # 15.0, and this bundle's Intel floor is 12.0 with STRICT_MIN_OS=1 — the wheel # would fail the guard and, shipped anyway, would refuse to load on Monterey # (issue #39). It is one C++ file with no SIMD and no dependencies, so the x64 -# branch compiles it directly, exactly as zsmooth splits for the same reason. +# branch compiles it directly, as zsmooth's x64 build does for the same reason. # Wheel 3.0 and git tag v3 are the same release; keep the two in step. DEDOT_VERSION="3.0" DEDOT_TAG="v3" @@ -2311,7 +2375,7 @@ cat > "$DEPS_DIR/version.json" << EOF "version": "$EXPECTED_DEPS_VERSION", "installedAt": "$(date -u +"%Y-%m-%dT%H:%M:%SZ")", "platform": "$PLATFORM_DIR", - "architecture": "$ARCH", + "architecture": "$ARCH",$([ -n "$TIER" ] && printf '\n "tier": "%s",' "$TIER") "buildType": "source" } EOF diff --git a/Scripts/download-deps-windows.ps1 b/Scripts/download-deps-windows.ps1 index 8141295..a6077f7 100644 --- a/Scripts/download-deps-windows.ps1 +++ b/Scripts/download-deps-windows.ps1 @@ -16,13 +16,20 @@ .PARAMETER TargetDir The target directory for dependencies. Default: deps/windows-x64 +.PARAMETER Tier + CPU tier to build: v3 (x86-64-v3, Haswell and later; the default) or v2 + (anything older). See the tier block below. + .EXAMPLE .\download-deps-windows.ps1 + .\download-deps-windows.ps1 -Tier v2 .\download-deps-windows.ps1 -TargetDir "C:\vapourbox\deps\windows-x64" #> param( - [string]$TargetDir = "deps\windows-x64" + [string]$TargetDir = "deps\windows-x64", + [ValidateSet("v3", "v2")] + [string]$Tier = "v3" ) $ErrorActionPreference = "Stop" @@ -33,10 +40,34 @@ $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $ProjectRoot = Split-Path -Parent $ScriptDir $FullTargetDir = Join-Path $ProjectRoot $TargetDir +# CPU tier (issue #92). The x86 bundle ships twice: v3 for x86-64-v3 CPUs +# (Haswell, 2013, and later) and v2 for anything older, chosen by the app from +# `vapourbox-worker --probe-cpu`. This block is the ONLY place the tier is +# interpreted; everything below reads these variables and never $Tier itself. +# $ZsmoothCpu zsmooth's baseline (upstream has no runtime dispatch) +# That is the whole difference on Windows: every other plugin in this bundle +# renders under Intel SDE emulating Sandy Bridge (AVX without AVX2) — see +# .github/workflows/probe-cpu-compat.yml. CPUs without AVX at all cannot be +# checked there (Microsoft's runtime trips SDE first), so they are unverified. +$ZsmoothCpu = if ($Tier -eq "v3") { "haswell" } else { "x86_64_v2" } + Write-Host "=== VapourBox Windows Dependency Downloader ===" -ForegroundColor Cyan Write-Host "Target directory: $FullTargetDir" +Write-Host "CPU tier: $Tier" Write-Host "" +# Both tiers build into the same directory (the tier lives in version.json, not +# the path), and every step skips what already exists. So building one tier +# over the other would keep the previous zsmooth and silently mix the bundles. +$ExistingVersionFile = Join-Path $FullTargetDir "version.json" +if (Test-Path $ExistingVersionFile) { + $ExistingTier = (Get-Content $ExistingVersionFile -Raw | ConvertFrom-Json).tier + if (-not $ExistingTier) { $ExistingTier = "v3" } + if ($ExistingTier -ne $Tier) { + throw "$FullTargetDir holds the $ExistingTier bundle; building $Tier over it would mix the two. Delete it first." + } +} + # Create directory structure $Directories = @( "$FullTargetDir\vapoursynth\vs-plugins", @@ -396,8 +427,8 @@ $PluginsZip = @( Url = "https://github.com/Khanattila/KNLMeansCL/releases/download/v1.1.1/KNLMeansCL-v1.1.1.zip" Check = "KNLMeansCL.dll" } - # zsmooth is NOT here: it ships as two CPU-specific builds outside the - # autoload directory. See section 4a below. + # zsmooth is NOT here: which build ships depends on the CPU tier. See + # section 4a below. ) foreach ($Plugin in $Plugins7z) { @@ -510,43 +541,37 @@ if ($BadArch.Count -gt 0) { Write-Host " All plugin DLLs are x64" -ForegroundColor Green # ============================================================================= -# 4a. zsmooth — one build per CPU baseline +# 4a. zsmooth — exactly one build, in vs-plugins # ============================================================================= # core.zsmooth.CCD (also Cnr4 and a set of RemoveGrain/TemporalMedian-family -# filters). Upstream publishes only `haswell` (an AVX2 baseline) and `znver4` -# builds, compiled throughout with NO runtime dispatch — so on a pre-2013 x86 -# CPU the DLL loads fine and then dies with an illegal instruction -# (0xC000001D) the instant a filter runs. That is issue #82, reported on a -# Celeron J4105 and a Core i7 870, and it is silent: vspipe prints nothing, so -# the encode surfaces as ffmpeg reading an empty pipe. +# filters). zsmooth has NO runtime dispatch: each build is compiled for one CPU +# baseline throughout, so a build above the machine's baseline loads fine and +# then dies with an illegal instruction (0xC000001D) the instant a filter runs. +# That is issue #82, reported on a Celeron J4105 and a Core i7 870, and it is +# silent: vspipe prints nothing, so the encode surfaces as ffmpeg reading an +# empty pipe. # -# Both builds are therefore shipped and the worker loads exactly one by path -# (DependencyLocator::zsmooth_plugin). They cannot both sit in vs-plugins — -# each registers the namespace `zsmooth`, so whichever autoloads second is -# rejected — hence the separate directory, which is deliberately not on the -# plugin path. -# -# Why not just ship the portable build for everyone: measured on this plugin at -# 720x576, `x86_64` is 2.0x slower than haswell on CCD and 3.0x on Cnr4 -# (`x86_64_v2` 1.4x and 1.4x). Paying that on every modern machine to serve the -# rare old one is the wrong trade; picking at runtime costs ~4 MB of zip. +# The tier block at the top picks $ZsmoothCpu: the author's haswell build for +# v3, or an x86_64_v2 build compiled here for v2 (SSE4.2/POPCNT, everything +# from Nehalem 2009 on; measured 1.4x slower than haswell on CCD and Cnr4, +# against 2-3x for plain x86_64). One build per bundle means it loads from +# vs-plugins like any other plugin; bundles up to 1.10.0 shipped both builds in +# a separate zsmooth\ directory for the worker to load by path. # # Keep ZSMOOTH_VERSION in step with download-deps-{macos,linux}.sh: a skew # would make the same job produce different chroma per OS. Write-Host "" -Write-Host "[4a/8] Installing zsmooth (per-CPU builds)..." -ForegroundColor Yellow +Write-Host "[4a/8] Installing zsmooth ($ZsmoothCpu)..." -ForegroundColor Yellow $ZsmoothVersion = "0.19.0" # Must satisfy zsmooth's build.zig.zon `minimum_zig_version`; 0.15.2 for 0.19.0. $ZigVersion = "0.15.2" -$ZsmoothDir = "$FullTargetDir\vapoursynth\zsmooth" -if (-not (Test-Path $ZsmoothDir)) { - New-Item -ItemType Directory -Force -Path $ZsmoothDir | Out-Null -} +$ZsmoothOut = "$PluginsDir\zsmooth.dll" +Remove-Item "$FullTargetDir\vapoursynth\zsmooth" -Recurse -Force -ErrorAction SilentlyContinue -# The AVX2 build comes pre-built from upstream. -$HaswellPath = "$ZsmoothDir\zsmooth-haswell.dll" -if (-not (Test-Path $HaswellPath)) { +if (Test-Path $ZsmoothOut) { + Write-Host " zsmooth.dll already installed" -ForegroundColor Gray +} elseif ($ZsmoothCpu -eq "haswell") { Write-Host " Downloading zsmooth $ZsmoothVersion (haswell/AVX2)..." -ForegroundColor Gray try { $ZsZip = Join-Path $TempDir "zsmooth-haswell.zip" @@ -555,24 +580,19 @@ if (-not (Test-Path $HaswellPath)) { Expand-Archive -Path $ZsZip -DestinationPath $ZsExtract -Force $Dll = Get-ChildItem -Path $ZsExtract -Recurse -Filter "zsmooth.dll" | Select-Object -First 1 if (-not $Dll) { throw "no zsmooth.dll in the upstream archive" } - Copy-Item $Dll.FullName $HaswellPath -Force + Copy-Item $Dll.FullName $ZsmoothOut -Force Remove-Item $ZsZip -Force -ErrorAction SilentlyContinue Remove-Item $ZsExtract -Recurse -Force -ErrorAction SilentlyContinue - Write-Host " Installed: zsmooth-haswell.dll" -ForegroundColor Gray + Write-Host " Installed: zsmooth.dll (haswell)" -ForegroundColor Gray } catch { Write-Host " Failed: $_" -ForegroundColor Red } } else { - Write-Host " zsmooth-haswell.dll already installed" -ForegroundColor Gray -} - -# The portable build has no upstream asset and must be compiled. This is the -# first from-source build in this script; Zig cross-compiles with its own libc -# and builds zsmooth's fftw dependency itself, so it needs no MSVC — only git -# and network access (zig build fetches zsmooth's own Zig dependencies). -$V2Path = "$ZsmoothDir\zsmooth-x86_64_v2.dll" -if (-not (Test-Path $V2Path)) { - Write-Host " Building zsmooth $ZsmoothVersion (x86_64_v2, runs without AVX2)..." -ForegroundColor Gray + # The portable build has no upstream asset and must be compiled. Zig + # cross-compiles with its own libc and builds zsmooth's fftw dependency + # itself, so it needs no MSVC — only git and network access (zig build + # fetches zsmooth's own Zig dependencies). + Write-Host " Building zsmooth $ZsmoothVersion ($ZsmoothCpu, runs without AVX2)..." -ForegroundColor Gray try { $ZigDir = Join-Path $TempDir "zig-toolchain" $ZigZip = Join-Path $TempDir "zig.zip" @@ -595,10 +615,9 @@ if (-not (Test-Path $V2Path)) { Push-Location $ZsSrc try { - # -Dcpu=x86_64_v2 is SSE4.2/POPCNT: everything from Nehalem (2009) - # on, which covers both CPUs in issue #82. Plain `x86_64` would add - # pre-2009 chips at roughly half the CCD/Cnr4 throughput again. - & $ZigExe build -Doptimize=ReleaseFast -Dtarget=x86_64-windows-gnu -Dcpu=x86_64_v2 + # Quoted: PowerShell passes a bare -Dcpu=$ZsmoothCpu to a native command + # literally, unexpanded. + & $ZigExe build -Doptimize=ReleaseFast -Dtarget=x86_64-windows-gnu "-Dcpu=$ZsmoothCpu" if ($LASTEXITCODE -ne 0) { throw "zig build failed (exit $LASTEXITCODE)" } } finally { Pop-Location @@ -606,26 +625,21 @@ if (-not (Test-Path $V2Path)) { $Built = Get-ChildItem -Path (Join-Path $ZsSrc "zig-out") -Recurse -Filter "zsmooth.dll" | Select-Object -First 1 if (-not $Built) { throw "zig build produced no zsmooth.dll" } - Copy-Item $Built.FullName $V2Path -Force + Copy-Item $Built.FullName $ZsmoothOut -Force Remove-Item $ZigZip -Force -ErrorAction SilentlyContinue Remove-Item $ZigDir, $ZsSrc -Recurse -Force -ErrorAction SilentlyContinue - Write-Host " Built: zsmooth-x86_64_v2.dll" -ForegroundColor Gray + Write-Host " Built: zsmooth.dll ($ZsmoothCpu)" -ForegroundColor Gray } catch { Write-Host " Failed: $_" -ForegroundColor Red } -} else { - Write-Host " zsmooth-x86_64_v2.dll already installed" -ForegroundColor Gray } -# A missing build here is not a warning to scroll past: without the AVX2 one -# every modern machine loses the pass, and without the portable one issue #82 -# comes straight back. deps-expected-plugins.json also covers both, so the -# packaging step would fail — this just fails nearer the cause. -$MissingZsmooth = @(@($HaswellPath, $V2Path) | Where-Object { -not (Test-Path $_) }) -if ($MissingZsmooth.Count -gt 0) { - throw "zsmooth build(s) missing: $(($MissingZsmooth | Split-Path -Leaf) -join ', ')" +# Fail near the cause rather than at packaging: without zsmooth the Chroma +# Denoise pass is gone on every machine. +if (-not (Test-Path $ZsmoothOut)) { + throw "zsmooth ($ZsmoothCpu) is missing" } -Write-Host " zsmooth: both CPU builds present" -ForegroundColor Green +Write-Host " zsmooth: $ZsmoothCpu build present" -ForegroundColor Green # ============================================================================= # 4b. FFTW Library (required by DFTTest) @@ -1135,9 +1149,10 @@ $ExpectedVersion = (Get-Content $DepsVersionJson -Raw | ConvertFrom-Json).versio version = $ExpectedVersion installedAt = (Get-Date).ToUniversalTime().ToString("o") platform = "windows-x64" + tier = $Tier buildType = "source" } | ConvertTo-Json | Set-Content -Path "$FullTargetDir\version.json" -Encoding utf8 -Write-Host " version.json written ($ExpectedVersion)" -ForegroundColor Green +Write-Host " version.json written ($ExpectedVersion, $Tier)" -ForegroundColor Green # ============================================================================= # 8. Cleanup diff --git a/Scripts/package-deps-linux.sh b/Scripts/package-deps-linux.sh index 2151e84..05bfd12 100755 --- a/Scripts/package-deps-linux.sh +++ b/Scripts/package-deps-linux.sh @@ -54,7 +54,17 @@ PACKAGE_INCOMPLETE="" package_arch() { local ARCH_NAME=$1 local DEPS_DIR="$PROJECT_ROOT/deps/linux-$ARCH_NAME" - local PACKAGE_NAME="VapourBox-deps-$VERSION-linux-$ARCH_NAME" + # The CPU tier (issue #92) is whatever download-deps-linux.sh stamped into + # the build's version.json; only x64 is tiered. v3 keeps the plain asset + # name, v2 is suffixed — the same rule as DependencyManager.assetIdFor. + local TIER="" + if [ "$ARCH_NAME" = "x64" ]; then + TIER=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('tier','v3'))" \ + "$DEPS_DIR/version.json" 2>/dev/null || echo "v3") + fi + local PLATFORM_KEY="linux-$ARCH_NAME" + [ "$TIER" = "v2" ] && PLATFORM_KEY="linux-$ARCH_NAME-v2" + local PACKAGE_NAME="VapourBox-deps-$VERSION-$PLATFORM_KEY" local PACKAGE_DIR="$DIST_DIR/$PACKAGE_NAME" echo "[1/4] Checking prerequisites for $ARCH_NAME..." @@ -121,7 +131,7 @@ package_arch() { echo " Creating version file..." cat > "$PACKAGE_DIR/version.json" << EOF { - "version": "$VERSION", + "version": "$VERSION",$([ -n "$TIER" ] && printf '\n "tier": "%s",' "$TIER") "installedAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" } EOF @@ -135,30 +145,27 @@ EOF # Completeness guard: every required plugin must be present before we zip, # so a silently-failed build/download can't ship an incomplete bundle. # Contract: Scripts/deps-expected-plugins.json. - echo " Verifying required plugins for $ARCH_NAME..." + echo " Verifying required plugins for $PLATFORM_KEY..." local MANIFEST="$PROJECT_ROOT/Scripts/deps-expected-plugins.json" local MISSING - MISSING=$(python3 - "$MANIFEST" "linux-$ARCH_NAME" "$PACKAGE_DIR/vapoursynth/plugins" "$PACKAGE_DIR" <<'PY' + MISSING=$(python3 - "$MANIFEST" "$PLATFORM_KEY" "$PACKAGE_DIR/vapoursynth/plugins" <<'PY' import json, os, sys -manifest, key, plugin_dir, bundle_root = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] -expected = json.load(open(manifest)).get(key, []) -# An entry with a "/" is bundle-root-relative, not a plugin-directory filename: -# zsmooth ships one build per CPU baseline OUTSIDE the autoload directory (both -# register the same namespace, so the worker loads exactly one by path), and a -# guard that only looked in plugins/ would stop covering it. -def target(f): - return os.path.join(bundle_root, *f.split("/")) if "/" in f else os.path.join(plugin_dir, f) -print("\n".join(f for f in expected if not os.path.isfile(target(f)))) +manifest, key, plugin_dir = sys.argv[1], sys.argv[2], sys.argv[3] +expected = json.load(open(manifest)).get(key) +if expected is None: + print(f"(no {key} entry in deps-expected-plugins.json)") + sys.exit() +print("\n".join(f for f in expected if not os.path.isfile(os.path.join(plugin_dir, f)))) PY ) if [ -n "$MISSING" ]; then - echo "ERROR: linux-$ARCH_NAME bundle is missing required plugins:" >&2 + echo "ERROR: $PLATFORM_KEY bundle is missing required plugins:" >&2 echo "$MISSING" | sed 's/^/ - /' >&2 echo "A plugin build/download likely failed - check the download-deps-linux.sh output." >&2 - PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE linux-$ARCH_NAME" + PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE $PLATFORM_KEY" return 1 fi - echo " All required plugins present for $ARCH_NAME" + echo " All required plugins present for $PLATFORM_KEY" echo "[4/4] Creating zip archive for $ARCH_NAME..." local ZIP_FILE="$DIST_DIR/$PACKAGE_NAME.zip" diff --git a/Scripts/package-deps-macos.sh b/Scripts/package-deps-macos.sh index 6ed9c78..f8851b9 100755 --- a/Scripts/package-deps-macos.sh +++ b/Scripts/package-deps-macos.sh @@ -54,7 +54,17 @@ PACKAGE_INCOMPLETE="" package_arch() { local ARCH_NAME=$1 local DEPS_DIR="$PROJECT_ROOT/deps/macos-$ARCH_NAME" - local PACKAGE_NAME="VapourBox-deps-$VERSION-macos-$ARCH_NAME" + # The CPU tier (issue #92) is whatever download-deps-macos.sh stamped into + # the build's version.json; only x64 is tiered. v3 keeps the plain asset + # name, v2 is suffixed — the same rule as DependencyManager.assetIdFor. + local TIER="" + if [ "$ARCH_NAME" = "x64" ]; then + TIER=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('tier','v3'))" \ + "$DEPS_DIR/version.json" 2>/dev/null || echo "v3") + fi + local PLATFORM_KEY="macos-$ARCH_NAME" + [ "$TIER" = "v2" ] && PLATFORM_KEY="macos-$ARCH_NAME-v2" + local PACKAGE_NAME="VapourBox-deps-$VERSION-$PLATFORM_KEY" local PACKAGE_DIR="$DIST_DIR/$PACKAGE_NAME" echo "[1/4] Checking prerequisites for $ARCH_NAME..." @@ -126,7 +136,7 @@ package_arch() { echo " Creating version file..." cat > "$PACKAGE_DIR/version.json" << EOF { - "version": "$VERSION", + "version": "$VERSION",$([ -n "$TIER" ] && printf '\n "tier": "%s",' "$TIER") "installedAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" } EOF @@ -142,30 +152,41 @@ EOF # Completeness guard: every required plugin must be present before we zip, # so a silently-failed build/download can't ship an incomplete bundle. # Contract: Scripts/deps-expected-plugins.json. - echo " Verifying required plugins for $ARCH_NAME..." + echo " Verifying required plugins for $PLATFORM_KEY..." local MANIFEST="$PROJECT_ROOT/Scripts/deps-expected-plugins.json" local MISSING - MISSING=$(python3 - "$MANIFEST" "macos-$ARCH_NAME" "$PACKAGE_DIR/vapoursynth/plugins" "$PACKAGE_DIR" <<'PY' + MISSING=$(python3 - "$MANIFEST" "$PLATFORM_KEY" "$PACKAGE_DIR/vapoursynth/plugins" <<'PY' import json, os, sys -manifest, key, plugin_dir, bundle_root = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] -expected = json.load(open(manifest)).get(key, []) -# An entry with a "/" is bundle-root-relative, not a plugin-directory filename: -# zsmooth ships one build per CPU baseline OUTSIDE the autoload directory (both -# register the same namespace, so the worker loads exactly one by path), and a -# guard that only looked in plugins/ would stop covering it. -def target(f): - return os.path.join(bundle_root, *f.split("/")) if "/" in f else os.path.join(plugin_dir, f) -print("\n".join(f for f in expected if not os.path.isfile(target(f)))) +manifest, key, plugin_dir = sys.argv[1], sys.argv[2], sys.argv[3] +expected = json.load(open(manifest)).get(key) +if expected is None: + print(f"(no {key} entry in deps-expected-plugins.json)") + sys.exit() +print("\n".join(f for f in expected if not os.path.isfile(os.path.join(plugin_dir, f)))) PY ) if [ -n "$MISSING" ]; then - echo "ERROR: macos-$ARCH_NAME bundle is missing required plugins:" >&2 + echo "ERROR: $PLATFORM_KEY bundle is missing required plugins:" >&2 echo "$MISSING" | sed 's/^/ - /' >&2 echo "A plugin build/download likely failed - check the download-deps-macos.sh output." >&2 - PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE macos-$ARCH_NAME" + PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE $PLATFORM_KEY" return 1 fi - echo " All required plugins present for $ARCH_NAME" + echo " All required plugins present for $PLATFORM_KEY" + + # The v2 bundle's whole promise is that it loads on a CPU without AVX, and + # macOS has no Intel SDE to prove that by running it. So prove statically + # that no plugin can execute a VEX instruction while it loads — the fault + # issue #92 was: MVTools' AVX2 static initializers SIGILL inside dlopen. + if [ "$TIER" = "v2" ]; then + echo " Checking no plugin runs AVX while loading (v2 tier)..." + if ! python3 "$PROJECT_ROOT/Scripts/check-load-time-simd.py" \ + "$PACKAGE_DIR"/vapoursynth/plugins/*.dylib; then + echo "ERROR: a plugin in the $PLATFORM_KEY bundle runs AVX at load time" >&2 + PACKAGE_INCOMPLETE="$PACKAGE_INCOMPLETE $PLATFORM_KEY" + return 1 + fi + fi echo "[4/4] Creating zip archive for $ARCH_NAME..." local ZIP_FILE="$DIST_DIR/$PACKAGE_NAME.zip" diff --git a/Scripts/package-deps-windows.ps1 b/Scripts/package-deps-windows.ps1 index c91ec44..3918a43 100644 --- a/Scripts/package-deps-windows.ps1 +++ b/Scripts/package-deps-windows.ps1 @@ -17,17 +17,29 @@ $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $ProjectRoot = Split-Path -Parent $ScriptDir $DistDir = Join-Path $ProjectRoot "dist" $AppName = "VapourBox" -$PackageName = "$AppName-deps-$Version-windows-x64" +$DepsDir = Join-Path $ProjectRoot "deps\windows-x64" + +# The CPU tier (issue #92) is whatever download-deps-windows.ps1 stamped into the +# build's version.json. v3 keeps the plain asset name, v2 is suffixed — the same +# rule as DependencyManager.assetIdFor. A build from before tiering is v3. +$Tier = "v3" +$BuiltVersionFile = Join-Path $DepsDir "version.json" +if (Test-Path $BuiltVersionFile) { + $Stamped = (Get-Content $BuiltVersionFile -Raw | ConvertFrom-Json).tier + if ($Stamped) { $Tier = $Stamped } +} +$PlatformKey = if ($Tier -eq "v2") { "windows-x64-v2" } else { "windows-x64" } +$PackageName = "$AppName-deps-$Version-$PlatformKey" $PackageDir = Join-Path $DistDir $PackageName Write-Host "=== Packaging VapourBox Dependencies for Windows ===" -ForegroundColor Cyan Write-Host "Version: $Version" +Write-Host "CPU tier: $Tier ($PlatformKey)" Write-Host "" # Check prerequisites Write-Host "[1/5] Checking prerequisites..." -ForegroundColor Yellow -$DepsDir = Join-Path $ProjectRoot "deps\windows-x64" if (-not (Test-Path $DepsDir)) { Write-Host "ERROR: Dependencies not found at $DepsDir" -ForegroundColor Red Write-Host "Run '.\Scripts\download-deps-windows.ps1' first" -ForegroundColor Red @@ -141,6 +153,7 @@ Copy-Item (Join-Path $DepsDir "ffmpeg\ffprobe.exe") "$PackageDir\ffmpeg\" -Error Write-Host " Creating version file..." $VersionInfo = @{ version = $Version + tier = $Tier installedAt = (Get-Date).ToString("o") } | ConvertTo-Json -Depth 10 Set-Content -Path "$PackageDir\version.json" -Value $VersionInfo @@ -150,16 +163,13 @@ Set-Content -Path "$PackageDir\version.json" -Value $VersionInfo # otherwise ship an incomplete bundle. Contract: Scripts/deps-expected-plugins.json. Write-Host "[4b/5] Verifying required plugins..." -ForegroundColor Yellow $ManifestPath = Join-Path $ProjectRoot "Scripts\deps-expected-plugins.json" -$ExpectedPlugins = (Get-Content $ManifestPath -Raw | ConvertFrom-Json)."windows-x64" +$ExpectedPlugins = (Get-Content $ManifestPath -Raw | ConvertFrom-Json).$PlatformKey +if (-not $ExpectedPlugins) { + Write-Host "ERROR: no $PlatformKey entry in deps-expected-plugins.json" -ForegroundColor Red + exit 1 +} $StagedPluginDir = Join-Path "$PackageDir\vapoursynth" "vs-plugins" -# An entry with a '/' is bundle-root-relative, not a plugin-directory filename. -# zsmooth ships one build per CPU baseline outside the autoload directory (both -# register the same namespace, so the worker loads exactly one by path), and a -# guard that only ever looked in vs-plugins would stop covering it. -$MissingPlugins = @($ExpectedPlugins | Where-Object { - $Target = if ($_ -match '/') { Join-Path $PackageDir ($_ -replace '/', '\') } else { Join-Path $StagedPluginDir $_ } - -not (Test-Path $Target) -}) +$MissingPlugins = @($ExpectedPlugins | Where-Object { -not (Test-Path (Join-Path $StagedPluginDir $_)) }) if ($MissingPlugins.Count -gt 0) { Write-Host "ERROR: bundle is missing $($MissingPlugins.Count) required plugin(s):" -ForegroundColor Red $MissingPlugins | ForEach-Object { Write-Host " - $_" -ForegroundColor Red } diff --git a/Scripts/patches/mvtools-v24-no-avx2.patch b/Scripts/patches/mvtools-v24-no-avx2.patch new file mode 100644 index 0000000..e19b26c --- /dev/null +++ b/Scripts/patches/mvtools-v24-no-avx2.patch @@ -0,0 +1,120 @@ +diff --git a/meson.build b/meson.build +index f24a406..63b0298 100644 +--- a/meson.build ++++ b/meson.build +@@ -150,18 +150,13 @@ if host_cpu_family.startswith('x86') + + + libavx2_sources = [ +- 'src/MaskFun_AVX2.cpp', +- 'src/MVDegrains_AVX2.cpp', +- 'src/MVFrame_AVX2.cpp', +- 'src/Overlap_AVX2.cpp', +- 'src/SADFunctions_AVX2.cpp', +- 'src/SimpleResize_AVX2.cpp', ++ 'src/AVX2_stubs.cpp', + ] + + helper_libs += static_library('avx2', + libavx2_sources, + dependencies: vapoursynth_dep, +- cpp_args: [cflags, '-mavx2', '-mtune=haswell'], ++ cpp_args: cflags, + install: false) + endif + +diff --git a/src/AVX2_stubs.cpp b/src/AVX2_stubs.cpp +new file mode 100644 +index 0000000..96a5503 +--- /dev/null ++++ b/src/AVX2_stubs.cpp +@@ -0,0 +1,77 @@ ++// VapourBox v2-tier build: the AVX2 translation units are replaced by these ++// stubs, compiled at the baseline like everything else (issue #92). ++// ++// Upstream builds six files with -mavx2. Their static initializers (the ++// kernel lookup tables) then contain VEX instructions, which run inside dlopen, ++// before any CPU check, and fault with SIGILL on every CPU without AVX. The ++// shared template code those files instantiate can also win the linker's ++// choice of copy and leak AVX2 into callers that are not guarded at all. ++// ++// So the v2 build contains no AVX2-compiled code, and EntryPoint.c masks ++// X264_CPU_AVX2 out of g_cpuinfo so no dispatch site ever selects these. They ++// abort rather than do nothing: reaching one means the mask was lost, and a ++// silent no-op would render blank frames instead of failing. ++ ++#include ++#include ++#include ++ ++#include "MaskFun.h" ++#include "MVDegrains.h" ++#include "Overlap.h" ++#include "SADFunctions.h" ++#include "SimpleResize.h" ++ ++[[noreturn]] static void mvtools_avx2_unreachable(const char *name) { ++ std::fprintf(stderr, "MVTools (VapourBox v2 build): AVX2 path %s reached; " ++ "the X264_CPU_AVX2 mask in EntryPoint.c is missing\n", name); ++ std::abort(); ++} ++ ++void selectFlowInterFunctions_AVX2(FlowInterSimpleFunction *, FlowInterFunction *, FlowInterExtraFunction *, int) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++DenoiseFunction selectDegrainFunctionAVX2(unsigned, unsigned, unsigned, unsigned) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++OverlapsFunction selectOverlapsFunctionAVX2(unsigned, unsigned, unsigned) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++SADFunction selectSADFunctionAVX2(unsigned, unsigned, unsigned) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void Average2_avx2(uint8_t *, const uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void VerticalBilinear_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void HorizontalBilinear_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void DiagonalBilinear_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void VerticalWiener_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void HorizontalWiener_avx2(uint8_t *, const uint8_t *, intptr_t, intptr_t, intptr_t, intptr_t) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void simpleResize_uint8_t_avx2(const SimpleResize *, uint8_t *, int, const uint8_t *, int, int) { ++ mvtools_avx2_unreachable(__func__); ++} ++ ++void simpleResize_int16_t_avx2(const SimpleResize *, int16_t *, int, const int16_t *, int, int) { ++ mvtools_avx2_unreachable(__func__); ++} +diff --git a/src/EntryPoint.c b/src/EntryPoint.c +index 6b34eab..ca30663 100644 +--- a/src/EntryPoint.c ++++ b/src/EntryPoint.c +@@ -44,5 +44,6 @@ VapourSynthPluginInit(VSConfigPlugin configFunc, VSRegisterFunction registerFunc + mvscdetectionRegister(registerFunc, plugin); + mvdepanRegister(registerFunc, plugin); + +- g_cpuinfo = cpu_detect(); ++ // VapourBox v2 tier: no AVX2 code is built (see AVX2_stubs.cpp). ++ g_cpuinfo = cpu_detect() & ~X264_CPU_AVX2; + } diff --git a/Scripts/probe-plugin-compat.py b/Scripts/probe-plugin-compat.py new file mode 100755 index 0000000..643d82c --- /dev/null +++ b/Scripts/probe-plugin-compat.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +"""Load and render every plugin in a VapourBox deps bundle, one per process. + +Answers "which bundled plugins can this CPU run?" (issue #92). Each plugin is +loaded by path into a core created with DISABLE_AUTO_LOADING, so one plugin +that faults while loading cannot take the others down with it. That masking is +exactly what made the first version of this probe report every plugin as +crashed: R78 autoloads the whole plugin directory, and MVTools faults in a +static initializer before any filter runs. + +The parent never imports vapoursynth. Every test is a child process, so a +SIGILL / 0xC000001D in a plugin is a result, not the end of the run. + +Usage: + python probe-plugin-compat.py [DEPS_DIR] [--report FILE] [--json FILE] + [--wrap "sde64 -nhm --"] [--extra FILE] + [--fail-on-crash] + +DEPS_DIR defaults to the installed bundle for this OS. Run it with the +bundle's own Python (see probe-plugin-compat.sh) so no system Python is needed. +""" + +import argparse +import json +import os +import platform +import shlex +import subprocess +import sys +from datetime import datetime + +# Namespace -> render call. `c` is a 640x480 YUV420P8 BlankClip, 12 frames long +# (temporal filters need neighbours). Calls mirror worker/templates where the +# pipeline uses the plugin, so a pass here means the shipped call runs. A +# namespace missing from this table is still load-tested. +RENDER = { + "mv": ("sup = core.mv.Super(c, pel=2, sharp=1)\n" + "bw = core.mv.Analyse(sup, isb=True, delta=1, blksize=8, overlap=4)\n" + "fw = core.mv.Analyse(sup, isb=False, delta=1, blksize=8, overlap=4)\n" + "c = core.mv.Degrain1(clip=c, super=sup, mvbw=bw, mvfw=fw)"), + "znedi3": "c = core.znedi3.nnedi3(c, field=1, dh=False)", + "nnedi3": "c = core.nnedi3.nnedi3(c, field=1, dh=False)", + "eedi3m": "c = core.eedi3m.EEDI3(c, field=1, dh=False)", + "fmtc": "c = core.fmtc.resample(c, w=320, h=240)", + "dfttest": "c = core.dfttest.DFTTest(c)", + "misc": "c = core.misc.SCDetect(c)", + "rgvs": "c = core.rgvs.RemoveGrain(c, mode=2)", + "grain": "c = core.grain.Add(c, var=4)", + "cas": "c = core.cas.CAS(c, sharpness=0.5)", + "dctf": "c = core.dctf.DCTFilter(c, factors=[1.0] * 8)", + "deblock": "c = core.deblock.Deblock(c, quant=25)", + "warp": "c = core.warp.AWarpSharp2(c, thresh=128, blur=3, type=0)", + # opt=2 (SSE2) is what script_generator::ctmf_opt picks on a CPU without + # AVX2, i.e. on every machine this probe exists for. + "ctmf": "c = core.ctmf.CTMF(c, radius=2, memsize=16777216, opt=2)", + "tcanny": "c = core.tcanny.TCanny(c, sigma=1.5, mode=-1)", + "tmedian": "c = core.tmedian.TemporalMedian(c, radius=1)", + "removedirt": "c = core.removedirt.RestoreMotionBlocks(c, restore=c)", + "lghost": "c = core.lghost.LGhost(c, mode=[1], shift=[2], intensity=[10])", + "bwdif": "c = core.bwdif.Bwdif(c, field=1)", + "zsmooth": "c = core.zsmooth.CCD(c, threshold=4, scale=1)", + "neo_f3kdb": "c = core.neo_f3kdb.Deband(c, y=64, cb=64, cr=64)", + "vivtc": "c = core.vivtc.VFM(c, order=1)", + "fb": "c = core.fb.FillBorders(c, left=2, right=2, top=2, bottom=2, mode='fillmargins')", + "descratch": "c = core.descratch.DeScratch(c)", + "fft3dfilter": "c = core.fft3dfilter.FFT3DFilter(c, sigma=2.0, bt=3)", + "ttmpsm": "c = core.ttmpsm.TTempSmooth(c)", + "flux": "c = core.flux.SmoothT(c, temporal_threshold=7)", + "dedot": "c = core.dedot.Dedot(c)", + "bifrost": "c = core.bifrost.Bifrost(c)", + "retinex": "c = core.retinex.MSRCP(core.std.ShufflePlanes(c, 0, vs.GRAY))", + "akarin": "c = core.akarin.Expr(c, 'x 1 +')", +} + +# OpenCL filters need a GPU to render, which says nothing about the CPU. +LOAD_ONLY = ("knlmeanscl", "nnedi3cl") + +# Plugins that call another plugin internally, by file-name substring. The +# dependency is loaded first; it is also tested on its own, so a crash that +# only shows up here is still attributable. +DEPENDS = {"ttempsmooth": ["miscfilters"]} + +# Exercises the parts of VapourSynth itself every job touches: zimg resizing +# and the std.Expr JIT, both of which pick SIMD code paths at runtime. +CORE_RENDER = ("c = core.resize.Bicubic(c, width=320, height=240, format=vs.YUV444P16)\n" + "c = core.std.Expr(c, 'x 1 +')") + +CHILD = r''' +import sys, vapoursynth as vs + +class _NoAutoload(vs.EnvironmentPolicy): + def on_policy_registered(self, api): + self._env = api.create_environment(vs.CoreCreationFlags.DISABLE_AUTO_LOADING) + def get_current_environment(self): + return self._env + def set_environment(self, env): + prev, self._env = self._env, env + return prev + +vs.register_policy(_NoAutoload()) +core = vs.core +path, render = sys.argv[1], sys.argv[2] +for dep in sys.argv[3:]: + core.std.LoadPlugin(dep) +new_ns = [] +if path != "-": + before = {p.namespace for p in core.plugins()} + try: + core.std.LoadPlugin(path) + except vs.Error as e: + print("NOTPLUGIN " + str(e).replace("\n", " "), flush=True) + sys.exit(3) + new_ns = sorted({p.namespace for p in core.plugins()} - before) +print("NAMESPACES " + ",".join(new_ns), flush=True) +if render == "@load": + sys.exit(0) +c = core.std.BlankClip(width=640, height=480, length=12, format=vs.YUV420P8) +if render == "@auto": + snippets = [RENDER[n] for n in new_ns if n in RENDER] + if not snippets: + print("NORENDER", flush=True) + sys.exit(0) + render = "\n".join(snippets) +exec(render) +for i in range(len(c)): + c.get_frame(i) +print("RENDERED", flush=True) +''' + +# Under SDE on Windows, Microsoft's runtime DLLs pick SIMD paths from what the +# *host* kernel reports (IsProcessorFeaturePresent reads shared kernel memory, +# which SDE cannot virtualise), so they execute AVX that SDE then flags — on +# code that is fine on real pre-AVX hardware. A fault in one of these says +# nothing about the plugin under test. +EMULATION_ARTIFACT_IMAGES = ("vcruntime140", "ucrtbase", "msvcp140", "ntdll", + "kernelbase", "kernel32") + +WINDOWS_CRASH_NAMES = { + 0xC000001D: "illegal instruction", + 0xC0000005: "access violation", + 0xC00000FD: "stack overflow", + 0xC0000094: "integer divide by zero", + 0xC0000409: "stack buffer overrun", +} + + +def default_deps_dir(): + system = platform.system() + home = os.path.expanduser("~") + if system == "Darwin": + arch = "macos-arm64" if platform.machine() == "arm64" else "macos-x64" + return os.path.join(home, "Library", "Application Support", "VapourBox", "deps", arch) + if system == "Linux": + arch = "linux-arm64" if platform.machine() == "aarch64" else "linux-x64" + base = os.environ.get("XDG_DATA_HOME") or os.path.join(home, ".local", "share") + return os.path.join(base, "VapourBox", "deps", arch) + return os.path.join(os.getcwd(), "deps", "windows-x64") + + +def bundle_layout(deps): + """Python executable, child environment and plugin files for a bundle.""" + env = {k: v for k, v in os.environ.items() + if k not in ("PYTHONHOME", "PYTHONPATH", "VAPOURSYNTH_EXTRA_PLUGIN_PATH")} + env["PYTHONNOUSERSITE"] = "1" + vs_dir = os.path.join(deps, "vapoursynth") + if platform.system() == "Windows": + python = os.path.join(vs_dir, "python.exe") + env["PYTHONHOME"] = vs_dir + env["PYTHONPATH"] = os.pathsep.join([ + os.path.join(deps, "python-packages"), + os.path.join(vs_dir, "Lib", "site-packages"), + ]) + env["PATH"] = os.pathsep.join([vs_dir, env.get("PATH", "")]) + plugin_dir, exts = os.path.join(vs_dir, "vs-plugins"), (".dll",) + else: + py_home = os.path.join(deps, "python") + python = os.path.join(py_home, "bin", "python3") + env["PYTHONHOME"] = py_home + env["PYTHONPATH"] = os.pathsep.join([ + os.path.join(deps, "python-packages"), + deps, + os.path.join(py_home, "lib", "python3.12", "site-packages"), + ]) + libs = [vs_dir, os.path.join(py_home, "lib"), os.path.join(deps, "lib")] + var = "DYLD_LIBRARY_PATH" if platform.system() == "Darwin" else "LD_LIBRARY_PATH" + env[var] = os.pathsep.join(libs + ([env[var]] if env.get(var) else [])) + plugin_dir = os.path.join(vs_dir, "plugins") + exts = (".dylib",) if platform.system() == "Darwin" else (".so",) + + files = [] + for d in (plugin_dir, os.path.join(vs_dir, "zsmooth")): + if os.path.isdir(d): + files += sorted(os.path.join(d, f) for f in os.listdir(d) + if f.lower().endswith(exts)) + return python, env, files + + +def classify(proc, sde_wrapped): + out, err, code = proc.stdout, proc.stderr, proc.returncode + namespaces = "" + for line in out.splitlines(): + if line.startswith("NAMESPACES "): + namespaces = line[len("NAMESPACES "):] + tail = "\n".join(err.strip().splitlines()[-6:]) + + if sde_wrapped and ("SDE-ERROR" in err or "not valid for specified chip" in err): + image = next((l.split("Image:", 1)[1].strip() for l in err.splitlines() + if "Image:" in l), "") + where = os.path.basename(image.replace("\\", "/")) or "unknown image" + if any(a in where.lower() for a in EMULATION_ARTIFACT_IMAGES): + return ("INCONCLUSIVE", + f"SDE flagged the OS runtime ({where}), not this plugin", + namespaces, tail) + return "CRASHED", f"illegal instruction in {where}", namespaces, tail + if code < 0: + sig = -code + try: + import signal + name = signal.Signals(sig).name + except (ValueError, ImportError): + name = f"signal {sig}" + return "CRASHED", name, namespaces, tail + if code & 0xFFFFFFFF in WINDOWS_CRASH_NAMES: + return "CRASHED", WINDOWS_CRASH_NAMES[code & 0xFFFFFFFF], namespaces, tail + if code == 3 and "NOTPLUGIN" in out: + return "SKIPPED", "not a VapourSynth plugin", namespaces, "" + if code == 0 and "NORENDER" in out: + return "LOADED", "no render test for this namespace", namespaces, "" + if code == 0 and "RENDERED" in out: + return "PASS", "", namespaces, "" + if code == 0: + return "LOADED", "load-only test", namespaces, "" + return "ERROR", f"exit {code}", namespaces, tail + + +def cpu_summary(): + lines = [f"Machine: {platform.system()} {platform.release()} ({platform.machine()})"] + system = platform.system() + try: + if system == "Darwin": + def sysctl(key): + r = subprocess.run(["sysctl", "-n", key], capture_output=True, text=True) + return r.stdout.strip() + feats = sysctl("machdep.cpu.features").split() + leaf7 = sysctl("machdep.cpu.leaf7_features").split() + lines.append(f"CPU: {sysctl('machdep.cpu.brand_string')}") + # macOS spells plain AVX as "AVX1.0". + lines.append(f"AVX: {int('AVX1.0' in feats)} AVX2: {int('AVX2' in leaf7)}") + elif system == "Linux": + with open("/proc/cpuinfo") as f: + info = f.read() + model = next((l.split(":", 1)[1].strip() for l in info.splitlines() + if l.startswith("model name")), "unknown") + flags = next((l.split(":", 1)[1].split() for l in info.splitlines() + if l.startswith("flags")), []) + lines.append(f"CPU: {model}") + lines.append(f"AVX: {int('avx' in flags)} AVX2: {int('avx2' in flags)}") + elif system == "Windows": + import ctypes + present = ctypes.windll.kernel32.IsProcessorFeaturePresent + lines.append(f"CPU: {platform.processor()}") + # PF_AVX_INSTRUCTIONS_AVAILABLE = 39, PF_AVX2_INSTRUCTIONS_AVAILABLE = 40 + lines.append(f"AVX: {int(bool(present(39)))} AVX2: {int(bool(present(40)))}") + except Exception as e: # the report is still useful without CPU details + lines.append(f"CPU: unavailable ({e})") + return lines + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("deps", nargs="?", default=None) + ap.add_argument("--report", default=None, help="text report path") + ap.add_argument("--json", default=None, help="machine-readable results path") + ap.add_argument("--wrap", default="", help='prefix for each test, e.g. "sde64 -nhm --"') + ap.add_argument("--extra", action="append", default=[], metavar="FILE", + help="also test this plugin file (e.g. a known-bad control build)") + ap.add_argument("--timeout", type=int, default=600) + ap.add_argument("--fail-on-crash", action="store_true", + help="exit 1 if anything crashed, errored or was inconclusive (for CI gates)") + args = ap.parse_args() + + deps = os.path.abspath(args.deps or default_deps_dir()) + if not os.path.isdir(os.path.join(deps, "vapoursynth")): + sys.exit(f"No VapourBox deps bundle found at {deps}\n" + "Pass the path to your deps folder as the first argument.") + python, env, files = bundle_layout(deps) + files += [os.path.abspath(f) for f in args.extra] + wrap = shlex.split(args.wrap) + child_src = f"RENDER = {RENDER!r}\n" + CHILD + + if args.report is None: + desktop = os.path.join(os.path.expanduser("~"), "Desktop") + where = desktop if os.path.isdir(desktop) else os.getcwd() + args.report = os.path.join(where, "vapourbox-plugin-compat-report.txt") + + version = "" + try: + with open(os.path.join(deps, "version.json")) as f: + version = json.dumps(json.load(f)) + except (OSError, ValueError): + pass + + header = ["VapourBox plugin compatibility report", + f"Generated: {datetime.now():%Y-%m-%d %H:%M:%S}", + f"Deps: {deps}", + f"Bundle: {version or 'unknown'}"] + header += cpu_summary() + if wrap: + header.append(f"Wrapped with: {' '.join(wrap)}") + header.append("=" * 72) + + report = open(args.report, "w") + + def emit(line=""): + print(line, flush=True) + report.write(line + "\n") + + for line in header: + emit(line) + + # "-" rather than "" for "no plugin": sde.exe drops empty arguments when + # it relaunches the child, which shifted every later argument. + tests = [("(core: VapourSynth, zimg, Expr)", "-", CORE_RENDER)] + for f in files: + stem = os.path.basename(f) + tests.append((stem, f, None)) + + def deps_for(path): + low = os.path.basename(path).lower() + wanted = next((v for k, v in DEPENDS.items() if k in low), []) + return [f for f in files + if any(w in os.path.basename(f).lower() for w in wanted)] + + results = [] + for label, path, render in tests: + extra = [] + if render is None: + # Unknown until the child reports its namespaces; OpenCL is load-only. + render = "@auto" + if any(k in os.path.basename(path).lower() for k in LOAD_ONLY): + render = "@load" + extra = deps_for(path) + cmd = wrap + [python, "-c", child_src, path, render] + extra + try: + proc = subprocess.run(cmd, env=env, capture_output=True, text=True, + timeout=args.timeout) + status, detail, ns, tail = classify(proc, bool(wrap)) + except subprocess.TimeoutExpired: + status, detail, ns, tail = "ERROR", f"timed out after {args.timeout}s", "", "" + results.append({"file": label, "namespaces": ns, "status": status, + "detail": detail}) + shown = f"{label} [{ns}]" if ns else label + emit(f"{shown:<46} {status}{' (' + detail + ')' if detail else ''}") + if tail and status in ("CRASHED", "ERROR"): + for t in tail.splitlines(): + emit(f" {t}") + + counts = {s: sum(r["status"] == s for r in results) + for s in ("PASS", "LOADED", "CRASHED", "ERROR", "INCONCLUSIVE", "SKIPPED")} + emit("=" * 72) + emit("Summary: " + ", ".join(f"{n} {s.lower()}" for s, n in counts.items())) + report.close() + print(f"\nReport written to: {args.report}") + + if args.json: + with open(args.json, "w") as f: + json.dump({"deps": deps, "bundle": version, "results": results}, f, indent=2) + if args.fail_on_crash and (counts["CRASHED"] or counts["ERROR"] or counts["INCONCLUSIVE"]): + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/Scripts/probe-plugin-compat.sh b/Scripts/probe-plugin-compat.sh new file mode 100755 index 0000000..e8d0469 --- /dev/null +++ b/Scripts/probe-plugin-compat.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Test which bundled VapourSynth plugins this machine's CPU can run (issue #92). +# +# bash probe-plugin-compat.sh [path-to-deps-folder] +# +# Runs probe-plugin-compat.py (which must sit next to this script) with the deps +# bundle's own Python, so no system Python is needed. Writes the report to +# ~/Desktop/vapourbox-plugin-compat-report.txt. + +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" + +if [ $# -ge 1 ]; then + DEPS="$1" +elif [ "$(uname)" = "Darwin" ]; then + [ "$(uname -m)" = "arm64" ] && ID=macos-arm64 || ID=macos-x64 + DEPS="$HOME/Library/Application Support/VapourBox/deps/$ID" +else + [ "$(uname -m)" = "aarch64" ] && ID=linux-arm64 || ID=linux-x64 + DEPS="${XDG_DATA_HOME:-$HOME/.local/share}/VapourBox/deps/$ID" +fi + +PY="$DEPS/python/bin/python3" +if [ ! -x "$PY" ]; then + echo "Could not find VapourBox's bundled Python at: $PY" + echo "Open VapourBox once so it can download its components, then try again." + exit 1 +fi + +exec "$PY" "$HERE/probe-plugin-compat.py" "$DEPS" "${@:2}" diff --git a/Scripts/release.sh b/Scripts/release.sh index 51b7c0a..12971ad 100755 --- a/Scripts/release.sh +++ b/Scripts/release.sh @@ -216,94 +216,25 @@ fi echo "" -# Step 1: Package and release dependencies if changed +# Step 1: Dependencies, if changed. +# +# Deps are built and published by CI only. The x64 bundles ship in two CPU +# tiers (issue #92), each built into the same deps/ directory, so a +# local tree only ever holds ONE tier of each — packaging it here would publish +# a release missing the other, and every machine on that tier would fail to +# download. (Windows deps cannot be built on macOS at all.) The build-deps-* +# workflows build every tier, gate the v2 bundles, and upload to the release. if $DEPS_CHANGED; then - echo -e "${BLUE}[1/6] Packaging dependencies...${NC}" - - # Package macOS deps - if [ -d "$PROJECT_ROOT/deps/macos-arm64" ] || [ -d "$PROJECT_ROOT/deps/macos-x64" ]; then - "$SCRIPT_DIR/package-deps-macos.sh" --version "$DEPS_VERSION" --arch both || true - fi - - # Package Windows deps (if on Windows or deps exist) - if [ -d "$PROJECT_ROOT/deps/windows-x64" ]; then - echo -e "${YELLOW}Windows deps found. Package manually on Windows or copy existing.${NC}" - # On macOS we can still create the zip if deps directory exists - WINDOWS_PACKAGE_DIR="$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-windows-x64" - rm -rf "$WINDOWS_PACKAGE_DIR" - mkdir -p "$WINDOWS_PACKAGE_DIR" - cp -r "$PROJECT_ROOT/deps/windows-x64/"* "$WINDOWS_PACKAGE_DIR/" - - # Create version file - cat > "$WINDOWS_PACKAGE_DIR/version.json" << EOF -{ - "version": "$DEPS_VERSION", - "installedAt": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" -} -EOF - - # Create zip - cd "$PROJECT_ROOT/dist" - zip -r -q "VapourBox-deps-$DEPS_VERSION-windows-x64.zip" "VapourBox-deps-$DEPS_VERSION-windows-x64" - rm -rf "$WINDOWS_PACKAGE_DIR" - echo "Created: dist/VapourBox-deps-$DEPS_VERSION-windows-x64.zip" - - # Integrity sidecar (matches package-deps-windows.ps1 output) so the app - # can verify the download. This manual path bypasses the PowerShell - # packager, so write the sidecar here too. - WIN_ZIP="$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-windows-x64.zip" - WIN_SHA=$(shasum -a 256 "$WIN_ZIP" | cut -d' ' -f1) - WIN_SIZE=$(stat -f%z "$WIN_ZIP" 2>/dev/null || stat -c%s "$WIN_ZIP") - cat > "$WIN_ZIP.sha256.json" << EOF -{ - "filename": "VapourBox-deps-$DEPS_VERSION-windows-x64.zip", - "sha256": "$WIN_SHA", - "size": $WIN_SIZE, - "version": "$DEPS_VERSION" -} -EOF - fi - - # Package Linux deps (if deps exist) - if [ -d "$PROJECT_ROOT/deps/linux-x64" ] || [ -d "$PROJECT_ROOT/deps/linux-arm64" ]; then - "$SCRIPT_DIR/package-deps-linux.sh" --version "$DEPS_VERSION" --arch both || true - fi - + echo -e "${RED}Dependencies changed: build and publish them with CI first.${NC}" echo "" - echo -e "${BLUE}[2/6] Creating deps release on GitHub...${NC}" - - # Create deps release - DEPS_NOTES="## VapourBox Dependencies $DEPS_VERSION - -This release contains pre-built dependencies for VapourBox. - -### Contents -- VapourSynth portable with plugins -- FFmpeg -- Python packages (havsfunc, mvsfunc, etc.) - -### Downloads -- \`VapourBox-deps-$DEPS_VERSION-windows-x64.zip\` - Windows x64 -- \`VapourBox-deps-$DEPS_VERSION-macos-arm64.zip\` - macOS Apple Silicon -- \`VapourBox-deps-$DEPS_VERSION-macos-x64.zip\` - macOS Intel - -These dependencies are automatically downloaded by the app on first launch." - - gh release create "$DEPS_TAG" \ - --repo "$GITHUB_REPO" \ - --title "Dependencies $DEPS_VERSION" \ - --notes "$DEPS_NOTES" \ - --latest=false \ - "$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-"*.zip \ - "$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-"*.zip.sha256.json 2>/dev/null || { - echo -e "${YELLOW}Uploading assets to existing release...${NC}" - for f in "$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-"*.zip \ - "$PROJECT_ROOT/dist/VapourBox-deps-$DEPS_VERSION-"*.zip.sha256.json; do - [ -f "$f" ] && gh release upload "$DEPS_TAG" "$f" --repo "$GITHUB_REPO" --clobber - done - } - - echo -e "${GREEN}Deps release created: $DEPS_TAG${NC}" + echo " gh release create $DEPS_TAG --repo $GITHUB_REPO --title \"VapourBox deps $DEPS_VERSION\" \\" + echo " --notes \"Dependencies $DEPS_VERSION\" --latest=false" + echo " gh workflow run build-deps-macos.yml -f version=$DEPS_VERSION -f release_tag=$DEPS_TAG -f arch=both" + echo " gh workflow run build-deps-windows.yml -f version=$DEPS_VERSION -f release_tag=$DEPS_TAG" + echo " gh workflow run build-deps-linux.yml -f version=$DEPS_VERSION -f release_tag=$DEPS_TAG -f arch=both" + echo "" + echo "Then re-run this script with --skip-deps-check." + exit 1 else echo -e "${BLUE}[1/6] Skipping deps packaging (unchanged)${NC}" echo -e "${BLUE}[2/6] Skipping deps release (unchanged)${NC}" diff --git a/app/assets/deps-version.json b/app/assets/deps-version.json index 4477365..ebe26f8 100644 --- a/app/assets/deps-version.json +++ b/app/assets/deps-version.json @@ -1,6 +1,6 @@ { - "version": "1.10.0", - "releaseTag": "deps-v1.10.0", - "releaseDate": "2026-08-28", + "version": "1.11.0", + "releaseTag": "deps-v1.11.0", + "releaseDate": "2026-09-25", "githubRepo": "StuartCameronCode/VapourBox" } diff --git a/app/lib/services/dependency_manager.dart b/app/lib/services/dependency_manager.dart index de68427..cad0dd9 100644 --- a/app/lib/services/dependency_manager.dart +++ b/app/lib/services/dependency_manager.dart @@ -10,6 +10,7 @@ import 'package:rhttp/rhttp.dart'; import 'package:path/path.dart' as path; import 'temp_directory_service.dart'; +import 'tool_locator.dart'; /// Status of the dependency installation. enum DependencyStatus { @@ -22,6 +23,13 @@ enum DependencyStatus { /// Dependencies are installed but older than this app expects outdated, + /// Installed deps were built for a different CPU tier than this machine's + /// (see [DependencyManager.depsTier]) — typically the install was carried + /// over from, or to, another computer. Replaced even when the installed + /// version is newer than expected: a bundle this CPU cannot run is not + /// "newer", it is unusable. + wrongTier, + /// Installed deps are NEWER than the version this app was built against. /// /// Treated as usable, not as something to "fix": replacing them would be a @@ -179,9 +187,14 @@ class InstalledDepsInfo { final String version; final DateTime? installedAt; + /// CPU tier the bundle was built for (`v2`/`v3`), or null. Bundles from + /// before tiering carry no tier; on x86 those are the v3 build. + final String? tier; + InstalledDepsInfo({ required this.version, this.installedAt, + this.tier, }); factory InstalledDepsInfo.fromJson(Map json) { @@ -190,6 +203,7 @@ class InstalledDepsInfo { installedAt: json['installedAt'] != null ? DateTime.tryParse(json['installedAt'] as String) : null, + tier: json['tier'] as String?, ); } @@ -197,6 +211,7 @@ class InstalledDepsInfo { return { 'version': version, 'installedAt': installedAt?.toIso8601String(), + if (tier != null) 'tier': tier, }; } } @@ -233,6 +248,98 @@ class DependencyManager { throw UnsupportedError('Unsupported platform'); } + /// Whether a platform's deps ship in CPU tiers. Only x86 does: the ARM + /// bundles have a single NEON baseline. + static bool isTiered(String platformId) => platformId.endsWith('-x64'); + + /// The release asset id for a platform and tier: the v3 bundle keeps the + /// plain platform name (so every pre-tiering release and install stays + /// valid), the v2 bundle gets a `-v2` suffix. The *install* directory is + /// always [platformId] — the tier lives only in which zip is downloaded and in + /// version.json — so the worker, dev paths and tests need no tier awareness. + static String assetIdFor(String platformId, String? tier) => + tier == 'v2' ? '$platformId-v2' : platformId; + + String? _cachedTier; + bool _tierResolved = false; + + /// Test seam: replaces the worker probe. Returns the probe's raw `tier` + /// value, or null to simulate a failed probe. + @visibleForTesting + Future Function()? tierProbeOverride; + + /// Forget the cached tier (tests only). + @visibleForTesting + void resetTierForTesting() { + _cachedTier = null; + _tierResolved = false; + } + + /// Which deps tier this machine needs: `v3` or `v2` on x86, null on ARM. + /// + /// The worker decides (`vapourbox-worker --probe-cpu`, see + /// worker/src/cpu.rs) so the app and the worker share one definition of the + /// tiers — re-deriving it here from sysctl or /proc would be a second + /// implementation, and sysctl is wrong under Rosetta anyway. + /// `VAPOURBOX_DEPS_TIER=v2|v3` overrides the probe: an escape hatch for a VM + /// that misreports its CPU, and for dev builds with no worker built yet. + /// + /// Anything short of a clear answer on x86 means `v2`. The v2 bundle runs on + /// every x86-64 CPU and only costs speed; guessing `v3` wrongly costs a crash + /// on every job (issue #92). + Future depsTier() async { + if (_tierResolved) return _cachedTier; + final id = platformId; + final override = Platform.environment['VAPOURBOX_DEPS_TIER']; + String? probed; + if (isTiered(id) && override != 'v2' && override != 'v3') { + probed = await (tierProbeOverride ?? _probeWorkerTier)(); + if (probed != 'v3' && probed != 'v2') { + print('DependencyManager: CPU tier probe gave no answer ' + '(${probed ?? 'failed'}) - using the v2 bundle, which runs anywhere'); + } + } + _cachedTier = resolveTier(platformId: id, override: override, probed: probed); + _tierResolved = true; + return _cachedTier; + } + + /// The tier decision itself, pure so it is testable on any host. + static String? resolveTier( + {required String platformId, String? override, String? probed}) { + if (!isTiered(platformId)) return null; + if (override == 'v2' || override == 'v3') return override; + return probed == 'v3' ? 'v3' : 'v2'; + } + + /// Whether an installed bundle's tier suits this machine. A bundle from + /// before tiering records no tier; on x86 that was always the v3 build. + static bool tierMatches({String? installed, String? machine}) => + (installed ?? (machine == null ? null : 'v3')) == machine; + + /// This machine's release asset id (see [assetIdFor]). + Future assetPlatformId() async => + assetIdFor(platformId, await depsTier()); + + Future _probeWorkerTier() async { + final worker = ToolLocator.findWorkerExecutable(); + if (worker == null) return null; + try { + final result = await Process.run(worker, ['--probe-cpu']) + .timeout(const Duration(seconds: 15)); + if (result.exitCode != 0) return null; + for (final line in result.stdout.toString().split('\n')) { + final trimmed = line.trim(); + if (!trimmed.startsWith('{')) continue; + final json = jsonDecode(trimmed); + if (json is Map && json['tier'] is String) return json['tier'] as String; + } + } catch (e) { + print('DependencyManager: CPU tier probe failed: $e'); + } + return null; + } + /// Get the dependencies directory path. Future getDepsDirectory() async { // Explicit override (used by `flutter test` in CI, where the executable is @@ -384,12 +491,25 @@ class DependencyManager { return DependencyStatus.missing; } + // A bundle for the wrong CPU tier is replaced before the version is even + // considered: the v3 bundle faults on every job on a CPU below x86-64-v3 + // (issue #92), and an install can outlive the machine it was chosen for + // (a migrated account, a restored backup, a CPU upgrade). This deliberately + // wins over newerThanExpected, so a newer wrong-tier install is replaced + // with the expected version — a downgrade, but one that runs. + final tier = await depsTier(); + if (!tierMatches(installed: installed.tier, machine: tier)) { + print('DependencyManager: Installed deps are the ' + '${installed.tier ?? 'v3'} bundle but this CPU needs $tier'); + return DependencyStatus.wrongTier; + } + // Check version match (per-platform: a platform may pin its own version). // Direction matters. Older than expected is an upgrade; newer is not a // fault at all, and treating it as one downgraded a deliberately newer // bundle back to the released one — destructively, since installing wipes // and replaces. - final expectedVersion = expected.versionFor(platformId); + final expectedVersion = expected.versionFor(assetIdFor(platformId, tier)); if (installed.version != expectedVersion) { final order = compareVersions(installed.version, expectedVersion); if (order > 0) { @@ -535,10 +655,13 @@ class DependencyManager { /// installation is done or fails. Future downloadAndInstall() async { final expected = await getExpectedVersion(); + final tier = await depsTier(); + final assetId = assetIdFor(platformId, tier); - // Construct download URL from release tag (filename is derived). - final downloadUrl = expected.getDownloadUrl(platformId); - final filename = expected.filenameFor(platformId); + // Construct download URL from release tag (filename is derived). The asset + // carries the tier; the install directory does not. + final downloadUrl = expected.getDownloadUrl(assetId); + final filename = expected.filenameFor(assetId); print('DependencyManager: Downloading from $downloadUrl'); @@ -552,7 +675,7 @@ class DependencyManager { // is best-effort: if the sidecar is missing/unreadable we still install (the // download is over HTTPS), matching prior behaviour when no hash was set. final expectedSha256 = - await _fetchExpectedSha256(expected.getManifestUrl(platformId)); + await _fetchExpectedSha256(expected.getManifestUrl(assetId)); // The zip is downloaded into a stable cache path rather than a throwaway // temp directory, and kept if anything after the download fails. Everything @@ -671,8 +794,8 @@ class DependencyManager { // Write version file (per-platform version, so the next check matches). // Still the last thing written into the tree, so a staged directory that // never gets swapped in can never look complete. - await _writeInstalledVersion(expected.versionFor(platformId), - depsDirOverride: target); + await _writeInstalledVersion(expected.versionFor(assetId), + tier: tier, depsDirOverride: target); // Swap. If the second rename fails we have already moved the old install // aside, so put it back rather than leaving the user with no deps at all. @@ -1133,12 +1256,13 @@ class DependencyManager { /// Write the installed version file. Future _writeInstalledVersion(String version, - {Directory? depsDirOverride}) async { + {String? tier, Directory? depsDirOverride}) async { final versionFile = await _getInstalledVersionFile(depsDirOverride: depsDirOverride); final info = InstalledDepsInfo( version: version, installedAt: DateTime.now(), + tier: tier, ); await versionFile.writeAsString( const JsonEncoder.withIndent(' ').convert(info.toJson()), diff --git a/app/lib/services/tool_locator.dart b/app/lib/services/tool_locator.dart index 3e0a4c2..2f75cc4 100644 --- a/app/lib/services/tool_locator.dart +++ b/app/lib/services/tool_locator.dart @@ -109,8 +109,14 @@ class ToolLocator { return File(p).existsSync() ? p : null; } - /// Resolve the vapourbox-worker executable path. - String? _resolveWorker() { + String? _resolveWorker() => findWorkerExecutable(); + + /// Locate the vapourbox-worker executable. + /// + /// Static and independent of the deps directory, because the worker is also + /// needed *before* any deps exist: `DependencyManager.depsTier()` asks it + /// which x86 bundle this CPU can run, and that decides what gets downloaded. + static String? findWorkerExecutable() { // Explicit override, mirroring VAPOURBOX_DEPS_DIR above. Under `flutter // test` the resolved executable is the test runner, not the app bundle, so // neither the production nor the dev path below can find the worker — which diff --git a/app/lib/views/dependency_download_dialog.dart b/app/lib/views/dependency_download_dialog.dart index 7eaab99..31c14e0 100644 --- a/app/lib/views/dependency_download_dialog.dart +++ b/app/lib/views/dependency_download_dialog.dart @@ -104,6 +104,10 @@ class _DependencyDownloadDialogState extends State { case DependencyStatus.outdated: return 'A new version of the processing components is available.\n\n' 'Updating to ensure compatibility.'; + case DependencyStatus.wrongTier: + return 'The installed processing components were built for a ' + 'different processor.\n\n' + 'Downloading the version for this computer.'; case DependencyStatus.corrupted: return 'Some processing components are damaged or incomplete.\n\n' 'Re-downloading to fix the issue.'; diff --git a/app/test/attribution_test.dart b/app/test/attribution_test.dart index 738d61b..983b3df 100644 --- a/app/test/attribution_test.dart +++ b/app/test/attribution_test.dart @@ -92,20 +92,10 @@ const _pluginToNotice = { const _mustNotAppear = ['ffms2', 'BestSource']; String _stem(String filename) { - // A manifest entry may be a bundle-relative path rather than a bare filename - // (zsmooth ships outside the autoload directory), and may carry a CPU-target - // suffix because it ships once per baseline. Credit is owed to the project, - // not to each build of it, so both are normalised away. - var s = filename.toLowerCase().split('/').last; + var s = filename.toLowerCase(); final dot = s.lastIndexOf('.'); if (dot > 0) s = s.substring(0, dot); if (s.startsWith('lib') && s.length > 3) s = s.substring(3); - for (final variant in const ['-haswell', '-x86_64_v2']) { - if (s.endsWith(variant)) { - s = s.substring(0, s.length - variant.length); - break; - } - } return s; } @@ -149,6 +139,24 @@ void main() { ); }); + test('both CPU tiers of an x64 bundle ship the same plugins', () { + // The v2 bundle (issue #92) differs from v3 in how some plugins are + // BUILT, never in which ones ship: a plugin missing from v2 would just + // be a pass that fails on older machines. The packaging guard checks + // each list separately, so only this keeps the two lists in step. + final manifest = jsonDecode( + File(p.join(root, 'Scripts', 'deps-expected-plugins.json')).readAsStringSync(), + ) as Map; + for (final platform in ['macos-x64', 'windows-x64', 'linux-x64']) { + expect(manifest['$platform-v2'], isNotNull, reason: 'no $platform-v2 list'); + expect( + (manifest['$platform-v2'] as List).toSet(), + (manifest[platform] as List).toSet(), + reason: '$platform-v2 must list exactly the plugins $platform does', + ); + } + }); + test('does not credit components that are no longer shipped', () { for (final gone in _mustNotAppear) { expect( diff --git a/app/test/deps_tier_test.dart b/app/test/deps_tier_test.dart new file mode 100644 index 0000000..ba0aca4 --- /dev/null +++ b/app/test/deps_tier_test.dart @@ -0,0 +1,131 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:vapourbox/services/dependency_manager.dart'; + +/// The x86 deps bundles ship in two CPU tiers (issue #92): v3 for x86-64-v3 +/// CPUs, v2 for anything older. These pin the rules that pick one, name its +/// release asset, and decide when an existing install is for the wrong CPU. +void main() { + group('resolveTier', () { + test('ARM platforms are not tiered, whatever the probe says', () { + for (final id in ['macos-arm64', 'linux-arm64']) { + expect(DependencyManager.resolveTier(platformId: id, probed: 'v3'), isNull); + expect(DependencyManager.resolveTier(platformId: id, override: 'v2'), isNull); + } + }); + + test('x86 follows the worker probe', () { + for (final id in ['macos-x64', 'windows-x64', 'linux-x64']) { + expect(DependencyManager.resolveTier(platformId: id, probed: 'v3'), 'v3'); + expect(DependencyManager.resolveTier(platformId: id, probed: 'v2'), 'v2'); + } + }); + + test('no clear answer on x86 means v2, which runs everywhere', () { + // Guessing v3 wrongly is a crash on every job; v2 only costs speed. + for (final probed in [null, '', 'v4', 'V3']) { + expect( + DependencyManager.resolveTier(platformId: 'macos-x64', probed: probed), + 'v2', + reason: 'probe "$probed"'); + } + }); + + test('VAPOURBOX_DEPS_TIER overrides the probe in both directions', () { + expect( + DependencyManager.resolveTier( + platformId: 'linux-x64', override: 'v2', probed: 'v3'), + 'v2'); + expect( + DependencyManager.resolveTier( + platformId: 'linux-x64', override: 'v3', probed: 'v2'), + 'v3'); + }); + + test('an invalid override is ignored rather than trusted', () { + expect( + DependencyManager.resolveTier( + platformId: 'linux-x64', override: 'fast', probed: 'v3'), + 'v3'); + }); + }); + + group('release assets', () { + test('v3 keeps the pre-tiering asset name; v2 is suffixed', () { + expect(DependencyManager.assetIdFor('macos-x64', 'v3'), 'macos-x64'); + expect(DependencyManager.assetIdFor('macos-x64', 'v2'), 'macos-x64-v2'); + expect(DependencyManager.assetIdFor('macos-arm64', null), 'macos-arm64'); + }); + + test('the v2 asset resolves to its own zip and sidecar', () { + // Must agree with what the package-deps-* scripts upload. + final info = DepsVersionInfo.fromJson({ + 'version': '1.11.0', + 'releaseTag': 'deps-v1.11.0', + 'githubRepo': 'StuartCameronCode/VapourBox', + }); + final id = DependencyManager.assetIdFor('windows-x64', 'v2'); + expect(info.filenameFor(id), 'VapourBox-deps-1.11.0-windows-x64-v2.zip'); + expect( + info.getManifestUrl(id), + 'https://github.com/StuartCameronCode/VapourBox/releases/download/' + 'deps-v1.11.0/VapourBox-deps-1.11.0-windows-x64-v2.zip.sha256.json'); + }); + }); + + group('tierMatches', () { + test('a pre-tiering install is the v3 bundle', () { + // Every x86 install before tiering is the v3 build — which is exactly + // what faults on the #92 Mac Pro, so it must read as the wrong tier there. + expect(DependencyManager.tierMatches(installed: null, machine: 'v3'), isTrue); + expect(DependencyManager.tierMatches(installed: null, machine: 'v2'), isFalse); + }); + + test('a recorded tier must equal the machine tier', () { + expect(DependencyManager.tierMatches(installed: 'v2', machine: 'v2'), isTrue); + expect(DependencyManager.tierMatches(installed: 'v3', machine: 'v3'), isTrue); + expect(DependencyManager.tierMatches(installed: 'v2', machine: 'v3'), isFalse); + expect(DependencyManager.tierMatches(installed: 'v3', machine: 'v2'), isFalse); + }); + + test('ARM installs never mismatch', () { + expect(DependencyManager.tierMatches(installed: null, machine: null), isTrue); + }); + }); + + group('version.json', () { + test('round-trips the tier', () { + final info = InstalledDepsInfo(version: '1.11.0', tier: 'v2'); + final back = InstalledDepsInfo.fromJson(info.toJson()); + expect(back.tier, 'v2'); + expect(back.version, '1.11.0'); + }); + + test('omits the tier where there is none, and reads old files', () { + expect(InstalledDepsInfo(version: '1.11.0').toJson().containsKey('tier'), + isFalse); + expect(InstalledDepsInfo.fromJson({'version': '1.10.0'}).tier, isNull); + }); + }); + + group('depsTier', () { + final manager = DependencyManager.instance; + tearDown(() { + manager.tierProbeOverride = null; + manager.resetTierForTesting(); + }); + + test('asks the probe only where bundles are tiered, and caches it', () async { + var calls = 0; + manager.tierProbeOverride = () async { + calls++; + return 'v3'; + }; + final tiered = DependencyManager.isTiered(manager.platformId); + final first = await manager.depsTier(); + final second = await manager.depsTier(); + expect(first, tiered ? 'v3' : isNull); + expect(second, first); + expect(calls, tiered ? 1 : 0); + }); + }); +} diff --git a/app/test/support/worker_harness.dart b/app/test/support/worker_harness.dart index 88f5d28..53aa932 100644 --- a/app/test/support/worker_harness.dart +++ b/app/test/support/worker_harness.dart @@ -244,27 +244,17 @@ class WorkerHarness { return '${probe['arch']} [${features.isEmpty ? 'none detected' : features.join(' ')}]'; } - /// The instruction set extensions the worker process can actually execute, - /// as a set. Empty when the probe could not run at all. - /// - /// A test that needs to know whether a CPU-specific binary is safe to load - /// must ask this rather than parse `/proc/cpuinfo` or `sysctl` itself: a - /// second implementation of the same decision is how the interface and the - /// pipeline come to disagree, and the worker's answer is the one that governs - /// what the pipeline loads. Used by the zsmooth build check in - /// `vapoursynth_integration_test`. - static Future> cpuFeatures() async { + /// The deps tier to fetch when downloading, decided exactly as the app does + /// (`DependencyManager.resolveTier`): `VAPOURBOX_DEPS_TIER` if set, else the + /// worker's `--probe-cpu` answer. The override is how CI tests the v2 bundle + /// on runners that all probe as v3. + static Future depsTier() async { final probe = await _probeCpu(); - if (probe == null) return {}; - return (probe['features'] as List).cast().toSet(); - } - - /// True when the worker reports an x86 architecture (so the x86-specific - /// zsmooth builds are the relevant ones). - static Future isX86() async { - final probe = await _probeCpu(); - final arch = probe?['arch'] as String?; - return arch != null && (arch.startsWith('x86') || arch == 'amd64'); + return DependencyManager.resolveTier( + platformId: platform, + override: Platform.environment['VAPOURBOX_DEPS_TIER'], + probed: probe?['tier'] as String?, + ); } /// Never throws: this is diagnostic, and it is asked for before @@ -773,11 +763,13 @@ class WorkerHarness { /// Download + extract the pinned deps zip into [destDir]. static Future _downloadDeps(String destDir) async { final info = _loadDepsVersion(); - final url = info.getDownloadUrl(platform); - final expectedSha = await _fetchSidecarSha(info.getManifestUrl(platform)); + final tier = await depsTier(); + final assetId = DependencyManager.assetIdFor(platform, tier); + final url = info.getDownloadUrl(assetId); + final expectedSha = await _fetchSidecarSha(info.getManifestUrl(assetId)); final tmp = await Directory.systemTemp.createTemp('vb_deps_'); - final zip = File(p.join(tmp.path, info.filenameFor(platform))); + final zip = File(p.join(tmp.path, info.filenameFor(assetId))); try { // ignore: avoid_print print('WorkerHarness: downloading $url'); @@ -830,8 +822,9 @@ class WorkerHarness { // Stamp installed version so a normal app run treats it as up-to-date. File(p.join(destDir, 'version.json')).writeAsStringSync( const JsonEncoder.withIndent(' ').convert({ - 'version': info.versionFor(platform), + 'version': info.versionFor(assetId), 'installedAt': DateTime.now().toIso8601String(), + if (tier != null) 'tier': tier, }), ); } finally { diff --git a/app/test/vapoursynth_integration_test.dart b/app/test/vapoursynth_integration_test.dart index a452920..a66fb71 100644 --- a/app/test/vapoursynth_integration_test.dart +++ b/app/test/vapoursynth_integration_test.dart @@ -10,7 +10,6 @@ import 'dart:io'; import 'package:flutter_test/flutter_test.dart'; import 'package:path/path.dart' as path; -import 'support/worker_harness.dart'; void main() { late String depsDir; @@ -49,8 +48,12 @@ void main() { 'Unsupported platform: ${Platform.operatingSystem}'); } - // Try different possible locations for deps + // $VAPOURBOX_DEPS_DIR first, as the worker harness and the app honour it — + // without that, a run pointed at an unreleased bundle silently tested + // whatever happened to be in the repo's deps/ instead. + final override = Platform.environment['VAPOURBOX_DEPS_DIR']; final possibleDepsPaths = [ + if (override != null && override.isNotEmpty) override, path.join(scriptDir, '..', 'deps', depsPlatform), path.join(scriptDir, 'deps', depsPlatform), ]; @@ -117,17 +120,15 @@ core = vs.core # for on the paths the app exposes. A plugin missing here is a FILTER that fails # at job time with "No attribute with the name exists" — which is what an # incomplete or stale deps install looks like from the user's side. `zsmooth` -# (Chroma Denoise / CCD) was added to the bundle after this list was written and -# went uncovered, so a bundle without it passed the suite and failed the filter. -# It is no longer in THIS list because it is deliberately not autoloaded — it -# ships once per CPU baseline and the worker loads one by path. The test below -# covers it. +# (Chroma Denoise / CCD) was once missed here, so a bundle without it passed the +# suite and failed the filter. Since the x86 bundles split by CPU tier (#92) +# each ships exactly one zsmooth build, autoloaded like the rest. required = ['std', 'resize', 'mv', 'znedi3', 'eedi3m', 'fmtc', 'dfttest', 'neo_f3kdb', 'cas', 'dctf', 'deblock', 'rgvs', 'ctmf', 'warp', 'misc', 'grain', 'tcanny', 'descratch', 'vivtc', 'ttmpsm', 'tmedian', 'fft3dfilter', 'flux', 'bifrost', 'retinex', - 'bwdif', 'fb', 'removedirt', 'dedot', 'lghost'] + 'bwdif', 'fb', 'removedirt', 'dedot', 'lghost', 'zsmooth'] # On ARM, `nnedi3` is load-bearing and `znedi3` is only the fallback: znedi3's # SIMD is x86-only, so the ARM bundles build it scalar and both the templates' @@ -173,66 +174,17 @@ else: expect(result.stdout.toString(), contains('All plugins loaded')); }); - test('the zsmooth build for this CPU loads and runs', () async { - // zsmooth is deliberately OUTSIDE the autoload directory: upstream builds - // it for an AVX2 baseline with no runtime dispatch, so that binary dies - // with an illegal instruction on a pre-2013 CPU the instant a filter runs - // (issue #82). The bundle ships one build per CPU baseline and the worker - // loads exactly one by path. - // - // The Rust side (test_154) proves the generated script asks for the right - // file on any hardware; only running it proves the file is there and - // executes — and only ever for the CPU that ran it, which is why the - // choice itself is asserted in Rust and not here. - final zsmoothDir = Directory(path.join(depsDir, 'vapoursynth', 'zsmooth')); - expect( - zsmoothDir.existsSync(), - isTrue, - reason: 'deps bundle has no vapoursynth/zsmooth directory: ' - '${zsmoothDir.path} — a bundle older than deps 1.10.0, or a failed ' - 'zsmooth build', - ); - - // Asked of the worker, never derived here: loading the AVX2 build on a - // CPU without AVX2 is the crash this whole split exists to prevent, so a - // second-guessed answer is worse than none. - final features = await WorkerHarness.cpuFeatures(); - final isX86 = await WorkerHarness.isX86(); - final ext = Platform.isWindows - ? 'dll' - : Platform.isMacOS - ? 'dylib' - : 'so'; - final prefix = Platform.isWindows ? '' : 'lib'; - // Same preference order as DependencyLocator::zsmooth_candidates. With no - // probe (worker not built), only the portable build is considered — it - // runs everywhere, so the check degrades rather than risking the crash. - final candidates = [ - if (isX86 && features.contains('avx2')) '${prefix}zsmooth-haswell.$ext', - if (isX86) '${prefix}zsmooth-x86_64_v2.$ext', - '${prefix}zsmooth.$ext', - ]; - final chosen = candidates - .map((f) => File(path.join(zsmoothDir.path, f))) - .where((f) => f.existsSync()) - .firstOrNull; - expect( - chosen, - isNotNull, - reason: 'no zsmooth build this CPU can run in ${zsmoothDir.path}: ' - 'looked for $candidates, found ' - '${zsmoothDir.listSync().map((e) => path.basename(e.path)).toList()} ' - '(CPU features: $features)', - ); - + test('the bundled zsmooth build runs on this CPU', () async { + // zsmooth has no runtime dispatch: a build above this CPU's baseline + // loads fine and then dies with an illegal instruction the instant a + // filter runs (issue #82). Each x86 bundle tier ships the one build its + // CPUs can run, so the autoloaded namespace must render here. + // Constructing the node is not enough — the fault is in the kernel, so a + // frame has to be rendered. final script = ''' import vapoursynth as vs core = vs.core -core.std.LoadPlugin(r"${chosen!.path}") clip = core.std.BlankClip(width=160, height=120, format=vs.YUV420P8, length=2) -# CCD is what the Chroma Denoise pass is made of and what #82 was reported -# against. Constructing the node is not enough: the fault is in the kernel, so a -# frame has to be rendered. clip = core.zsmooth.CCD(clip, threshold=4, scale=1) clip.get_frame(0) print("zsmooth OK") diff --git a/docs/BUILDING.md b/docs/BUILDING.md index 36edc90..cce5a0f 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -116,14 +116,17 @@ come pre-built from ./Scripts/download-deps-macos.sh --force ``` -To build the x64 deps on Apple Silicon instead, run it under Rosetta 2 with an -Intel Homebrew prefix (`macos-13` Intel runners were retired Dec 2025): - -```bash -softwareupdate --install-rosetta --agree-to-license -arch -x86_64 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" -arch -x86_64 /bin/bash -lc 'PATH=/usr/local/bin:$PATH ./Scripts/download-deps-macos.sh --force' -``` +The x64 deps can no longer be built on Apple Silicon: the script needs an Intel +Homebrew prefix, and Homebrew's installer now refuses to create one under +Rosetta ("Homebrew on macOS is only supported on Apple Silicon processors!"). +Build them in CI (`build-deps-macos.yml`, natively on `macos-15-intel`), or +unzip a published or artifact `macos-x64` bundle into `deps/macos-x64`. + +**CPU tiers (x64 only).** Every x64 bundle exists in two tiers: `v3` (the +default; x86-64-v3 CPUs) and `v2` (older CPUs). Choose with `--tier v2` +(`-Tier v2` on Windows). Both write to the same `deps/`, so switching +tier needs `--force` (Windows: delete the directory first) — the scripts refuse +to build one tier over the other rather than produce a mixture. In CI, deps are produced by **Build macOS Deps** (`build-deps-macos.yml`, arm64 on `macos-15` and x64 natively on `macos-15-intel`), **Build Linux Deps** diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index f11740c..8d1ad12 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -384,7 +384,102 @@ Note the version parser accepts `n9.0.1` and `9.0.1` and deliberately **rejects a `master` build** (`N-125978-...`), so reverting any platform to an unpinned master URL is a red build rather than a silent regression. -### zsmooth ships once per CPU baseline, and is loaded by path (issue #82, 2026-08-28) +### x86 bundles split into CPU tiers (issue #92, 2026-09-25) + +**The report.** A Mac Pro 5,1 (Xeon X5690, Westmere — SSE4.2, no AVX at all) +failed every preview at once with `vspipe exited with signal 4` (SIGILL), even +with every pass disabled. The reporter's `.ips` put the fault in +`__GLOBAL__sub_I_MVDegrains_AVX2.cpp` inside `libmvtools.dylib`, called from +`dlopen` ← `VSCore::loadAllPluginsInPath` ← `createCore`. + +**Why every job.** Stefan-Olt's macOS MVTools compiles six files with +`-mavx2`. Three of them (`MVDegrains_AVX2`, `Overlap_AVX2`, `SADFunctions_AVX2`) +build a `static const std::unordered_map` of kernels at load time, and the +compiler used VEX instructions for it. Static initializers run inside `dlopen`, +before any plugin code can check the CPU, and R78 autoloads every plugin when a +core is created — so one plugin faulting at load kills every job, whatever it +asks for. Runtime dispatch in the filters is irrelevant; they never get to run. + +**How it was pinned down, and the traps on the way.** +- A disassembly scan of initializers *by name* (`_GLOBAL__sub_I_*`) came back + clean on the Linux bundle only because GCC's LTO renames them + (`__static_initialization_and_destruction_0v.lto_priv.N`). Name-based scans + are not evidence. `Scripts/check-load-time-simd.py` finds initializers from + `__init_offsets` / `__mod_init_func` instead, follows direct calls (never + into `__stubs`), and flags the stock 1.10.0 MVTools and nothing else in the + bundle. +- The first real-hardware probe reported all 18 prebuilt plugins as crashing, + because each test autoloaded the whole directory, MVTools included. + `Scripts/probe-plugin-compat.py` loads one plugin per process into a + `CoreCreationFlags.DISABLE_AUTO_LOADING` core instead. +- Rosetta 2 translates AVX/AVX2 on macOS 15+ (not AVX-512), so the crashing + MVTools ran perfectly on Apple Silicon. Nothing about pre-AVX2 behaviour can + be learned there. +- The reporter's Ivy Bridge MacBook (AVX, no AVX2) loaded everything: the + initializer VEX is plain AVX, so it is the *pre-AVX* machines that fail. + (macOS spells the feature `AVX1.0` in `machdep.cpu.features`.) + +**Surveyed under Intel SDE** (`probe-cpu-compat.yml`), with three controls: the +runner itself must pass everything; a plugin-free core must pass under SDE; and +upstream's AVX2-only zsmooth must crash *in its own image*. +- **Linux at Westmere:** every plugin in the 1.10.0 bundle renders, MVTools + included (GCC's build keeps VEX out of that load path), except the zsmooth + haswell build — which the worker never loaded there anyway. +- **Windows at Westmere is unmeasurable:** every test, including the core + control, faults in `VCRUNTIME140.dll`'s `memcpy` on a `vmovdqu ymm`. Microsoft's + runtime takes its AVX path from what the *host kernel* reports + (`IsProcessorFeaturePresent` reads shared kernel memory), which SDE cannot + virtualise, and python.org's runtime runs fine on real pre-AVX Windows. Without + the core control this run looked valid, because the positive control also + "crashed" — in vcruntime. **Windows at Sandy Bridge** (AVX, no AVX2) is valid, + and there everything but zsmooth-haswell renders. +- `sde.exe` drops empty arguments when relaunching the child, which once made a + probe bug look like a CPU finding. + +**The design.** Rather than more per-plugin special cases (zsmooth's two +builds loaded by path was the first), each x86 platform ships two bundles: +`v3` (x86-64-v3; the pre-existing asset names) and `v2` (`-v2`). One gateway +chooses — the app downloads by `vapourbox-worker --probe-cpu`'s `tier`, and the +tier lives only in the asset name and `version.json`, never the install path, +so the worker, dev paths and tests are untouched. v3 needs the whole +x86-64-v3 feature set (a v3 build may use FMA/BMI2/MOVBE as well as AVX2). +Startup re-checks the tier, so an install that moved machines is replaced. + +What actually differs between the tiers is only what the measurements above +proved must: zsmooth's baseline on all three platforms, and on macOS MVTools. +The v2 MVTools is v24 from source with `patches/mvtools-v24-no-avx2.patch`: + +- It replaces the six AVX2 files with stubs compiled at the baseline. With no + `-mavx2` code in the binary, neither the initializer VEX nor the other + hazard remains: an `-mavx2` file's copy of shared template code can win the + linker's choice and put AVX2 into unguarded callers. +- It masks `X264_CPU_AVX2` out of `g_cpuinfo`, the single point where the CPU + flags are set. Every call into the AVX2 code is gated on that flag. +- The stubs `abort()` rather than return nothing. They are unreachable, and a + silent no-op would render blank frames on an AVX2 machine running the v2 + bundle if the mask were ever lost. +- Every AVX2 selector call site null-checks (`if (tmp) degrain = tmp;`). +- The nasm assembly stays, and is still runtime-dispatched. +- The patched binary has no compiler-generated VEX outside the asm kernels. + +**Gates.** A v2 bundle cannot be published without passing: SDE in +`build-deps-{linux,windows}.yml` (the upload job waits for it), the static +check inside `package-deps-macos.sh`. `release.sh` stopped packaging deps +locally — a checkout holds one tier per platform, so it would have published +half a release. + +**Open.** Pre-AVX *Windows* remains unverified (no SDE run is possible, and +`check-load-time-simd.py` reads Mach-O only). MVTools' Windows build is the +official MSVC v24 — if it turns out to have the same initializer problem, the +Windows v2 tier would need an MVTools source build, which the Windows deps +script has no toolchain for today. + +### zsmooth ships once per CPU baseline, and is loaded by path (issue #82, 2026-08-28) — superseded by the CPU tiers above + +Since deps 1.11.0 each tier's bundle carries exactly one zsmooth build, in the +autoload directory; the two-builds-loaded-by-path mechanism below was removed. +The measurements still hold and are why v2 uses `x86_64_v2` rather than +`x86_64`. A plugin can also have **no** dispatch at all. zsmooth is compiled for a whole CPU baseline — upstream publishes only `haswell` (AVX2) and `znver4` for x86, diff --git a/worker/src/cpu.rs b/worker/src/cpu.rs new file mode 100644 index 0000000..50d3f55 --- /dev/null +++ b/worker/src/cpu.rs @@ -0,0 +1,120 @@ +//! Which x86 deps bundle this CPU can run (issue #92). +//! +//! The x86 bundles ship in two capability tiers. `v3` is built for the +//! x86-64-v3 psABI level (Haswell, 2013, and later); `v2` runs on anything +//! older. This is the single place that decides between them: the app asks +//! `vapourbox-worker --probe-cpu` which bundle to download, and the worker's own +//! CPU-dependent choices (`script_generator::ctmf_opt`) derive from the same +//! answer, so the two can never disagree about what this machine is. +//! +//! Detection is a runtime CPUID query that also checks the OS has enabled the +//! AVX register state, so it reports what the process can really execute — +//! including under Rosetta, which exposes AVX2 on macOS 15+ and nothing earlier. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CpuTier { + V2, + V3, +} + +impl CpuTier { + pub fn as_str(self) -> &'static str { + match self { + CpuTier::V2 => "v2", + CpuTier::V3 => "v3", + } + } +} + +/// The x86-64-v3 feature set, as compilers define `-march=x86-64-v3`. +/// +/// All of it, not just AVX2: zsmooth's haswell build and the other v3 plugins +/// are compiled for the whole level, so a CPU with AVX2 but without (say) MOVBE +/// can still fault in them. Such CPUs are rare; the cost of sending one to v2 +/// is throughput, while the cost of the reverse is a crash. +pub const V3_FEATURES: [&str; 8] = ["avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "lzcnt", "movbe"]; + +/// The tier for a CPU, given a feature test. Pure, so it is testable on any host. +pub fn tier_from_features(has: impl Fn(&str) -> bool) -> CpuTier { + if V3_FEATURES.iter().all(|f| has(f)) { + CpuTier::V3 + } else { + CpuTier::V2 + } +} + +/// This machine's tier, or `None` off x86 (ARM bundles are not tiered). +pub fn cpu_tier() -> Option { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + Some(tier_from_features(has_feature)) + } + #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] + { + None + } +} + +/// Runtime feature test by name. `is_x86_feature_detected!` only takes +/// literals, hence the match; an unknown name is reported as absent. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +pub fn has_feature(name: &str) -> bool { + match name { + "sse2" => std::is_x86_feature_detected!("sse2"), + "sse4.1" => std::is_x86_feature_detected!("sse4.1"), + "sse4.2" => std::is_x86_feature_detected!("sse4.2"), + "avx" => std::is_x86_feature_detected!("avx"), + "avx2" => std::is_x86_feature_detected!("avx2"), + "bmi1" => std::is_x86_feature_detected!("bmi1"), + "bmi2" => std::is_x86_feature_detected!("bmi2"), + "f16c" => std::is_x86_feature_detected!("f16c"), + "fma" => std::is_x86_feature_detected!("fma"), + "lzcnt" => std::is_x86_feature_detected!("lzcnt"), + "movbe" => std::is_x86_feature_detected!("movbe"), + "avx512f" => std::is_x86_feature_detected!("avx512f"), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_v3_feature_is_required() { + assert_eq!(tier_from_features(|_| true), CpuTier::V3); + for missing in V3_FEATURES { + assert_eq!( + tier_from_features(|f| f != missing), + CpuTier::V2, + "a CPU without {missing} must not be offered the v3 bundle" + ); + } + } + + #[test] + fn a_pre_avx_cpu_is_v2() { + // Westmere, the Mac Pro 5,1 from issue #92: SSE4.2, no AVX at all. + let westmere = ["sse2", "sse4.1", "sse4.2"]; + assert_eq!(tier_from_features(|f| westmere.contains(&f)), CpuTier::V2); + } + + #[test] + fn tier_is_reported_exactly_where_bundles_are_tiered() { + let tier = cpu_tier(); + if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { + assert!(tier.is_some(), "x86 must always resolve to a tier"); + } else { + assert_eq!(tier, None, "ARM bundles are not tiered"); + } + } + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + #[test] + fn the_runtime_answer_agrees_with_the_feature_test() { + assert_eq!(cpu_tier(), Some(tier_from_features(has_feature))); + // Guard the literal match: a misspelt arm would silently read "absent" + // and push every machine to v2. + assert!(has_feature("sse2"), "sse2 is architectural on x86_64"); + } +} diff --git a/worker/src/dependency_locator.rs b/worker/src/dependency_locator.rs index 4ad660d..23de2eb 100644 --- a/worker/src/dependency_locator.rs +++ b/worker/src/dependency_locator.rs @@ -574,83 +574,6 @@ impl DependencyLocator { } } - /// Directory holding the zsmooth builds, which are deliberately NOT in the - /// autoload directory. - /// - /// Upstream publishes zsmooth only for `haswell` (an AVX2 baseline, no - /// runtime dispatch) and `znver4`, so the bundled binary hard-crashes with - /// an illegal instruction on any pre-2013 x86 CPU the moment a zsmooth - /// filter runs — issue #82, on a Celeron J4105 and a Core i7 870. A second - /// `x86_64_v2` build covers those machines, and it cannot simply sit beside - /// the first: both register the namespace `zsmooth`, so whichever autoloads - /// second is rejected. Exactly one is therefore loaded explicitly, by - /// [`Self::zsmooth_plugin`], from here. - pub fn zsmooth_dir(&self) -> PathBuf { - self.platform_dir().join("vapoursynth").join("zsmooth") - } - - /// The zsmooth build this machine can actually execute, or `None` when the - /// bundle predates the split. - /// - /// `None` is not a failure: deps bundles up to 1.9.0 ship a single zsmooth - /// inside the autoload directory, and on those the generated script must - /// emit no `LoadPlugin` at all and let autoload do what it has always done. - /// That keeps a newer worker working against an older bundle, which matters - /// because the app can be upgraded before the deps download completes. - /// - /// The choice is made here, in the worker, rather than in the script for the - /// same reason as `script_generator::ctmf_opt`: it is a property of the - /// machine, and no preceding pass can change the answer. - pub fn zsmooth_plugin(&self) -> Option { - let dir = self.zsmooth_dir(); - for name in Self::zsmooth_candidates() { - let path = dir.join(name); - if path.exists() { - return Some(path); - } - } - None - } - - /// Candidate filenames in preference order: the fastest build this CPU can - /// run first, then the one that runs anywhere. - /// - /// Never fall back the other way. Choosing haswell where AVX2 is absent is - /// not a slow job, it is a dead one. - fn zsmooth_candidates() -> Vec { - let ext = if cfg!(target_os = "windows") { - "dll" - } else if cfg!(target_os = "macos") { - "dylib" - } else { - "so" - }; - let prefix = if cfg!(target_os = "windows") { "" } else { "lib" }; - - let mut variants: Vec<&str> = Vec::new(); - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - if std::is_x86_feature_detected!("avx2") { - variants.push("haswell"); - } - variants.push("x86_64_v2"); - } - // A single build on non-x86: aarch64 has one NEON baseline and upstream - // publishes no variants for it. - variants.push(""); - - variants - .into_iter() - .map(|v| { - if v.is_empty() { - format!("{}zsmooth.{}", prefix, ext) - } else { - format!("{}zsmooth-{}.{}", prefix, v, ext) - } - }) - .collect() - } - /// Get the NNEDI3CL weights path. pub fn nnedi3cl_weights_path(&self) -> PathBuf { #[cfg(target_os = "windows")] @@ -927,62 +850,6 @@ impl DependencyLocator { mod tests { use super::*; - #[test] - fn zsmooth_never_offers_a_build_this_cpu_cannot_run() { - // The whole point of the split: naming the AVX2 build on a machine - // without AVX2 is not a slow job, it is `0xC000001D` and a dead one - // (issue #82, on a Celeron J4105 and a Core i7 870). This assertion - // runs on every platform whatever hardware CI draws, which is the - // durable half of the guard — the end-to-end test can only confirm - // opportunistically, and GitHub's fleet is a mixed draw. - let candidates = DependencyLocator::zsmooth_candidates(); - assert!(!candidates.is_empty(), "there must always be a candidate"); - - let haswell = candidates.iter().any(|c| c.contains("haswell")); - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - assert_eq!( - haswell, - std::is_x86_feature_detected!("avx2"), - "the haswell build may be offered only where AVX2 exists" - ); - // The fallback has to be present on x86 regardless, or a non-AVX2 - // machine has nothing to load. - assert!( - candidates.iter().any(|c| c.contains("x86_64_v2")), - "x86 must always offer the x86_64_v2 fallback: {candidates:?}" - ); - if haswell { - assert!( - candidates[0].contains("haswell"), - "where AVX2 exists the fastest build must be preferred: {candidates:?}" - ); - } - } - #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] - { - assert!(!haswell, "no x86 build may be offered off x86: {candidates:?}"); - } - } - - #[test] - fn zsmooth_candidates_use_this_platforms_library_naming() { - // A wrong prefix or extension makes every candidate miss, which - // degrades silently to the autoload path — i.e. to the bug. - let candidates = DependencyLocator::zsmooth_candidates(); - let (prefix, ext) = if cfg!(target_os = "windows") { - ("", ".dll") - } else if cfg!(target_os = "macos") { - ("lib", ".dylib") - } else { - ("lib", ".so") - }; - for c in &candidates { - assert!(c.starts_with(&format!("{prefix}zsmooth")), "bad prefix: {c}"); - assert!(c.ends_with(ext), "bad extension: {c}"); - } - } - #[test] fn test_platform_suffix() { let locator = DependencyLocator { diff --git a/worker/src/lib.rs b/worker/src/lib.rs index edff205..b97214d 100644 --- a/worker/src/lib.rs +++ b/worker/src/lib.rs @@ -3,6 +3,7 @@ //! Provides video processing functionality using VapourSynth. pub mod models; +pub mod cpu; pub mod dependency_locator; pub mod dvd_reader; pub mod filter_registry; diff --git a/worker/src/main.rs b/worker/src/main.rs index 5a73020..d3430f0 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -19,6 +19,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; mod models; +mod cpu; mod dependency_locator; mod dvd_reader; mod pipeline_executor; @@ -203,55 +204,38 @@ fn run_probe_opencl() -> ExitCode { } -/// Report the CPU architecture and the dispatch-relevant instruction set -/// extensions as JSON. +/// Report the CPU architecture, the dispatch-relevant instruction set +/// extensions and, on x86, the deps bundle tier, as JSON. /// -/// This is diagnostic, not a decision: nothing in the pipeline reads it. It -/// exists so a test run can *state* which hardware it tested, because several -/// bundled plugins select a code path from these bits and a green run is -/// otherwise silent about which path it took. +/// `tier` is a decision: the app reads it to choose which x86 deps bundle to +/// download (see `cpu::cpu_tier`). It is absent off x86, where bundles are not +/// tiered, and the app treats a missing or unreadable answer on x86 as `v2`. /// -/// The motivating case: `ctmf.CTMF`'s AVX-512 kernel for 8-bit input crashes -/// the process, and GitHub's hosted Windows runners are a mixed fleet — so the -/// nightly passed for days on non-AVX-512 machines, went red the night it drew -/// an AVX-512 one, and looked like a spontaneous failure against an unchanged -/// tree. Printing this next to the result turns "it passed" into "it passed on -/// this hardware". +/// `features` is diagnostic. It lets a test run *state* which hardware it +/// tested, because several bundled plugins select a code path from these bits +/// and a green run is otherwise silent about which path it took — the CTMF +/// AVX-512 crash passed for days on non-AVX-512 runners before drawing one. fn run_probe_cpu() -> ExitCode { - let features = detected_cpu_features(); - println!( - "{}", - serde_json::json!({ - "arch": std::env::consts::ARCH, - "features": features, - }) - ); + let mut out = serde_json::json!({ + "arch": std::env::consts::ARCH, + "features": detected_cpu_features(), + }); + if let Some(tier) = cpu::cpu_tier() { + out["tier"] = serde_json::json!(tier.as_str()); + } + println!("{out}"); ExitCode::SUCCESS } -/// The instruction set extensions that bundled plugins actually dispatch on. -/// -/// Deliberately a short list rather than everything detectable: these are the -/// ones that change which kernel a plugin runs here. Detection is a runtime -/// CPUID query, so it reports what the process can really execute — including -/// under emulation, where an x86_64 worker on Apple Silicon correctly reports -/// whatever Rosetta exposes rather than what the binary was compiled for. +/// The instruction set extensions that bundled plugins dispatch on, plus the +/// whole x86-64-v3 set that decides the bundle tier. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] fn detected_cpu_features() -> Vec<&'static str> { - let mut features = Vec::new(); - for (name, present) in [ - ("sse2", std::is_x86_feature_detected!("sse2")), - ("sse4.1", std::is_x86_feature_detected!("sse4.1")), - ("avx", std::is_x86_feature_detected!("avx")), - ("avx2", std::is_x86_feature_detected!("avx2")), - ("fma", std::is_x86_feature_detected!("fma")), - ("avx512f", std::is_x86_feature_detected!("avx512f")), - ] { - if present { - features.push(name); - } - } - features + let names = ["sse2", "sse4.1", "sse4.2"] + .into_iter() + .chain(cpu::V3_FEATURES) + .chain(["avx512f"]); + names.filter(|n| cpu::has_feature(n)).collect() } #[cfg(target_arch = "aarch64")] @@ -523,8 +507,7 @@ fn run_worker( let knlm_available = deps.as_ref().map(|d| d.knlm_available()).unwrap_or(true); let script_generator = ScriptGenerator::new()? .with_opencl_available(opencl_available) - .with_knlm_available(knlm_available) - .with_zsmooth_plugin(deps.as_ref().and_then(|d| d.zsmooth_plugin())); + .with_knlm_available(knlm_available); let script_path = script_generator .generate(&job) .with_context(|| "Failed to generate VapourSynth script")?; diff --git a/worker/src/pipeline_executor.rs b/worker/src/pipeline_executor.rs index 505b6f3..4c74104 100644 --- a/worker/src/pipeline_executor.rs +++ b/worker/src/pipeline_executor.rs @@ -1252,8 +1252,7 @@ impl PipelineExecutor { // FPS as rational let script_generator = ScriptGenerator::new()? .with_opencl_available(self.deps.opencl_available()) - .with_knlm_available(self.deps.knlm_available()) - .with_zsmooth_plugin(self.deps.zsmooth_plugin()); + .with_knlm_available(self.deps.knlm_available()); let (fps_num, fps_den) = script_generator.frame_rate_to_rational(frame_rate); let preview_params = PreviewParams { diff --git a/worker/src/script_generator.rs b/worker/src/script_generator.rs index 3bcbcea..ab54825 100644 --- a/worker/src/script_generator.rs +++ b/worker/src/script_generator.rs @@ -28,32 +28,23 @@ use crate::models::{ /// /// The plugin does **not** verify that the CPU can run the level it is handed, /// so this has to be a real capability query rather than a constant: 3 (AVX2) -/// where the CPU has it, otherwise 2 (SSE2, which every x86-64 CPU has by -/// definition). The three non-AVX-512 levels are bit-identical, so this costs -/// throughput and nothing else. Non-x86 builds of the plugin compile the -/// dispatch out and ignore the value. +/// on a v3-tier CPU, otherwise 2 (SSE2, which every x86-64 CPU has by +/// definition). It follows `cpu::cpu_tier`, the same answer that chose the deps +/// bundle, so the worker has one notion of what this machine is. The three +/// non-AVX-512 levels are bit-identical, so a v2 machine that happens to have +/// AVX2 loses throughput and nothing else. Non-x86 builds of the plugin compile +/// the dispatch out and ignore the value. /// /// This belongs in the worker rather than in the script because it is a /// property of the machine, not of the clip — unlike the depth scalings, no /// preceding pass can change the answer. pub fn ctmf_opt() -> u8 { - if cpu_has_avx2() { - 3 - } else { - 2 + match crate::cpu::cpu_tier() { + Some(crate::cpu::CpuTier::V3) => 3, + _ => 2, } } -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -fn cpu_has_avx2() -> bool { - std::is_x86_feature_detected!("avx2") -} - -#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] -fn cpu_has_avx2() -> bool { - false -} - /// Generates VapourSynth scripts from templates. pub struct ScriptGenerator { template: String, @@ -69,12 +60,6 @@ pub struct ScriptGenerator { /// `knlm.KNLMeansCL: CL_INVALID_VALUE` / a missing-namespace error. Defaults /// to true; callers set it from `DependencyLocator::knlm_available()`. knlm_available: bool, - /// Absolute path to the zsmooth build this machine can execute, from - /// `DependencyLocator::zsmooth_plugin()`. `None` means the bundle predates - /// the per-CPU split and still autoloads a single zsmooth, so no - /// `LoadPlugin` is emitted — see that method for why that has to keep - /// working. - zsmooth_plugin: Option, } /// Parameters for preview script generation. @@ -106,7 +91,6 @@ impl ScriptGenerator { preview_template, opencl_available: true, knlm_available: true, - zsmooth_plugin: None, }) } @@ -125,18 +109,6 @@ impl ScriptGenerator { self } - /// Set the zsmooth build to load explicitly (from - /// `DependencyLocator::zsmooth_plugin()`). - /// - /// Both the encode and the preview path must be given the same value: they - /// are separate scripts, and a preview that loaded a different build than - /// the render would show a different picture than it produced — the same - /// class of split the field-order derivation exists to prevent. - pub fn with_zsmooth_plugin(mut self, plugin: Option) -> Self { - self.zsmooth_plugin = plugin; - self - } - /// Generate a .vpy script file for the given job. pub fn generate(&self, job: &VideoJob) -> Result { let pipeline = job.effective_pipeline(); @@ -160,7 +132,6 @@ impl ScriptGenerator { // Start with preview template and substitute preview-specific params let mut script = self.preview_template.clone(); - script = self.substitute_zsmooth(script); // Pipe source directory (same as main pipeline) let pipe_source_dir = Self::pipe_source_dir().unwrap_or_else(|_| env::temp_dir()); @@ -281,30 +252,9 @@ impl ScriptGenerator { } } - /// Emit (or elide) the explicit `LoadPlugin` for zsmooth. - /// - /// One function for both scripts on purpose: the encode and the preview must - /// load the same build, and doing this twice is how they would drift. - fn substitute_zsmooth(&self, script: String) -> String { - match self.zsmooth_plugin.as_ref() { - Some(path) => { - // The template uses r"..." so backslashes are literal, exactly - // as {{PIPE_SOURCE_DIR}} relies on. - let script = script.replace("{{ZSMOOTH_PLUGIN}}", &path.to_string_lossy()); - script - .replace("{{#LOAD_ZSMOOTH}}\n", "") - .replace("{{/LOAD_ZSMOOTH}}\n", "") - .replace("{{#LOAD_ZSMOOTH}}", "") - .replace("{{/LOAD_ZSMOOTH}}", "") - } - None => remove_block("{{#LOAD_ZSMOOTH}}", "{{/LOAD_ZSMOOTH}}", script), - } - } - /// Substitute parameters in a script string. fn substitute_parameters(&self, template: &str, job: &VideoJob, pipeline: &ProcessingPipeline, _input_path: &str) -> String { let mut script = template.to_string(); - script = self.substitute_zsmooth(script); // Pipe source parameters — FFmpeg decodes, pipes raw frames to VapourSynth via stdin let pipe_source_dir = Self::pipe_source_dir().unwrap_or_else(|_| env::temp_dir()); diff --git a/worker/templates/pipeline_template.vpy b/worker/templates/pipeline_template.vpy index 17a2b0d..eb1fd4b 100644 --- a/worker/templates/pipeline_template.vpy +++ b/worker/templates/pipeline_template.vpy @@ -15,16 +15,6 @@ core = vs.core # 1GB default, can be adjusted based on system memory core.max_cache_size = 1024 -{{#LOAD_ZSMOOTH}} -# zsmooth is loaded here rather than autoloaded, because the bundle ships more -# than one build of it and only one may register the namespace. Upstream -# compiles it for an AVX2 baseline with no runtime dispatch, so that build dies -# with an illegal instruction on a pre-2013 CPU (issue #82); the worker picks the -# fastest build this machine can execute and substitutes its path. The block is -# absent on a deps bundle old enough to autoload a single zsmooth. -core.std.LoadPlugin(r"{{ZSMOOTH_PLUGIN}}") -{{/LOAD_ZSMOOTH}} - # Load input video from stdin pipe (FFmpeg decodes → raw frames → VapourSynth) # This eliminates FFMS2 indexing which blocks on large/NAS files. import os diff --git a/worker/templates/preview_template.vpy b/worker/templates/preview_template.vpy index 2642ab7..f85d56d 100644 --- a/worker/templates/preview_template.vpy +++ b/worker/templates/preview_template.vpy @@ -14,13 +14,6 @@ core = vs.core # Configure cache size for optimal performance with temporal filters core.max_cache_size = 1024 -{{#LOAD_ZSMOOTH}} -# Same explicit zsmooth load as the encode template, from the same worker-side -# choice — a preview that loaded a different build than the render would show a -# different picture than it produced. See pipeline_template.vpy for why. -core.std.LoadPlugin(r"{{ZSMOOTH_PLUGIN}}") -{{/LOAD_ZSMOOTH}} - # Load raw frames piped from FFmpeg via stdin sys.path.insert(0, r"{{PIPE_SOURCE_DIR}}") from pipe_source import create_pipe_clip diff --git a/worker/tests/filter_integration_test.rs b/worker/tests/filter_integration_test.rs index cec5d92..5715a58 100644 --- a/worker/tests/filter_integration_test.rs +++ b/worker/tests/filter_integration_test.rs @@ -5655,53 +5655,14 @@ fn test_153_ctmf_opt_is_a_level_the_cpu_can_actually_run() { // The plugin does NOT verify that the CPU supports the level it is handed — // opt=3 on a pre-AVX2 machine installs the AVX2 kernels and crashes exactly // as opt=4 does on this one. So this has to stay a real capability query, - // not a constant: 3 only where AVX2 was detected, otherwise 2 (SSE2, which - // every x86-64 CPU has by definition). Both are bit-identical to the C path. + // not a constant: 3 only on a v3-tier CPU, otherwise 2 (SSE2, which every + // x86-64 CPU has by definition). Both are bit-identical to the C path. It + // follows the same tier that chose the deps bundle (issue #92), so the + // worker has one answer to "what is this machine". + use vapourbox_worker::cpu::{cpu_tier, CpuTier}; let opt = vapourbox_worker::script_generator::ctmf_opt(); - assert!( - opt == 2 || opt == 3, - "opt must be SSE2 or AVX2, got {opt}" - ); - - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - let expected = if std::is_x86_feature_detected!("avx2") { 3 } else { 2 }; - assert_eq!(opt, expected, "opt must follow what the CPU actually has"); - } -} - -/// Generate both scripts with an explicit zsmooth build selected, the way the -/// worker does once a bundle carries the per-CPU split. -fn generate_both_scripts_with_zsmooth( - job: &VideoJob, - plugin: Option, -) -> (String, String) { - let generator = ScriptGenerator::new() - .expect("create generator") - .with_zsmooth_plugin(plugin); - let encode_path = generator.generate(job).expect("generate encode script"); - let encode = std::fs::read_to_string(&encode_path).expect("read encode script"); - - let params = PreviewParams { - width: job.input_width.unwrap_or(720), - height: job.input_height.unwrap_or(480), - pix_fmt: job - .input_pixel_format - .clone() - .unwrap_or_else(|| "yuv420p".to_string()), - num_frames: 11, - fps_num: 30000, - fps_den: 1001, - output_index: 5, - }; - let preview_path = generator - .generate_preview(job, ¶ms) - .expect("generate preview script"); - let preview = std::fs::read_to_string(&preview_path).expect("read preview script"); - - let _ = std::fs::remove_file(&encode_path); - let _ = std::fs::remove_file(&preview_path); - (encode, preview) + let expected = if cpu_tier() == Some(CpuTier::V3) { 3 } else { 2 }; + assert_eq!(opt, expected, "opt must follow the CPU tier"); } fn chroma_denoise_job(id: &str) -> VideoJob { @@ -5719,69 +5680,34 @@ fn chroma_denoise_job(id: &str) -> VideoJob { } #[test] -fn test_154_zsmooth_is_loaded_explicitly_from_the_chosen_build() { - // zsmooth is bundled twice — upstream builds it for an AVX2 baseline with no - // runtime dispatch, so that binary dies with an illegal instruction - // (0xC000001D) on a pre-2013 CPU the instant a filter runs (issue #82). Both - // builds register the namespace `zsmooth`, so neither may autoload and - // exactly one is loaded by path. +fn test_154_zsmooth_is_autoloaded_from_the_tiered_bundle() { + // zsmooth used to ship twice per x86 bundle (an AVX2-only haswell build and + // an x86_64_v2 fallback, issue #82) and was loaded by explicit path. Since + // the bundles are split by CPU tier (issue #92), each carries exactly one + // build in the autoload directory, so the scripts must load nothing + // themselves. A LoadPlugin here would fail every job: the zsmooth/ + // directory it pointed into no longer exists. // // Both scripts, because the reporter in #82 hit the preview first. create_output_dir(); - let job = chroma_denoise_job("test_154_zsmooth_load"); - let chosen = std::path::PathBuf::from("/deps/vapoursynth/zsmooth/zsmooth-x86_64_v2.dll"); - let (encode, preview) = generate_both_scripts_with_zsmooth(&job, Some(chosen.clone())); + let job = chroma_denoise_job("test_154_zsmooth_autoload"); + let (encode, preview) = generate_both_scripts(&job); for (name, script) in [("encode", &encode), ("preview", &preview)] { assert!( - script.contains(&format!("core.std.LoadPlugin(r\"{}\")", chosen.display())), - "{name} script must load the chosen zsmooth build explicitly" + !script.contains("LoadPlugin"), + "{name} script must leave zsmooth to autoload" ); assert!( script.contains("core.zsmooth.CCD("), "{name} script should still call the filter" ); - // An unsubstituted marker is valid Python nowhere and would fail the job + // A leftover marker is valid Python nowhere and would fail the job // with a SyntaxError that reads like a template bug. - for leftover in ["{{#LOAD_ZSMOOTH}}", "{{/LOAD_ZSMOOTH}}", "{{ZSMOOTH_PLUGIN}}"] { - assert!( - !script.contains(leftover), - "{name} script left {leftover} unsubstituted" - ); - } - // The load has to precede the first use, or the namespace is missing - // when the filter is constructed. - let load = script.find("core.std.LoadPlugin(r\"").expect("load present"); - let use_ = script.find("core.zsmooth.CCD(").expect("call present"); - assert!(load < use_, "{name} script loads zsmooth after using it"); - } -} - -#[test] -fn test_155_a_bundle_without_the_split_still_autoloads_zsmooth() { - // Deps bundles up to 1.9.0 ship one zsmooth inside the autoload directory. - // A worker that emitted a LoadPlugin for a path those bundles do not have - // would fail every job on them — and the app can be upgraded before the - // deps download finishes, so that window is real. No path selected must - // therefore mean no LoadPlugin, leaving the script as it was before the - // split existed. - create_output_dir(); - let job = chroma_denoise_job("test_155_zsmooth_autoload"); - let (encode, preview) = generate_both_scripts_with_zsmooth(&job, None); - - for (name, script) in [("encode", &encode), ("preview", &preview)] { - assert!( - !script.contains("LoadPlugin"), - "{name} script must not load a plugin the bundle may not have" - ); - assert!( - script.contains("core.zsmooth.CCD("), - "{name} script should still call the filter, via autoload" - ); - for leftover in ["{{#LOAD_ZSMOOTH}}", "{{/LOAD_ZSMOOTH}}", "{{ZSMOOTH_PLUGIN}}"] { + for leftover in ["LOAD_ZSMOOTH", "ZSMOOTH_PLUGIN"] { assert!( !script.contains(leftover), - "{name} script left {leftover} unsubstituted" + "{name} script still carries {leftover}" ); } }