From 7133efa79f9f4148309370e4d5e6ba83e5cf25a3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 8 Aug 2026 03:41:06 +0000 Subject: [PATCH] Apply the open H3 fixes to the prebuilts we publish The prebuilt pipeline builds leejet's source at an aged release tag, not this fork's master, so the three MiniMax-H3 fixes on master reach nobody: every Studio user installs a binary that still aborts on the default cfg-scale, still aborts on --vae-on-cpu, and still quantizes H3's 1-D norms into an output uncorrelated with its own bf16 reference. Building from master instead would throw away the reason the pipeline is shaped this way, which is that what we publish should be traceable to a specific upstream release. So keep the upstream tag as the base and carry the delta explicitly: - patches/ holds one file per fix, each with its upstream pull request in the header. All three are open on leejet: #1861, #1862, #1863. - resolve applies them to the checked-out tag, after running git apply --check over the whole set so a stale patch stops the run before the tree is half modified. That failure is the signal to delete the patch (upstream merged it) or refresh it (upstream moved the code). - a non-empty set moves the published tag to -u, where id is the sha256 prefix of the concatenated patches. The tag then says whether a box is stock, and a changed patch set republishes rather than matching an existing release and skipping. - the manifest and the release notes both record the applied list. An empty patches/ leaves the tag and every asset name exactly as they are today. Verified by running the resolve step against master-813-bfbef5b with gh stubbed: all three patches apply, the tag becomes master-813-bfbef5b-u, and the stamped source tarball contains the fixes. --- .github/workflows/unsloth-sd-prebuilt.yml | 61 ++++++++- ...e-1d-norm-weights-from-blanket-quant.patch | 34 +++++ ...02-h3-cfg-scale-and-audio-vae-on-cpu.patch | 121 ++++++++++++++++++ patches/0003-h3-img-gen-mode-guard.patch | 31 +++++ patches/README.md | 41 ++++++ scripts/unsloth/assemble_metadata.py | 12 ++ 6 files changed, 294 insertions(+), 6 deletions(-) create mode 100644 patches/0001-spare-1d-norm-weights-from-blanket-quant.patch create mode 100644 patches/0002-h3-cfg-scale-and-audio-vae-on-cpu.patch create mode 100644 patches/0003-h3-img-gen-mode-guard.patch create mode 100644 patches/README.md diff --git a/.github/workflows/unsloth-sd-prebuilt.yml b/.github/workflows/unsloth-sd-prebuilt.yml index 36d0d0534..56ce42609 100644 --- a/.github/workflows/unsloth-sd-prebuilt.yml +++ b/.github/workflows/unsloth-sd-prebuilt.yml @@ -58,6 +58,8 @@ jobs: contents: read outputs: tag: ${{ steps.r.outputs.tag }} + upstream_tag: ${{ steps.r.outputs.upstream_tag }} + patch_list: ${{ steps.r.outputs.patch_list }} commit: ${{ steps.r.outputs.commit }} exists: ${{ steps.r.outputs.exists }} source_artifact: ${{ steps.r.outputs.source_artifact }} @@ -87,18 +89,56 @@ jobs: fi printf '%s' "$TAG" | grep -qE '^master-[0-9]+-[0-9a-f]+$' \ || { echo "refusing non-release tag '$TAG'" >&2; exit 1; } + UPSTREAM_TAG="$TAG" - # Does OUR published release already exist? (drafts don't count.) - EXISTS=false - if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft 2>/dev/null || true)" = "false" ]; then - EXISTS=true - fi + # Stash patches/ before the upstream checkout: it is tracked HERE and absent from + # leejet's tree, so checking out their tag in this worktree would remove it. + PATCHES="${RUNNER_TEMP}/patches" + rm -rf "$PATCHES"; mkdir -p "$PATCHES" + cp -a patches/. "$PATCHES"/ 2>/dev/null || true # Fetch leejet's source AT THAT TAG and tar it (no .git); the build legs # extract this identical tree. Our own scripts come from the tooling checkout. git remote add upstream https://github.com/leejet/stable-diffusion.cpp.git || true git fetch -q --depth 1 upstream "refs/tags/${TAG}:refs/tags/${TAG}" git checkout -q --detach "refs/tags/${TAG}" + + # Apply the local patch set (see patches/README.md). Each one is a fix that is open + # upstream but not yet in a release; the set is normally empty. A non-empty set moves + # the published tag to -u, so the tag says what the box carries and a + # changed set republishes instead of matching an existing release and skipping. + # + # --check runs over ALL of them first, so a stale patch stops the run before the tree + # is half-modified. That failure is the intended signal: upstream either merged the fix + # (delete the patch) or moved the code under it (refresh and re-verify it). + PATCH_LIST="" + FILES=() + while IFS= read -r f; do FILES+=("$f"); done < <(find "$PATCHES" -maxdepth 1 -name '*.patch' | sort) + if [ "${#FILES[@]}" -gt 0 ]; then + for p in "${FILES[@]}"; do + git apply --check "$p" || { + echo "ERROR: $(basename "$p") no longer applies to ${UPSTREAM_TAG}." >&2 + echo " Upstream merged it (delete the file) or moved the code under it" >&2 + echo " (refresh it). See patches/README.md." >&2 + exit 1 + } + done + for p in "${FILES[@]}"; do + git apply "$p" + PATCH_LIST="${PATCH_LIST}${PATCH_LIST:+,}$(basename "$p")" + echo "applied $(basename "$p")" + done + PATCH_ID="$(cat "${FILES[@]}" | sha256sum | cut -c1-7)" + TAG="${UPSTREAM_TAG}-u${PATCH_ID}" + echo "patched build: ${UPSTREAM_TAG} + ${PATCH_LIST} -> ${TAG}" + fi + + # Does OUR published release already exist? (drafts don't count.) Checked against the + # final, patch-aware tag so a patch-set change is never mistaken for an existing build. + EXISTS=false + if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft 2>/dev/null || true)" = "false" ]; then + EXISTS=true + fi # ggml is a REQUIRED submodule; the server frontend + libwebp/libwebm are not # (we build with SD_SERVER_BUILD_FRONTEND / SD_WEBP / SD_WEBM off), so fetch only # ggml to keep the source tarball small and the build self-contained. @@ -114,6 +154,8 @@ jobs: { echo "tag=$TAG" + echo "upstream_tag=$UPSTREAM_TAG" + echo "patch_list=$PATCH_LIST" echo "commit=$COMMIT" echo "exists=$EXISTS" echo "source_artifact=$SRC_ARTIFACT" @@ -324,6 +366,8 @@ jobs: set -eux python3 tooling/scripts/unsloth/assemble_metadata.py \ --tag '${{ needs.resolve.outputs.tag }}' \ + --upstream-tag '${{ needs.resolve.outputs.upstream_tag }}' \ + --patches '${{ needs.resolve.outputs.patch_list }}' \ --source-repo 'leejet/stable-diffusion.cpp' \ --commit '${{ needs.resolve.outputs.commit }}' \ --dist dist --out dist \ @@ -358,8 +402,13 @@ jobs: run: | set -eux TAG='${{ needs.resolve.outputs.tag }}' + UPSTREAM_TAG='${{ needs.resolve.outputs.upstream_tag }}' + PATCH_LIST='${{ needs.resolve.outputs.patch_list }}' REPO="$GITHUB_REPOSITORY" - NOTES="Automated Unsloth stable-diffusion.cpp CPU + Apple prebuild (sd-cli + sd-server) for upstream [${TAG}](https://github.com/leejet/stable-diffusion.cpp/releases/tag/${TAG}). GPU hosts use diffusers/torch; this native engine targets CPU (Linux/WSL/Windows) and Apple (Metal)." + NOTES="Automated Unsloth stable-diffusion.cpp CPU + Apple prebuild (sd-cli + sd-server) for upstream [${UPSTREAM_TAG}](https://github.com/leejet/stable-diffusion.cpp/releases/tag/${UPSTREAM_TAG}). GPU hosts use diffusers/torch; this native engine targets CPU (Linux/WSL/Windows) and Apple (Metal)." + if [ -n "$PATCH_LIST" ]; then + NOTES="${NOTES}"$'\n\n'"Not a stock upstream build. It carries the patch set in [patches/](https://github.com/${REPO}/tree/master/patches), each one a fix that is open upstream but not yet in a release, which is what the \`-u\` suffix on the tag marks:"$'\n\n'"$(printf '%s' "$PATCH_LIST" | tr ',' '\n' | sed 's/^/- `/; s/$/`/')"$'\n\n'"\`sd-prebuilt-manifest.json\` records the same list." + fi if [ "$(gh release view "$TAG" --repo "$REPO" --json isDraft --jq .isDraft 2>/dev/null || true)" = "true" ]; then gh release delete "$TAG" --repo "$REPO" --yes fi diff --git a/patches/0001-spare-1d-norm-weights-from-blanket-quant.patch b/patches/0001-spare-1d-norm-weights-from-blanket-quant.patch new file mode 100644 index 000000000..3c4065dce --- /dev/null +++ b/patches/0001-spare-1d-norm-weights-from-blanket-quant.patch @@ -0,0 +1,34 @@ +Spare 1-D norm weights from a blanket --type + +Upstream: https://github.com/leejet/stable-diffusion.cpp/pull/1861 + +A 1-D weight is a per-channel gain, never a matmul weight. These survived a blanket --type +only when their length did not divide the block size; MiniMax-H3's [5376] norms do divide it, +so 106 of them were crushed to 4 bits. The model still loads and still renders something +plausible, which is what makes it worth guarding: measured against a bf16 render of the same +prompt and seed, PSNR 9.87 / SSIM 0.074 / LPIPS 0.981, against 22.22 / 0.841 / 0.292 with the +rule in place. + +diff --git a/src/model_loader.cpp b/src/model_loader.cpp +index 047c9d9..b286f2a 100644 +--- a/src/model_loader.cpp ++++ b/src/model_loader.cpp +@@ -1511,6 +1511,18 @@ bool ModelLoader::tensor_should_be_converted(const TensorStorage& tensor_storage + if (type != GGML_TYPE_COUNT) { + if (ggml_is_quantized(type) && tensor_storage.ne[0] % ggml_blck_size(type) != 0) { + // Pass, do not convert ++ } else if (ggml_is_quantized(type) && tensor_storage.n_dims <= 1) { ++ // Pass, do not convert. A 1-D weight is a per-channel scale (LayerNorm/RMSNorm gain), ++ // never a matmul weight, so quantizing it buys almost nothing and costs a lot: every ++ // channel of the block shares one scale and one min, and a gain vector has no reason ++ // to be locally smooth. Until now these survived only by accident, when their length ++ // did not divide the block size (FLUX q_norm/k_norm are [128] and 128 % 256 != 0) or ++ // when a name rule above happened to match. A model whose norms DO divide the block ++ // size, such as MiniMax-H3 with [5376] and 5376 % 256 == 0, had 106 norm scales ++ // crushed to 4 bits by a blanket --type. The result still loads and still renders a ++ // plausible image, so a "does it run" check passes it, while measured against a bf16 ++ // render of the same prompt and seed it is destroyed: PSNR 9.87 / SSIM 0.074 / ++ // LPIPS 0.981, against 22.22 / 0.841 / 0.292 with this rule in place. + } else if (ends_with(name, ".bias")) { + // Pass, do not convert + } else if (ends_with(name, ".scale")) { diff --git a/patches/0002-h3-cfg-scale-and-audio-vae-on-cpu.patch b/patches/0002-h3-cfg-scale-and-audio-vae-on-cpu.patch new file mode 100644 index 000000000..d3660a889 --- /dev/null +++ b/patches/0002-h3-cfg-scale-and-audio-vae-on-cpu.patch @@ -0,0 +1,121 @@ +Stop MiniMax-H3 aborting on the default cfg-scale and on --vae-on-cpu + +Upstream: https://github.com/leejet/stable-diffusion.cpp/pull/1862 + +H3 aborts at any --cfg-scale above 1.0, and --cfg-scale defaults to 7.0, so a vid_gen command +that omits the flag crashes rather than rendering. Separately, --vae-on-cpu aborts on the +AUDIO VAE: ggml_conv_1d builds an F16 im2col and the CPU path asserts the kernel is F16, while +the audio VAE kernels arrive as F32. + +diff --git a/src/model/vae/ltx_audio_vae.hpp b/src/model/vae/ltx_audio_vae.hpp +index 3319a5c..e385aad 100644 +--- a/src/model/vae/ltx_audio_vae.hpp ++++ b/src/model/vae/ltx_audio_vae.hpp +@@ -182,6 +182,17 @@ namespace LTXV { + } + }; + ++ // ggml_conv_1d / ggml_conv_1d_dw build an F16 im2col, and the CPU backend ++ // additionally requires the kernel itself to be F16. F32 kernels (as loaded ++ // from F32 checkpoints) work on GPU backends but abort on CPU, so cast them ++ // in-graph here instead of degrading the weights on every backend. ++ static ggml_tensor* conv1d_kernel_for_backend(GGMLRunnerContext* runner_ctx, ggml_tensor* kernel) { ++ if (kernel->type == GGML_TYPE_F32 && sd_backend_is_cpu(runner_ctx->backend)) { ++ return ggml_cast(runner_ctx->ggml_ctx, kernel, GGML_TYPE_F16); ++ } ++ return kernel; ++ } ++ + static ggml_tensor* compute_log_mel_spectrogram(GGMLRunnerContext* runner_ctx, + ggml_tensor* waveform, + ggml_tensor* forward_basis, +@@ -218,7 +229,8 @@ namespace LTXV { + x = ggml_ext_pad_ext(ctx, runner_ctx->backend, x, static_cast(left_pad), 0, 0, 0, 0, 0, 0, 0); + } + +- auto frames = ggml_conv_1d(ctx, forward_basis, x, hop_length, 0, 1); ++ auto frames = ggml_conv_1d(ctx, conv1d_kernel_for_backend(runner_ctx, forward_basis), ++ x, hop_length, 0, 1); + GGML_ASSERT(frames->ne[0] == frame_count); + GGML_ASSERT(frames->ne[1] == stft_channels); + GGML_ASSERT(frames->ne[2] == channels * batch); +@@ -319,7 +331,8 @@ namespace LTXV { + int padding) { + auto ctx = runner_ctx->ggml_ctx; + GGML_ASSERT(x->ne[3] == 1); +- auto tiled = tile_depthwise_filter_1d(runner_ctx, filter, x->ne[1]); ++ auto tiled = conv1d_kernel_for_backend(runner_ctx, ++ tile_depthwise_filter_1d(runner_ctx, filter, x->ne[1])); + auto out = ggml_conv_1d_dw(ctx, tiled, x, stride, padding, 1); + return ggml_reshape_4d(ctx, out, out->ne[0], out->ne[1], 1, 1); + } +@@ -339,10 +352,11 @@ namespace LTXV { + return reversed; + } + +- static ggml_tensor* depthwise_conv_transpose1d(ggml_context* ctx, ++ static ggml_tensor* depthwise_conv_transpose1d(GGMLRunnerContext* runner_ctx, + ggml_tensor* x, + ggml_tensor* filter, + int stride) { ++ auto ctx = runner_ctx->ggml_ctx; + GGML_ASSERT(x->ne[2] == 1 && x->ne[3] == 1); + GGML_ASSERT(filter->ne[1] == 1); + GGML_ASSERT(filter->ne[2] == 1 && filter->ne[3] == 1); +@@ -364,7 +378,8 @@ namespace LTXV { + x_flat = ggml_reshape_3d(ctx, x_flat, time * stride, 1, channels); + + auto reversed_filter = reverse_1d_filter(ctx, filter); +- auto out = ggml_conv_1d(ctx, reversed_filter, x_flat, 1, static_cast(kernel_size - 1), 1); ++ auto out = ggml_conv_1d(ctx, conv1d_kernel_for_backend(runner_ctx, reversed_filter), ++ x_flat, 1, static_cast(kernel_size - 1), 1); + if (out->ne[0] > out_time) { + out = ggml_ext_slice(ctx, out, 0, 0, out_time); + } +@@ -404,7 +419,7 @@ namespace LTXV { + + auto x = ggml_reshape_3d(ctx, waveform, time, channels * batch, 1); + x = replicate_pad_1d(runner_ctx, x, pad, pad); +- x = depthwise_conv_transpose1d(ctx, x, filter, ratio); ++ x = depthwise_conv_transpose1d(runner_ctx, x, filter, ratio); + x = ggml_ext_slice(ctx, x, 0, pad_left, x->ne[0] - pad_right); + return ggml_reshape_3d(ctx, x, x->ne[0], channels, batch); + } +@@ -553,7 +568,7 @@ namespace LTXV { + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { +- x = ggml_conv_1d(ctx->ggml_ctx, params["weight"], x, stride, padding, dilation); ++ x = ggml_conv_1d(ctx->ggml_ctx, conv1d_kernel_for_backend(ctx, params["weight"]), x, stride, padding, dilation); + if (bias) { + auto b = ggml_reshape_4d(ctx->ggml_ctx, params["bias"], 1, params["bias"]->ne[0], 1, 1); + x = ggml_add_inplace(ctx->ggml_ctx, x, b); +@@ -670,7 +685,7 @@ namespace LTXV { + int up_pad_right = up_pad * up_ratio + (up_kernel_size - up_ratio + 1) / 2; + + x = replicate_pad_1d(ctx, x, up_pad, up_pad); +- x = depthwise_conv_transpose1d(ctx->ggml_ctx, x, up_filter, up_ratio); ++ x = depthwise_conv_transpose1d(ctx, x, up_filter, up_ratio); + x = ggml_ext_slice(ctx->ggml_ctx, x, 0, up_pad_left, x->ne[0] - up_pad_right); + + x = act->forward(ctx, x); +diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp +index f2b69cc..be74ea5 100644 +--- a/src/stable-diffusion.cpp ++++ b/src/stable-diffusion.cpp +@@ -4222,6 +4222,15 @@ struct GenerationRequest { + guidance->img_cfg = 1.f; + } + ++ // MiniMax-H3 is distilled with guidance baked in and has no negative ++ // prompt semantics; its empty uncond prompt cannot even be encoded ++ // (zero tokens), so any cfg != 1 would abort deep inside ggml. ++ if (sd_version_is_minimax_h3(sd_ctx->sd->version) && guidance->txt_cfg != 1.f) { ++ LOG_WARN("MiniMax-H3 is a distilled, CFG-free model; forcing cfg-scale from %.2f to 1.0", ++ guidance->txt_cfg); ++ guidance->txt_cfg = 1.f; ++ } ++ + if (guidance->img_cfg != guidance->txt_cfg) { + *use_uncond = true; + } diff --git a/patches/0003-h3-img-gen-mode-guard.patch b/patches/0003-h3-img-gen-mode-guard.patch new file mode 100644 index 000000000..811354dc3 --- /dev/null +++ b/patches/0003-h3-img-gen-mode-guard.patch @@ -0,0 +1,31 @@ +Fail with a message when MiniMax-H3 is run in img_gen mode + +Upstream: https://github.com/leejet/stable-diffusion.cpp/pull/1863 + +Only generate_video computes H3's audio length, so an H3 checkpoint reaching generate_image is +guaranteed to die on GGML_ASSERT(!audio_input_cache.empty()), as a core dump with no hint that +--mode vid_gen is what is missing. + +diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp +index f2b69cc..5986163 100644 +--- a/src/stable-diffusion.cpp ++++ b/src/stable-diffusion.cpp +@@ -5584,6 +5584,18 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, + return false; + } + ++ // MiniMax-H3 is video-only. Its denoiser always splits the packed latent into a video and an ++ // audio half, and only generate_video ever computes the audio length, so reaching this ++ // function with an H3 checkpoint is guaranteed to die on ++ // GGML_ASSERT(!audio_input_cache.empty()) with a core dump, after the several minutes it ++ // takes to load the weights, and with nothing in the output pointing at the missing --mode. ++ // (The AnimateDiff path below routes vid_gen back through here, but that is SD1.5 plus a ++ // motion module, never H3.) ++ if (sd_version_is_minimax_h3(sd_ctx->sd->version)) { ++ LOG_ERROR("MiniMax-H3 is a video model and cannot be run in img_gen mode; use --mode vid_gen"); ++ return false; ++ } ++ + sd_ctx->sd->reset_cancel_flag(); + + int64_t t0 = ggml_time_ms(); diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 000000000..1eae60b65 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,41 @@ +# Patches applied to the published prebuilts + +`.github/workflows/unsloth-sd-prebuilt.yml` builds leejet's source **at a published release +tag**, aged for at least six hours, rather than this fork's `master`. That is deliberate: what we +publish should be traceable to a specific upstream release rather than to whatever a fork happened +to contain that day. + +Every `*.patch` in this directory is applied to that tree, in filename order, before the build. A +release built with a non-empty patch set is tagged `-u`, where `` is the +first seven hex digits of the sha256 of the concatenated patch files. So: + +- an unpatched build keeps the plain upstream tag, exactly as before, +- changing, adding or removing a patch changes the tag, which is what makes the pipeline rebuild + and republish rather than see an existing release and skip, +- the tag on the box says whether the box is stock, and which patch set it carries. + +`sd-prebuilt-manifest.json` in each release records the patch filenames and the id. + +## Rules + +**Every patch must be open upstream.** These exist to close the gap between "fixed" and +"released", not to carry a private fork. Put the upstream pull request in the header comment of +the patch file. + +**A patch that no longer applies fails the build.** This is intentional, not a bug to route +around. `git apply --check` runs on all of them before any is applied, and the run stops with the +patch name. That happens for exactly two reasons: upstream merged it, in which case delete the +file, or upstream changed the surrounding code, in which case refresh the patch against the new +tag and re-verify it. + +**Delete on merge.** Once the fix ships in an upstream release the patch is dead weight, and +leaving it in place means the next release silently carries a duplicate of code upstream already +has. + +## Current set + +| patch | upstream PR | what it fixes | +|---|---|---| +| `0001-spare-1d-norm-weights-from-blanket-quant.patch` | leejet/stable-diffusion.cpp#1861 | a blanket `--type` quantizes 1-D norm gains whose length divides the block size, which silently destroys MiniMax-H3 output (LPIPS 0.981 against its own bf16 render) | +| `0002-h3-cfg-scale-and-audio-vae-on-cpu.patch` | leejet/stable-diffusion.cpp#1862 | H3 aborts on the default `--cfg-scale 7.0`, and aborts again on `--vae-on-cpu` because the audio VAE reaches the CPU conv1d path with F32 kernels | +| `0003-h3-img-gen-mode-guard.patch` | leejet/stable-diffusion.cpp#1863 | an H3 checkpoint run without `--mode vid_gen` core dumps on a raw ggml assert instead of saying which flag is missing | diff --git a/scripts/unsloth/assemble_metadata.py b/scripts/unsloth/assemble_metadata.py index 1017c6719..bb65ed29b 100755 --- a/scripts/unsloth/assemble_metadata.py +++ b/scripts/unsloth/assemble_metadata.py @@ -39,6 +39,16 @@ def main() -> int: ap.add_argument("--tag", required = True) ap.add_argument("--source-repo", required = True) ap.add_argument("--commit", required = True) + # A patched build is tagged -u, so the two differ exactly when patches were + # applied. Both are recorded: the tag identifies the artifact, upstream_tag and patches say + # what it is made of, so a consumer can tell a stock build from a patched one without + # unpacking it. See patches/README.md. + ap.add_argument("--upstream-tag", default = "") + ap.add_argument( + "--patches", + default = "", + help = "comma-separated patch filenames applied to the upstream tree (empty = stock)", + ) ap.add_argument("--dist", required = True, help = "dir holding the sd-*.zip bundles") ap.add_argument("--out", required = True, help = "dir to write the metadata json into") ap.add_argument("--publish-repo", required = True) @@ -70,6 +80,8 @@ def main() -> int: manifest = { "schema": 1, "tag": args.tag, + "upstream_tag": args.upstream_tag or args.tag, + "patches": [p for p in args.patches.split(",") if p], "source_repo": args.source_repo, "source_commit": args.commit, "publish_repo": args.publish_repo,