diff --git a/.github/workflows/unsloth-sd-prebuilt.yml b/.github/workflows/unsloth-sd-prebuilt.yml index 36d0d0534..56ce42609 100644 --- a/.github/workflows/unsloth-sd-prebuilt.yml +++ b/.github/workflows/unsloth-sd-prebuilt.yml @@ -58,6 +58,8 @@ jobs: contents: read outputs: tag: ${{ steps.r.outputs.tag }} + upstream_tag: ${{ steps.r.outputs.upstream_tag }} + patch_list: ${{ steps.r.outputs.patch_list }} commit: ${{ steps.r.outputs.commit }} exists: ${{ steps.r.outputs.exists }} source_artifact: ${{ steps.r.outputs.source_artifact }} @@ -87,18 +89,56 @@ jobs: fi printf '%s' "$TAG" | grep -qE '^master-[0-9]+-[0-9a-f]+$' \ || { echo "refusing non-release tag '$TAG'" >&2; exit 1; } + UPSTREAM_TAG="$TAG" - # Does OUR published release already exist? (drafts don't count.) - EXISTS=false - if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft 2>/dev/null || true)" = "false" ]; then - EXISTS=true - fi + # Stash patches/ before the upstream checkout: it is tracked HERE and absent from + # leejet's tree, so checking out their tag in this worktree would remove it. + PATCHES="${RUNNER_TEMP}/patches" + rm -rf "$PATCHES"; mkdir -p "$PATCHES" + cp -a patches/. "$PATCHES"/ 2>/dev/null || true # Fetch leejet's source AT THAT TAG and tar it (no .git); the build legs # extract this identical tree. Our own scripts come from the tooling checkout. git remote add upstream https://github.com/leejet/stable-diffusion.cpp.git || true git fetch -q --depth 1 upstream "refs/tags/${TAG}:refs/tags/${TAG}" git checkout -q --detach "refs/tags/${TAG}" + + # Apply the local patch set (see patches/README.md). Each one is a fix that is open + # upstream but not yet in a release; the set is normally empty. A non-empty set moves + # the published tag to -u, so the tag says what the box carries and a + # changed set republishes instead of matching an existing release and skipping. + # + # --check runs over ALL of them first, so a stale patch stops the run before the tree + # is half-modified. That failure is the intended signal: upstream either merged the fix + # (delete the patch) or moved the code under it (refresh and re-verify it). + PATCH_LIST="" + FILES=() + while IFS= read -r f; do FILES+=("$f"); done < <(find "$PATCHES" -maxdepth 1 -name '*.patch' | sort) + if [ "${#FILES[@]}" -gt 0 ]; then + for p in "${FILES[@]}"; do + git apply --check "$p" || { + echo "ERROR: $(basename "$p") no longer applies to ${UPSTREAM_TAG}." >&2 + echo " Upstream merged it (delete the file) or moved the code under it" >&2 + echo " (refresh it). See patches/README.md." >&2 + exit 1 + } + done + for p in "${FILES[@]}"; do + git apply "$p" + PATCH_LIST="${PATCH_LIST}${PATCH_LIST:+,}$(basename "$p")" + echo "applied $(basename "$p")" + done + PATCH_ID="$(cat "${FILES[@]}" | sha256sum | cut -c1-7)" + TAG="${UPSTREAM_TAG}-u${PATCH_ID}" + echo "patched build: ${UPSTREAM_TAG} + ${PATCH_LIST} -> ${TAG}" + fi + + # Does OUR published release already exist? (drafts don't count.) Checked against the + # final, patch-aware tag so a patch-set change is never mistaken for an existing build. + EXISTS=false + if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft 2>/dev/null || true)" = "false" ]; then + EXISTS=true + fi # ggml is a REQUIRED submodule; the server frontend + libwebp/libwebm are not # (we build with SD_SERVER_BUILD_FRONTEND / SD_WEBP / SD_WEBM off), so fetch only # ggml to keep the source tarball small and the build self-contained. @@ -114,6 +154,8 @@ jobs: { echo "tag=$TAG" + echo "upstream_tag=$UPSTREAM_TAG" + echo "patch_list=$PATCH_LIST" echo "commit=$COMMIT" echo "exists=$EXISTS" echo "source_artifact=$SRC_ARTIFACT" @@ -324,6 +366,8 @@ jobs: set -eux python3 tooling/scripts/unsloth/assemble_metadata.py \ --tag '${{ needs.resolve.outputs.tag }}' \ + --upstream-tag '${{ needs.resolve.outputs.upstream_tag }}' \ + --patches '${{ needs.resolve.outputs.patch_list }}' \ --source-repo 'leejet/stable-diffusion.cpp' \ --commit '${{ needs.resolve.outputs.commit }}' \ --dist dist --out dist \ @@ -358,8 +402,13 @@ jobs: run: | set -eux TAG='${{ needs.resolve.outputs.tag }}' + UPSTREAM_TAG='${{ needs.resolve.outputs.upstream_tag }}' + PATCH_LIST='${{ needs.resolve.outputs.patch_list }}' REPO="$GITHUB_REPOSITORY" - NOTES="Automated Unsloth stable-diffusion.cpp CPU + Apple prebuild (sd-cli + sd-server) for upstream [${TAG}](https://github.com/leejet/stable-diffusion.cpp/releases/tag/${TAG}). GPU hosts use diffusers/torch; this native engine targets CPU (Linux/WSL/Windows) and Apple (Metal)." + NOTES="Automated Unsloth stable-diffusion.cpp CPU + Apple prebuild (sd-cli + sd-server) for upstream [${UPSTREAM_TAG}](https://github.com/leejet/stable-diffusion.cpp/releases/tag/${UPSTREAM_TAG}). GPU hosts use diffusers/torch; this native engine targets CPU (Linux/WSL/Windows) and Apple (Metal)." + if [ -n "$PATCH_LIST" ]; then + NOTES="${NOTES}"$'\n\n'"Not a stock upstream build. It carries the patch set in [patches/](https://github.com/${REPO}/tree/master/patches), each one a fix that is open upstream but not yet in a release, which is what the \`-u\` suffix on the tag marks:"$'\n\n'"$(printf '%s' "$PATCH_LIST" | tr ',' '\n' | sed 's/^/- `/; s/$/`/')"$'\n\n'"\`sd-prebuilt-manifest.json\` records the same list." + fi if [ "$(gh release view "$TAG" --repo "$REPO" --json isDraft --jq .isDraft 2>/dev/null || true)" = "true" ]; then gh release delete "$TAG" --repo "$REPO" --yes fi diff --git a/patches/0001-spare-1d-norm-weights-from-blanket-quant.patch b/patches/0001-spare-1d-norm-weights-from-blanket-quant.patch new file mode 100644 index 000000000..3c4065dce --- /dev/null +++ b/patches/0001-spare-1d-norm-weights-from-blanket-quant.patch @@ -0,0 +1,34 @@ +Spare 1-D norm weights from a blanket --type + +Upstream: https://github.com/leejet/stable-diffusion.cpp/pull/1861 + +A 1-D weight is a per-channel gain, never a matmul weight. These survived a blanket --type +only when their length did not divide the block size; MiniMax-H3's [5376] norms do divide it, +so 106 of them were crushed to 4 bits. The model still loads and still renders something +plausible, which is what makes it worth guarding: measured against a bf16 render of the same +prompt and seed, PSNR 9.87 / SSIM 0.074 / LPIPS 0.981, against 22.22 / 0.841 / 0.292 with the +rule in place. + +diff --git a/src/model_loader.cpp b/src/model_loader.cpp +index 047c9d9..b286f2a 100644 +--- a/src/model_loader.cpp ++++ b/src/model_loader.cpp +@@ -1511,6 +1511,18 @@ bool ModelLoader::tensor_should_be_converted(const TensorStorage& tensor_storage + if (type != GGML_TYPE_COUNT) { + if (ggml_is_quantized(type) && tensor_storage.ne[0] % ggml_blck_size(type) != 0) { + // Pass, do not convert ++ } else if (ggml_is_quantized(type) && tensor_storage.n_dims <= 1) { ++ // Pass, do not convert. A 1-D weight is a per-channel scale (LayerNorm/RMSNorm gain), ++ // never a matmul weight, so quantizing it buys almost nothing and costs a lot: every ++ // channel of the block shares one scale and one min, and a gain vector has no reason ++ // to be locally smooth. Until now these survived only by accident, when their length ++ // did not divide the block size (FLUX q_norm/k_norm are [128] and 128 % 256 != 0) or ++ // when a name rule above happened to match. A model whose norms DO divide the block ++ // size, such as MiniMax-H3 with [5376] and 5376 % 256 == 0, had 106 norm scales ++ // crushed to 4 bits by a blanket --type. The result still loads and still renders a ++ // plausible image, so a "does it run" check passes it, while measured against a bf16 ++ // render of the same prompt and seed it is destroyed: PSNR 9.87 / SSIM 0.074 / ++ // LPIPS 0.981, against 22.22 / 0.841 / 0.292 with this rule in place. + } else if (ends_with(name, ".bias")) { + // Pass, do not convert + } else if (ends_with(name, ".scale")) { diff --git a/patches/0002-h3-cfg-scale-and-audio-vae-on-cpu.patch b/patches/0002-h3-cfg-scale-and-audio-vae-on-cpu.patch new file mode 100644 index 000000000..d3660a889 --- /dev/null +++ b/patches/0002-h3-cfg-scale-and-audio-vae-on-cpu.patch @@ -0,0 +1,121 @@ +Stop MiniMax-H3 aborting on the default cfg-scale and on --vae-on-cpu + +Upstream: https://github.com/leejet/stable-diffusion.cpp/pull/1862 + +H3 aborts at any --cfg-scale above 1.0, and --cfg-scale defaults to 7.0, so a vid_gen command +that omits the flag crashes rather than rendering. Separately, --vae-on-cpu aborts on the +AUDIO VAE: ggml_conv_1d builds an F16 im2col and the CPU path asserts the kernel is F16, while +the audio VAE kernels arrive as F32. + +diff --git a/src/model/vae/ltx_audio_vae.hpp b/src/model/vae/ltx_audio_vae.hpp +index 3319a5c..e385aad 100644 +--- a/src/model/vae/ltx_audio_vae.hpp ++++ b/src/model/vae/ltx_audio_vae.hpp +@@ -182,6 +182,17 @@ namespace LTXV { + } + }; + ++ // ggml_conv_1d / ggml_conv_1d_dw build an F16 im2col, and the CPU backend ++ // additionally requires the kernel itself to be F16. F32 kernels (as loaded ++ // from F32 checkpoints) work on GPU backends but abort on CPU, so cast them ++ // in-graph here instead of degrading the weights on every backend. ++ static ggml_tensor* conv1d_kernel_for_backend(GGMLRunnerContext* runner_ctx, ggml_tensor* kernel) { ++ if (kernel->type == GGML_TYPE_F32 && sd_backend_is_cpu(runner_ctx->backend)) { ++ return ggml_cast(runner_ctx->ggml_ctx, kernel, GGML_TYPE_F16); ++ } ++ return kernel; ++ } ++ + static ggml_tensor* compute_log_mel_spectrogram(GGMLRunnerContext* runner_ctx, + ggml_tensor* waveform, + ggml_tensor* forward_basis, +@@ -218,7 +229,8 @@ namespace LTXV { + x = ggml_ext_pad_ext(ctx, runner_ctx->backend, x, static_cast(left_pad), 0, 0, 0, 0, 0, 0, 0); + } + +- auto frames = ggml_conv_1d(ctx, forward_basis, x, hop_length, 0, 1); ++ auto frames = ggml_conv_1d(ctx, conv1d_kernel_for_backend(runner_ctx, forward_basis), ++ x, hop_length, 0, 1); + GGML_ASSERT(frames->ne[0] == frame_count); + GGML_ASSERT(frames->ne[1] == stft_channels); + GGML_ASSERT(frames->ne[2] == channels * batch); +@@ -319,7 +331,8 @@ namespace LTXV { + int padding) { + auto ctx = runner_ctx->ggml_ctx; + GGML_ASSERT(x->ne[3] == 1); +- auto tiled = tile_depthwise_filter_1d(runner_ctx, filter, x->ne[1]); ++ auto tiled = conv1d_kernel_for_backend(runner_ctx, ++ tile_depthwise_filter_1d(runner_ctx, filter, x->ne[1])); + auto out = ggml_conv_1d_dw(ctx, tiled, x, stride, padding, 1); + return ggml_reshape_4d(ctx, out, out->ne[0], out->ne[1], 1, 1); + } +@@ -339,10 +352,11 @@ namespace LTXV { + return reversed; + } + +- static ggml_tensor* depthwise_conv_transpose1d(ggml_context* ctx, ++ static ggml_tensor* depthwise_conv_transpose1d(GGMLRunnerContext* runner_ctx, + ggml_tensor* x, + ggml_tensor* filter, + int stride) { ++ auto ctx = runner_ctx->ggml_ctx; + GGML_ASSERT(x->ne[2] == 1 && x->ne[3] == 1); + GGML_ASSERT(filter->ne[1] == 1); + GGML_ASSERT(filter->ne[2] == 1 && filter->ne[3] == 1); +@@ -364,7 +378,8 @@ namespace LTXV { + x_flat = ggml_reshape_3d(ctx, x_flat, time * stride, 1, channels); + + auto reversed_filter = reverse_1d_filter(ctx, filter); +- auto out = ggml_conv_1d(ctx, reversed_filter, x_flat, 1, static_cast(kernel_size - 1), 1); ++ auto out = ggml_conv_1d(ctx, conv1d_kernel_for_backend(runner_ctx, reversed_filter), ++ x_flat, 1, static_cast(kernel_size - 1), 1); + if (out->ne[0] > out_time) { + out = ggml_ext_slice(ctx, out, 0, 0, out_time); + } +@@ -404,7 +419,7 @@ namespace LTXV { + + auto x = ggml_reshape_3d(ctx, waveform, time, channels * batch, 1); + x = replicate_pad_1d(runner_ctx, x, pad, pad); +- x = depthwise_conv_transpose1d(ctx, x, filter, ratio); ++ x = depthwise_conv_transpose1d(runner_ctx, x, filter, ratio); + x = ggml_ext_slice(ctx, x, 0, pad_left, x->ne[0] - pad_right); + return ggml_reshape_3d(ctx, x, x->ne[0], channels, batch); + } +@@ -553,7 +568,7 @@ namespace LTXV { + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { +- x = ggml_conv_1d(ctx->ggml_ctx, params["weight"], x, stride, padding, dilation); ++ x = ggml_conv_1d(ctx->ggml_ctx, conv1d_kernel_for_backend(ctx, params["weight"]), x, stride, padding, dilation); + if (bias) { + auto b = ggml_reshape_4d(ctx->ggml_ctx, params["bias"], 1, params["bias"]->ne[0], 1, 1); + x = ggml_add_inplace(ctx->ggml_ctx, x, b); +@@ -670,7 +685,7 @@ namespace LTXV { + int up_pad_right = up_pad * up_ratio + (up_kernel_size - up_ratio + 1) / 2; + + x = replicate_pad_1d(ctx, x, up_pad, up_pad); +- x = depthwise_conv_transpose1d(ctx->ggml_ctx, x, up_filter, up_ratio); ++ x = depthwise_conv_transpose1d(ctx, x, up_filter, up_ratio); + x = ggml_ext_slice(ctx->ggml_ctx, x, 0, up_pad_left, x->ne[0] - up_pad_right); + + x = act->forward(ctx, x); +diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp +index f2b69cc..be74ea5 100644 +--- a/src/stable-diffusion.cpp ++++ b/src/stable-diffusion.cpp +@@ -4222,6 +4222,15 @@ struct GenerationRequest { + guidance->img_cfg = 1.f; + } + ++ // MiniMax-H3 is distilled with guidance baked in and has no negative ++ // prompt semantics; its empty uncond prompt cannot even be encoded ++ // (zero tokens), so any cfg != 1 would abort deep inside ggml. ++ if (sd_version_is_minimax_h3(sd_ctx->sd->version) && guidance->txt_cfg != 1.f) { ++ LOG_WARN("MiniMax-H3 is a distilled, CFG-free model; forcing cfg-scale from %.2f to 1.0", ++ guidance->txt_cfg); ++ guidance->txt_cfg = 1.f; ++ } ++ + if (guidance->img_cfg != guidance->txt_cfg) { + *use_uncond = true; + } diff --git a/patches/0003-h3-img-gen-mode-guard.patch b/patches/0003-h3-img-gen-mode-guard.patch new file mode 100644 index 000000000..811354dc3 --- /dev/null +++ b/patches/0003-h3-img-gen-mode-guard.patch @@ -0,0 +1,31 @@ +Fail with a message when MiniMax-H3 is run in img_gen mode + +Upstream: https://github.com/leejet/stable-diffusion.cpp/pull/1863 + +Only generate_video computes H3's audio length, so an H3 checkpoint reaching generate_image is +guaranteed to die on GGML_ASSERT(!audio_input_cache.empty()), as a core dump with no hint that +--mode vid_gen is what is missing. + +diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp +index f2b69cc..5986163 100644 +--- a/src/stable-diffusion.cpp ++++ b/src/stable-diffusion.cpp +@@ -5584,6 +5584,18 @@ SD_API bool generate_image(sd_ctx_t* sd_ctx, + return false; + } + ++ // MiniMax-H3 is video-only. Its denoiser always splits the packed latent into a video and an ++ // audio half, and only generate_video ever computes the audio length, so reaching this ++ // function with an H3 checkpoint is guaranteed to die on ++ // GGML_ASSERT(!audio_input_cache.empty()) with a core dump, after the several minutes it ++ // takes to load the weights, and with nothing in the output pointing at the missing --mode. ++ // (The AnimateDiff path below routes vid_gen back through here, but that is SD1.5 plus a ++ // motion module, never H3.) ++ if (sd_version_is_minimax_h3(sd_ctx->sd->version)) { ++ LOG_ERROR("MiniMax-H3 is a video model and cannot be run in img_gen mode; use --mode vid_gen"); ++ return false; ++ } ++ + sd_ctx->sd->reset_cancel_flag(); + + int64_t t0 = ggml_time_ms(); diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 000000000..1eae60b65 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,41 @@ +# Patches applied to the published prebuilts + +`.github/workflows/unsloth-sd-prebuilt.yml` builds leejet's source **at a published release +tag**, aged for at least six hours, rather than this fork's `master`. That is deliberate: what we +publish should be traceable to a specific upstream release rather than to whatever a fork happened +to contain that day. + +Every `*.patch` in this directory is applied to that tree, in filename order, before the build. A +release built with a non-empty patch set is tagged `-u`, where `` is the +first seven hex digits of the sha256 of the concatenated patch files. So: + +- an unpatched build keeps the plain upstream tag, exactly as before, +- changing, adding or removing a patch changes the tag, which is what makes the pipeline rebuild + and republish rather than see an existing release and skip, +- the tag on the box says whether the box is stock, and which patch set it carries. + +`sd-prebuilt-manifest.json` in each release records the patch filenames and the id. + +## Rules + +**Every patch must be open upstream.** These exist to close the gap between "fixed" and +"released", not to carry a private fork. Put the upstream pull request in the header comment of +the patch file. + +**A patch that no longer applies fails the build.** This is intentional, not a bug to route +around. `git apply --check` runs on all of them before any is applied, and the run stops with the +patch name. That happens for exactly two reasons: upstream merged it, in which case delete the +file, or upstream changed the surrounding code, in which case refresh the patch against the new +tag and re-verify it. + +**Delete on merge.** Once the fix ships in an upstream release the patch is dead weight, and +leaving it in place means the next release silently carries a duplicate of code upstream already +has. + +## Current set + +| patch | upstream PR | what it fixes | +|---|---|---| +| `0001-spare-1d-norm-weights-from-blanket-quant.patch` | leejet/stable-diffusion.cpp#1861 | a blanket `--type` quantizes 1-D norm gains whose length divides the block size, which silently destroys MiniMax-H3 output (LPIPS 0.981 against its own bf16 render) | +| `0002-h3-cfg-scale-and-audio-vae-on-cpu.patch` | leejet/stable-diffusion.cpp#1862 | H3 aborts on the default `--cfg-scale 7.0`, and aborts again on `--vae-on-cpu` because the audio VAE reaches the CPU conv1d path with F32 kernels | +| `0003-h3-img-gen-mode-guard.patch` | leejet/stable-diffusion.cpp#1863 | an H3 checkpoint run without `--mode vid_gen` core dumps on a raw ggml assert instead of saying which flag is missing | diff --git a/scripts/unsloth/assemble_metadata.py b/scripts/unsloth/assemble_metadata.py index 1017c6719..bb65ed29b 100755 --- a/scripts/unsloth/assemble_metadata.py +++ b/scripts/unsloth/assemble_metadata.py @@ -39,6 +39,16 @@ def main() -> int: ap.add_argument("--tag", required = True) ap.add_argument("--source-repo", required = True) ap.add_argument("--commit", required = True) + # A patched build is tagged -u, so the two differ exactly when patches were + # applied. Both are recorded: the tag identifies the artifact, upstream_tag and patches say + # what it is made of, so a consumer can tell a stock build from a patched one without + # unpacking it. See patches/README.md. + ap.add_argument("--upstream-tag", default = "") + ap.add_argument( + "--patches", + default = "", + help = "comma-separated patch filenames applied to the upstream tree (empty = stock)", + ) ap.add_argument("--dist", required = True, help = "dir holding the sd-*.zip bundles") ap.add_argument("--out", required = True, help = "dir to write the metadata json into") ap.add_argument("--publish-repo", required = True) @@ -70,6 +80,8 @@ def main() -> int: manifest = { "schema": 1, "tag": args.tag, + "upstream_tag": args.upstream_tag or args.tag, + "patches": [p for p in args.patches.split(",") if p], "source_repo": args.source_repo, "source_commit": args.commit, "publish_repo": args.publish_repo,