From 13d46c5291922cb79b723c082211932a531ca67a Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Sat, 3 Oct 2026 06:09:18 +1000 Subject: [PATCH 1/3] fix(ci): build the musl @cipherstash/auth binary in Alpine, and check every binary's C library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The linux-x64-musl leg of _build-auth-artifacts.yml targeted x86_64-unknown-linux-musl on the Ubuntu runner but named no musl linker, so cargo linked the binary against the runner's glibc. Such a binary fails to load on musl systems such as Alpine. auth-preflight's libc check caught it: "linux-x64-musl links glibc — it is the gnu binary". The suite's published @cipherstash/auth-linux-x64-musl 0.44.0 has the same fault, because its workflow built musl the same way and never ran such a check. Build that leg inside node:22-alpine, pinned by digest: Alpine is a musl system, so its compiler and runtime libraries are musl's own. The container installs Rust at the version mise pins and pnpm at the root packageManager version, installs only the auth package's dependencies, and runs the same napi build, with -crt-static off so the binary links musl at load time. Two other ways failed on 2 October 2026: Ubuntu's musl-gcc cannot link a Rust shared library ("cannot find libgcc_s.so.1"), and the musl.cc toolchain that _build-ffi-artifacts.yml uses timed out from GitHub's runners. Also check which C library each Linux binary links as it is built, before the package is packed. Until now only auth-preflight.yml read it, and nothing runs the preflight automatically, so a release could have published a glibc-linked musl binary. The gnu binaries must need libc.so.6, and the musl binary must not. Tests in auth-build-artifacts.test.mjs cover the Alpine build, the skipped host steps for the musl leg, and the C library check coming before the pack. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PaY5xYydZUWhv8Nex9Sw8a --- .github/workflows/_build-auth-artifacts.yml | 72 +++++++++++++++++-- .../__tests__/auth-build-artifacts.test.mjs | 44 +++++++++++- 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/.github/workflows/_build-auth-artifacts.yml b/.github/workflows/_build-auth-artifacts.yml index 4f408b31f..8021cc8ee 100644 --- a/.github/workflows/_build-auth-artifacts.yml +++ b/.github/workflows/_build-auth-artifacts.yml @@ -65,13 +65,6 @@ jobs: ref: ${{ inputs.ref }} persist-credentials: false - - name: Install musl tools (linux-x64-musl) - if: ${{ matrix.platform == 'linux-x64-musl' }} - run: | - set -euo pipefail - sudo apt-get update - sudo apt-get install -y musl-tools - # Rust 1.94.1 from the root mise.toml, the toolchain the crate is tested # with. `cache: false` is required here, not a default. - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 @@ -104,17 +97,56 @@ jobs: run: npm install -g node-gyp - name: Install dependencies + if: ${{ matrix.platform != 'linux-x64-musl' }} run: pnpm install --frozen-lockfile # `--js false` is load-bearing. With `--platform`, napi v2 also writes its # own `index.js` loader over the committed one, which is a published, # frozen file (release-gate.mjs FROZEN_ARTEFACT_DIGESTS). - name: Build the native binding + if: ${{ matrix.platform != 'linux-x64-musl' }} working-directory: languages/typescript/packages/auth env: TARGET: ${{ matrix.target }} run: mise x -- pnpm exec napi build --platform --release --target "$TARGET" --strip --dts native.d.ts --js false + # The musl binary is built inside Alpine Linux, a musl system, so the + # compiler and its runtime libraries are musl's own. Built on the Ubuntu + # runner, it linked the runner's glibc and failed to load on musl; the + # suite's 0.44.0 has that fault. Ubuntu's musl-gcc cannot link a Rust + # shared library (it has no musl libgcc_s), and the musl.cc toolchain + # that _build-ffi-artifacts.yml downloads timed out from GitHub's runners + # on 2 October 2026. The image is pinned by digest because its output is + # published with provenance. `-crt-static` keeps the binary linked + # against musl at load time, which a Node.js native module needs, and + # which the C library check below reads. The same napi flags as the host + # build, for the same reasons. + - name: Build the native binding in Alpine (linux-x64-musl) + if: ${{ matrix.platform == 'linux-x64-musl' }} + env: + TARGET: ${{ matrix.target }} + ALPINE_NODE_IMAGE: node:22-alpine@sha256:0a7108bf6c7bf5de370ffb1a3ed6be93d405b43ff159f681a8d18c0e2bc2e402 + run: | + set -euo pipefail + rust_version=$(mise current rust) + pnpm_spec=$(node -p "require('./package.json').packageManager") + docker run --rm \ + -v "$GITHUB_WORKSPACE:/build" -w /build \ + -e TARGET -e RUST_VERSION="$rust_version" -e PNPM_SPEC="$pnpm_spec" \ + -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ + -e RUSTFLAGS="-C target-feature=-crt-static" \ + "$ALPINE_NODE_IMAGE" sh -euc ' + apk add --no-cache build-base cmake perl linux-headers git curl rustup + rustup-init -y --profile minimal --default-toolchain "$RUST_VERSION" --target "$TARGET" + . "$HOME/.cargo/env" + corepack enable + corepack prepare "$PNPM_SPEC" --activate + pnpm install --frozen-lockfile --ignore-scripts --filter "@cipherstash/auth..." + cd languages/typescript/packages/auth + pnpm exec napi build --platform --release --target "$TARGET" --strip --dts native.d.ts --js false + chown -R "$HOST_UID:$HOST_GID" /build + ' + # The build must leave the tracked tree alone: `native.d.ts` is # regenerated and has to equal the committed copy, and nothing else may # change. @@ -130,6 +162,32 @@ jobs: mv "stack-auth-node.${PLATFORM}.node" "platforms/${PLATFORM}/" test -s "platforms/${PLATFORM}/stack-auth-node.${PLATFORM}.node" + # Every build checks which C library its binary links, so a release + # cannot publish a musl binary that links glibc even if nobody ran + # auth-preflight first. The same check as auth-preflight.yml's smoke test. + # Into a variable, never into `grep -q`, so readelf cannot die on EPIPE. + - name: Check which C library the binary links + if: ${{ runner.os == 'Linux' }} + working-directory: languages/typescript/packages/auth + env: + PLATFORM: ${{ matrix.platform }} + run: | + set -euo pipefail + dynamic=$(readelf -d "platforms/${PLATFORM}/stack-auth-node.${PLATFORM}.node") + case "$PLATFORM" in + linux-x64-gnu|linux-arm64-gnu) + grep -q 'NEEDED.*libc\.so\.6' <<< "$dynamic" || { + echo "::error::$PLATFORM does not link glibc"; exit 1; } ;; + linux-x64-musl) + if grep -q 'NEEDED.*libc\.so\.6' <<< "$dynamic" ; then + echo "::error::linux-x64-musl links glibc — it is the gnu binary" + exit 1 + fi ;; + *) + echo "::error::no C library rule for $PLATFORM"; exit 1 ;; + esac + echo "$PLATFORM: links the expected C library" + # `npm pack`, not `pnpm pack`: a platform package has no `workspace:` # dependency to rewrite. The wrapper does, and is packed with pnpm below. - name: Pack the platform package diff --git a/scripts/__tests__/auth-build-artifacts.test.mjs b/scripts/__tests__/auth-build-artifacts.test.mjs index fdaee354d..cdc56d463 100644 --- a/scripts/__tests__/auth-build-artifacts.test.mjs +++ b/scripts/__tests__/auth-build-artifacts.test.mjs @@ -51,7 +51,8 @@ describe('_build-auth-artifacts.yml', () => { it('builds each binding without writing over the committed loader', () => { const build = runs(binaries).filter((run) => /\bnapi build\b/.test(run)) - expect(build).toHaveLength(1) + // One host build, and one inside Alpine for the musl leg. + expect(build).toHaveLength(2) for (const flag of [ '--platform', '--release', @@ -60,7 +61,7 @@ describe('_build-auth-artifacts.yml', () => { '--dts native.d.ts', '--js false', ]) { - expect(build[0]).toContain(flag) + for (const run of build) expect(run).toContain(flag) } }) @@ -73,6 +74,45 @@ describe('_build-auth-artifacts.yml', () => { expect(check).toBeGreaterThan(build) }) + it('builds the musl binding inside Alpine, from an image pinned by digest', () => { + // Built on the Ubuntu runner, the musl binary linked glibc and failed to + // load on musl. Alpine is a musl system, so its compiler links musl. + const steps = binaries?.steps ?? [] + const musl = steps.filter((step) => + String(step?.if ?? '').includes("== 'linux-x64-musl'"), + ) + const run = musl.map((step) => String(step?.run ?? '')).join('\n') + const env = Object.assign({}, ...musl.map((step) => step?.env ?? {})) + expect(env.ALPINE_NODE_IMAGE).toMatch(/-alpine@sha256:[0-9a-f]{64}$/) + expect(run).toContain('docker run') + expect(run).toContain('RUSTFLAGS="-C target-feature=-crt-static"') + // The host build and the host dependency install skip the musl leg. + for (const name of ['Build the native binding', 'Install dependencies']) { + const step = steps.find((s) => s?.name === name) + expect(String(step?.if ?? '')).toContain("!= 'linux-x64-musl'") + } + }) + + it('checks the C library of every Linux binary before it is packed', () => { + // Without this, only a hand-run auth-preflight sees a glibc-linked musl + // binary, and a release would publish it. + const steps = binaries?.steps ?? [] + const check = steps.findIndex((step) => + String(step?.run ?? '').includes('readelf -d'), + ) + const pack = steps.findIndex((step) => + /\bnpm pack\b/.test(String(step?.run ?? '')), + ) + expect(check).toBeGreaterThan(-1) + expect(check).toBeLessThan(pack) + const run = String(steps[check].run) + expect(String(steps[check].if)).toContain("runner.os == 'Linux'") + expect(run).toMatch( + /linux-x64-gnu\|linux-arm64-gnu\)[\s\S]*libc\\\.so\\\.6/, + ) + expect(run).toContain('linux-x64-musl links glibc') + }) + it('packs the wrapper with pnpm, which rewrites its workspace peers', () => { const packs = runs(wrapper).filter((run) => /\bpack\b/.test(run)) expect(packs.some((run) => /\bpnpm\b.*\bpack\b/.test(run))).toBe(true) From ea9dc4a008e31ab53438574ae25b3c68d2a68938 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Sat, 3 Oct 2026 06:35:33 +1000 Subject: [PATCH 2/3] ci: require musl in the musl binary, and load it inside Alpine cipherstash-bot's review of #1018, verified: - The linux-x64-musl rule only refused glibc, so a statically linked binary, with no libc entry at all, passed it. Node.js cannot load that as a native module, and no job loaded the musl binary: auth-preflight installs only the runner's own linux-x64-gnu package. Both the build and the preflight now require a NEEDED entry for musl's libc, and the preflight installs the wrapper and the musl package inside the pinned Alpine image and loads them. That step loads the fixed build, and fails on the suite's glibc-linked 0.44.0 with "Failed to load native binding". - The container's chown ran last, so a failed build left root-owned files in the workspace. A trap now runs it on every exit. - The musl leg no longer adds a host Rust target or installs node-gyp, which only the host build uses. Tests now cover: the tracked-file check coming after both builds; docker run using the pinned image variable, not a tag; the trap; musl being required in the build and the preflight; every rejection exiting 1; a rule for every Linux platform in the matrix; and the Alpine load step using the build's image. Each fails when its part is removed. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PaY5xYydZUWhv8Nex9Sw8a --- .github/workflows/_build-auth-artifacts.yml | 11 +++- .github/workflows/auth-preflight.yml | 30 +++++++++- .../__tests__/auth-build-artifacts.test.mjs | 59 +++++++++++++++++-- 3 files changed, 91 insertions(+), 9 deletions(-) diff --git a/.github/workflows/_build-auth-artifacts.yml b/.github/workflows/_build-auth-artifacts.yml index 8021cc8ee..36e18a967 100644 --- a/.github/workflows/_build-auth-artifacts.yml +++ b/.github/workflows/_build-auth-artifacts.yml @@ -79,6 +79,7 @@ jobs: # `macos-latest` is arm64, so without it `darwin-x64` ships an arm64 # binary. - name: Add the Rust target + if: ${{ matrix.platform != 'linux-x64-musl' }} env: TARGET: ${{ matrix.target }} run: mise x -- rustup target add "$TARGET" @@ -94,6 +95,7 @@ jobs: package-manager-cache: false - name: Install node-gyp + if: ${{ matrix.platform != 'linux-x64-musl' }} run: npm install -g node-gyp - name: Install dependencies @@ -136,6 +138,7 @@ jobs: -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ -e RUSTFLAGS="-C target-feature=-crt-static" \ "$ALPINE_NODE_IMAGE" sh -euc ' + trap "chown -R \"\$HOST_UID:\$HOST_GID\" /build" EXIT apk add --no-cache build-base cmake perl linux-headers git curl rustup rustup-init -y --profile minimal --default-toolchain "$RUST_VERSION" --target "$TARGET" . "$HOME/.cargo/env" @@ -144,7 +147,6 @@ jobs: pnpm install --frozen-lockfile --ignore-scripts --filter "@cipherstash/auth..." cd languages/typescript/packages/auth pnpm exec napi build --platform --release --target "$TARGET" --strip --dts native.d.ts --js false - chown -R "$HOST_UID:$HOST_GID" /build ' # The build must leave the tracked tree alone: `native.d.ts` is @@ -182,7 +184,12 @@ jobs: if grep -q 'NEEDED.*libc\.so\.6' <<< "$dynamic" ; then echo "::error::linux-x64-musl links glibc — it is the gnu binary" exit 1 - fi ;; + fi + # A static binary has no libc entry at all, and Node.js cannot + # load it as a native module, so musl must be named. + grep -q 'NEEDED.*libc\.musl-' <<< "$dynamic" || { + echo "::error::linux-x64-musl has no musl libc NEEDED entry — it is linked statically" + exit 1; } ;; *) echo "::error::no C library rule for $PLATFORM"; exit 1 ;; esac diff --git a/.github/workflows/auth-preflight.yml b/.github/workflows/auth-preflight.yml index 9bf48e831..aa50f0e77 100644 --- a/.github/workflows/auth-preflight.yml +++ b/.github/workflows/auth-preflight.yml @@ -92,7 +92,10 @@ jobs: echo "::error::linux-x64-musl links glibc — it is the gnu binary" exit 1 fi - echo "linux-x64-musl: no glibc NEEDED entry" ;; + grep -q 'NEEDED.*libc\.musl-' <<< "$dynamic" || { + echo "::error::linux-x64-musl has no musl libc NEEDED entry — it is linked statically" + exit 1; } + echo "linux-x64-musl: links musl, not glibc" ;; esac checked=$((checked + 1)) done @@ -130,3 +133,28 @@ jobs: console.log('smoke OK') EOF node smoke.mjs + + # The host step above installs only linux-x64-gnu, the runner's own + # platform. The musl binary loads only where musl is the C library, so it + # is installed and loaded inside Alpine, from the image the build uses. + - name: Smoke-test the musl artifact inside Alpine + env: + ALPINE_NODE_IMAGE: node:22-alpine@sha256:0a7108bf6c7bf5de370ffb1a3ed6be93d405b43ff159f681a8d18c0e2bc2e402 + run: | + set -euo pipefail + wrapper=$(basename "$(ls "$GITHUB_WORKSPACE"/auth-dist/cipherstash-auth-[0-9]*.tgz)") + musl=$(basename "$(ls "$GITHUB_WORKSPACE"/auth-dist/cipherstash-auth-linux-x64-musl-*.tgz)") + docker run --rm -v "$GITHUB_WORKSPACE/auth-dist:/dist:ro" \ + -e WRAPPER="$wrapper" -e MUSL="$musl" \ + "$ALPINE_NODE_IMAGE" sh -euc ' + mkdir -p /tmp/smoke && cd /tmp/smoke + echo "{\"name\":\"smoke\",\"version\":\"1.0.0\",\"private\":true}" > package.json + npm install --no-audit --no-fund "/dist/$WRAPPER" "/dist/$MUSL" + node -e " + const auth = require(\"@cipherstash/auth\") + for (const name of [\"AccessKeyStrategy\", \"AutoStrategy\", \"OidcFederationStrategy\"]) { + if (typeof auth[name] !== \"function\") throw new Error(\"no \" + name) + } + console.log(\"musl smoke OK\") + " + ' diff --git a/scripts/__tests__/auth-build-artifacts.test.mjs b/scripts/__tests__/auth-build-artifacts.test.mjs index cdc56d463..3d7930eaa 100644 --- a/scripts/__tests__/auth-build-artifacts.test.mjs +++ b/scripts/__tests__/auth-build-artifacts.test.mjs @@ -65,13 +65,16 @@ describe('_build-auth-artifacts.yml', () => { } }) - it('fails the leg when the build changed a tracked file', () => { + it('fails the leg when either build changed a tracked file', () => { const all = runs(binaries) - const build = all.findIndex((run) => /\bnapi build\b/.test(run)) + const builds = all + .map((_, index) => index) + .filter((index) => /\bnapi build\b/.test(all[index])) const check = all.findIndex((run) => new RegExp(`git diff --exit-code\\b.*${AUTH_DIR}`).test(run), ) - expect(check).toBeGreaterThan(build) + expect(builds).toHaveLength(2) + for (const build of builds) expect(check).toBeGreaterThan(build) }) it('builds the musl binding inside Alpine, from an image pinned by digest', () => { @@ -84,10 +87,19 @@ describe('_build-auth-artifacts.yml', () => { const run = musl.map((step) => String(step?.run ?? '')).join('\n') const env = Object.assign({}, ...musl.map((step) => step?.env ?? {})) expect(env.ALPINE_NODE_IMAGE).toMatch(/-alpine@sha256:[0-9a-f]{64}$/) - expect(run).toContain('docker run') + // The pinned image is the one that runs, not a mutable tag. + expect(run).toMatch(/docker run[\s\S]*"\$ALPINE_NODE_IMAGE"/) + expect(run).not.toMatch(/\bnode:\d+-alpine(?!@)/) expect(run).toContain('RUSTFLAGS="-C target-feature=-crt-static"') - // The host build and the host dependency install skip the musl leg. - for (const name of ['Build the native binding', 'Install dependencies']) { + // The container hands its files back even when the build fails. + expect(run).toMatch(/^\s*trap "chown -R .*\/build" EXIT$/m) + // The host steps that only the host build uses skip the musl leg. + for (const name of [ + 'Add the Rust target', + 'Install node-gyp', + 'Install dependencies', + 'Build the native binding', + ]) { const step = steps.find((s) => s?.name === name) expect(String(step?.if ?? '')).toContain("!= 'linux-x64-musl'") } @@ -111,6 +123,41 @@ describe('_build-auth-artifacts.yml', () => { /linux-x64-gnu\|linux-arm64-gnu\)[\s\S]*libc\\\.so\\\.6/, ) expect(run).toContain('linux-x64-musl links glibc') + // A static binary has no libc entry, so musl must be named, not only + // glibc refused. + expect(run).toMatch(/linux-x64-musl\)[\s\S]*libc\\\.musl-/) + // Every rejection fails the job, so the binary is never packed. + const errors = run.match(/::error::/g) ?? [] + expect(errors.length).toBeGreaterThan(0) + expect((run.match(/exit 1/g) ?? []).length).toBe(errors.length) + // Every Linux platform in the matrix has a rule. + const linux = (binaries?.strategy?.matrix?.include ?? []) + .filter((leg) => String(leg.os).startsWith('ubuntu')) + .map((leg) => leg.platform) + expect(linux.length).toBeGreaterThan(0) + for (const platform of linux) { + expect(run).toMatch(new RegExp(`(^|[|\\s])${platform}[|)]`, 'm')) + } + }) + + it('auth-preflight requires musl, and loads the musl artifact inside Alpine', () => { + const preflight = readWorkflow('.github/workflows/auth-preflight.yml') + const steps = preflight?.jobs?.smoke?.steps ?? [] + const verify = steps.map((step) => String(step?.run ?? '')).join('\n') + expect(verify).toMatch(/linux-x64-musl\)[\s\S]*libc\\\.musl-/) + // The host smoke test installs only the runner's own platform, so without + // this step no job loads the musl binary. + const alpine = steps.find((step) => + /\bdocker run\b/.test(String(step?.run ?? '')), + ) + expect(alpine).toBeDefined() + expect(String(alpine.run)).toContain('linux-x64-musl') + expect(String(alpine.run)).toMatch(/docker run[\s\S]*"\$ALPINE_NODE_IMAGE"/) + // The same image as the build, so the load test matches the build. + const build = (binaries?.steps ?? []).find( + (step) => step?.env?.ALPINE_NODE_IMAGE, + ) + expect(alpine.env?.ALPINE_NODE_IMAGE).toBe(build?.env?.ALPINE_NODE_IMAGE) }) it('packs the wrapper with pnpm, which rewrites its workspace peers', () => { From 6ba1a0dd2fd641d477a59cb5a0a62f16a7db18cc Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Sat, 3 Oct 2026 06:41:17 +1000 Subject: [PATCH 3/3] ci: load the musl auth binary inside the build container readelf only infers that the musl binary will load. Require it with Node.js inside the Alpine container straight after the build, so every build proves it, release builds included, and a missing runtime library or an unresolved symbol fails the job before the package is packed. A statically linked binary fails here too. freshtonic suggested this in review of #1018. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PaY5xYydZUWhv8Nex9Sw8a --- .github/workflows/_build-auth-artifacts.yml | 4 ++++ scripts/__tests__/auth-build-artifacts.test.mjs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/_build-auth-artifacts.yml b/.github/workflows/_build-auth-artifacts.yml index 36e18a967..c92ba3ab0 100644 --- a/.github/workflows/_build-auth-artifacts.yml +++ b/.github/workflows/_build-auth-artifacts.yml @@ -147,6 +147,10 @@ jobs: pnpm install --frozen-lockfile --ignore-scripts --filter "@cipherstash/auth..." cd languages/typescript/packages/auth pnpm exec napi build --platform --release --target "$TARGET" --strip --dts native.d.ts --js false + # Load it here, on musl: readelf only infers that it will load, + # and this also catches a missing runtime library or an + # unresolved symbol, in every build, release builds included. + node -e "require(process.argv[1])" "$PWD/stack-auth-node.linux-x64-musl.node" ' # The build must leave the tracked tree alone: `native.d.ts` is diff --git a/scripts/__tests__/auth-build-artifacts.test.mjs b/scripts/__tests__/auth-build-artifacts.test.mjs index 3d7930eaa..af9c311a3 100644 --- a/scripts/__tests__/auth-build-artifacts.test.mjs +++ b/scripts/__tests__/auth-build-artifacts.test.mjs @@ -91,6 +91,10 @@ describe('_build-auth-artifacts.yml', () => { expect(run).toMatch(/docker run[\s\S]*"\$ALPINE_NODE_IMAGE"/) expect(run).not.toMatch(/\bnode:\d+-alpine(?!@)/) expect(run).toContain('RUSTFLAGS="-C target-feature=-crt-static"') + // The container loads the binary it built, on musl. + expect(run).toMatch( + /napi build[\s\S]*node -e "require\(process\.argv\[1\]\)" "\$PWD\/stack-auth-node\.linux-x64-musl\.node"/, + ) // The container hands its files back even when the build fails. expect(run).toMatch(/^\s*trap "chown -R .*\/build" EXIT$/m) // The host steps that only the host build uses skip the musl leg.