From 1e679369ec5a6e8e1ee7b40b918c2630cdac41ec Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Sat, 3 Oct 2026 09:36:58 +1000 Subject: [PATCH 1/3] test: share the workflow condition evaluator, and model cancelled() Move the GitHub Actions expression evaluator out of workflow-dispatch-job-conditions.test.mjs into lib/expressions.mjs, so a second guard can walk the release job graph with the same reading of a condition rather than a copy of it. Model `cancelled()` from a run state the caller passes, false by default. The release EQL jobs are about to be written `!cancelled() && ...`, and the evaluator must not refuse them. `success()` and `failure()` still throw: they depend on the whole needs chain, which these contexts do not carry. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PaY5xYydZUWhv8Nex9Sw8a --- scripts/__tests__/lib/expressions.mjs | 256 ++++++++++++++++++ .../workflow-dispatch-job-conditions.test.mjs | 253 ++--------------- 2 files changed, 274 insertions(+), 235 deletions(-) create mode 100644 scripts/__tests__/lib/expressions.mjs diff --git a/scripts/__tests__/lib/expressions.mjs b/scripts/__tests__/lib/expressions.mjs new file mode 100644 index 000000000..34eef8ea9 --- /dev/null +++ b/scripts/__tests__/lib/expressions.mjs @@ -0,0 +1,256 @@ +/** + * A small evaluator for the GitHub Actions expression subset used by job `if:`. + * + * Extracted from `workflow-dispatch-job-conditions.test.mjs` when a second + * guard needed it: `eql-release-assets.test.mjs` walks the release job graph + * through the same conditions. Two evaluators would be two opinions about + * what a condition means, and the guard that disagreed with GitHub would be the + * one nobody noticed. + * + * The grammar is small on purpose and it THROWS on anything outside it — a + * condition this module cannot reason about fails loudly rather than being + * waved through, because "the evaluator did not understand it" and "the job + * runs" must not produce the same green. + */ + +export class UnsupportedExpression extends Error {} + +/** + * GitHub's loose equality: "if the types do not match, GitHub coerces the type + * to a number", with null -> 0, booleans -> 0/1, non-numeric strings and + * objects -> NaN. That coercion is why the original condition failed silently + * rather than erroring — `null == 'cipherstash/stack'` is `0 == NaN`, false. + */ +function toNumber(value) { + if (value === null || value === undefined) return 0 + if (typeof value === 'boolean') return value ? 1 : 0 + if (typeof value === 'number') return value + if (typeof value === 'string') { + if (value.trim() === '') return 0 + return Number(value) + } + return Number.NaN +} + +export function looseEquals(a, b) { + // GitHub compares strings case-insensitively. + if (typeof a === 'string' && typeof b === 'string') { + return a.toLowerCase() === b.toLowerCase() + } + if (typeof a === 'boolean' && typeof b === 'boolean') return a === b + const [x, y] = [toNumber(a), toNumber(b)] + return Number.isNaN(x) || Number.isNaN(y) ? false : x === y +} + +/** GitHub truthiness: null, false, 0 and the empty string are false. */ +function truthy(value) { + if (value === null || value === undefined) return false + if (typeof value === 'string') return value !== '' + if (typeof value === 'number') return value !== 0 + if (typeof value === 'boolean') return value + return true +} + +function tokenize(expression) { + const tokens = [] + let i = 0 + while (i < expression.length) { + const ch = expression[i] + if (/\s/.test(ch)) { + i++ + continue + } + if (ch === "'") { + // `''` is GitHub's escape for a literal quote inside a string. + let value = '' + i++ + while (i < expression.length) { + if (expression[i] === "'") { + if (expression[i + 1] === "'") { + value += "'" + i += 2 + continue + } + i++ + break + } + value += expression[i] + i++ + } + tokens.push({ type: 'string', value }) + continue + } + const two = expression.slice(i, i + 2) + if (two === '==' || two === '!=' || two === '&&' || two === '||') { + tokens.push({ type: 'operator', value: two }) + i += 2 + continue + } + if (ch === '(' || ch === ')' || ch === '!') { + tokens.push({ type: 'operator', value: ch }) + i++ + continue + } + const path = /^[A-Za-z_][A-Za-z0-9_.-]*/.exec(expression.slice(i)) + if (path) { + tokens.push({ type: 'path', value: path[0] }) + i += path[0].length + continue + } + const number = /^\d+(\.\d+)?/.exec(expression.slice(i)) + if (number) { + tokens.push({ type: 'number', value: Number(number[0]) }) + i += number[0].length + continue + } + throw new UnsupportedExpression( + `unexpected character ${JSON.stringify(ch)} at offset ${i}`, + ) + } + return tokens +} + +/** Walk a dotted path; a missing property yields null, as GitHub does. */ +function lookup(path, context) { + let current = context + for (const segment of path.split('.')) { + if (current === null || typeof current !== 'object') return null + current = segment in current ? current[segment] : null + } + return current === undefined ? null : current +} + +/** + * The only status functions this evaluator will accept, and only in their + * zero-argument form. + * + * `always()` is total: it is true on every event, for every job, whatever its + * `needs:` reported. So it can be modelled exactly rather than guessed at, + * which is the bar the rest of this file sets. An aggregator job — EQL's + * `ci-required`, the single required status check its merge queue references — + * cannot be written without it, and refusing the whole grammar would have meant + * either no such job in a dispatchable workflow, or this guard skipping the + * workflow that contains one. + * + * `cancelled()` is a fact about the run, not about any job, so it comes from + * the caller's `run` argument and is false unless that says `cancelled: true`. + * + * Everything else still throws. `success()` and `failure()` depend on the + * results of the whole `needs:` chain, which the contexts here do not carry, + * and `contains()` / `startsWith` take arguments the parser below deliberately + * cannot evaluate. + */ +const SUPPORTED_FUNCTIONS = new Map([ + ['always', () => true], + ['cancelled', (run) => run.cancelled === true], +]) + +/** + * Recursive descent over `|| && ! == != ()`, string / number / boolean + * literals, context paths, and the zero-argument functions in + * `SUPPORTED_FUNCTIONS`. Any other call throws: guessing at a function's + * verdict is worse than refusing it, because "the evaluator did not understand + * it" and "the job runs" must not produce the same green. + */ +function evaluate(expression, context, run) { + const tokens = tokenize(expression) + let position = 0 + + const peek = () => tokens[position] + const eat = (value) => { + if (peek()?.value === value) { + position++ + return true + } + return false + } + + const parsePrimary = () => { + const token = peek() + if (!token) throw new UnsupportedExpression('unexpected end of expression') + if (eat('(')) { + const value = parseOr() + if (!eat(')')) throw new UnsupportedExpression('unbalanced parentheses') + return value + } + position++ + if (token.type === 'string' || token.type === 'number') return token.value + if (token.type === 'path') { + if (peek()?.value === '(') { + const fn = SUPPORTED_FUNCTIONS.get(token.value) + // Zero-argument only: `(` must be followed directly by `)`. A call with + // arguments falls through to the throw, even for a supported name. + if (fn && tokens[position + 1]?.value === ')') { + position += 2 + return fn(run) + } + throw new UnsupportedExpression( + `function call ${token.value}() is not understood`, + ) + } + if (token.value === 'true') return true + if (token.value === 'false') return false + if (token.value === 'null') return null + return lookup(token.value, context) + } + throw new UnsupportedExpression(`unexpected token ${token.value}`) + } + + const parseUnary = () => { + if (eat('!')) return !truthy(parseUnary()) + return parsePrimary() + } + + const parseEquality = () => { + let left = parseUnary() + for (;;) { + if (eat('==')) left = looseEquals(left, parseUnary()) + else if (eat('!=')) left = !looseEquals(left, parseUnary()) + else return left + } + } + + const parseAnd = () => { + let left = parseEquality() + while (eat('&&')) { + const right = parseEquality() + left = truthy(left) ? right : left + } + return left + } + + function parseOr() { + let left = parseAnd() + while (eat('||')) { + const right = parseAnd() + left = truthy(left) ? left : right + } + return left + } + + const value = parseOr() + if (position !== tokens.length) { + throw new UnsupportedExpression( + `trailing input from token ${position}: ${tokens + .slice(position) + .map((token) => token.value) + .join(' ')}`, + ) + } + return truthy(value) +} + +/** Strip the optional `${{ … }}` wrapper GitHub allows around a condition. */ +export function unwrap(condition) { + const trimmed = String(condition).trim() + const match = /^\$\{\{([\s\S]*)\}\}$/.exec(trimmed) + return (match ? match[1] : trimmed).trim() +} + +/** `context` is `{ github, needs, vars }`; `run` is `{ cancelled }`. */ +export function runsWhen(condition, context, run = {}) { + return evaluate(unwrap(condition), context, run) +} + +/** `${{ body }}`, as a parsed workflow holds it, without a JS placeholder lint. */ +export const expr = (body) => `\${{ ${body} }}` diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index d97e8a892..79f11e15f 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -1,4 +1,10 @@ import { describe, expect, it } from 'vitest' +import { + looseEquals, + runsWhen, + UnsupportedExpression, + unwrap, +} from './lib/expressions.mjs' import { readWorkflow, workflowFiles } from './lib/workflows.mjs' /** @@ -154,241 +160,6 @@ function extractForkClause(condition) { return flat.includes(CANONICAL_FORK_CLAUSE) ? CANONICAL_FORK_CLAUSE : null } -// --------------------------------------------------------------------------- -// A small evaluator for the GitHub Actions expression subset used by job `if:` -// --------------------------------------------------------------------------- - -class UnsupportedExpression extends Error {} - -/** - * GitHub's loose equality: "if the types do not match, GitHub coerces the type - * to a number", with null -> 0, booleans -> 0/1, non-numeric strings and - * objects -> NaN. That coercion is why the original condition failed silently - * rather than erroring — `null == 'cipherstash/stack'` is `0 == NaN`, false. - */ -function toNumber(value) { - if (value === null || value === undefined) return 0 - if (typeof value === 'boolean') return value ? 1 : 0 - if (typeof value === 'number') return value - if (typeof value === 'string') { - if (value.trim() === '') return 0 - return Number(value) - } - return Number.NaN -} - -function looseEquals(a, b) { - // GitHub compares strings case-insensitively. - if (typeof a === 'string' && typeof b === 'string') { - return a.toLowerCase() === b.toLowerCase() - } - if (typeof a === 'boolean' && typeof b === 'boolean') return a === b - const [x, y] = [toNumber(a), toNumber(b)] - return Number.isNaN(x) || Number.isNaN(y) ? false : x === y -} - -/** GitHub truthiness: null, false, 0 and the empty string are false. */ -function truthy(value) { - if (value === null || value === undefined) return false - if (typeof value === 'string') return value !== '' - if (typeof value === 'number') return value !== 0 - if (typeof value === 'boolean') return value - return true -} - -function tokenize(expression) { - const tokens = [] - let i = 0 - while (i < expression.length) { - const ch = expression[i] - if (/\s/.test(ch)) { - i++ - continue - } - if (ch === "'") { - // `''` is GitHub's escape for a literal quote inside a string. - let value = '' - i++ - while (i < expression.length) { - if (expression[i] === "'") { - if (expression[i + 1] === "'") { - value += "'" - i += 2 - continue - } - i++ - break - } - value += expression[i] - i++ - } - tokens.push({ type: 'string', value }) - continue - } - const two = expression.slice(i, i + 2) - if (two === '==' || two === '!=' || two === '&&' || two === '||') { - tokens.push({ type: 'operator', value: two }) - i += 2 - continue - } - if (ch === '(' || ch === ')' || ch === '!') { - tokens.push({ type: 'operator', value: ch }) - i++ - continue - } - const path = /^[A-Za-z_][A-Za-z0-9_.-]*/.exec(expression.slice(i)) - if (path) { - tokens.push({ type: 'path', value: path[0] }) - i += path[0].length - continue - } - const number = /^\d+(\.\d+)?/.exec(expression.slice(i)) - if (number) { - tokens.push({ type: 'number', value: Number(number[0]) }) - i += number[0].length - continue - } - throw new UnsupportedExpression( - `unexpected character ${JSON.stringify(ch)} at offset ${i}`, - ) - } - return tokens -} - -/** Walk a dotted path; a missing property yields null, as GitHub does. */ -function lookup(path, context) { - let current = context - for (const segment of path.split('.')) { - if (current === null || typeof current !== 'object') return null - current = segment in current ? current[segment] : null - } - return current === undefined ? null : current -} - -/** - * The only status function this evaluator will accept, and only in its - * zero-argument form. - * - * `always()` is total: it is true on every event, for every job, whatever its - * `needs:` reported. So it can be modelled exactly rather than guessed at, - * which is the bar the rest of this file sets. An aggregator job — EQL's - * `ci-required`, the single required status check its merge queue references — - * cannot be written without it, and refusing the whole grammar would have meant - * either no such job in a dispatchable workflow, or this guard skipping the - * workflow that contains one. - * - * Everything else still throws. `success()`, `failure()` and `cancelled()` - * depend on run state this file does not model, and `contains()` / `startsWith` - * take arguments the parser below deliberately cannot evaluate. - */ -const SUPPORTED_FUNCTIONS = new Map([['always', () => true]]) - -/** - * Recursive descent over `|| && ! == != ()`, string / number / boolean - * literals, context paths, and the zero-argument functions in - * `SUPPORTED_FUNCTIONS`. Any other call throws: guessing at a function's - * verdict is worse than refusing it, because "the evaluator did not understand - * it" and "the job runs" must not produce the same green. - */ -function evaluate(expression, context) { - const tokens = tokenize(expression) - let position = 0 - - const peek = () => tokens[position] - const eat = (value) => { - if (peek()?.value === value) { - position++ - return true - } - return false - } - - const parsePrimary = () => { - const token = peek() - if (!token) throw new UnsupportedExpression('unexpected end of expression') - if (eat('(')) { - const value = parseOr() - if (!eat(')')) throw new UnsupportedExpression('unbalanced parentheses') - return value - } - position++ - if (token.type === 'string' || token.type === 'number') return token.value - if (token.type === 'path') { - if (peek()?.value === '(') { - const fn = SUPPORTED_FUNCTIONS.get(token.value) - // Zero-argument only: `(` must be followed directly by `)`. A call with - // arguments falls through to the throw, even for a supported name. - if (fn && tokens[position + 1]?.value === ')') { - position += 2 - return fn() - } - throw new UnsupportedExpression( - `function call ${token.value}() is not understood`, - ) - } - if (token.value === 'true') return true - if (token.value === 'false') return false - if (token.value === 'null') return null - return lookup(token.value, context) - } - throw new UnsupportedExpression(`unexpected token ${token.value}`) - } - - const parseUnary = () => { - if (eat('!')) return !truthy(parseUnary()) - return parsePrimary() - } - - const parseEquality = () => { - let left = parseUnary() - for (;;) { - if (eat('==')) left = looseEquals(left, parseUnary()) - else if (eat('!=')) left = !looseEquals(left, parseUnary()) - else return left - } - } - - const parseAnd = () => { - let left = parseEquality() - while (eat('&&')) { - const right = parseEquality() - left = truthy(left) ? right : left - } - return left - } - - function parseOr() { - let left = parseAnd() - while (eat('||')) { - const right = parseAnd() - left = truthy(left) ? left : right - } - return left - } - - const value = parseOr() - if (position !== tokens.length) { - throw new UnsupportedExpression( - `trailing input from token ${position}: ${tokens - .slice(position) - .map((token) => token.value) - .join(' ')}`, - ) - } - return truthy(value) -} - -/** Strip the optional `${{ … }}` wrapper GitHub allows around a condition. */ -function unwrap(condition) { - const trimmed = String(condition).trim() - const match = /^\$\{\{([\s\S]*)\}\}$/.exec(trimmed) - return (match ? match[1] : trimmed).trim() -} - -function runsWhen(condition, context) { - return evaluate(unwrap(condition), context) -} - // --------------------------------------------------------------------------- // The four contexts a condition has to land correctly in // --------------------------------------------------------------------------- @@ -727,6 +498,18 @@ describe('the expression evaluator this guard depends on', () => { ) }) + it('reads `cancelled()` from the run state, false unless told otherwise', () => { + // The release EQL jobs are `!cancelled() && …`. Every context above is a + // run nobody cancelled, which is the only state this guard asserts about. + for (const context of Object.values(CONTEXTS)) { + expect(runsWhen('!cancelled()', context)).toBe(true) + expect(runsWhen('!cancelled()', context, { cancelled: true })).toBe(false) + } + expect(() => runsWhen('cancelled(true)', CONTEXTS.push)).toThrow( + UnsupportedExpression, + ) + }) + it('applies GitHub null coercion rather than JavaScript equality', () => { // `null == 'cipherstash/stack'` is `0 == NaN`. This is the coercion that // turned a missing `pull_request` payload into a silent `false` instead of From 99ed095bed9f1a46bbca8d369c495ff7bdbf2a3c Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Sat, 3 Oct 2026 09:38:24 +1000 Subject: [PATCH 2/3] fix(release): wait for npm to list the native packages before changeset publish publish-ffi and publish-auth publish their seven tarballs each with `npm publish`. The release job then runs `changeset publish`, which publishes every version `npm info` does not list. npm lists a publish minutes after accepting it, so changesets published some native packages a second time, from the workspace and with restricted access, and npm refused them with E402. That failed the release job for protect-ffi 0.33.0 (run 36978691809) and @cipherstash/auth 0.44.1 (run 37071820363). Both publish jobs now export the name@version list they write to published.txt. A new step in the release job, before changesets/action, polls `npm view versions` until every listed version appears, for up to 15 minutes. A skipped publish job exports an empty list, so an ordinary JS release does not wait. A timeout fails the job before `changeset publish`, so nothing is published, and a re-run waits again. Refs: CIP-4276 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PaY5xYydZUWhv8Nex9Sw8a --- .github/workflows/release.yml | 30 +- AGENTS.md | 7 + .../__tests__/wait-for-npm-versions.test.mjs | 332 ++++++++++++++++++ scripts/wait-for-npm-versions.mjs | 127 +++++++ 4 files changed, 494 insertions(+), 2 deletions(-) create mode 100644 scripts/__tests__/wait-for-npm-versions.test.mjs create mode 100644 scripts/wait-for-npm-versions.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4bebad20..f253e4b8c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -220,6 +220,10 @@ jobs: permissions: contents: write # the seven git tags and the GitHub release id-token: write # npm OIDC trusted publishing + # `release` waits for npm to serve every one of these before it runs + # `changeset publish`. Empty when this job is skipped. + outputs: + published: ${{ steps.publish.outputs.published }} steps: - uses: actions/download-artifact@v4 with: @@ -289,7 +293,10 @@ jobs: published+=("${name}@${version}") done printf '%s\n' "${published[@]}" > published.txt - echo "version=$(meta "$wrapper" version)" >> "$GITHUB_OUTPUT" + { + echo "version=$(meta "$wrapper" version)" + echo "published=${published[*]}" + } >> "$GITHUB_OUTPUT" # Changesets tags only what IT published — `tagPublish` receives # `publishedPackages.filter(p => p.result === "published")` — and it skips @@ -369,6 +376,9 @@ jobs: permissions: contents: write # the seven git tags and the GitHub release id-token: write # npm OIDC trusted publishing + # Waited for by `release`, as publish-ffi's is. + outputs: + published: ${{ steps.publish.outputs.published }} steps: - uses: actions/download-artifact@v4 with: @@ -420,7 +430,10 @@ jobs: published+=("${name}@${version}") done printf '%s\n' "${published[@]}" > published.txt - echo "version=$(meta "$wrapper" version)" >> "$GITHUB_OUTPUT" + { + echo "version=$(meta "$wrapper" version)" + echo "published=${published[*]}" + } >> "$GITHUB_OUTPUT" # Changesets tags only what it published itself, so without this an auth # release has no git tag and no GitHub release. The same idempotent tag @@ -591,6 +604,19 @@ jobs: add_shims_to_path: false env: false + # BEFORE `changeset publish`. npm accepts a publish minutes before its + # package document lists the version, and `changeset publish` publishes + # every version that document does not list — the native packages again, + # from the workspace, with `restricted` access, which npm refuses with + # E402. That failed this job for protect-ffi 0.33.0 and for + # @cipherstash/auth 0.44.1. See scripts/wait-for-npm-versions.mjs. + - name: Wait for npm to list the native packages + timeout-minutes: 20 + env: + FFI_PUBLISHED: ${{ needs.publish-ffi.outputs.published }} + AUTH_PUBLISHED: ${{ needs.publish-auth.outputs.published }} + run: node scripts/wait-for-npm-versions.mjs + - name: Publish to npm id: changesets uses: changesets/action@v1.9.0 diff --git a/AGENTS.md b/AGENTS.md index 430d142d7..f10885b77 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -207,6 +207,13 @@ stays optional for everyone else. bumps nothing is a no-op for all seven. `ffi-preflight.yml` is the dry run (`changeset publish` has no `--dry-run`); dispatch it against the Version Packages branch before merging a release that moves an FFI version. + **The `release` job waits for npm to list all seven** (and the seven + `@cipherstash/auth` packages `publish-auth` publishes the same way) before + `changeset publish`, through `scripts/wait-for-npm-versions.mjs`. npm lists + a publish minutes after accepting it, and `changeset publish` publishes any + version npm does not list yet a second time — with `restricted` access, so + npm refuses it with E402 and the job fails. That happened to protect-ffi + 0.33.0 and @cipherstash/auth 0.44.1 before the wait existed. - **Trusted publishing binds to (repository, workflow filename).** Keep `release.yml` as the single npm entry point; a rename silently invalidates all seven publisher configurations. Each one must also list `npm publish` under diff --git a/scripts/__tests__/wait-for-npm-versions.test.mjs b/scripts/__tests__/wait-for-npm-versions.test.mjs new file mode 100644 index 000000000..1a2e97457 --- /dev/null +++ b/scripts/__tests__/wait-for-npm-versions.test.mjs @@ -0,0 +1,332 @@ +import { spawnSync } from 'node:child_process' +import { + chmodSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { delimiter, join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { parseSpecs, waitForVersions } from '../wait-for-npm-versions.mjs' +import { expr } from './lib/expressions.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { readWorkflow } from './lib/workflows.mjs' + +/** + * The wait between the native publish jobs and `changeset publish`. + * + * Without it `changeset publish` asks npm about versions npm accepted a minute + * earlier and does not list yet, publishes them a second time, and fails the + * `release` job with E402 — which is what happened to protect-ffi 0.33.0 and + * to @cipherstash/auth 0.44.1 on 2 October 2026. + */ + +const SCRIPT = join(REPO_ROOT, 'scripts/wait-for-npm-versions.mjs') +const RELEASE = '.github/workflows/release.yml' + +const FFI = [ + '@cipherstash/protect-ffi-darwin-arm64@0.33.0', + '@cipherstash/protect-ffi-linux-x64-musl@0.33.0', + '@cipherstash/protect-ffi@0.33.0', +].join(' ') +const AUTH = [ + '@cipherstash/auth-darwin-x64@0.44.1', + '@cipherstash/auth-win32-x64-msvc@0.44.1', + '@cipherstash/auth@0.44.1', +].join(' ') + +/** + * A registry whose package documents start listing a version after `hiddenFor` + * lookups of that name, with a clock that only moves when the wait sleeps. + */ +function fakeRegistry(packages) { + const calls = new Map() + let clock = 0 + const sleeps = [] + return { + calls, + sleeps, + now: () => clock, + wait: async (ms) => { + sleeps.push(ms) + clock += ms + }, + lookup: (name) => { + calls.set(name, (calls.get(name) ?? 0) + 1) + const entry = packages[name] + if (entry === undefined) throw new Error(`unexpected lookup of ${name}`) + if (entry instanceof Error) throw entry + return calls.get(name) > (entry.hiddenFor ?? 0) + ? entry.after + : entry.before + }, + } +} + +const quiet = { log: () => {} } + +describe('the published lists', () => { + it('read as nothing for a skipped publish job, whose output is empty', () => { + expect(parseSpecs('')).toEqual([]) + expect(parseSpecs(undefined)).toEqual([]) + expect(parseSpecs(' \n ')).toEqual([]) + }) + + it('split each entry on its last `@`, so the scope survives', () => { + expect(parseSpecs(`${AUTH}\n`)).toEqual([ + { name: '@cipherstash/auth-darwin-x64', version: '0.44.1' }, + { name: '@cipherstash/auth-win32-x64-msvc', version: '0.44.1' }, + { name: '@cipherstash/auth', version: '0.44.1' }, + ]) + }) + + it('refuse an entry with no version rather than waiting for nothing', () => { + expect(() => parseSpecs('@cipherstash/auth')).toThrow(/name@version/) + expect(() => parseSpecs('@cipherstash/auth@')).toThrow(/name@version/) + }) +}) + +describe('the wait', () => { + it('asks npm nothing when both publish jobs were skipped', async () => { + const registry = fakeRegistry({}) + await waitForVersions([], { ...registry, ...quiet }) + expect(registry.calls.size).toBe(0) + expect(registry.sleeps).toEqual([]) + }) + + it('waits for the FFI and the auth packages, each until npm lists it', async () => { + // The shape of both failures: some of the seven were listed in time and + // some were not. + const registry = fakeRegistry({ + '@cipherstash/protect-ffi-darwin-arm64': { after: ['0.32.0', '0.33.0'] }, + '@cipherstash/protect-ffi-linux-x64-musl': { + before: ['0.32.0'], + after: ['0.32.0', '0.33.0'], + hiddenFor: 2, + }, + '@cipherstash/protect-ffi': { + before: ['0.32.0'], + after: ['0.32.0', '0.33.0'], + hiddenFor: 1, + }, + '@cipherstash/auth-darwin-x64': { + before: ['0.44.0'], + after: ['0.44.0', '0.44.1'], + hiddenFor: 3, + }, + '@cipherstash/auth-win32-x64-msvc': { after: ['0.44.0', '0.44.1'] }, + '@cipherstash/auth': { + before: ['0.44.0'], + after: ['0.44.0', '0.44.1'], + hiddenFor: 1, + }, + }) + await waitForVersions([...parseSpecs(FFI), ...parseSpecs(AUTH)], { + ...registry, + ...quiet, + intervalMs: 15_000, + }) + // Three sleeps: the slowest package was hidden for three lookups. + expect(registry.sleeps).toEqual([15_000, 15_000, 15_000]) + // A package is not asked about again once npm lists it. + expect(Object.fromEntries(registry.calls)).toEqual({ + '@cipherstash/protect-ffi-darwin-arm64': 1, + '@cipherstash/protect-ffi-linux-x64-musl': 3, + '@cipherstash/protect-ffi': 2, + '@cipherstash/auth-darwin-x64': 4, + '@cipherstash/auth-win32-x64-msvc': 1, + '@cipherstash/auth': 2, + }) + }) + + it('reads a 404 as not listed yet: a first publish of a new package', async () => { + const registry = fakeRegistry({ + '@cipherstash/auth': { before: null, after: ['0.1.0'], hiddenFor: 2 }, + }) + await waitForVersions(parseSpecs('@cipherstash/auth@0.1.0'), { + ...registry, + ...quiet, + }) + expect(registry.calls.get('@cipherstash/auth')).toBe(3) + }) + + it('is satisfied by the exact version only', async () => { + const registry = fakeRegistry({ + '@cipherstash/auth': { after: ['0.44.10', '0.44.1-rc.0'] }, + }) + await expect( + waitForVersions(parseSpecs('@cipherstash/auth@0.44.1'), { + ...registry, + ...quiet, + timeoutMs: 30_000, + intervalMs: 15_000, + }), + ).rejects.toThrow(/@cipherstash\/auth@0\.44\.1/) + }) + + it('times out naming every version npm never listed, and only those', async () => { + const registry = fakeRegistry({ + '@cipherstash/auth': { after: ['0.44.1'] }, + '@cipherstash/auth-darwin-x64': { after: ['0.44.0'] }, + }) + const waiting = waitForVersions( + parseSpecs( + '@cipherstash/auth@0.44.1 @cipherstash/auth-darwin-x64@0.44.1', + ), + { ...registry, ...quiet, timeoutMs: 60_000, intervalMs: 15_000 }, + ) + await expect(waiting).rejects.toThrow( + /within 60s: @cipherstash\/auth-darwin-x64@0\.44\.1\. /, + ) + // It polled until the deadline, not once. + expect(registry.calls.get('@cipherstash/auth-darwin-x64')).toBe(5) + }) + + it('retries a registry error until the deadline, then reports it', async () => { + const registry = fakeRegistry({ + '@cipherstash/auth': new Error( + 'npm view @cipherstash/auth failed: ETIMEDOUT', + ), + }) + await expect( + waitForVersions(parseSpecs('@cipherstash/auth@0.44.1'), { + ...registry, + ...quiet, + timeoutMs: 30_000, + intervalMs: 15_000, + }), + ).rejects.toThrow(/@cipherstash\/auth@0\.44\.1 \(npm view .* ETIMEDOUT\)/) + expect(registry.calls.get('@cipherstash/auth')).toBe(3) + }) +}) + +/** + * The script as the workflow runs it, against an `npm` shim on PATH that + * answers `npm view versions --json` and counts the calls. + */ +describe('the script', () => { + let dir + afterEach(() => dir && rmSync(dir, { recursive: true, force: true })) + + function run(env, packages) { + dir = mkdtempSync(join(tmpdir(), 'wait-for-npm-')) + const state = join(dir, 'state.json') + writeFileSync(state, JSON.stringify({ packages, calls: {} })) + writeFileSync( + join(dir, 'npm'), + '#!/usr/bin/env node\n' + + "const fs = require('node:fs')\n" + + "const state = JSON.parse(fs.readFileSync(process.env.FAKE_NPM_STATE, 'utf8'))\n" + + 'const [command, name] = process.argv.slice(2)\n' + + "if (command !== 'view') process.exit(2)\n" + + 'state.calls[name] = (state.calls[name] ?? 0) + 1\n' + + 'fs.writeFileSync(process.env.FAKE_NPM_STATE, JSON.stringify(state))\n' + + 'const entry = state.packages[name]\n' + + 'const listed = entry && state.calls[name] > (entry.hiddenFor ?? 0) ? entry.after : null\n' + + "if (!listed) { process.stderr.write('npm error code E404\\n'); process.exit(1) }\n" + + 'process.stdout.write(JSON.stringify(listed))\n', + ) + chmodSync(join(dir, 'npm'), 0o755) + const result = spawnSync('node', [SCRIPT], { + cwd: REPO_ROOT, + encoding: 'utf8', + env: { + ...process.env, + PATH: `${dir}${delimiter}${process.env.PATH}`, + FAKE_NPM_STATE: state, + FFI_PUBLISHED: '', + AUTH_PUBLISHED: '', + NPM_WAIT_INTERVAL_SECONDS: '0', + ...env, + }, + }) + const calls = JSON.parse(readFileSync(state, 'utf8')).calls + return { ...result, calls } + } + + it('returns at once when both publish jobs were skipped', () => { + const { status, stdout, calls } = run({}, {}) + expect(status).toBe(0) + expect(stdout).toContain('no wait') + expect(calls).toEqual({}) + }) + + it('waits for the auth packages when publish-ffi was skipped', () => { + const { status, stdout, calls } = run( + { + AUTH_PUBLISHED: + '@cipherstash/auth-darwin-x64@0.44.1 @cipherstash/auth@0.44.1', + }, + { + '@cipherstash/auth-darwin-x64': { after: ['0.44.1'], hiddenFor: 2 }, + '@cipherstash/auth': { after: ['0.44.1'] }, + }, + ) + expect(status).toBe(0) + expect(stdout).toContain('npm lists all 2.') + expect(calls).toEqual({ + '@cipherstash/auth-darwin-x64': 3, + '@cipherstash/auth': 1, + }) + }) + + it('fails before `changeset publish` when npm never lists a version', () => { + const { status, stderr } = run( + { + FFI_PUBLISHED: '@cipherstash/protect-ffi@0.33.0', + NPM_WAIT_TIMEOUT_SECONDS: '0', + }, + {}, + ) + expect(status).toBe(1) + expect(stderr).toMatch( + /::error::npm did not list these within 0s: @cipherstash\/protect-ffi@0\.33\.0\. /, + ) + }) +}) + +describe('release.yml waits for both native publish jobs', () => { + const jobs = readWorkflow(RELEASE).jobs + + for (const name of ['publish-ffi', 'publish-auth']) { + it(`${name} exports the list it published`, () => { + const job = jobs[name] + expect(job.outputs?.published).toBe( + expr('steps.publish.outputs.published'), + ) + const publish = job.steps.find((step) => step.id === 'publish') + // Every tarball it handled, published now or before: the same list it + // writes to published.txt for the tags. + expect(publish.run).toContain(`echo "published=\${published[*]}"`) + expect(publish.run).toContain('>> "$GITHUB_OUTPUT"') + }) + } + + it('runs the wait in `release`, before `changeset publish`, on both lists', () => { + const release = jobs.release + expect(release.needs).toEqual( + expect.arrayContaining(['publish-ffi', 'publish-auth']), + ) + const steps = release.steps + const wait = steps.findIndex((step) => + String(step.run ?? '').includes('scripts/wait-for-npm-versions.mjs'), + ) + const publish = steps.findIndex((step) => + String(step.uses ?? '').startsWith('changesets/action'), + ) + expect( + wait, + 'no step runs scripts/wait-for-npm-versions.mjs', + ).toBeGreaterThan(-1) + expect(wait, 'the wait must come before changesets/action').toBeLessThan( + publish, + ) + expect(steps[wait].if, 'the wait must not be skippable').toBeUndefined() + expect(steps[wait].env).toEqual({ + FFI_PUBLISHED: expr('needs.publish-ffi.outputs.published'), + AUTH_PUBLISHED: expr('needs.publish-auth.outputs.published'), + }) + }) +}) diff --git a/scripts/wait-for-npm-versions.mjs b/scripts/wait-for-npm-versions.mjs new file mode 100644 index 000000000..117c2576a --- /dev/null +++ b/scripts/wait-for-npm-versions.mjs @@ -0,0 +1,127 @@ +/** + * Wait until npm lists every version the native publish jobs just published. + * + * `release.yml`'s `publish-ffi` and `publish-auth` publish with `npm publish`, + * then the `release` job runs `changeset publish`, which publishes every + * version `npm info ` does not list. npm accepts a publish minutes + * before it lists it, so changesets publishes the native packages again — + * from the workspace, with `restricted` access — and npm refuses with E402. + * That failed the `release` job for `@cipherstash/protect-ffi` 0.33.0 and + * `@cipherstash/auth` 0.44.1 on 2 October 2026. This polls the same `versions` + * list, through the same npm, before changesets asks. + * + * A lookup error is retried rather than thrown: a 404 is a first publish npm + * does not list yet, and a registry error says nothing about the publish. The + * deadline bounds both. A timeout fails the step BEFORE `changeset publish`, + * so nothing is published, and a re-run waits again. + */ +import process from 'node:process' +import { setTimeout as sleep } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' +import { npmVersions } from './release-gate.mjs' + +/** npm says "a few minutes". The two failed runs had waited 1 to 2. */ +export const DEFAULT_TIMEOUT_SECONDS = 900 +export const DEFAULT_INTERVAL_SECONDS = 15 + +/** + * Space-separated `name@version` entries, as the publish jobs export them. A + * skipped job exports an empty string. The LAST `@` splits, so a scope survives. + */ +export function parseSpecs(text) { + return String(text ?? '') + .split(/\s+/) + .filter(Boolean) + .map((entry) => { + const at = entry.lastIndexOf('@') + const [name, version] = [entry.slice(0, at), entry.slice(at + 1)] + if (at <= 0 || !version) { + throw new Error(`\`${entry}\` is not a name@version entry`) + } + return { name, version } + }) +} + +const label = ({ name, version }) => `${name}@${version}` + +/** `lookup` answers like `npmVersions`: the versions, or `null` for a 404. */ +export async function waitForVersions( + specs, + { + lookup = npmVersions, + timeoutMs = DEFAULT_TIMEOUT_SECONDS * 1000, + intervalMs = DEFAULT_INTERVAL_SECONDS * 1000, + now = Date.now, + wait = sleep, + log = console.log, + } = {}, +) { + const deadline = now() + timeoutMs + const lastError = new Map() + let missing = specs + + for (;;) { + missing = missing.filter((spec) => { + try { + const listed = [lookup(spec.name) ?? []].flat().includes(spec.version) + lastError.delete(label(spec)) + if (listed) log(`${label(spec)} is listed on npm`) + return !listed + } catch (err) { + lastError.set(label(spec), err.message) + return true + } + }) + if (missing.length === 0) return + + if (now() >= deadline) { + const detail = missing.map((spec) => { + const error = lastError.get(label(spec)) + return error ? `${label(spec)} (${error})` : label(spec) + }) + throw new Error( + `npm did not list these within ${Math.round(timeoutMs / 1000)}s: ${detail.join(', ')}. ` + + '`changeset publish` would publish them again and fail. Re-run the job once ' + + '`npm view versions` lists them.', + ) + } + log(`waiting for npm to list ${missing.map(label).join(', ')}`) + await wait(intervalMs) + } +} + +function seconds(name, fallback) { + const raw = process.env[name] + if (raw === undefined || raw === '') return fallback + const value = Number(raw) + if (!Number.isFinite(value) || value < 0) { + throw new Error(`${name} must be a number of seconds, not \`${raw}\``) + } + return value +} + +async function main() { + const specs = [ + ...parseSpecs(process.env.FFI_PUBLISHED), + ...parseSpecs(process.env.AUTH_PUBLISHED), + ] + if (specs.length === 0) { + console.log('publish-ffi and publish-auth published nothing; no wait.') + return + } + // The workflow sets neither; they exist for the process tests. + await waitForVersions(specs, { + timeoutMs: + seconds('NPM_WAIT_TIMEOUT_SECONDS', DEFAULT_TIMEOUT_SECONDS) * 1000, + intervalMs: + seconds('NPM_WAIT_INTERVAL_SECONDS', DEFAULT_INTERVAL_SECONDS) * 1000, + }) + console.log(`npm lists all ${specs.length}.`) +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error(`::error::${err.message}`) + process.exit(1) + }) +} From d4a95551a1cc90e311f1e6755222f745305be5c2 Mon Sep 17 00:00:00 2001 From: Lindsay Holmwood Date: Sat, 3 Oct 2026 09:39:01 +1000 Subject: [PATCH 3/3] fix(release): decide the EQL release assets from npm and the tags The SQL, docs and image jobs ran only when the release job's `eql_published` output was true. A step after `changeset publish` set it, so when the publish failed the step never ran, even though EQL had been published, and a re-run found nothing to publish. That is why EQL 3.0.6 has no eql-3.0.6 tag, GitHub release or image. A new eql-assets job runs scripts/eql-release-assets.mjs after the release job, whatever its result. The assets are owed when npm carries the tree's EQL version and the eql- tag does not exist. This run's publishedPackages also counts, because npm lists a new version minutes after accepting it; the release job now exports it straight from the changesets step. The assets are built at the commit the @cipherstash/eql@ tag names, which is where npm's tarball came from, so a later run repairs an old release from that release's source. The four EQL jobs now use `!cancelled()` instead of the implicit `success()`. That is false when any job up the needs chain was skipped (actions/runner#2205), and publish-ffi and publish-auth are skipped on most releases, so an EQL release without an FFI and an auth release in the same run would also have skipped every EQL asset job. Refs: CIP-4276 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PaY5xYydZUWhv8Nex9Sw8a --- .github/workflows/release.yml | 134 +++-- AGENTS.md | 11 + scripts/__tests__/eql-release-assets.test.mjs | 567 ++++++++++++++++++ .../workflow-dispatch-job-conditions.test.mjs | 12 +- scripts/eql-release-assets.mjs | 177 ++++++ 5 files changed, 844 insertions(+), 57 deletions(-) create mode 100644 scripts/__tests__/eql-release-assets.test.mjs create mode 100644 scripts/eql-release-assets.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f253e4b8c..3b44ac698 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -501,12 +501,12 @@ jobs: id-token: write # npm OIDC trusted publishing contents: write # changesets commits and pushes the Version Packages branch pull-requests: write # …and opens/updates the PR for it - # `published` alone is not enough: this job publishes every unpublished JS - # package, and a `@cipherstash/stack` release must not fire an EQL one. + # Read by `eql-assets`. Straight from the step, with no step in between: + # changesets/action sets it for whatever it published even when + # `changeset publish` then fails, and a job output is evaluated when the + # job ends, whatever its result. outputs: - eql_published: ${{ steps.eql.outputs.eql_published }} - eql_version: ${{ steps.eql.outputs.eql_version }} - eql_prerelease: ${{ steps.eql.outputs.eql_prerelease }} + published_packages: ${{ steps.changesets.outputs.publishedPackages }} steps: - name: Checkout Repo uses: actions/checkout@v6 @@ -646,84 +646,108 @@ jobs: # gh variable set STASH_POSTHOG_KEY --repo cipherstash/stack --body '' STASH_POSTHOG_KEY: ${{ vars.STASH_POSTHOG_KEY }} - # Read from `publishedPackages` rather than from the tree, so the SQL - # release, docs bundle and image tag agree with what reached npm. A re-run - # against an already-published version finds nothing and the EQL branch - # skips, which is correct. - - name: Resolve the published EQL version - id: eql - if: steps.changesets.outputs.published == 'true' - env: - PUBLISHED: ${{ steps.changesets.outputs.publishedPackages }} - run: | - set -euo pipefail - version="$(node -e "const p=JSON.parse(process.env.PUBLISHED);const e=p.find(x=>x.name==='@cipherstash/eql');console.log(e ? e.version : '')")" - if [ -z "$version" ]; then - echo "@cipherstash/eql was not part of this release" - exit 0 - fi - if [[ "$version" == *-* ]]; then - prerelease=true - else - prerelease=false - fi - echo "@cipherstash/eql@${version} published (prerelease=${prerelease})" - { - echo "eql_published=true" - echo "eql_version=${version}" - echo "eql_prerelease=${prerelease}" - } >> "$GITHUB_OUTPUT" - # ---- The EQL release line: production ------------------------------------ # # EQL ships as five artefacts at one version. Changesets publishes the npm # package (above) and release-plz.yml the crate on the same push; the rest are - # built here, in the run that published, so they cannot drift from it. + # built here. + # + # `eql-assets` decides from npm and the tags, not from this run, so a run + # whose `changeset publish` failed still builds them, and so does the next + # push to main if that run never got this far. See + # scripts/eql-release-assets.mjs. + # + # `!cancelled()`, NOT the implicit `success()`, on all four: `success()` + # is false when ANY job up the `needs:` chain was skipped or failed, and + # `publish-ffi` and `publish-auth` are skipped on most releases. Each job + # names the results it does need instead. `always()` would also do that, and + # would keep them running after somebody cancelled the run. # # The `needs:` chain is load-bearing: `eql-docs` attaches to the release # `eql-sql` creates, and `eql-image` dispatches against the tag it produced. + eql-assets: + name: Does EQL still need its release assets? + needs: [classify, gate, release] + if: >- + !cancelled() && + needs.classify.outputs.mode == 'production' && + needs.gate.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + needed: ${{ steps.eql.outputs.needed }} + version: ${{ steps.eql.outputs.version }} + prerelease: ${{ steps.eql.outputs.prerelease }} + ref: ${{ steps.eql.outputs.ref }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 22 + package-manager-cache: false + + # No pnpm install: the script imports node builtins only, as `gate` does. + - name: Ask npm and the tags + id: eql + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PUBLISHED_PACKAGES: ${{ needs.release.outputs.published_packages }} + run: node scripts/eql-release-assets.mjs + eql-sql: name: Build and attach the EQL SQL release - needs: [classify, gate, eql-armed, release] + needs: [classify, eql-armed, eql-assets] if: >- + !cancelled() && needs.classify.outputs.mode == 'production' && needs.eql-armed.outputs.armed == 'true' && - needs.release.outputs.eql_published == 'true' + needs.eql-assets.outputs.needed == 'true' permissions: contents: write # creates the eql- tag and release uses: ./.github/workflows/_build-eql-sql.yml with: - ref: ${{ github.sha }} - tag: eql-${{ needs.release.outputs.eql_version }} + # The commit npm's tarball was built from, which is this run's commit + # unless this run is repairing an older release. + ref: ${{ needs.eql-assets.outputs.ref }} + tag: eql-${{ needs.eql-assets.outputs.version }} attach: true - target_commitish: ${{ github.sha }} - prerelease: ${{ needs.release.outputs.eql_prerelease == 'true' }} + target_commitish: ${{ needs.eql-assets.outputs.ref }} + prerelease: ${{ needs.eql-assets.outputs.prerelease == 'true' }} eql-docs: name: Build and attach the EQL docs bundle - needs: [classify, gate, eql-armed, release, eql-sql] + needs: [classify, eql-armed, eql-assets, eql-sql] if: >- + !cancelled() && needs.classify.outputs.mode == 'production' && needs.eql-armed.outputs.armed == 'true' && - needs.release.outputs.eql_published == 'true' + needs.eql-assets.outputs.needed == 'true' && + needs.eql-sql.result == 'success' permissions: contents: write # attaches to the release eql-sql just created uses: ./.github/workflows/_build-eql-docs.yml with: - ref: ${{ github.sha }} - tag: eql-${{ needs.release.outputs.eql_version }} + ref: ${{ needs.eql-assets.outputs.ref }} + tag: eql-${{ needs.eql-assets.outputs.version }} eql-image: name: Dispatch the Postgres + EQL image build - needs: [classify, gate, eql-armed, release, eql-sql, eql-docs] + needs: [classify, eql-armed, eql-assets, eql-sql, eql-docs] # Production finals only: the floating :latest / : tags must not # move for a prerelease. An alpha image is still buildable on demand. if: >- + !cancelled() && needs.classify.outputs.mode == 'production' && needs.eql-armed.outputs.armed == 'true' && - needs.release.outputs.eql_published == 'true' && - needs.release.outputs.eql_prerelease == 'false' + needs.eql-assets.outputs.needed == 'true' && + needs.eql-assets.outputs.prerelease == 'false' && + needs.eql-sql.result == 'success' && + needs.eql-docs.result == 'success' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -736,7 +760,7 @@ jobs: # the released source even if main has advanced. env: GH_TOKEN: ${{ github.token }} - VERSION: ${{ needs.release.outputs.eql_version }} + VERSION: ${{ needs.eql-assets.outputs.version }} run: | set -euo pipefail # `--repo` is required: no checkout, so gh cannot infer it. @@ -940,13 +964,14 @@ jobs: - eql-armed # `gate` and `release` are not EQL jobs, and they are the two that most # often decide an EQL run does nothing. `gate` exits non-zero for a frozen - # publisher — which is the NORMAL inert state — and skips `release`, whose - # `eql_published` output then gates the production EQL chain. Without them - # here every job below reads `skipped` and the table cannot separate - # "correctly inert" from "the gate refused this release" from "changesets - # failed", which is the distinction this job exists to draw. + # publisher — which is the NORMAL inert state — and skips `release` and + # `eql-assets`, whose `needed` output gates the production EQL chain. + # Without them here every job below reads `skipped` and the table cannot + # separate "correctly inert" from "the gate refused this release" from + # "changesets failed", which is the distinction this job exists to draw. - gate - release + - eql-assets - eql-sql - eql-docs - eql-image @@ -965,6 +990,8 @@ jobs: ARMED: ${{ needs.eql-armed.outputs.armed }} GATE: ${{ needs.gate.result }} RELEASE: ${{ needs.release.result }} + EQL_ASSETS: ${{ needs.eql-assets.result }} + EQL_NEEDED: ${{ needs.eql-assets.outputs.needed }} EQL_SQL: ${{ needs.eql-sql.result }} EQL_DOCS: ${{ needs.eql-docs.result }} EQL_IMAGE: ${{ needs.eql-image.result }} @@ -985,6 +1012,7 @@ jobs: echo "| --- | --- |" echo "| gate | ${GATE} |" echo "| release (changesets) | ${RELEASE} |" + echo "| eql-assets (needed: \`${EQL_NEEDED:-n/a}\`) | ${EQL_ASSETS} |" echo "| eql-sql | ${EQL_SQL} |" echo "| eql-docs | ${EQL_DOCS} |" echo "| eql-image | ${EQL_IMAGE} |" diff --git a/AGENTS.md b/AGENTS.md index f10885b77..21d983f92 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -368,6 +368,17 @@ monorepo, which is where the silent failures are. | `lint-release.yml` | merged into the root file of the same name | | ~~`rebuild-docs.yml`~~ | **not ported.** It targeted the retired docs site through the deprecated `DOCS_WEBHOOK_URL`; versioned docs artifacts are still built by `_build-eql-docs.yml` | + **The SQL, docs and image jobs key on the registry and the tags, not on the + run that published.** `release.yml`'s `eql-assets` job runs + `scripts/eql-release-assets.mjs`, which reports the assets as owed when npm + carries the tree's EQL version and the `eql-` tag does not exist, + and builds them at the commit the `@cipherstash/eql@` tag names. So + a run whose `changeset publish` failed still builds them, and a later push + repairs a release that never got them — EQL 3.0.6 was the first. The four + jobs are `!cancelled() && …` because the implicit `success()` is false when + any job up the `needs:` chain was skipped, and `publish-ffi` and + `publish-auth` are skipped on most releases. + **Inertness is a derived switch, not a flag somebody flips.** The one piece of state is `FROZEN_PUBLISHERS` in `scripts/release-gate.mjs` — the existing map recording "this package lives here but is published elsewhere" — and diff --git a/scripts/__tests__/eql-release-assets.test.mjs b/scripts/__tests__/eql-release-assets.test.mjs new file mode 100644 index 000000000..2d2672488 --- /dev/null +++ b/scripts/__tests__/eql-release-assets.test.mjs @@ -0,0 +1,567 @@ +import { spawnSync } from 'node:child_process' +import { + chmodSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { delimiter, join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + assetTag, + changesetsTag, + EQL_MANIFEST, + eqlAssets, + parsePublished, + tagCommitVia, +} from '../eql-release-assets.mjs' +import { expr, runsWhen } from './lib/expressions.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { readWorkflow } from './lib/workflows.mjs' + +/** + * Whether EQL still owes its GitHub assets, and the release.yml wiring that + * acts on the answer. + * + * EQL 3.0.6 reached npm on 2 October 2026 in a run whose `changeset publish` + * then failed, and the step that would have reported the publish never ran. + * No `eql-3.0.6` tag, release or image was built, and no later run would have + * built them. These hold the replacement: an answer from npm and the tags, + * and EQL jobs that still run when the `release` job fails. + */ + +const SCRIPT = join(REPO_ROOT, 'scripts/eql-release-assets.mjs') +const RELEASE = '.github/workflows/release.yml' +const EQL = '@cipherstash/eql' +const TREE_VERSION = JSON.parse( + readFileSync(join(REPO_ROOT, EQL_MANIFEST), 'utf8'), +).version + +const HEAD = 'cb58a7b993646173578e2bb3c35d8e22ed1a7944' +const PUBLISHED_AT = '23e9af3f8e28c6d0495fdafcff096f58cbf0f4f7' + +/** Lookups that record what they were asked. */ +function world({ npm = [], tags = {} } = {}) { + const asked = [] + return { + asked, + npmHas: (version) => { + asked.push(`npm ${version}`) + return npm.includes(version) + }, + tagCommit: (tag) => { + asked.push(`tag ${tag}`) + return tags[tag] ?? null + }, + } +} + +const decide = (overrides, lookups) => + eqlAssets({ + version: '3.0.6', + published: [], + headSha: HEAD, + ...lookups, + ...overrides, + }) + +describe('the decision', () => { + it('owes the assets of a version on npm with no eql- tag, at the commit that published it', () => { + // EQL 3.0.6 today: published by an earlier run, never tagged. + const result = decide( + {}, + world({ + npm: ['3.0.5', '3.0.6'], + tags: { '@cipherstash/eql@3.0.6': PUBLISHED_AT }, + }), + ) + expect(result).toMatchObject({ + needed: true, + version: '3.0.6', + prerelease: false, + ref: PUBLISHED_AT, + }) + }) + + it("counts this run's publish before npm lists it", () => { + // A `changeset publish` that published EQL and then failed: the action + // still reports EQL in `publishedPackages`. + const lookups = world({ tags: { '@cipherstash/eql@3.0.6': HEAD } }) + const result = decide( + { + published: [ + { name: '@cipherstash/stack', version: '1.2.0' }, + { name: EQL, version: '3.0.6' }, + ], + }, + lookups, + ) + expect(result).toMatchObject({ needed: true, ref: HEAD }) + expect(lookups.asked).not.toContain('npm 3.0.6') + }) + + it("builds at this run's commit when this run published and pushed no tag", () => { + const result = decide( + { published: [{ name: EQL, version: '3.0.6' }] }, + world(), + ) + expect(result).toMatchObject({ needed: true, ref: HEAD }) + }) + + it('owes nothing once the eql- tag exists, and asks npm nothing', () => { + const lookups = world({ + npm: ['3.0.6'], + tags: { 'eql-3.0.6': PUBLISHED_AT }, + }) + const result = decide( + { published: [{ name: EQL, version: '3.0.6' }] }, + lookups, + ) + expect(result).toMatchObject({ needed: false, ref: '' }) + expect(result.reason).toContain('eql-3.0.6 already exists') + expect(lookups.asked).toEqual(['tag eql-3.0.6']) + }) + + it('owes nothing for a version that is not on npm', () => { + // The tree's version did not publish: there is nothing to attach to. + const result = decide({}, world({ npm: ['3.0.5'] })) + expect(result).toMatchObject({ needed: false }) + expect(result.reason).toContain('is not on npm') + }) + + it('does not count another package, or another EQL version, as this run publishing EQL', () => { + const result = decide( + { + published: [ + { name: '@cipherstash/eql-bindings', version: '3.0.6' }, + { name: EQL, version: '3.0.5' }, + ], + }, + world(), + ) + expect(result.needed).toBe(false) + }) + + it('refuses a version on npm when nothing says which commit published it', () => { + expect(() => decide({}, world({ npm: ['3.0.6'] }))).toThrow( + /no @cipherstash\/eql@3\.0\.6 tag says which commit published it/, + ) + }) + + it('marks a prerelease, so eql-image keeps the floating tags where they are', () => { + const result = decide( + { + version: '3.1.0-rc.1', + published: [{ name: EQL, version: '3.1.0-rc.1' }], + }, + world(), + ) + expect(result).toMatchObject({ needed: true, prerelease: true }) + }) + + it('refuses a version that is not one, since it becomes a tag and a dispatch input', () => { + expect(() => decide({ version: '3.0.6; rm -rf /' }, world())).toThrow( + /not X\.Y\.Z/, + ) + }) + + it('owes nothing, and asks nothing, while EQL is a frozen publisher', () => { + const lookups = world({ npm: ['3.0.6'] }) + const result = decide({ armed: false }, lookups) + expect(result).toMatchObject({ needed: false }) + expect(result.reason).toContain('not armed') + expect(lookups.asked).toEqual([]) + }) + + it('names the tags it reads', () => { + expect(assetTag('3.0.6')).toBe('eql-3.0.6') + expect(changesetsTag('3.0.6')).toBe('@cipherstash/eql@3.0.6') + }) +}) + +describe('the inputs', () => { + it('reads an empty publishedPackages as nothing published', () => { + // changesets/action sets no output when it published nothing, and the + // release job may not have run at all. + expect(parsePublished('')).toEqual([]) + expect(parsePublished(undefined)).toEqual([]) + expect(parsePublished('[{"name":"a","version":"1.0.0"}]')).toEqual([ + { name: 'a', version: '1.0.0' }, + ]) + }) + + it('refuses a publishedPackages it cannot read', () => { + expect(() => parsePublished('{')).toThrow() + expect(() => parsePublished('{"name":"a"}')).toThrow(/not an array/) + }) + + it('matches the tag exactly, not by prefix', () => { + const call = () => [ + { ref: 'refs/tags/eql-3.0.60', object: { type: 'commit', sha: 'wrong' } }, + ] + expect(tagCommitVia('o/r', call)('eql-3.0.6')).toBeNull() + }) + + it('follows an annotated tag to its commit', () => { + const answers = { + 'repos/o/r/git/matching-refs/tags/eql-3.0.6': [ + { ref: 'refs/tags/eql-3.0.6', object: { type: 'tag', sha: 'tagobj' } }, + ], + 'repos/o/r/git/tags/tagobj': { + object: { type: 'commit', sha: PUBLISHED_AT }, + }, + } + expect(tagCommitVia('o/r', (path) => answers[path])('eql-3.0.6')).toBe( + PUBLISHED_AT, + ) + }) + + it('lets a failed tag lookup throw rather than read as "no tag"', () => { + const call = () => { + throw new Error('HTTP 401') + } + expect(() => tagCommitVia('o/r', call)('eql-3.0.6')).toThrow('HTTP 401') + }) +}) + +/** + * The script as the workflow runs it, over the real tree's version, with `npm` + * and `gh` shimmed on PATH. + */ +describe('the script', () => { + let dir + afterEach(() => dir && rmSync(dir, { recursive: true, force: true })) + + function shim(name, body) { + writeFileSync(join(dir, name), `#!/usr/bin/env node\n${body}`) + chmodSync(join(dir, name), 0o755) + } + + function run({ npm = null, api = {}, published = '' }) { + dir = mkdtempSync(join(tmpdir(), 'eql-assets-')) + const output = join(dir, 'output') + writeFileSync(output, '') + writeFileSync(join(dir, 'npm.json'), JSON.stringify(npm)) + writeFileSync(join(dir, 'api.json'), JSON.stringify(api)) + shim( + 'npm', + "const versions = require(process.env.FAKE_DIR + '/npm.json')\n" + + "if (versions === null) { process.stderr.write('npm error code E404\\n'); process.exit(1) }\n" + + 'process.stdout.write(JSON.stringify(versions))\n', + ) + shim( + 'gh', + "const api = require(process.env.FAKE_DIR + '/api.json')\n" + + 'const path = process.argv[3]\n' + + "if (api[path] === 'fail') { process.stderr.write('HTTP 502\\n'); process.exit(1) }\n" + + "process.stdout.write(JSON.stringify(api[path] ?? (path.includes('/matching-refs/') ? [] : null)))\n", + ) + const result = spawnSync('node', [SCRIPT], { + cwd: REPO_ROOT, + encoding: 'utf8', + env: { + ...process.env, + PATH: `${dir}${delimiter}${process.env.PATH}`, + FAKE_DIR: dir, + GITHUB_OUTPUT: output, + GITHUB_SHA: HEAD, + REPO: 'cipherstash/stack', + PUBLISHED_PACKAGES: published, + }, + }) + return { ...result, output: readFileSync(output, 'utf8') } + } + + const refs = (tag, sha) => ({ + [`repos/cipherstash/stack/git/matching-refs/tags/${tag}`]: [ + { ref: `refs/tags/${tag}`, object: { type: 'commit', sha } }, + ], + }) + + it('owes the assets of an already-published version with no tag', () => { + const { status, stdout, output } = run({ + npm: [TREE_VERSION], + api: refs(changesetsTag(TREE_VERSION), PUBLISHED_AT), + }) + expect(status).toBe(0) + expect(stdout).toContain(`building at ${PUBLISHED_AT}`) + expect(output).toBe( + `needed=true\nversion=${TREE_VERSION}\n` + + `prerelease=${TREE_VERSION.includes('-')}\nref=${PUBLISHED_AT}\n`, + ) + }) + + it('owes them after a failed `changeset publish` that published EQL, before npm lists it', () => { + const { status, output } = run({ + npm: null, + published: JSON.stringify([{ name: EQL, version: TREE_VERSION }]), + }) + expect(status).toBe(0) + expect(output).toContain('needed=true\n') + expect(output).toContain(`ref=${HEAD}\n`) + }) + + it('owes nothing once the tag exists', () => { + const { status, output } = run({ + npm: [TREE_VERSION], + api: refs(assetTag(TREE_VERSION), PUBLISHED_AT), + }) + expect(status).toBe(0) + expect(output).toContain('needed=false\n') + }) + + it('fails, writing nothing, when it cannot read the tags', () => { + const { status, stderr, output } = run({ + npm: [TREE_VERSION], + api: { + [`repos/cipherstash/stack/git/matching-refs/tags/${assetTag(TREE_VERSION)}`]: + 'fail', + }, + }) + expect(status).toBe(1) + expect(stderr).toContain('::error::') + expect(output).toBe('') + }) +}) + +// --------------------------------------------------------------------------- +// The release job graph +// --------------------------------------------------------------------------- + +const STATUS_FUNCTION = /\b(always|cancelled|success|failure)\s*\(/ + +/** + * Which jobs of release.yml run, for given job results and outputs, in a run + * somebody cancelled while `cancelledDuring` was running, if it is set. + * + * A condition with no status function is ANDed with `success()`, and GitHub + * evaluates that over every job up the `needs:` chain, not only the direct + * ones: a skipped or failed ancestor anywhere skips the job + * (https://github.com/actions/runner/issues/2205). That rule is the one that + * matters here, because `publish-ffi` and `publish-auth` are skipped in most + * runs. + */ +function simulate({ results = {}, outputs = {}, cancelledDuring } = {}) { + const jobs = readWorkflow(RELEASE).jobs + const needsOf = (name) => [jobs[name].needs ?? []].flat() + const ancestors = (name, seen = new Set()) => { + for (const parent of needsOf(name)) { + if (!seen.has(parent)) { + seen.add(parent) + ancestors(parent, seen) + } + } + return seen + } + + const state = {} + const pending = new Set(Object.keys(jobs)) + while (pending.size > 0) { + const ready = [...pending].filter((name) => + needsOf(name).every((parent) => parent in state), + ) + if (ready.length === 0) throw new Error('release.yml has a needs cycle') + for (const name of ready) { + pending.delete(name) + if (name === cancelledDuring) { + state[name] = { result: 'cancelled', outputs: outputs[name] ?? {} } + continue + } + // Jobs that had already finished are unaffected by the cancel. + const cancelled = ancestors(name).has(cancelledDuring) + const condition = jobs[name].if === undefined ? '' : String(jobs[name].if) + const context = { + github: { event_name: 'push', ref: 'refs/heads/main' }, + vars: {}, + needs: Object.fromEntries(needsOf(name).map((n) => [n, state[n]])), + } + const runs = STATUS_FUNCTION.test(condition) + ? runsWhen(condition, context, { cancelled }) + : !cancelled && + [...ancestors(name)].every((a) => state[a].result === 'success') && + (condition === '' || runsWhen(condition, context)) + state[name] = runs + ? { result: results[name] ?? 'success', outputs: outputs[name] ?? {} } + : { result: 'skipped', outputs: {} } + } + } + return state +} + +const OWED = { + needed: 'true', + version: '3.0.6', + prerelease: 'false', + ref: PUBLISHED_AT, +} + +/** A production push where the gate found the given lines unpublished. */ +function scenario({ + ffi = false, + auth = false, + results, + eql = OWED, + cancelledDuring, + published = '', +}) { + return simulate({ + cancelledDuring, + results, + outputs: { + classify: { mode: 'production', version: '' }, + 'eql-armed': { armed: 'true' }, + gate: { ffi: String(ffi), auth: String(auth) }, + release: { published_packages: published }, + 'eql-assets': eql, + }, + }) +} + +const EQL_JOBS = ['eql-assets', 'eql-sql', 'eql-docs', 'eql-image'] +const ran = (state, names) => + Object.fromEntries( + names.map((name) => [name, state[name].result !== 'skipped']), + ) + +describe('release.yml builds the EQL assets whatever happened to `release`', () => { + it('runs them on a release with no FFI or auth in it', () => { + // publish-ffi and publish-auth are skipped, so under the implicit + // `success()` every EQL job would be skipped with them. + const state = scenario({}) + expect(state['publish-ffi'].result).toBe('skipped') + expect(state['publish-auth'].result).toBe('skipped') + expect(state.release.result).toBe('success') + expect(ran(state, EQL_JOBS)).toEqual({ + 'eql-assets': true, + 'eql-sql': true, + 'eql-docs': true, + 'eql-image': true, + }) + }) + + it('runs them when `changeset publish` fails', () => { + // The 3.0.6 run: FFI published, then `changeset publish` failed with E402. + const state = scenario({ + ffi: true, + results: { release: 'failure' }, + published: JSON.stringify([{ name: EQL, version: '3.0.6' }]), + }) + expect(state['publish-ffi'].result).toBe('success') + expect(state.release.result).toBe('failure') + expect(ran(state, EQL_JOBS)).toEqual({ + 'eql-assets': true, + 'eql-sql': true, + 'eql-docs': true, + 'eql-image': true, + }) + }) + + it('still repairs an older release when a native build fails and skips `release`', () => { + const state = scenario({ + auth: true, + results: { 'auth-artifacts': 'failure' }, + }) + expect(state.release.result).toBe('skipped') + expect(ran(state, EQL_JOBS)['eql-sql']).toBe(true) + }) + + it('builds nothing when nothing is owed', () => { + const state = scenario({ eql: { ...OWED, needed: 'false', ref: '' } }) + expect(ran(state, EQL_JOBS)).toEqual({ + 'eql-assets': true, + 'eql-sql': false, + 'eql-docs': false, + 'eql-image': false, + }) + }) + + it('builds the SQL and docs of a prerelease, and leaves the image alone', () => { + const state = scenario({ + eql: { ...OWED, version: '3.1.0-rc.1', prerelease: 'true' }, + }) + expect(ran(state, EQL_JOBS)).toEqual({ + 'eql-assets': true, + 'eql-sql': true, + 'eql-docs': true, + 'eql-image': false, + }) + }) + + it('stops at the first EQL job that fails', () => { + const state = scenario({ results: { 'eql-sql': 'failure' } }) + expect(ran(state, ['eql-docs', 'eql-image'])).toEqual({ + 'eql-docs': false, + 'eql-image': false, + }) + }) + + it('does nothing EQL when the gate fails', () => { + const state = scenario({ results: { gate: 'failure' } }) + expect(ran(state, EQL_JOBS)).toEqual({ + 'eql-assets': false, + 'eql-sql': false, + 'eql-docs': false, + 'eql-image': false, + }) + }) + + it('does nothing EQL in a run cancelled during `release`', () => { + const state = scenario({ cancelledDuring: 'release' }) + expect(state.gate.result).toBe('success') + expect(ran(state, EQL_JOBS)).toEqual({ + 'eql-assets': false, + 'eql-sql': false, + 'eql-docs': false, + 'eql-image': false, + }) + }) +}) + +describe('release.yml feeds the decision and acts on it', () => { + const jobs = readWorkflow(RELEASE).jobs + + it("hands `eql-assets` this run's publishedPackages straight from the step", () => { + // A job output is evaluated when the job ends, failed or not; a step in + // between would be skipped by the failure it exists to survive. + expect(jobs.release.outputs).toEqual({ + published_packages: expr('steps.changesets.outputs.publishedPackages'), + }) + const step = jobs['eql-assets'].steps.find((s) => + String(s.run ?? '').includes('scripts/eql-release-assets.mjs'), + ) + expect(step?.env?.PUBLISHED_PACKAGES).toBe( + expr('needs.release.outputs.published_packages'), + ) + expect(jobs['eql-assets'].outputs).toEqual( + Object.fromEntries( + ['needed', 'version', 'prerelease', 'ref'].map((key) => [ + key, + expr(`steps.${step.id}.outputs.${key}`), + ]), + ), + ) + }) + + it('builds the version, at the commit, that `eql-assets` names', () => { + expect(jobs['eql-sql'].with).toMatchObject({ + ref: expr('needs.eql-assets.outputs.ref'), + target_commitish: expr('needs.eql-assets.outputs.ref'), + tag: `eql-${expr('needs.eql-assets.outputs.version')}`, + prerelease: expr("needs.eql-assets.outputs.prerelease == 'true'"), + }) + expect(jobs['eql-docs'].with).toEqual({ + ref: expr('needs.eql-assets.outputs.ref'), + tag: `eql-${expr('needs.eql-assets.outputs.version')}`, + }) + expect(jobs['eql-image'].steps[0].env.VERSION).toBe( + expr('needs.eql-assets.outputs.version'), + ) + }) + + it('reads none of the outputs `release` no longer has', () => { + const text = readFileSync(join(REPO_ROOT, RELEASE), 'utf8') + expect(text).not.toMatch(/needs\.release\.outputs\.eql_/) + }) +}) diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index 79f11e15f..dd0aeb83a 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -188,16 +188,20 @@ const PERMISSIVE_NEEDS = { gate: { result: 'success', outputs: { ffi: 'true', auth: 'true' } }, 'publish-ffi': { result: 'success' }, 'publish-auth': { result: 'success' }, - release: { + release: { result: 'success', outputs: { published_packages: '' } }, + 'eql-assets': { result: 'success', outputs: { - eql_published: 'true', - eql_version: '3.0.6', + needed: 'true', + version: '3.0.6', // A final: `eql-image` runs only for one. The prerelease half of the // file is in DISPATCH_SKIPPED_JOBS instead. - eql_prerelease: 'false', + prerelease: 'false', + ref: 'cb58a7b993646173578e2bb3c35d8e22ed1a7944', }, }, + 'eql-sql': { result: 'success' }, + 'eql-docs': { result: 'success' }, // release-postgres-eql-image.yml's `promote-latest` reads this from its own // `build-sql` job, which copies the dispatch input through. 'build-sql': { outputs: { update_floating_tags: 'true' } }, diff --git a/scripts/eql-release-assets.mjs b/scripts/eql-release-assets.mjs new file mode 100644 index 000000000..e70579210 --- /dev/null +++ b/scripts/eql-release-assets.mjs @@ -0,0 +1,177 @@ +/** + * Does EQL still owe the GitHub assets for the version in the tree? + * + * `release.yml`'s `eql-sql`, `eql-docs` and `eql-image` build the `eql-` + * tag and release with the SQL bundle, the docs bundle and the `postgres-eql` + * image. They ran when a step after `changeset publish` reported it had just + * published `@cipherstash/eql`. A failed `changeset publish` never reached that + * step, even when it HAD published EQL, and a re-run found nothing left to + * publish. That is why EQL 3.0.6 is on npm with no tag, release or image. + * + * So the answer comes from state that outlives a run: the assets are owed when + * npm carries the tree's version and the `eql-` tag does not exist. + * This run's own publish also counts, because npm lists a new version minutes + * after accepting it. + * + * THE ASSETS ARE BUILT AT THE COMMIT NPM'S TARBALL CAME FROM. changesets/action + * pushes `@cipherstash/eql@` at the commit it published, so a run + * repairing an older release builds that release's source rather than main's. + * With no such tag, only this run's publish can say which commit it was; a + * version on npm with neither is refused rather than guessed. + */ +import { execFileSync } from 'node:child_process' +import { appendFileSync, readFileSync } from 'node:fs' +import { join, resolve } from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' +import { EQL_PACKAGE, eqlPipelineArmed } from './eql-pipeline-armed.mjs' +import { npmVersions } from './release-gate.mjs' + +const REPO_ROOT = resolve(import.meta.dirname, '..') + +/** Two levels down: the subtree root carries no package.json. */ +export const EQL_MANIFEST = 'packages/eql/packages/eql/package.json' + +/** + * The versions this may name. It ends up in a tag, a release name and a + * `gh workflow run -f` argument, so anything else is refused here. + */ +const VERSION = /^\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?$/ + +export const assetTag = (version) => `eql-${version}` +export const changesetsTag = (version) => `${EQL_PACKAGE}@${version}` + +/** changesets/action's `publishedPackages`: a JSON array, or empty. */ +export function parsePublished(text) { + if (!text) return [] + const parsed = JSON.parse(text) + if (!Array.isArray(parsed)) { + throw new Error(`publishedPackages is not an array: ${text}`) + } + return parsed +} + +/** + * `npmHas(version)` and `tagCommit(tag)` are asked lazily, in the order + * below, so a version that is already tagged costs no registry call. + * + * `armed` first: while EQL is a frozen publisher its versions reach npm from + * another repository, with no changesets tag here, and the refusal below would + * fail every push to main. + */ +export function eqlAssets({ + version, + published, + npmHas, + tagCommit, + headSha, + armed = eqlPipelineArmed(), +}) { + if (!VERSION.test(version)) { + throw new Error( + `${EQL_MANIFEST} carries version \`${version}\`, which is not X.Y.Z or X.Y.Z-pre`, + ) + } + const prerelease = version.includes('-') + const none = (reason) => ({ + needed: false, + version, + prerelease, + ref: '', + reason, + }) + + if (!armed) return none('the EQL release line is not armed') + + const tagged = tagCommit(assetTag(version)) + if (tagged) { + return none(`${assetTag(version)} already exists at ${tagged}`) + } + + const thisRun = published.some( + (p) => p.name === EQL_PACKAGE && p.version === version, + ) + if (!thisRun && !npmHas(version)) { + return none(`${EQL_PACKAGE}@${version} is not on npm`) + } + + const publishedAt = tagCommit(changesetsTag(version)) + const ref = publishedAt ?? (thisRun ? headSha : null) + if (!ref) { + throw new Error( + `${EQL_PACKAGE}@${version} is on npm and ${assetTag(version)} does not exist, ` + + `but no ${changesetsTag(version)} tag says which commit published it. Build the ` + + 'assets by hand from that commit, or push the tag there.', + ) + } + return { + needed: true, + version, + prerelease, + ref, + reason: `${EQL_PACKAGE}@${version} is ${thisRun ? 'published by this run' : 'on npm'} and ${assetTag(version)} does not exist; building at ${ref}`, + } +} + +function gh(...args) { + return JSON.parse(execFileSync('gh', ['api', ...args], { encoding: 'utf8' })) +} + +/** + * The commit a tag names, or `null` when there is no such tag. Errors throw: + * reading a failed lookup as "no tag" would build a second release. + * + * `matching-refs`, not `git/ref`: see `publish-ffi` in release.yml. An + * annotated tag points at a tag object, which names the commit. + */ +export function tagCommitVia(repo, call = gh) { + return (tag) => { + const ref = call(`repos/${repo}/git/matching-refs/tags/${tag}`).find( + (entry) => entry.ref === `refs/tags/${tag}`, + ) + if (!ref) return null + let object = ref.object + for (let depth = 0; object.type === 'tag'; depth++) { + if (depth === 5) throw new Error(`${tag} nests tag objects too deeply`) + object = call(`repos/${repo}/git/tags/${object.sha}`).object + } + return object.sha + } +} + +function required(name) { + const value = process.env[name] + if (!value) throw new Error(`${name} is not set`) + return value +} + +function main() { + const { version } = JSON.parse( + readFileSync(join(REPO_ROOT, EQL_MANIFEST), 'utf8'), + ) + const result = eqlAssets({ + version, + published: parsePublished(process.env.PUBLISHED_PACKAGES), + npmHas: (v) => [npmVersions(EQL_PACKAGE) ?? []].flat().includes(v), + tagCommit: tagCommitVia(required('REPO')), + headSha: required('GITHUB_SHA'), + }) + + console.log(result.reason) + if (process.env.GITHUB_OUTPUT) { + appendFileSync( + process.env.GITHUB_OUTPUT, + `needed=${result.needed}\nversion=${result.version}\n` + + `prerelease=${result.prerelease}\nref=${result.ref}\n`, + ) + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + main() + } catch (err) { + console.error(`::error::${err.message}`) + process.exit(1) + } +}