diff --git a/AGENTS.md b/AGENTS.md index 430d142d7..064ee5985 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -767,6 +767,13 @@ Every command and flag named in `skills/stash-cli/SKILL.md` must resolve against manifest (the deprecated `db install` / `db upgrade` / `db status` aliases excepted — they're intentionally absent from the registry). +**The version step moves the release-train pins.** `scripts/sync-skill-pins.mjs` +runs from the root `version` script after `changeset version`. It rewrites every +exact pin of a release-train package in `skills/`, such as +`npx --package=stash@X.Y.Z` and `npm:@cipherstash/stack@X.Y.Z/wasm-inline`, to the +stable version in the tree, so the Version Packages PR carries them. Name an older +release in prose without the `name@X.Y.Z` form, or the script moves it too. + Skills must not contain Linear issue IDs; they're public. GitHub issue numbers are fine. ## Supply Chain Security diff --git a/package.json b/package.json index 1989ad5fe..4a24e2a4a 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "lint:typecheck-scope": "node scripts/lint-typecheck-scope.mjs", "lint:workflow-cache": "node scripts/lint-no-workflow-caching.mjs", "release:gate": "node scripts/release-gate.mjs", - "version": "changeset version && node scripts/sync-lockstep-versions.mjs", + "version": "changeset version && node scripts/sync-skill-pins.mjs && node scripts/sync-lockstep-versions.mjs", "release": "pnpm run build && changeset publish", "test": "turbo test --filter './languages/typescript/packages/**' --filter './packages/**'", "test:e2e": "turbo run test:e2e", diff --git a/packages/eql/docs/development/releasing.md b/packages/eql/docs/development/releasing.md index 75362279e..7265db192 100644 --- a/packages/eql/docs/development/releasing.md +++ b/packages/eql/docs/development/releasing.md @@ -39,9 +39,12 @@ at one commit. The root `version` script is the mechanism: ``` -pnpm run version == changeset version && node scripts/sync-lockstep-versions.mjs +pnpm run version == changeset version && node scripts/sync-skill-pins.mjs && node scripts/sync-lockstep-versions.mjs ``` +`scripts/sync-skill-pins.mjs` moves the `stash` release-train pins in `skills/` +and does not touch EQL. + 1. `changeset version` consumes the pending `.changeset/*.md` files and bumps `packages/eql/package.json` to the next `V`. 2. `scripts/sync-lockstep-versions.mjs` reads that `V` and propagates it: diff --git a/scripts/__tests__/sync-lockstep-versions.test.mjs b/scripts/__tests__/sync-lockstep-versions.test.mjs index baac703d1..0e71184be 100644 --- a/scripts/__tests__/sync-lockstep-versions.test.mjs +++ b/scripts/__tests__/sync-lockstep-versions.test.mjs @@ -1,6 +1,7 @@ import { spawnSync } from 'node:child_process' import { createHash } from 'node:crypto' import { + chmodSync, copyFileSync, cpSync, mkdirSync, @@ -16,10 +17,11 @@ import { afterAll, describe, expect, test } from 'vitest' import { bumpCargoPackageVersion, cargoLockWorkspaces, - LOCKED_CRATE, + pathLockedVersion, prepareBindingAssets, readEqlVersion, refreshCargoLock, + refreshCargoLocks, } from '../sync-lockstep-versions.mjs' import { REPO_ROOT } from './lib/repo-root.mjs' import { readWorkflow } from './lib/workflows.mjs' @@ -137,22 +139,35 @@ describe('lockstep Cargo.lock refresh', () => { expect(args.slice(0, 3)).toEqual(['exec', '--', 'cargo']) expect(options.cwd).toBe('/repo/packages/eql') - expect(args).toContain('--package') - expect(args).toContain(LOCKED_CRATE) + expect(args).toContain('--workspace') expect(args).toContain('--manifest-path') expect(args).toContain( '/repo/languages/typescript/packages/protect-ffi/Cargo.toml', ) }) + /** + * `--package eql-bindings` re-picks every edge whose requirement spans + * several semver-incompatible `windows-sys` versions, and each run moves + * them: on `main`'s lock it rewrites three, and on its own output it rewrites + * them again. Version Packages #1020 shipped one such line with no EQL bump. + * Measured with cargo 1.90.0, 1.94.1 and 1.99.0, which agree; `--workspace` + * changes only the `eql-bindings` version line, with or without a bump. + */ + test('does not pass --package, which rewrites unrelated windows-sys edges', () => { + const [, args] = captureRefresh() + expect(args).not.toContain('--package') + expect(args).not.toContain('-p') + }) + /** * NOT `--offline`, and the reason is a property of the job this runs in. * * `--offline` reads as free here — `eql-bindings` resolves from a path, so - * why would refreshing it need a registry? Because `cargo update -p X` does - * not update X in isolation: it re-resolves the WHOLE graph and rewrites a - * complete lock, and in offline mode every other package has to come from - * the local registry cache. `languages/typescript/packages/protect-ffi` has 167 of them. + * why would refreshing it need a registry? Because `cargo update` re-resolves + * the WHOLE graph, even with `--workspace`, and in offline mode every other + * package has to come from the local registry cache. + * `languages/typescript/packages/protect-ffi` has 167 of them. * * The release job has no such cache. `jdx/mise-action` runs there with * `install: true, cache: false` — it installs toolchains and populates @@ -169,10 +184,9 @@ describe('lockstep Cargo.lock refresh', () => { * half-applied-release failure the mise-install step above exists to prevent. * * The measurement that made `--offline` look safe was taken on a developer - * machine with a warm `~/.cargo`. Both resolutions agree — verified on the - * 3.0.4 -> 3.0.5 bump, byte-identical including the `windows-sys` edges cargo - * repaired along the way — so dropping the flag changes nothing except - * whether the step can run at all where it actually runs. + * machine with a warm `~/.cargo`. Both resolutions agree where both can run, + * so dropping the flag changes nothing except whether the step can run at + * all where it actually runs. */ test('does not pass --offline, which cannot resolve on a cold registry', () => { const [, args] = captureRefresh() @@ -221,6 +235,141 @@ describe('lockstep Cargo.lock refresh', () => { }) }) +/** A `Cargo.lock` with `eql-bindings` from a path at `version`, and a registry crate. */ +const lockAt = (version) => `# This file is automatically @generated by Cargo. +version = 4 + +[[package]] +name = "eql-bindings" +version = "${version}" +dependencies = [ + "serde", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +` + +describe('step 3 runs cargo only for a lock that needs it', () => { + test('reads the version a lock records for the crate from a path', () => { + expect(pathLockedVersion(lockAt('3.0.6'))).toBe('3.0.6') + // From a registry it is a different crate as far as this script cares. + expect( + pathLockedVersion( + '[[package]]\nname = "eql-bindings"\nversion = "3.0.6"\nsource = "registry+x"\n', + ), + ).toBeNull() + expect(pathLockedVersion('version = 4\n')).toBeNull() + }) + + /** A tree of workspaces, each with a `Cargo.lock` at the given version. */ + function tree(locks) { + const root = mkdtempSync(join(tmpdir(), 'lockstep-locks-')) + for (const [workspace, version] of Object.entries(locks)) { + mkdirSync(join(root, workspace), { recursive: true }) + writeFileSync(join(root, workspace, 'Cargo.lock'), lockAt(version)) + } + return root + } + + /** A `run` that records each workspace cargo was pointed at, and bumps its lock. */ + function fakeCargo(writes = true) { + const calls = [] + const run = (_command, args) => { + const manifest = args[args.indexOf('--manifest-path') + 1] + calls.push(manifest) + if (writes) { + writeFileSync(join(dirname(manifest), 'Cargo.lock'), lockAt('3.0.7')) + } + } + return { calls, run } + } + + test('runs no cargo when every lock already records the version', () => { + // A release that does not bump EQL: nothing may be rewritten. + const root = tree({ 'a/ffi': '3.0.6', eql: '3.0.6' }) + try { + const cargo = fakeCargo() + const result = refreshCargoLocks({ + root, + eqlRoot: join(root, 'eql'), + version: '3.0.6', + run: cargo.run, + log: () => {}, + }) + expect(cargo.calls).toEqual([]) + expect(result).toEqual({ workspaces: ['a/ffi', 'eql'], refreshed: [] }) + expect(readFileSync(join(root, 'a/ffi/Cargo.lock'), 'utf8')).toBe( + lockAt('3.0.6'), + ) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + test('runs cargo for each lock at another version, and only those', () => { + const root = tree({ 'a/ffi': '3.0.6', eql: '3.0.7' }) + try { + const cargo = fakeCargo() + const result = refreshCargoLocks({ + root, + eqlRoot: join(root, 'eql'), + version: '3.0.7', + run: cargo.run, + log: () => {}, + }) + expect(cargo.calls).toEqual([join(root, 'a/ffi/Cargo.toml')]) + expect(result.refreshed).toEqual(['a/ffi']) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + test('fails when cargo ran and the lock still records another version', () => { + const root = tree({ ffi: '3.0.6' }) + try { + expect(() => + refreshCargoLocks({ + root, + eqlRoot: root, + version: '3.0.7', + run: fakeCargo(false).run, + log: () => {}, + }), + ).toThrow(/records eql-bindings 3\.0\.6, not 3\.0\.7/) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + + test('runs no cargo over this tree at its own EQL version', () => { + // The committed locks against the committed version: what the release job + // sees on every release that leaves EQL alone. + const version = JSON.parse( + readFileSync( + join(REPO_ROOT, 'packages/eql/packages/eql/package.json'), + 'utf8', + ), + ).version + // A recorder that writes nothing: this tree's locks are real files. + const cargo = fakeCargo(false) + try { + refreshCargoLocks({ + root: REPO_ROOT, + eqlRoot: join(REPO_ROOT, 'packages/eql'), + version, + run: cargo.run, + log: () => {}, + }) + } catch { + // The after-check throws once cargo was called; the call is the finding. + } + expect(cargo.calls).toEqual([]) + }) +}) + /** * Step 4, and the release-stopper it used to arm. * @@ -490,6 +639,96 @@ describe('lockstep main-guard', () => { }) }) +/** + * Step 3 through the real script, with `mise` replaced on PATH by a recorder + * that bumps the lock it is pointed at. Step 4 then fails on the fixture's + * missing SQL assets; only what step 3 did is asserted. + */ +describe('the script runs cargo only for a stale lock', () => { + function runWithLock(lockedVersion) { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'lockstep-step3-'))) + const bin = join(root, 'bin') + const lockPath = join(root, 'languages/ffi/Cargo.lock') + for (const dir of [ + bin, + join(root, 'scripts'), + join(root, 'packages/eql/crates/eql-bindings'), + join(root, 'packages/eql/packages/eql'), + dirname(lockPath), + ]) { + mkdirSync(dir, { recursive: true }) + } + copyFileSync( + join(REPO_ROOT, 'scripts/sync-lockstep-versions.mjs'), + join(root, 'scripts/sync-lockstep-versions.mjs'), + ) + writeFileSync( + join(root, 'packages/eql/packages/eql/package.json'), + JSON.stringify({ name: '@cipherstash/eql', version: '9.9.9' }), + ) + writeFileSync( + join(root, 'packages/eql/crates/eql-bindings/Cargo.toml'), + '[package]\nname = "eql-bindings"\nversion = "9.9.8"\n', + ) + writeFileSync(lockPath, lockAt(lockedVersion)) + writeFileSync( + join(bin, 'mise'), + '#!/usr/bin/env node\n' + + "const fs = require('node:fs'), path = require('node:path')\n" + + 'const args = process.argv.slice(2)\n' + + "fs.appendFileSync(process.env.MISE_LOG, args.join(' ') + '\\n')\n" + + "const at = args.indexOf('--manifest-path')\n" + + 'if (at !== -1) {\n' + + " const lock = path.join(path.dirname(args[at + 1]), 'Cargo.lock')\n" + + ' fs.writeFileSync(lock, fs.readFileSync(lock, \'utf8\').replace(/^version = "9\\.9\\.8"$/m, \'version = "9.9.9"\'))\n' + + '}\n', + ) + chmodSync(join(bin, 'mise'), 0o755) + const miseLog = join(root, 'mise.log') + writeFileSync(miseLog, '') + const { stdout } = spawnSync( + process.execPath, + [join(root, 'scripts/sync-lockstep-versions.mjs')], + { + cwd: root, + encoding: 'utf8', + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + MISE_LOG: miseLog, + }, + }, + ) + const result = { + stdout, + calls: readFileSync(miseLog, 'utf8').split('\n').filter(Boolean), + lock: readFileSync(lockPath, 'utf8'), + } + rmSync(root, { recursive: true, force: true }) + return result + } + + test('leaves a lock that already records the version untouched', () => { + const { stdout, calls, lock } = runWithLock('9.9.9') + // Proof that step 3 ran and chose to skip, rather than never being reached. + expect(stdout).toContain( + 'languages/ffi/Cargo.lock already records eql-bindings 9.9.9', + ) + expect(calls.filter((call) => call.includes('cargo'))).toEqual([]) + expect(lock).toBe(lockAt('9.9.9')) + }) + + test('refreshes a lock at the previous version', () => { + const { calls, lock } = runWithLock('9.9.8') + expect(calls.filter((call) => call.includes('cargo'))).toEqual([ + expect.stringMatching( + /^exec -- cargo update --workspace --manifest-path .*\/languages\/ffi\/Cargo\.toml$/, + ), + ]) + expect(pathLockedVersion(lock)).toBe('9.9.9') + }) +}) + /** * The invariant the whole script exists to maintain, asserted against the * committed tree. diff --git a/scripts/__tests__/sync-skill-pins.test.mjs b/scripts/__tests__/sync-skill-pins.test.mjs new file mode 100644 index 000000000..e6cd74a01 --- /dev/null +++ b/scripts/__tests__/sync-skill-pins.test.mjs @@ -0,0 +1,196 @@ +import { spawnSync } from 'node:child_process' +import { + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { RELEASE_TRAIN_MANIFESTS } from '../../languages/typescript/packages/cli/src/release-train.ts' +import { + releaseTrain, + stableVersion, + syncPins, + syncSkillPins, + trainVersions, +} from '../sync-skill-pins.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' + +/** + * The version step moves the skill pins, so a Version Packages PR cannot leave + * them behind. #1020 did, because no CI ran on it, and `main` then failed + * `release-train.test.ts` until #1029 moved them by hand. + */ + +const CONFIG = JSON.parse( + readFileSync(join(REPO_ROOT, '.changeset/config.json'), 'utf8'), +) +const VERSIONS = new Map([ + ['stash', '1.2.1'], + ['@cipherstash/stack', '1.2.1'], +]) + +describe('the release train', () => { + it('is the fixed group that holds stash, and the CLI release train', () => { + // Two lists of one set: changesets moves these together, and + // release-train.test.ts checks their pins. + expect(releaseTrain(CONFIG).sort()).toEqual( + Object.keys(RELEASE_TRAIN_MANIFESTS).sort(), + ) + }) + + it('refuses a config with no group holding stash', () => { + expect(() => releaseTrain({ fixed: [['@cipherstash/auth']] })).toThrow( + /holds `stash`/, + ) + }) + + it('reads each version from the manifest the CLI embeds', () => { + const versions = trainVersions(REPO_ROOT, releaseTrain(CONFIG)) + for (const [name, rel] of Object.entries(RELEASE_TRAIN_MANIFESTS)) { + const manifest = JSON.parse( + readFileSync( + join(REPO_ROOT, 'languages/typescript/packages/cli', rel), + 'utf8', + ), + ) + expect(versions.get(name), name).toBe(manifest.version) + } + }) + + it('refuses a train package with no manifest', () => { + expect(() => + trainVersions(REPO_ROOT, ['stash', '@cipherstash/no-such-package']), + ).toThrow(/@cipherstash\/no-such-package/) + }) +}) + +describe('the pins', () => { + it('move the three forms the skills use', () => { + const before = [ + "npx --package=stash@1.2.0 stash eql install --database-url 'postgres://...'", + "} from 'npm:@cipherstash/stack@1.2.0/wasm-inline'", + '"@cipherstash/stack/wasm-inline": "npm:@cipherstash/stack@1.2.0/wasm-inline"', + ].join('\n') + const { text, changes } = syncPins(before, VERSIONS) + expect(text).toBe( + [ + "npx --package=stash@1.2.1 stash eql install --database-url 'postgres://...'", + "} from 'npm:@cipherstash/stack@1.2.1/wasm-inline'", + '"@cipherstash/stack/wasm-inline": "npm:@cipherstash/stack@1.2.1/wasm-inline"', + ].join('\n'), + ) + expect(changes).toHaveLength(3) + }) + + it('pin the stable version during a prerelease', () => { + expect(stableVersion('1.3.0-rc.0')).toBe('1.3.0') + const { text } = syncPins( + 'npx --package=stash@1.2.1 stash', + new Map([['stash', '1.3.0-rc.0']]), + ) + expect(text).toBe('npx --package=stash@1.3.0 stash') + }) + + it('leave everything that is not a release-train pin alone', () => { + const text = [ + '@cipherstash/eql@3.0.6', + '@cipherstash/protect-ffi@0.33.0', + '"stash": "^1.0.0"', + 'cipherstash@1.0.0', + 'mystash@1.0.0', + 'stash@latest', + ].join('\n') + expect(syncPins(text, VERSIONS)).toEqual({ text, changes: [] }) + }) + + it('change nothing that is already current', () => { + const text = 'npx --package=stash@1.2.1 stash eql install' + expect(syncPins(text, VERSIONS)).toEqual({ text, changes: [] }) + }) + + it('in this tree already name the release-train versions', () => { + // What the next Version Packages PR starts from. A failure here names the + // file: run `node scripts/sync-skill-pins.mjs`. + expect(syncSkillPins({ root: REPO_ROOT, write: false })).toEqual([]) + }) +}) + +/** The real script over a throwaway tree with one stale skill. */ +describe('the script', () => { + let root + afterEach(() => root && rmSync(root, { recursive: true, force: true })) + + function fixture() { + root = realpathSync(mkdtempSync(join(tmpdir(), 'skill-pins-'))) + const write = (rel, text) => { + mkdirSync(dirname(join(root, rel)), { recursive: true }) + writeFileSync(join(root, rel), text) + } + write( + '.changeset/config.json', + JSON.stringify({ fixed: [['stash', '@cipherstash/stack']] }), + ) + write( + 'languages/typescript/packages/cli/package.json', + JSON.stringify({ name: 'stash', version: '9.9.9' }), + ) + write( + 'languages/typescript/packages/stack/package.json', + JSON.stringify({ name: '@cipherstash/stack', version: '9.9.9' }), + ) + write( + 'skills/stash-cli/SKILL.md', + 'npx --package=stash@9.9.8 stash eql install\n', + ) + write('skills/other/SKILL.md', 'nothing pinned here\n') + mkdirSync(join(root, 'scripts')) + copyFileSync( + join(REPO_ROOT, 'scripts/sync-skill-pins.mjs'), + join(root, 'scripts/sync-skill-pins.mjs'), + ) + return () => + spawnSync(process.execPath, [join(root, 'scripts/sync-skill-pins.mjs')], { + cwd: root, + encoding: 'utf8', + }) + } + + it('moves a stale pin, and a second run changes nothing', () => { + const run = fixture() + const first = run() + expect(first.status).toBe(0) + expect(first.stdout).toContain( + 'skills/stash-cli/SKILL.md: stash@9.9.8 -> stash@9.9.9', + ) + expect(readFileSync(join(root, 'skills/stash-cli/SKILL.md'), 'utf8')).toBe( + 'npx --package=stash@9.9.9 stash eql install\n', + ) + + const second = run() + expect(second.status).toBe(0) + expect(second.stdout).toContain('already name the release-train versions') + }) +}) + +describe('the root version script moves the pins', () => { + const version = JSON.parse( + readFileSync(join(REPO_ROOT, 'package.json'), 'utf8'), + ).scripts.version + + it('runs the pin sync after `changeset version`, which sets the versions', () => { + expect(version).toContain('node scripts/sync-skill-pins.mjs') + expect(version.indexOf('changeset version')).toBeLessThan( + version.indexOf('sync-skill-pins.mjs'), + ) + // `&&`, so a failed sync stops the release rather than shipping stale pins. + expect(version).toMatch( + /changeset version && .*node scripts\/sync-skill-pins\.mjs &&/, + ) + }) +}) diff --git a/scripts/__tests__/workflow-mise-setup.test.mjs b/scripts/__tests__/workflow-mise-setup.test.mjs index d19d330b0..2f42e8dd3 100644 --- a/scripts/__tests__/workflow-mise-setup.test.mjs +++ b/scripts/__tests__/workflow-mise-setup.test.mjs @@ -18,7 +18,7 @@ import { readWorkflow, workflowFiles } from './lib/workflows.mjs' * is three hops deep and entirely invisible in the workflow file: * * version: pnpm run version (release.yml) - * -> "changeset version && node scripts/sync-lockstep-versions.mjs" + * -> "changeset version && … && node scripts/sync-lockstep-versions.mjs" * (root package.json) * -> execFileSync('mise', ['run', 'release:prepare_bindings_assets'…]) * (that script) diff --git a/scripts/sync-lockstep-versions.mjs b/scripts/sync-lockstep-versions.mjs index 99a3f0884..97b27f718 100644 --- a/scripts/sync-lockstep-versions.mjs +++ b/scripts/sync-lockstep-versions.mjs @@ -11,7 +11,8 @@ // It (paths relative to the monorepo root): // 1. reads V from packages/eql/packages/eql/package.json, // 2. sets packages/eql/crates/eql-bindings/Cargo.toml [package] version = V, -// 3. re-resolves that crate in every Cargo.lock that records it from a path, +// 3. re-resolves that crate in every Cargo.lock that records it from a path +// at another version — a lock already at V is left alone, // 4. runs `mise run release:prepare_bindings_assets --version V`, which builds // the exact-version SQL and writes it (+ release manifests) into both the // crate and the npm package — UNLESS the tree already carries those assets @@ -96,21 +97,28 @@ export function cargoLockWorkspaces(root) { } if (entry.name !== 'Cargo.lock') continue const lock = readFileSync(join(abs, entry.name), 'utf8') - const records = lock - .split(/^\[\[package\]\]$/m) - .slice(1) - .some( - (block) => - new RegExp(`^name = "${LOCKED_CRATE}"$`, 'm').test(block) && - !/^source = "/m.test(block), - ) - if (records) found.push(relative(root, abs).split(sep).join('/')) + if (pathLockedVersion(lock) !== null) { + found.push(relative(root, abs).split(sep).join('/')) + } } } walk(root) return found.sort() } +/** The version a `Cargo.lock` records for `LOCKED_CRATE` from a path, or null. */ +export function pathLockedVersion(lock) { + for (const block of lock.split(/^\[\[package\]\]$/m).slice(1)) { + if ( + new RegExp(`^name = "${LOCKED_CRATE}"$`, 'm').test(block) && + !/^source = "/m.test(block) + ) { + return /^version = "([^"]*)"$/m.exec(block)?.[1] ?? null + } + } + return null +} + /** * Re-resolve `LOCKED_CRATE` in one workspace's `Cargo.lock`. * @@ -122,22 +130,23 @@ export function cargoLockWorkspaces(root) { * `eql-codegen` for the SQL build a few lines below, so there is one Rust here * rather than two. * - * NOT `--offline`, however tempting it looks. `eql-bindings` resolves from a - * path, so the flag reads as free — but `cargo update -p X` does not update X - * in isolation. It re-resolves the whole graph and rewrites a complete lock, - * and offline that means every OTHER package has to be served from the local - * registry cache; `languages/typescript/packages/protect-ffi` has 167 of them. The release job has - * no such cache — `jdx/mise-action` runs there with `cache: false`, installing - * toolchains and populating nothing under `~/.cargo/registry`, and - * `scripts/lint-no-workflow-caching.mjs` forbids a cache restore anywhere an - * artifact is published. So `--offline` died on the first call with - * `error: no matching package named \`chrono\` found`, taking `pnpm run version` - * with it AFTER `changeset version` had rewritten every manifest and CHANGELOG. + * `--workspace`, NOT `--package eql-bindings`. A path dependency's version is + * read from its manifest, so `--workspace` records the bump and keeps every + * other locked edge. `--package` also re-picks the edges whose requirement + * spans several semver-incompatible `windows-sys` versions (`winapi-util` + * 0.1.11 asks for `>=0.48.0, <=0.61`). Every pick is valid, so `--locked` + * passes either way, but each run moves them: Version Packages #1020 rewrote + * one with no EQL bump at all. cargo 1.90.0, 1.94.1 and 1.99.0 all do it. * - * The two resolutions agree where both can run — verified byte-identical on the - * 3.0.4 -> 3.0.5 bump, `windows-sys` edges included — so this is a change of - * where the step works, not of what it produces. Asserted, with the CI - * precondition it depends on, in scripts/__tests__/sync-lockstep-versions.test.mjs. + * NOT `--offline`, however tempting it looks. `cargo update` re-resolves the + * whole graph, and offline that means every OTHER package has to be served + * from the local registry cache; `languages/typescript/packages/protect-ffi` + * has 167 of them. The release job has no such cache — `jdx/mise-action` runs + * there with `cache: false`, and `scripts/lint-no-workflow-caching.mjs` + * forbids a cache restore anywhere an artifact is published. So `--offline` + * died on the first call with `error: no matching package named \`chrono\` + * found`, taking `pnpm run version` with it AFTER `changeset version` had + * rewritten every manifest and CHANGELOG. * * `--manifest-path` rather than a second `cwd`, so every cargo invocation in * this script runs from the one directory whose mise config is trusted. @@ -158,8 +167,7 @@ export function refreshCargoLock({ '--', 'cargo', 'update', - '--package', - LOCKED_CRATE, + '--workspace', '--manifest-path', join(root, workspace, 'Cargo.toml'), ], @@ -167,6 +175,53 @@ export function refreshCargoLock({ ) } +/** + * Step 3: refresh each lock that records `LOCKED_CRATE` at another version. + * + * A lock already at `version` is not handed to cargo at all, so a release that + * does not bump EQL leaves every `Cargo.lock` byte-for-byte as it was. A lock + * that still disagrees after cargo ran fails the release here, before the SQL + * build, rather than in `cargo-lock-freshness.test.mjs` on the next PR. + */ +export function refreshCargoLocks({ + root, + eqlRoot, + version, + run = execFileSync, + log = console.log, +}) { + const workspaces = cargoLockWorkspaces(root) + if (workspaces.length === 0) { + throw new Error( + `no Cargo.lock under ${root} records \`${LOCKED_CRATE}\` as a path dependency — ` + + 'the scan that finds the locks to refresh has stopped matching, so the bump above ' + + 'would leave every one of them stale.', + ) + } + const lockedIn = (workspace) => + pathLockedVersion(readFileSync(join(root, workspace, 'Cargo.lock'), 'utf8')) + + const refreshed = [] + for (const workspace of workspaces) { + if (lockedIn(workspace) === version) { + log( + `${workspace}/Cargo.lock already records ${LOCKED_CRATE} ${version}; not running cargo`, + ) + continue + } + refreshCargoLock({ root, eqlRoot, workspace, run }) + const after = lockedIn(workspace) + if (after !== version) { + throw new Error( + `cargo update ran in ${workspace} and its Cargo.lock records ${LOCKED_CRATE} ` + + `${after}, not ${version}.`, + ) + } + refreshed.push(workspace) + } + return { workspaces, refreshed } +} + /** * The EQL subtree, repo-relative. * @@ -517,17 +572,11 @@ function main() { // Before the SQL build, not after: this is cheap, so a cargo that cannot run // should stop the release in seconds rather than after a full eql-codegen // compile. - const workspaces = cargoLockWorkspaces(stackRoot) - if (workspaces.length === 0) { - throw new Error( - `no Cargo.lock under ${stackRoot} records \`${LOCKED_CRATE}\` as a path dependency — ` + - 'the scan that finds the locks to refresh has stopped matching, so the bump above ' + - 'would leave every one of them stale.', - ) - } - for (const workspace of workspaces) { - refreshCargoLock({ root: stackRoot, eqlRoot, workspace }) - } + const { workspaces, refreshed } = refreshCargoLocks({ + root: stackRoot, + eqlRoot, + version, + }) // Build the exact-version SQL and copy it (+ manifests) into both packages — // unless the tree already carries them at this version. @@ -535,7 +584,8 @@ function main() { console.log( `synced EQL lockstep version ${version} to Cargo.toml, ` + - `${workspaces.length} Cargo.lock (${workspaces.join(', ')}) + bundled SQL assets ` + + `${refreshed.length} of ${workspaces.length} Cargo.lock refreshed ` + + `(${refreshed.join(', ') || 'none needed it'}) + bundled SQL assets ` + `(${assets.action})`, ) } diff --git a/scripts/sync-skill-pins.mjs b/scripts/sync-skill-pins.mjs new file mode 100644 index 000000000..0b967021f --- /dev/null +++ b/scripts/sync-skill-pins.mjs @@ -0,0 +1,133 @@ +/** + * Move the release-train version pins in `skills/` to the versions in the tree. + * + * `skills/` ships inside the `stash` tarball verbatim, and no build step + * rewrites a version inside it, so a pin such as `npx --package=stash@1.2.0` + * keeps telling customers to install the previous release. + * `release-train.test.ts` fails a tree whose `stash-cli` pin is not the + * `stash` version. People moved the pins by hand in each Version Packages PR: + * #928 and #938 did, and #1020 did not, because no CI ran on it. + * + * This runs from the root `version` script, right after `changeset version`, + * so the Version Packages PR carries the pins with the versions they name. + * + * The packages are the changesets `fixed` group that holds `stash`: the ones + * `changeset version` moves together. A test holds that group to the CLI's + * `RELEASE_TRAIN_MANIFESTS`, which is the set `release-train.test.ts` checks. + * A pin takes the STABLE version, because a prerelease must not be pinned in a + * customer's repo; that is the test's rule too. + */ +import { readdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join, relative, resolve, sep } from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' + +const REPO_ROOT = resolve(import.meta.dirname, '..') + +export const SKILLS_DIR = 'skills' +export const PACKAGES_DIR = 'languages/typescript/packages' + +/** + * `release-train.test.ts`'s pin form: an exact `name@X.Y.Z[-pre]`, optionally + * `npm:`-prefixed. A range such as `^1.0.0` is not a pin. The lookbehind stops + * `stash` matching inside a longer name. + */ +const PIN = + /(? version.split('-')[0] + +/** The names in the changesets `fixed` group that holds `stash`. */ +export function releaseTrain(changesetConfig) { + const group = (changesetConfig.fixed ?? []).find((names) => + names.includes('stash'), + ) + if (!group) { + throw new Error( + 'no `fixed` group in .changeset/config.json holds `stash`, so there is no ' + + 'release train to pin the skills to', + ) + } + return [...group] +} + +/** name -> version for each `names` entry, from the package manifests. */ +export function trainVersions(root, names) { + const found = new Map() + const dir = join(root, PACKAGES_DIR) + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue + let manifest + try { + manifest = JSON.parse( + readFileSync(join(dir, entry.name, 'package.json'), 'utf8'), + ) + } catch (err) { + if (err.code === 'ENOENT') continue + throw err + } + if (names.includes(manifest.name)) + found.set(manifest.name, manifest.version) + } + const missing = names.filter((name) => !found.has(name)) + if (missing.length > 0) { + throw new Error( + `no manifest under ${PACKAGES_DIR}/ names ${missing.join(', ')}, so their skill ` + + 'pins cannot be moved', + ) + } + return found +} + +/** Rewrite each train pin in `text` to its stable version. */ +export function syncPins(text, versions) { + const changes = [] + const out = text.replace(PIN, (whole, prefix, pkg, pinned) => { + if (!versions.has(pkg)) return whole + const to = stableVersion(versions.get(pkg)) + if (pinned === to) return whole + changes.push({ pkg, from: pinned, to }) + return `${prefix}${pkg}@${to}` + }) + return { text: out, changes } +} + +function markdownFiles(dir) { + return readdirSync(dir, { withFileTypes: true, recursive: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith('.md')) + .map((entry) => join(entry.parentPath, entry.name)) + .sort() +} + +/** Every changed file, repo-relative, with its changes. Writes unless `write` is false. */ +export function syncSkillPins({ root = REPO_ROOT, write = true } = {}) { + const config = JSON.parse( + readFileSync(join(root, '.changeset/config.json'), 'utf8'), + ) + const versions = trainVersions(root, releaseTrain(config)) + const changed = [] + for (const file of markdownFiles(join(root, SKILLS_DIR))) { + const before = readFileSync(file, 'utf8') + const { text, changes } = syncPins(before, versions) + if (changes.length === 0) continue + if (write) writeFileSync(file, text) + changed.push({ file: relative(root, file).split(sep).join('/'), changes }) + } + return changed +} + +function main() { + const changed = syncSkillPins() + for (const { file, changes } of changed) { + for (const { pkg, from, to } of changes) { + console.log(`${file}: ${pkg}@${from} -> ${pkg}@${to}`) + } + } + console.log( + changed.length === 0 + ? 'skill pins already name the release-train versions' + : `moved the release-train pins in ${changed.length} skill file(s)`, + ) +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) main()