From 7ed43360f9def66e2c7f6c1f11b990df3ac54908 Mon Sep 17 00:00:00 2001 From: bing Date: Tue, 6 Oct 2026 11:41:51 +0800 Subject: [PATCH] feat: build and publish preset binaries Add an optional zapi.presets list to package.json. Each preset is a build of the same step with extra zig build args. The 1st preset is the mainnet preset (the default) and keeps .node. The 2nd preset is the minimal preset -minimal.node `requireNapiLibrary(dir, preset)` selects one at load time. `zapi build --preset ` renames a local build to match. A preset is bundled into the target packages unless it is marked standalone, in which case prepublish and publish emit it as its own -- packages that are left out of the main package's `optionalDependencies`, so consumers download it only when they depend on it explicitly. Motivation: lodestar-z compiles the consensus preset in, and lodestar's minimal-preset spec tests need a minimal binary that production installs should not have to download. --- README.md | 35 ++++- examples/presets/mod.test.ts | 292 +++++++++++++++++++++++++++++++++++ ts/build.ts | 67 ++++++-- ts/buildArtifacts.ts | 116 ++++++++------ ts/cli.ts | 10 +- ts/config.ts | 130 ++++++++++++++-- ts/lib.ts | 16 +- ts/prepublish.ts | 66 ++++---- ts/publish.ts | 41 ++--- 9 files changed, 642 insertions(+), 131 deletions(-) create mode 100644 examples/presets/mod.test.ts diff --git a/README.md b/README.md index 806dca9..56cacff 100644 --- a/README.md +++ b/README.md @@ -751,11 +751,26 @@ Add a `zapi` field to your `package.json`: "x86_64-apple-darwin", "aarch64-apple-darwin", "x86_64-pc-windows-msvc" + ], + "presets": [ + { "name": "default" }, + { "name": "small", "args": ["-Dsmall=true"] }, + { "name": "debug", "args": ["-Ddebug-hooks=true"], "standalone": true } ] } } ``` +`presets` is optional. Each preset is a separate build of the same step with its own extra +`zig build` args. The first preset is the default and keeps the `.node` file name, +so a config without `presets` behaves exactly as before. Every later preset is built as +`-.node` and selected at runtime with `requireNapiLibrary(dir, "")`. + +By default a preset's file is bundled into the same target packages as the default preset. +A preset marked `"standalone": true` is published instead as its own `--` +packages, which are not added to the main package's `optionalDependencies`, so consumers only +download it when they depend on it explicitly. + ### Supported Targets | Target | Platform | Arch | ABI | @@ -790,11 +805,14 @@ zapi build [options] | `--step` | Zig build step | `zapi.step` from package.json | | `--target` | Target triple | Current platform | | `--optimize` | `Debug`, `ReleaseSafe`, `ReleaseFast`, `ReleaseSmall` | - | +| `--preset` | Preset name from `zapi.presets` | First preset | | `--zig-cwd` | Working directory for zig build | `.` | +A non-default preset is renamed to `zig-out/lib/-.node` after the build. + #### `zapi build-artifacts` -Build for all configured targets and collect artifacts. +Build every preset for all configured targets and collect artifacts. ```bash zapi build-artifacts [options] @@ -821,8 +839,8 @@ zapi build-artifacts [options] #### `zapi prepublish` Prepare npm packages for publishing: -- Creates `npm//` directories for each target -- Moves compiled `.node` binaries from artifacts into target packages +- Creates `npm//` directories for each target, and `npm/-/` for each standalone preset +- Moves every preset's compiled `.node` binary from artifacts into its package - Generates `package.json` for each target package (with correct `os`, `cpu`, `libc`) - Updates the main `package.json` with `optionalDependencies` @@ -893,9 +911,10 @@ Set `DEBUG=1` for full stack traces on errors. ## Runtime Loading -### `requireNapiLibrary(packageDir)` +### `requireNapiLibrary(packageDir, preset?)` -Load the native addon, automatically selecting the correct binary for the current platform: +Load the native addon, automatically selecting the correct binary for the current platform. +`preset` is one of `zapi.presets[].name` and defaults to the first preset: ```typescript import { requireNapiLibrary } from "@chainsafe/zapi"; @@ -906,9 +925,9 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const addon = requireNapiLibrary(__dirname); ``` -Resolution order: -1. Local build: `zig-out/lib/.node` -2. Published package: `-` +Resolution order (`` is `.node` for the default preset, `-.node` otherwise): +1. Local build: `zig-out/lib/` +2. Published package: `-/`, or `--/` for a standalone preset --- diff --git a/examples/presets/mod.test.ts b/examples/presets/mod.test.ts new file mode 100644 index 0000000..60439e4 --- /dev/null +++ b/examples/presets/mod.test.ts @@ -0,0 +1,292 @@ +import {chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; +import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; +import {build, buildLocal} from "../../ts/build.js"; +import {buildArtifacts} from "../../ts/buildArtifacts.js"; +import {type PkgJson, parsePkgJson, resolvePreset, targetPackages} from "../../ts/config.js"; +import {getTarget, requireNapiLibrary} from "../../ts/lib.js"; +import {createNpmDirs, moveArtifacts, updateOptionalDependencies, updateTargetPkgJsons} from "../../ts/prepublish.js"; +import {publish} from "../../ts/publish.js"; + +const target = "aarch64-apple-darwin"; +const otherTarget = "x86_64-unknown-linux-gnu"; + +const pkgJson: PkgJson = { + name: "@scope/addon", + version: "1.2.3", + zapi: { + binaryName: "addon", + step: "lib", + targets: [target, otherTarget], + presets: [ + {name: "mainnet", args: ["-Dpreset=mainnet"]}, + {name: "minimal", args: ["-Dpreset=minimal"]}, + ], + }, +}; + +const standalonePkgJson: PkgJson = { + ...pkgJson, + zapi: { + ...pkgJson.zapi!, + presets: [ + {name: "mainnet", args: ["-Dpreset=mainnet"]}, + {name: "minimal", args: ["-Dpreset=minimal"], standalone: true}, + ], + }, +}; + +// Concurrent builds append to one log, so each call must land as a single write. +const FAKE_ZIG = `#!/bin/sh +prefix="" +prev="" +line="" +for arg in "$@"; do + if [ "$prev" = "--prefix" ]; then prefix="$arg"; fi + line="$line$arg " + prev="$arg" +done +printf '%s\\n' "$line" >> "$ZAPI_FAKE_ZIG_LOG" +libdir="\${prefix:-zig-out}/lib" +mkdir -p "$libdir" +: > "$libdir/addon.node" +`; + +let tmp: string; +let zigLog: string; +const savedEnv = {...process.env}; + +function zigCalls(): string[] { + return readFileSync(zigLog, "utf-8").trim().split("\n").map((line) => line.trim()); +} + +function withPresets(presets: unknown): PkgJson { + return {...pkgJson, zapi: {...pkgJson.zapi!, presets: presets as NonNullable["presets"]}}; +} + +beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "zapi-presets-")); + zigLog = join(tmp, "zig-calls.log"); + const bin = join(tmp, "bin"); + mkdirSync(bin); + writeFileSync(join(bin, "zig"), FAKE_ZIG); + chmodSync(join(bin, "zig"), 0o755); + process.env.PATH = `${bin}:${savedEnv.PATH}`; + process.env.ZAPI_FAKE_ZIG_LOG = zigLog; +}); + +afterEach(() => { + process.env.PATH = savedEnv.PATH; + delete process.env.ZAPI_FAKE_ZIG_LOG; + vi.restoreAllMocks(); +}); + +describe("presets config", () => { + it("defaults to a single unsuffixed preset", () => { + const config = parsePkgJson({name: "addon", version: "0.0.0", zapi: {binaryName: "addon", targets: [target]}}); + expect(config.presets).toEqual([{name: "default", args: [], file: "addon.node", standalone: false}]); + }); + + it("suffixes every preset after the first", () => { + const config = parsePkgJson(pkgJson); + expect(config.presets).toEqual([ + {name: "mainnet", args: ["-Dpreset=mainnet"], file: "addon.node", standalone: false}, + {name: "minimal", args: ["-Dpreset=minimal"], file: "addon-minimal.node", standalone: false}, + ]); + }); + + it("parses the standalone flag", () => { + const config = parsePkgJson(standalonePkgJson); + expect(config.presets.map((preset) => preset.standalone)).toEqual([false, true]); + }); + + it("rejects duplicate preset names", () => { + expect(() => parsePkgJson(withPresets([{name: "a"}, {name: "a"}]))).toThrow("zapi.presets"); + }); + + it("rejects preset names that are not file-name safe", () => { + expect(() => parsePkgJson(withPresets([{name: "min/imal"}]))).toThrow("zapi.presets"); + }); + + it("rejects a standalone default preset", () => { + expect(() => parsePkgJson(withPresets([{name: "a", standalone: true}]))).toThrow("zapi.presets[0]"); + }); + + it("rejects a non-boolean standalone flag", () => { + expect(() => parsePkgJson(withPresets([{name: "a"}, {name: "b", standalone: "yes"}]))).toThrow( + "zapi.presets[].standalone" + ); + }); + + it("resolves the first preset when none is requested", () => { + const config = parsePkgJson(pkgJson); + expect(resolvePreset(config).name).toBe("mainnet"); + expect(resolvePreset(config, "minimal").file).toBe("addon-minimal.node"); + }); + + it("rejects unknown preset names", () => { + const config = parsePkgJson(pkgJson); + expect(() => resolvePreset(config, "gnosis")).toThrow(/gnosis/); + }); + + it("lists a standalone preset as its own package", () => { + const packages = targetPackages(standalonePkgJson, parsePkgJson(standalonePkgJson)); + expect(packages.map((pkg) => pkg.name)).toEqual([ + `@scope/addon-${target}`, + `@scope/addon-minimal-${target}`, + `@scope/addon-${otherTarget}`, + `@scope/addon-minimal-${otherTarget}`, + ]); + expect(packages[0]).toMatchObject({dir: target, files: ["addon.node"], optional: true}); + expect(packages[1]).toMatchObject({dir: `minimal-${target}`, files: ["addon-minimal.node"], optional: false}); + }); +}); + +describe("build", () => { + it("passes preset args to zig build", async () => { + await build({extraArgs: ["-Dpreset=minimal"], quiet: true, step: "lib", target, zigCwd: tmp}); + expect(zigCalls()).toEqual(["build lib -Dtarget=aarch64-macos-none -Dpreset=minimal"]); + }); + + it("renames a local non-default preset build to its preset file", async () => { + const config = parsePkgJson(pkgJson); + await buildLocal({config, preset: resolvePreset(config, "minimal"), target, zigCwd: tmp}); + expect(existsSync(join(tmp, "zig-out/lib/addon-minimal.node"))).toBe(true); + expect(existsSync(join(tmp, "zig-out/lib/addon.node"))).toBe(false); + }); + + it("keeps the default preset file name for a local build", async () => { + const config = parsePkgJson(pkgJson); + await buildLocal({config, preset: resolvePreset(config), target, zigCwd: tmp}); + expect(existsSync(join(tmp, "zig-out/lib/addon.node"))).toBe(true); + }); +}); + +describe("build-artifacts", () => { + it("builds every preset for every target", async () => { + const config = parsePkgJson(pkgJson); + const artifactsDir = join(tmp, "artifacts"); + await buildArtifacts({artifactsDir, concurrency: 2, config, step: "lib", zigCwd: tmp}); + + for (const t of [target, otherTarget]) { + expect(existsSync(join(artifactsDir, t, "addon.node")), `${t} default`).toBe(true); + expect(existsSync(join(artifactsDir, t, "addon-minimal.node")), `${t} minimal`).toBe(true); + } + const calls = zigCalls(); + expect(calls).toHaveLength(4); + expect(calls.filter((c) => c.includes("-Dpreset=minimal"))).toHaveLength(2); + }); +}); + +describe("prepublish", () => { + const opts = () => ({"artifacts-dir": join(tmp, "artifacts"), concurrency: 1, "npm-dir": join(tmp, "npm")}); + + function writeArtifacts(): void { + for (const t of [target, otherTarget]) { + mkdirSync(join(tmp, "artifacts", t), {recursive: true}); + writeFileSync(join(tmp, "artifacts", t, "addon.node"), ""); + writeFileSync(join(tmp, "artifacts", t, "addon-minimal.node"), ""); + } + } + + it("moves every bundled preset file into the target package", async () => { + const config = parsePkgJson(pkgJson); + writeArtifacts(); + await createNpmDirs(pkgJson, config, opts()); + await moveArtifacts(pkgJson, config, opts()); + expect(existsSync(join(tmp, "npm", target, "addon.node"))).toBe(true); + expect(existsSync(join(tmp, "npm", target, "addon-minimal.node"))).toBe(true); + }); + + it("lists every bundled preset file in the target package.json", async () => { + const config = parsePkgJson(pkgJson); + await createNpmDirs(pkgJson, config, opts()); + await updateTargetPkgJsons(pkgJson, config, opts()); + const written = JSON.parse(readFileSync(join(tmp, "npm", target, "package.json"), "utf-8")); + expect(written.files).toEqual(["addon.node", "addon-minimal.node"]); + expect(written.main).toBe("addon.node"); + }); + + it("puts a standalone preset in its own package", async () => { + const config = parsePkgJson(standalonePkgJson); + writeArtifacts(); + await createNpmDirs(standalonePkgJson, config, opts()); + await moveArtifacts(standalonePkgJson, config, opts()); + await updateTargetPkgJsons(standalonePkgJson, config, opts()); + + expect(existsSync(join(tmp, "npm", `minimal-${target}`, "addon-minimal.node"))).toBe(true); + expect(existsSync(join(tmp, "npm", target, "addon-minimal.node"))).toBe(false); + + const regular = JSON.parse(readFileSync(join(tmp, "npm", target, "package.json"), "utf-8")); + expect(regular.files).toEqual(["addon.node"]); + + const standalone = JSON.parse(readFileSync(join(tmp, "npm", `minimal-${target}`, "package.json"), "utf-8")); + expect(standalone).toMatchObject({ + cpu: ["arm64"], + files: ["addon-minimal.node"], + main: "addon-minimal.node", + name: `@scope/addon-minimal-${target}`, + os: ["darwin"], + version: "1.2.3", + }); + }); + + it("leaves standalone packages out of optionalDependencies", () => { + const config = parsePkgJson(standalonePkgJson); + const updated = updateOptionalDependencies(structuredClone(standalonePkgJson), config); + expect(Object.keys(updated.optionalDependencies as object)).toEqual([ + `@scope/addon-${target}`, + `@scope/addon-${otherTarget}`, + ]); + }); +}); + +describe("publish", () => { + it("publishes standalone packages", async () => { + const dir = join(tmp, "pkg"); + mkdirSync(dir); + writeFileSync(join(dir, "package.json"), JSON.stringify(standalonePkgJson)); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + await publish({concurrency: 1, cwd: dir, "dry-run": true, "npm-dir": "npm", npmArgs: []}); + + const lines = log.mock.calls.map((call) => call.join(" ")); + expect(lines.some((line) => line.includes(`Would publish @scope/addon-minimal-${target}`))).toBe(true); + expect(lines.some((line) => line.includes(join(dir, "npm", `minimal-${target}`)))).toBe(true); + expect(lines.some((line) => line.includes("Would publish 4 package(s)"))).toBe(true); + }); +}); + +describe("requireNapiLibrary", () => { + function writePackage(json: PkgJson): string { + const dir = join(tmp, "pkg"); + mkdirSync(dir); + writeFileSync(join(dir, "package.json"), JSON.stringify(json)); + return dir; + } + + it("rejects unknown presets before loading", () => { + const dir = writePackage(pkgJson); + expect(() => requireNapiLibrary(dir, "gnosis")).toThrow(/gnosis/); + }); + + it("loads a bundled preset from the target package file", () => { + const dir = writePackage(pkgJson); + const current = getTarget(process.platform, process.arch); + expect(() => requireNapiLibrary(dir, "minimal")).toThrow(`@scope/addon-${current}/addon-minimal.node`); + }); + + it("loads a standalone preset from its own package", () => { + const dir = writePackage(standalonePkgJson); + const current = getTarget(process.platform, process.arch); + expect(() => requireNapiLibrary(dir, "minimal")).toThrow(`@scope/addon-minimal-${current}/addon-minimal.node`); + }); + + it("prefers a local preset build", () => { + const dir = writePackage(pkgJson); + mkdirSync(join(dir, "zig-out/lib"), {recursive: true}); + writeFileSync(join(dir, "zig-out/lib/addon-minimal.node"), ""); + expect(() => requireNapiLibrary(dir, "minimal")).toThrow(join(dir, "zig-out/lib/addon-minimal.node")); + }); +}); diff --git a/ts/build.ts b/ts/build.ts index 4357539..30aec64 100644 --- a/ts/build.ts +++ b/ts/build.ts @@ -1,14 +1,18 @@ import {spawn} from "node:child_process"; +import {promises as fs} from "node:fs"; +import {join} from "node:path"; import {type ParseArgsOptionsConfig, parseArgs} from "node:util"; -import {loadConfig} from "./config.js"; +import {type Config, type Preset, loadConfig, resolvePreset} from "./config.js"; import {type Optimize, type Target, getZigTriple, validateOptimize, validateTarget} from "./lib.js"; -import {logInfo, logSuccess} from "./log.js"; +import {logDetail, logInfo, logSuccess} from "./log.js"; export type BuildOptions = { target: Target; optimize?: Optimize; zigCwd: string; step: string; + /** Extra `zig build` args, e.g. a preset's `-D` options */ + extraArgs?: string[]; prefix?: string; cacheDir?: string; globalCacheDir?: string; @@ -26,6 +30,8 @@ export async function build(opts: BuildOptions): Promise { args.push(`-Doptimize=${opts.optimize}`); } + args.push(...(opts.extraArgs ?? [])); + if (opts.prefix) { args.push("--prefix", opts.prefix); } @@ -66,10 +72,52 @@ export async function build(opts: BuildOptions): Promise { } } +export function resolveStep(config: Config, step?: string): string { + const resolved = step ?? config.step; + if (!resolved) { + throw new Error("--step is required (or set zapi.step in package.json)"); + } + return resolved; +} + +export type BuildLocalOptions = { + config: Config; + preset: Preset; + target: Target; + optimize?: Optimize; + zigCwd: string; + step?: string; +}; + +/** + * Build one preset into `zig-out/lib` and give it the preset's file name, which is where + * `requireNapiLibrary` looks for a local build of that preset. + */ +export async function buildLocal(opts: BuildLocalOptions): Promise { + await build({ + extraArgs: opts.preset.args, + optimize: opts.optimize, + step: resolveStep(opts.config, opts.step), + target: opts.target, + zigCwd: opts.zigCwd, + }); + + const libDir = join(opts.zigCwd, "zig-out", "lib"); + const built = join(libDir, `${opts.config.binaryName}.node`); + const wanted = join(libDir, opts.preset.file); + if (built !== wanted) { + await fs.rename(built, wanted); + logDetail(`Renamed to ${wanted}`); + } +} + const buildCliOptions = { optimize: { type: "string", }, + preset: { + type: "string", + }, step: { type: "string", }, @@ -89,19 +137,16 @@ export async function buildCli(): Promise { }); const {config} = await loadConfig(); - const step = values.step ?? config.step; - if (!step) { - throw new Error("--step is required (or set zapi.step in package.json)"); - } const target = validateTarget(values.target as string | undefined); const optimize = validateOptimize(values.optimize as string | undefined); + const preset = resolvePreset(config, values.preset as string | undefined); - const buildOptions: BuildOptions = { + await buildLocal({ + config, optimize, - step, + preset, + step: values.step as string | undefined, target, zigCwd: values["zig-cwd"], - }; - - await build(buildOptions); + }); } diff --git a/ts/buildArtifacts.ts b/ts/buildArtifacts.ts index ab2187d..5680b9c 100644 --- a/ts/buildArtifacts.ts +++ b/ts/buildArtifacts.ts @@ -2,9 +2,9 @@ import {promises as fs} from "node:fs"; import {availableParallelism} from "node:os"; import {join} from "node:path"; import {type ParseArgsOptionsConfig, parseArgs} from "node:util"; -import {build} from "./build.js"; -import {loadConfig} from "./config.js"; -import {type Target, parsePositiveIntOption, runWithConcurrency, validateOptimize} from "./lib.js"; +import {build, resolveStep} from "./build.js"; +import {type Config, loadConfig} from "./config.js"; +import {type Optimize, type Target, parsePositiveIntOption, runWithConcurrency, validateOptimize} from "./lib.js"; import {logDetail, logInfo, logStep, logSuccess} from "./log.js"; const defaultConcurrency = String(Math.max(1, Math.min(availableParallelism(), 4))); @@ -35,6 +35,8 @@ type MoveArtifactOpts = { artifactsDir: string; target: Target; binaryName: string; + /** Destination file name inside `artifactsDir//` */ + file: string; buildPrefix?: string; }; @@ -44,7 +46,66 @@ export async function moveArtifact(opts: MoveArtifactOpts): Promise { const outputLibDir = opts.buildPrefix ? join(opts.buildPrefix, "lib") : join(opts.zigCwd, "zig-out", "lib"); - await fs.rename(join(outputLibDir, `${opts.binaryName}.node`), join(destDir, `${opts.binaryName}.node`)); + await fs.rename(join(outputLibDir, `${opts.binaryName}.node`), join(destDir, opts.file)); +} + +export type BuildArtifactsOptions = { + config: Config; + step?: string; + optimize?: Optimize; + zigCwd: string; + artifactsDir: string; + concurrency: number; +}; + +export async function buildArtifacts(opts: BuildArtifactsOptions): Promise { + const {config, artifactsDir, zigCwd} = opts; + const step = resolveStep(config, opts.step); + const presetCount = config.presets.length; + const total = config.targets.length * presetCount; + const buildRoot = join(artifactsDir, ".zapi-build"); + + const presetsNote = presetCount > 1 ? ` x ${presetCount} preset(s)` : ""; + logInfo( + `Building ${config.binaryName} for ${config.targets.length} target(s)${presetsNote} with concurrency ${opts.concurrency}...` + ); + + await runWithConcurrency(config.targets, opts.concurrency, async (target, index) => { + const targetBuildDir = join(buildRoot, target); + const prefix = join(targetBuildDir, "prefix"); + const cacheDir = join(targetBuildDir, "cache"); + const globalCacheDir = join(targetBuildDir, "global-cache"); + + // Presets share one prefix dir, so each build must be moved out before the next starts. + for (const [presetIndex, preset] of config.presets.entries()) { + const presetNote = presetCount > 1 ? ` (${preset.name})` : ""; + logStep(index * presetCount + presetIndex + 1, total, `Building for ${target}${presetNote}...`); + + await build({ + cacheDir, + extraArgs: preset.args, + globalCacheDir, + optimize: opts.optimize, + prefix, + quiet: true, + step, + target, + zigCwd, + }); + + logDetail(`Moving artifact to ${join(artifactsDir, target, preset.file)}`); + await moveArtifact({ + artifactsDir, + binaryName: config.binaryName, + buildPrefix: prefix, + file: preset.file, + target, + zigCwd, + }); + } + }); + + logSuccess(`Built ${total} artifact(s) to ${artifactsDir}/`); } export async function buildArtifactsCli(): Promise { @@ -54,52 +115,19 @@ export async function buildArtifactsCli(): Promise { }); const optimize = validateOptimize(values.optimize as string | undefined); - const {config} = await loadConfig(); - const step = values.step ?? config.step; - if (!step) { - throw new Error("--step is required (or set zapi.step in package.json)"); - } - const concurrency = parsePositiveIntOption( "concurrency", values.concurrency as string | undefined, Number(defaultConcurrency) ); - const total = config.targets.length; - const artifactsDir = values["artifacts-dir"]; - const buildRoot = join(artifactsDir, ".zapi-build"); - - logInfo(`Building ${config.binaryName} for ${total} target(s) with concurrency ${concurrency}...`); - await runWithConcurrency(config.targets, concurrency, async (target, index) => { - const targetBuildDir = join(buildRoot, target); - const prefix = join(targetBuildDir, "prefix"); - const cacheDir = join(targetBuildDir, "cache"); - const globalCacheDir = join(targetBuildDir, "global-cache"); - - logStep(index + 1, total, `Building for ${target}...`); - - await build({ - cacheDir, - globalCacheDir, - optimize, - prefix, - quiet: true, - step, - target, - zigCwd: values["zig-cwd"], - }); - - logDetail(`Moving artifact to ${join(artifactsDir, target)}`); - await moveArtifact({ - artifactsDir, - binaryName: config.binaryName, - buildPrefix: prefix, - target, - zigCwd: values["zig-cwd"], - }); + await buildArtifacts({ + artifactsDir: values["artifacts-dir"], + concurrency, + config, + optimize, + step: values.step as string | undefined, + zigCwd: values["zig-cwd"], }); - - logSuccess(`Built ${total} artifact(s) to ${artifactsDir}/`); } diff --git a/ts/cli.ts b/ts/cli.ts index 975ebbb..05e1bdc 100644 --- a/ts/cli.ts +++ b/ts/cli.ts @@ -21,9 +21,10 @@ Commands: --target Target triple (default: current platform) --optimize Optimization level: Debug, ReleaseSafe, ReleaseFast, ReleaseSmall --step Zig build step (required) + --preset Preset name from zapi.presets (default: first preset) --zig-cwd Working directory for zig build (default: .) - build-artifacts Build for all configured targets + build-artifacts Build every preset for all configured targets --optimize Optimization level --step Zig build step (required) --zig-cwd Working directory for zig build (default: .) @@ -50,9 +51,14 @@ Configuration: { "zapi": { "binaryName": "my-addon", - "targets": ["x86_64-unknown-linux-gnu", "aarch64-apple-darwin", ...] + "targets": ["x86_64-unknown-linux-gnu", "aarch64-apple-darwin", ...], + "presets": [{"name": "default"}, {"name": "small", "args": ["-Dsmall=true"]}] } } + "presets" is optional. The first preset is the default and keeps the + .node file name; later ones are published as + -.node inside the same target packages, or as their + own -- packages when marked "standalone": true. `.trim(); export async function cli(): Promise { diff --git a/ts/config.ts b/ts/config.ts index dfc29ae..0d2457a 100644 --- a/ts/config.ts +++ b/ts/config.ts @@ -2,12 +2,20 @@ import {promises as fs} from "node:fs"; import {basename, join} from "node:path"; import {TARGETS, type Target} from "./lib.js"; -const BINARY_NAME_RE = /^[A-Za-z0-9._-]+$/; +const FILE_NAME_RE = /^[A-Za-z0-9._-]+$/; + +export type Preset = { + name: string; + args: string[]; + file: string; + standalone: boolean; +}; export type Config = { binaryName: string; targets: Target[]; step?: string; + presets: Preset[]; }; export type PkgJson = { @@ -19,6 +27,7 @@ export type PkgJson = { binaryName: string; targets: string[]; step?: string; + presets?: {name: string; args?: string[]; standalone?: boolean}[]; }; [key: string]: unknown; }; @@ -28,6 +37,18 @@ export type LoadedConfig = { config: Config; }; +/** One npm package produced by `prepublish`: either a target package or a standalone preset package. */ +export type TargetPackage = { + name: string; + /** Directory under `--npm-dir` */ + dir: string; + target: Target; + /** Binary files it ships; the first one is `main` */ + files: string[]; + /** Listed in the main package's optionalDependencies */ + optional: boolean; +}; + /** * Load and parse package.json and zapi config from a directory. * @param cwd - Directory containing package.json (default: current directory) @@ -39,18 +60,63 @@ export async function loadConfig(cwd = "."): Promise { return {config, pkgJson}; } -function validateBinaryName(binaryName: string): void { - if (!BINARY_NAME_RE.test(binaryName)) { - throw new Error("zapi.binaryName must match /^[A-Za-z0-9._-]+$/"); +function validateFileNameSegment(field: string, value: string): void { + if (!FILE_NAME_RE.test(value)) { + throw new Error(`${field} must match /^[A-Za-z0-9._-]+$/`); } - if (binaryName !== basename(binaryName)) { - throw new Error("zapi.binaryName must not include path segments"); + if (value !== basename(value)) { + throw new Error(`${field} must not include path segments`); } - if (binaryName.includes("/") || binaryName.includes("\\") || binaryName.includes("..")) { - throw new Error("zapi.binaryName must not contain path separators or '..'"); + if (value.includes("/") || value.includes("\\") || value.includes("..")) { + throw new Error(`${field} must not contain path separators or '..'`); } } +/** + * Presets are alternative builds of the same binary, each with its own extra `zig build` args. + * The first preset is the default and keeps the plain `.node` file name so a config + * without presets publishes exactly what it did before. A standalone preset is published as its + * own `--` packages instead of being bundled into the target packages. + */ +function parsePresets(binaryName: string, raw: unknown): Preset[] { + if (raw == null) { + return [{args: [], file: `${binaryName}.node`, name: "default", standalone: false}]; + } + if (!Array.isArray(raw) || raw.length === 0) { + throw new Error("zapi.presets must be a non-empty array"); + } + const names = new Set(); + return raw.map((entry: unknown, index) => { + if (typeof entry !== "object" || entry === null) { + throw new Error("zapi.presets entries must be objects"); + } + const {name, args = [], standalone = false} = entry as {name?: unknown; args?: unknown; standalone?: unknown}; + if (typeof name !== "string") { + throw new Error("zapi.presets[].name must be a string"); + } + validateFileNameSegment("zapi.presets[].name", name); + if (names.has(name)) { + throw new Error(`zapi.presets names must be unique: "${name}"`); + } + names.add(name); + if (!Array.isArray(args) || args.some((arg) => typeof arg !== "string")) { + throw new Error("zapi.presets[].args must be an array of strings"); + } + if (typeof standalone !== "boolean") { + throw new Error("zapi.presets[].standalone must be a boolean"); + } + if (index === 0 && standalone) { + throw new Error("zapi.presets[0] is the default preset and cannot be standalone"); + } + return { + args: args as string[], + file: index === 0 ? `${binaryName}.node` : `${binaryName}-${name}.node`, + name, + standalone, + }; + }); +} + export function parsePkgJson(pkgJson: PkgJson): Config { const napi = pkgJson.zapi; if (typeof napi !== "object" || napi === null) { @@ -60,7 +126,7 @@ export function parsePkgJson(pkgJson: PkgJson): Config { if (typeof binaryName !== "string") { throw new Error("zapi.binaryName must be a string"); } - validateBinaryName(binaryName); + validateFileNameSegment("zapi.binaryName", binaryName); const targets = napi.targets; if (!Array.isArray(targets) || targets.length === 0) { @@ -81,11 +147,57 @@ export function parsePkgJson(pkgJson: PkgJson): Config { return { binaryName, + presets: parsePresets(binaryName, napi.presets), step, targets: targets as Target[], }; } +/** + * Pick a preset by name, or the default (first) preset when no name is given. + */ +export function resolvePreset(config: Config, name?: string): Preset { + if (name == null) { + return config.presets[0]; + } + const preset = config.presets.find((candidate) => candidate.name === name); + if (!preset) { + const valid = config.presets.map((candidate) => candidate.name).join(", "); + throw new Error(`Unknown preset "${name}". Valid values: ${valid}`); + } + return preset; +} + +/** npm package that ships `preset` for `target`. */ +export function presetPackageName(pkgJson: PkgJson, preset: Preset, target: Target): string { + return preset.standalone ? `${pkgJson.name}-${preset.name}-${target}` : `${pkgJson.name}-${target}`; +} + +export function targetPackages(pkgJson: PkgJson, config: Config): TargetPackage[] { + const bundled = config.presets.filter((preset) => !preset.standalone); + const packages: TargetPackage[] = []; + for (const target of config.targets) { + packages.push({ + dir: target, + files: bundled.map((preset) => preset.file), + name: `${pkgJson.name}-${target}`, + optional: true, + target, + }); + for (const preset of config.presets) { + if (!preset.standalone) continue; + packages.push({ + dir: `${preset.name}-${target}`, + files: [preset.file], + name: presetPackageName(pkgJson, preset, target), + optional: false, + target, + }); + } + } + return packages; +} + export function getTargetParts(target: Target): {platform: NodeJS.Platform; arch: NodeJS.Architecture; abi?: string} { let platform: NodeJS.Platform; let arch: NodeJS.Architecture; diff --git a/ts/lib.ts b/ts/lib.ts index fb3e74e..3adbe00 100644 --- a/ts/lib.ts +++ b/ts/lib.ts @@ -2,7 +2,7 @@ import {existsSync, readFileSync} from "node:fs"; import {createRequire} from "node:module"; import {join} from "node:path"; -import {parsePkgJson} from "./config.js"; +import {parsePkgJson, presetPackageName, resolvePreset} from "./config.js"; const require = createRequire(import.meta.url); @@ -208,19 +208,21 @@ export function getTarget(platform: NodeJS.Platform, arch: NodeJS.Architecture): * First, it checks the local build path, then it checks published paths. * * Eg: - * - built locally, `zig-out/lib/${binaryName}.node` - * - published, `${targetPackageName}` + * - built locally, `zig-out/lib/${preset.file}` + * - published, `${presetPackageName}/${preset.file}` + * + * @param presetName - One of `zapi.presets[].name`; defaults to the first (default) preset */ -export function requireNapiLibrary(packageDir: string): unknown { +export function requireNapiLibrary(packageDir: string, presetName?: string): unknown { const pkgJson = JSON.parse(readFileSync(join(packageDir, "package.json"), "utf-8")); const config = parsePkgJson(pkgJson); - const localPath = join(packageDir, `zig-out/lib/${config.binaryName}.node`); + const preset = resolvePreset(config, presetName); + const localPath = join(packageDir, "zig-out", "lib", preset.file); if (existsSync(localPath)) { return require(localPath); } const platform = process.platform; const arch = process.arch; const target = getTarget(platform, arch); - const targetPkgName = `${pkgJson.name}-${target}`; - return require(targetPkgName); + return require(`${presetPackageName(pkgJson, preset, target)}/${preset.file}`); } diff --git a/ts/prepublish.ts b/ts/prepublish.ts index ae60fae..be9ca32 100644 --- a/ts/prepublish.ts +++ b/ts/prepublish.ts @@ -2,7 +2,7 @@ import {promises as fs} from "node:fs"; import {availableParallelism} from "node:os"; import {join} from "node:path"; import {type ParseArgsOptionsConfig, parseArgs} from "node:util"; -import {type Config, type PkgJson, getTargetParts, loadConfig} from "./config.js"; +import {type Config, type PkgJson, getTargetParts, loadConfig, targetPackages} from "./config.js"; import {parsePositiveIntOption, runWithConcurrency} from "./lib.js"; import {logDetail, logInfo, logSuccess, logWarn} from "./log.js"; @@ -13,64 +13,68 @@ export type CreateNpmDirsOpts = { concurrency: number; }; -export async function createNpmDirs(config: Config, opts: CreateNpmDirsOpts): Promise { - await runWithConcurrency(config.targets, opts.concurrency, async (target) => { - await fs.mkdir(join(opts["npm-dir"], target), {recursive: true}); +export async function createNpmDirs(pkgJson: PkgJson, config: Config, opts: CreateNpmDirsOpts): Promise { + await runWithConcurrency(targetPackages(pkgJson, config), opts.concurrency, async (pkg) => { + await fs.mkdir(join(opts["npm-dir"], pkg.dir), {recursive: true}); }); } -export async function moveArtifacts(_pkgJson: PkgJson, config: Config, opts: PrepublishOpts): Promise { +export async function moveArtifacts(pkgJson: PkgJson, config: Config, opts: PrepublishOpts): Promise { logInfo("Moving artifacts to npm packages..."); - await runWithConcurrency(config.targets, opts.concurrency, async (target) => { - const artifactPath = join(opts["artifacts-dir"], target, `${config.binaryName}.node`); - const destPath = join(opts["npm-dir"], target, `${config.binaryName}.node`); - - await fs.rename(artifactPath, destPath).catch((err) => { - if (err.code === "ENOENT") { - logWarn(`Artifact not found: ${artifactPath}`); - return; - } - throw err; - }); - logDetail(`${target} → ${destPath}`); + await runWithConcurrency(targetPackages(pkgJson, config), opts.concurrency, async (pkg) => { + for (const file of pkg.files) { + const artifactPath = join(opts["artifacts-dir"], pkg.target, file); + const destPath = join(opts["npm-dir"], pkg.dir, file); + + await fs.rename(artifactPath, destPath).catch((err) => { + if (err.code === "ENOENT") { + logWarn(`Artifact not found: ${artifactPath}`); + return; + } + throw err; + }); + logDetail(`${pkg.target} → ${destPath}`); + } }); } export async function updateTargetPkgJsons(pkgJson: PkgJson, config: Config, opts: PrepublishOpts): Promise { logInfo("Generating target package.json files..."); - await runWithConcurrency(config.targets, opts.concurrency, async (target) => { - const {platform, arch, abi} = getTargetParts(target); + await runWithConcurrency(targetPackages(pkgJson, config), opts.concurrency, async (pkg) => { + const {platform, arch, abi} = getTargetParts(pkg.target); const libc = platform !== "linux" ? undefined : abi === "gnu" ? "glibc" : abi === "musl" ? "musl" : undefined; - const targetDir = join(opts["npm-dir"], target); + const pkgDir = join(opts["npm-dir"], pkg.dir); const targetPkgJson = { cpu: [arch], - files: [`${config.binaryName}.node`], + files: pkg.files, license: pkgJson.license, - main: `${config.binaryName}.node`, - name: `${pkgJson.name}-${target}`, + main: pkg.files[0], + name: pkg.name, os: [platform], repository: pkgJson.repository, version: pkgJson.version, ...(libc ? {libc: [libc]} : {}), }; - await fs.writeFile(join(targetDir, "package.json"), JSON.stringify(targetPkgJson, null, 2)); + await fs.writeFile(join(pkgDir, "package.json"), JSON.stringify(targetPkgJson, null, 2)); await fs.writeFile( - join(targetDir, "README.md"), + join(pkgDir, "README.md"), `# \`${targetPkgJson.name}\`\n -This is the ${target} target package for ${pkgJson.name}. +This is the ${pkg.target} target package for ${pkgJson.name}. ` ); - logDetail(`Created ${join(targetDir, "package.json")}`); + logDetail(`Created ${join(pkgDir, "package.json")}`); }); } export function updateOptionalDependencies(pkgJson: PkgJson, config: Config): PkgJson { const optionalDependencies: Record = {}; - for (const target of config.targets) { - optionalDependencies[`${pkgJson.name}-${target}`] = pkgJson.version; + for (const pkg of targetPackages(pkgJson, config)) { + if (pkg.optional) { + optionalDependencies[pkg.name] = pkgJson.version; + } } pkgJson.optionalDependencies = optionalDependencies; return pkgJson; @@ -118,7 +122,7 @@ export async function prepublishCli(): Promise { "npm-dir": values["npm-dir"], }; - await createNpmDirs(config, opts); + await createNpmDirs(pkgJson, config, opts); await moveArtifacts(pkgJson, config, opts); await updateTargetPkgJsons(pkgJson, config, opts); @@ -126,5 +130,5 @@ export async function prepublishCli(): Promise { const updatedPkgJson = await updateOptionalDependencies(pkgJson, config); await fs.writeFile("package.json", JSON.stringify(updatedPkgJson, null, 2)); - logSuccess(`Prepared ${config.targets.length} target package(s) in ${values["npm-dir"]}/`); + logSuccess(`Prepared ${targetPackages(pkgJson, config).length} package(s) in ${values["npm-dir"]}/`); } diff --git a/ts/publish.ts b/ts/publish.ts index 120e81c..8f54785 100644 --- a/ts/publish.ts +++ b/ts/publish.ts @@ -1,7 +1,7 @@ import {spawn} from "node:child_process"; import {join} from "node:path"; import {type ParseArgsOptionsConfig, parseArgs} from "node:util"; -import {loadConfig} from "./config.js"; +import {loadConfig, targetPackages} from "./config.js"; import {parsePositiveIntOption, runWithConcurrency} from "./lib.js"; import {logDetail, logInfo, logStep, logSuccess} from "./log.js"; @@ -24,6 +24,10 @@ export type PublishOpts = { "npm-dir": string; "dry-run": boolean; concurrency: number; + /** Package root holding package.json and the npm dir (default: process.cwd()) */ + cwd?: string; + /** Extra args appended to every `npm publish` (default: everything after `--` on the CLI) */ + npmArgs?: string[]; }; function extraPublishArgs(): string[] { @@ -59,45 +63,44 @@ async function runNpm(args: string[], cwd: string): Promise { } export async function publish(opts: PublishOpts): Promise { - const extraArgs = extraPublishArgs(); + const cwd = opts.cwd ?? process.cwd(); + const extraArgs = opts.npmArgs ?? extraPublishArgs(); const publishArgv = ["publish", ...extraArgs]; - const {config} = await loadConfig(); - const total = config.targets.length + 1; // +1 for main package + const {pkgJson, config} = await loadConfig(cwd); + const packages = targetPackages(pkgJson, config); + const total = packages.length + 1; // +1 for main package if (opts["dry-run"]) { logInfo( - `[DRY RUN] Would publish ${config.targets.length} target package(s) + main package with concurrency ${opts.concurrency}` + `[DRY RUN] Would publish ${packages.length} package(s) + main package with concurrency ${opts.concurrency}` ); logDetail(`Extra npm args: ${extraArgs.length > 0 ? extraArgs.join(" ") : "(none)"}`); - for (let i = 0; i < config.targets.length; i++) { - const target = config.targets[i]; - const cwd = join(process.cwd(), opts["npm-dir"], target); - logStep(i + 1, total, `Would publish ${target}`); - logDetail(`Directory: ${cwd}`); + for (let i = 0; i < packages.length; i++) { + const pkg = packages[i]; + logStep(i + 1, total, `Would publish ${pkg.name}`); + logDetail(`Directory: ${join(cwd, opts["npm-dir"], pkg.dir)}`); } logStep(total, total, "Would publish main package"); - logDetail(`Directory: ${process.cwd()}`); + logDetail(`Directory: ${cwd}`); logSuccess(`[DRY RUN] ${total} package(s) would be published`); return; } - logInfo( - `Publishing ${config.targets.length} target package(s) + main package with concurrency ${opts.concurrency}...` - ); + logInfo(`Publishing ${packages.length} package(s) + main package with concurrency ${opts.concurrency}...`); let started = 0; - await runWithConcurrency(config.targets, opts.concurrency, async (target) => { + await runWithConcurrency(packages, opts.concurrency, async (pkg) => { started += 1; - logStep(started, total, `Publishing ${target}...`); - await runNpm(publishArgv, join(process.cwd(), opts["npm-dir"], target)); + logStep(started, total, `Publishing ${pkg.name}...`); + await runNpm(publishArgv, join(cwd, opts["npm-dir"], pkg.dir)); }); logStep(total, total, "Publishing main package..."); - await runNpm(publishArgv, process.cwd()); + await runNpm(publishArgv, cwd); logSuccess(`Published ${total} package(s) successfully!`); } @@ -110,7 +113,7 @@ export async function publishCli(): Promise { }); await publish({ - concurrency: parsePositiveIntOption("concurrency", values["concurrency"] as string | undefined, 1), + concurrency: parsePositiveIntOption("concurrency", values.concurrency as string | undefined, 1), "dry-run": values["dry-run"] as boolean, "npm-dir": values["npm-dir"] as string, });