perf(tinybench): batch fast walltime invocations into rounds

Calibrate approximately 1 ms rounds from warmup and share the iteration count across isolated forks. Normalize latency statistics per invocation while retaining raw total time. Keep per-invocation hooks and concurrent tasks unbatched, and cover shared, forked, synchronous, and asynchronous execution.

Author: not-matthias

packages/core/src/isolation.ts

@@ -22,6 +22,13 @@ export const DEFAULT_WALLTIME_FORKS = 3;
 export const ISOLATED_BENCHMARK_ENV = "CODSPEED_ISOLATED_BENCHMARK";
 /** Set on each fork: the file where it writes its round samples. */
 export const FORK_SAMPLES_FILE_ENV = "CODSPEED_FORK_SAMPLES_FILE";
+/**
+ * Set on the forks after the first one by {@link runForks}: the number of
+ * iterations that the first fork measured per round. Every fork of a benchmark
+ * MUST batch the same way, because the samples of all forks are pooled.
+ */
+export const FORK_ITERATIONS_PER_ROUND_ENV =
+  "CODSPEED_FORK_ITERATIONS_PER_ROUND";
 /**
  * JSON array `[executable, ...args]` of the command that runs the benchmarks
  * again. Integrations whose benchmarks run in a worker process (vitest) set it
@@ -45,8 +52,16 @@ export type IsolationMode =
   | { kind: "SharedProcess" }
   /** Every benchmark is measured in `forks` fresh processes, one after the other. */
   | { kind: "Forks"; forks: number }
-  /** A fork: measures only `uri` and writes its samples to `samplesFile`. */
-  | { kind: "Fork"; uri: string; samplesFile: string };
+  /**
+   * A fork: measures only `uri` and writes its samples to `samplesFile`. The
+   * first fork picks `iterationsPerRound` itself; the ones after it inherit it.
+   */
+  | {
+      kind: "Fork";
+      uri: string;
+      samplesFile: string;
+      iterationsPerRound?: number;
+    };
 
 /**
  * The isolation of a walltime run, from the environment. `forks` is the
@@ -61,7 +76,16 @@ export function walltimeIsolation(forks?: number): IsolationMode {
         `[CodSpeed] ${ISOLATED_BENCHMARK_ENV} is set without ${FORK_SAMPLES_FILE_ENV}`,
       );
     }
-    return { kind: "Fork", uri, samplesFile };
+    const iterationsPerRound = process.env[FORK_ITERATIONS_PER_ROUND_ENV];
+    return {
+      kind: "Fork",
+      uri,
+      samplesFile,
+      iterationsPerRound:
+        iterationsPerRound === undefined || iterationsPerRound === ""
+          ? undefined
+          : validPositiveInt(iterationsPerRound, FORK_ITERATIONS_PER_ROUND_ENV),
+    };
   }
   const isolation = process.env[PROCESS_ISOLATION_ENV];
   if (isolation === "0" || isolation === "false") {
@@ -82,22 +106,22 @@ export function walltimeIsolation(forks?: number): IsolationMode {
 export function resolveWalltimeForks(option?: number): number {
   const env = process.env[WALLTIME_FORKS_ENV];
   if (env !== undefined && env !== "") {
-    return validForks(env, WALLTIME_FORKS_ENV);
+    return validPositiveInt(env, WALLTIME_FORKS_ENV);
   }
   if (option !== undefined) {
-    return validForks(option, "forks");
+    return validPositiveInt(option, "forks");
   }
   return DEFAULT_WALLTIME_FORKS;
 }
 
-function validForks(raw: string | number, source: string): number {
-  const forks = Number(raw);
-  if (!Number.isInteger(forks) || forks < 1) {
+function validPositiveInt(raw: string | number, source: string): number {
+  const value = Number(raw);
+  if (!Number.isInteger(value) || value < 1) {
     throw new Error(
       `[CodSpeed] ${source} must be a positive integer, got \`${raw}\``,
     );
   }
-  return forks;
+  return value;
 }
 
 /**
@@ -122,44 +146,78 @@ function isolationCommand(): string[] {
   return JSON.parse(recorded) as string[];
 }
 
+/** The pooled samples of the forks of one benchmark. */
+export interface ForkSamples {
+  /** Round samples (milliseconds), one per measured round. */
+  samples: number[];
+  /** Benchmark iterations one round measured; every fork MUST agree. */
+  iterationsPerRound: number;
+}
+
 /**
  * Runs the benchmark `uri` in `forks` fresh processes, one after the other,
  * and returns the round samples of all of them, in milliseconds. Throws when
  * a fork fails, without running the next one.
  *
+ * The first fork picks how many iterations one round batches; the ones after
+ * it inherit that count, so the pooled samples mean the same thing.
+ *
  * Each fork re-runs the benchmark command, not fork(): Node can't fork, and a
  * fork would only copy the calling thread. The fork's event loop stays free
  * while it runs: a Vitest worker must keep answering its main process, whose
  * calls time out otherwise.
  */
-export async function runForks(uri: string, forks: number): Promise<number[]> {
-  let samples: number[] = [];
+export async function runForks(
+  uri: string,
+  forks: number,
+): Promise<ForkSamples> {
+  let result: ForkSamples | undefined;
   for (let fork = 0; fork < forks; fork++) {
-    // Not push(...): spreading a time-based run's samples overflows the stack.
-    samples = samples.concat(await runFork(uri));
+    const next = await runFork(uri, result?.iterationsPerRound);
+    result = poolForkSamples(uri, result, next);
   }
-  return samples;
+  return result ?? { samples: [], iterationsPerRound: 1 };
 }
 
 /** {@link runForks} for synchronous runs, which block anyway. */
-export function runForksSync(uri: string, forks: number): number[] {
-  let samples: number[] = [];
+export function runForksSync(uri: string, forks: number): ForkSamples {
+  let result: ForkSamples | undefined;
   for (let fork = 0; fork < forks; fork++) {
-    samples = samples.concat(runForkSync(uri));
+    const next = runForkSync(uri, result?.iterationsPerRound);
+    result = poolForkSamples(uri, result, next);
   }
-  return samples;
+  return result ?? { samples: [], iterationsPerRound: 1 };
+}
+
+function poolForkSamples(
+  uri: string,
+  pooled: ForkSamples | undefined,
+  next: ForkSamples,
+): ForkSamples {
+  if (!pooled) return next;
+  if (pooled.iterationsPerRound !== next.iterationsPerRound) {
+    throw new Error(
+      `[CodSpeed] the forks of ${uri} measured a different number of iterations per round (${pooled.iterationsPerRound} and ${next.iterationsPerRound})`,
+    );
+  }
+  // Not push(...): spreading a time-based run's samples overflows the stack.
+  for (const sample of next.samples) pooled.samples.push(sample);
+  return pooled;
 }
 
 /** Forks run one after the other, so one file per parent process suffices. */
 const SAMPLES_FILE = join(tmpdir(), `codspeed-samples-${process.pid}.v8.gz`);
 
 /** Runs one fork of `uri` and returns the samples it wrote. */
-async function runFork(uri: string): Promise<number[]> {
+async function runFork(
+  uri: string,
+  iterationsPerRound?: number,
+): Promise<ForkSamples> {
   rmSync(SAMPLES_FILE, { force: true }); // a leftover file must not pass as this fork's
   const [executable, ...args] = isolationCommand();
   const child = spawn(executable, args, {
     stdio: "inherit",
-    env: forkEnv(uri),
+    env: forkEnv(uri, iterationsPerRound),
   });
   const { promise, resolve, reject } =
     Promise.withResolvers<[number | null, NodeJS.Signals | null]>();
@@ -169,24 +227,25 @@ async function runFork(uri: string): Promise<number[]> {
   return forkSamples(uri, status, signal);
 }
 
-function runForkSync(uri: string): number[] {
+function runForkSync(uri: string, iterationsPerRound?: number): ForkSamples {
   rmSync(SAMPLES_FILE, { force: true });
   const [executable, ...args] = isolationCommand();
   const result = spawnSync(executable, args, {
     stdio: "inherit",
-    env: forkEnv(uri),
+    env: forkEnv(uri, iterationsPerRound),
   });
   if (result.error !== undefined) {
     throw startError(uri, result.error);
   }
   return forkSamples(uri, result.status, result.signal);
 }
 
-function forkEnv(uri: string): NodeJS.ProcessEnv {
+function forkEnv(uri: string, iterationsPerRound?: number): NodeJS.ProcessEnv {
   return {
     ...process.env,
     [ISOLATED_BENCHMARK_ENV]: uri,
     [FORK_SAMPLES_FILE_ENV]: SAMPLES_FILE,
+    [FORK_ITERATIONS_PER_ROUND_ENV]: iterationsPerRound?.toString(),
   };
 }
 
@@ -201,15 +260,15 @@ function forkSamples(
   uri: string,
   status: number | null,
   signal: NodeJS.Signals | null,
-): number[] {
+): ForkSamples {
   try {
     if (status !== 0) {
       throw new Error(
         `[CodSpeed] the process for ${uri} failed with ${signal ?? `exit code ${status}`}`,
       );
     }
     try {
-      return deserialize(gunzipSync(readFileSync(SAMPLES_FILE))) as number[];
+      return deserialize(gunzipSync(readFileSync(SAMPLES_FILE))) as ForkSamples;
     } catch (error) {
       throw new Error(
         `[CodSpeed] the process for ${uri} did not report its samples: ${(error as Error).message}`,
@@ -220,10 +279,19 @@ function forkSamples(
   }
 }
 
-/** Hands round samples (milliseconds) to the parent as gzip-compressed V8 data. */
-export function writeForkSamples(samplesFile: string, samples: number[]) {
+/**
+ * Hands round samples (milliseconds) to the parent as gzip-compressed V8 data,
+ * along with the number of benchmark iterations one round measured.
+ */
+export function writeForkSamples(
+  samplesFile: string,
+  samples: number[],
+  iterationsPerRound = 1,
+) {
   writeFileSync(
     samplesFile,
-    gzipSync(serialize(samples), { level: constants.Z_BEST_SPEED }),
+    gzipSync(serialize({ samples, iterationsPerRound }), {
+      level: constants.Z_BEST_SPEED,
+    }),
   );
 }

packages/core/tests/fixtures/isolation.cjs

@@ -7,6 +7,8 @@ const {
 } = require("../../dist/index.cjs.js");
 
 const mode = walltimeIsolation();
+/** The count a first fork picks; the forks after it inherit it from the parent. */
+const CALIBRATED = Number(process.env.ISOLATION_FIXTURE_ITERATIONS ?? 1);
 
 function measure() {
   console.log(`ran ${process.pid} ${JSON.stringify(process.execArgv)}`);
@@ -16,7 +18,21 @@ function measure() {
     case "no-samples":
       process.exit(0);
   }
-  writeForkSamples(mode.samplesFile, [Number(process.pid), 1]);
+  const inherited = mode.iterationsPerRound;
+  // Tells the test whether this fork picked the count or reused the first one.
+  console.log(
+    `${inherited === undefined ? "calibrated" : "reused"} ${inherited ?? CALIBRATED}`,
+  );
+  const iterationsPerRound =
+    inherited !== undefined &&
+    process.env.ISOLATION_FIXTURE_ACTION === "mismatch"
+      ? inherited + 1 // batches differently than the first fork
+      : (inherited ?? CALIBRATED);
+  writeForkSamples(
+    mode.samplesFile,
+    [Number(process.pid), 1],
+    iterationsPerRound,
+  );
 }
 
 async function main() {
@@ -25,11 +41,12 @@ async function main() {
     return;
   }
   const forks = Number(process.env.ISOLATION_FIXTURE_FORKS);
-  const samples =
+  const result =
     process.env.ISOLATION_FIXTURE_SYNC !== undefined
       ? runForksSync("a", forks)
       : await runForks("a", forks);
-  console.log(`samples ${samples.length}`);
+  console.log(`samples ${result.samples.length}`);
+  console.log(`iterations ${result.iterationsPerRound}`);
 }
 
 // A rejection ends the process with exit code 1.