From c20e481a709996ed83e36ed92965d9884758cd51 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 00:52:31 +0800 Subject: [PATCH 01/23] test(fspy): benchmark tracking overhead Add a single multithreaded open-and-close workload, a Linux static variant, and three-platform Namespace reporting with PR/main baselines. Co-authored-by: GPT-5.6 Codex --- .github/scripts/fspy-benchmark.mjs | 264 ++++++++++++++++++ .github/scripts/fspy-benchmark.test.mjs | 38 +++ .../workflows/fspy-benchmark-fork-report.yml | 88 ++++++ .github/workflows/fspy-benchmark.yml | 203 ++++++++++++++ Cargo.lock | 112 +++++++- Cargo.toml | 3 + crates/fspy_benchmark/Cargo.toml | 36 +++ crates/fspy_benchmark/README.md | 26 ++ crates/fspy_benchmark/benches/fspy.rs | 144 ++++++++++ crates/fspy_benchmark/src/lib.rs | 1 + .../fspy_benchmark_static_target/Cargo.toml | 23 ++ .../fspy_benchmark_static_target/src/main.rs | 1 + crates/fspy_benchmark_target/Cargo.toml | 20 ++ crates/fspy_benchmark_target/src/main.rs | 53 ++++ justfile | 3 + 15 files changed, 1013 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/fspy-benchmark.mjs create mode 100644 .github/scripts/fspy-benchmark.test.mjs create mode 100644 .github/workflows/fspy-benchmark-fork-report.yml create mode 100644 .github/workflows/fspy-benchmark.yml create mode 100644 crates/fspy_benchmark/Cargo.toml create mode 100644 crates/fspy_benchmark/README.md create mode 100644 crates/fspy_benchmark/benches/fspy.rs create mode 100644 crates/fspy_benchmark/src/lib.rs create mode 100644 crates/fspy_benchmark_static_target/Cargo.toml create mode 100644 crates/fspy_benchmark_static_target/src/main.rs create mode 100644 crates/fspy_benchmark_target/Cargo.toml create mode 100644 crates/fspy_benchmark_target/src/main.rs diff --git a/.github/scripts/fspy-benchmark.mjs b/.github/scripts/fspy-benchmark.mjs new file mode 100644 index 000000000..67cf184e2 --- /dev/null +++ b/.github/scripts/fspy-benchmark.mjs @@ -0,0 +1,264 @@ +#!/usr/bin/env node + +import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { arch, cpus, platform as osPlatform, release } from 'node:os'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const THREAD_COUNT = 4; +const FILES_PER_THREAD = 2048; + +function parseArgs(args) { + const parsed = new Map(); + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if (!flag?.startsWith('--') || value === undefined) { + throw new Error(`invalid arguments near ${flag ?? ''}`); + } + parsed.set(flag.slice(2), value); + } + return parsed; +} + +function required(args, name) { + const value = args.get(name); + if (value === undefined || value === '') { + throw new Error(`missing --${name}`); + } + return value; +} + +async function githubJson(path) { + const token = requiredEnv('GITHUB_TOKEN'); + const response = await fetch(`https://api.github.com${path}`, { + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'vite-task-fspy-benchmark', + }, + }); + if (!response.ok) { + throw new Error(`GitHub API ${response.status}: ${await response.text()}`); + } + return response.json(); +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`missing ${name}`); + return value; +} + +async function findLatestArtifact(name, currentRunId) { + const repository = requiredEnv('GITHUB_REPOSITORY'); + const response = await githubJson( + `/repos/${repository}/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`, + ); + return response.artifacts + .filter( + (artifact) => !artifact.expired && String(artifact.workflow_run?.id) !== String(currentRunId), + ) + .sort((left, right) => Date.parse(right.created_at) - Date.parse(left.created_at))[0]; +} + +async function resolveBaseline(args) { + const outputPath = requiredEnv('GITHUB_OUTPUT'); + const platform = required(args, 'platform'); + const context = required(args, 'context'); + const currentRunId = requiredEnv('GITHUB_RUN_ID'); + const currentArtifactName = `fspy-benchmark-${platform}-${context}`; + const candidates = [currentArtifactName]; + if (context !== 'main') candidates.push(`fspy-benchmark-${platform}-main`); + + let artifact; + let kind; + for (const candidate of candidates) { + artifact = await findLatestArtifact(candidate, currentRunId); + if (artifact) { + kind = candidate === currentArtifactName ? 'previous PR run' : 'main'; + break; + } + } + + const outputs = [ + `current-artifact-name=${currentArtifactName}`, + `found=${artifact ? 'true' : 'false'}`, + ]; + if (artifact) { + outputs.push( + `artifact-name=${artifact.name}`, + `run-id=${artifact.workflow_run.id}`, + `kind=${kind}`, + ); + } + await appendFile(outputPath, `${outputs.join('\n')}\n`); +} + +async function readEstimate(criterionRoot, target, mode) { + const path = join(criterionRoot, target, mode, 'new', 'estimates.json'); + const estimates = JSON.parse(await readFile(path, 'utf8')); + return { + meanNs: estimates.mean.point_estimate, + meanLowerNs: estimates.mean.confidence_interval.lower_bound, + meanUpperNs: estimates.mean.confidence_interval.upper_bound, + medianNs: estimates.median.point_estimate, + }; +} + +async function collect(args) { + const criterionRoot = required(args, 'criterion-root'); + const output = required(args, 'output'); + const platform = required(args, 'platform'); + const expectStatic = args.get('expect-static') === 'true'; + const benchmarks = { + 'dynamic/untracked': await readEstimate(criterionRoot, 'dynamic', 'untracked'), + 'dynamic/tracked': await readEstimate(criterionRoot, 'dynamic', 'tracked'), + }; + if (expectStatic) { + benchmarks['static/untracked'] = await readEstimate(criterionRoot, 'static', 'untracked'); + benchmarks['static/tracked'] = await readEstimate(criterionRoot, 'static', 'tracked'); + } + + const result = { + schemaVersion: 1, + platform, + architecture: process.env.RUNNER_ARCH ?? arch(), + os: { + platform: osPlatform(), + release: release(), + cpu: cpus()[0]?.model ?? 'unknown', + }, + runner: args.get('runner') ?? '', + commit: required(args, 'commit'), + runId: required(args, 'run-id'), + runAttempt: args.get('run-attempt') ?? '1', + event: required(args, 'event'), + pullRequest: args.get('pull-request') ?? '', + workload: { + threadCount: THREAD_COUNT, + filesPerThread: FILES_PER_THREAD, + totalOpens: THREAD_COUNT * FILES_PER_THREAD, + }, + benchmarks, + }; + + await mkdir(dirname(output), { recursive: true }); + await writeFile(output, `${JSON.stringify(result, null, 2)}\n`); +} + +function ratio(result, target) { + return ( + result.benchmarks[`${target}/tracked`].meanNs / result.benchmarks[`${target}/untracked`].meanNs + ); +} + +function percentage(value) { + const sign = value > 0 ? '+' : ''; + return `${sign}${(value * 100).toFixed(2)}%`; +} + +function duration(nanoseconds) { + if (nanoseconds >= 1e9) return `${(nanoseconds / 1e9).toFixed(3)} s`; + if (nanoseconds >= 1e6) return `${(nanoseconds / 1e6).toFixed(3)} ms`; + if (nanoseconds >= 1e3) return `${(nanoseconds / 1e3).toFixed(3)} µs`; + return `${nanoseconds.toFixed(1)} ns`; +} + +function resultLink(result) { + const repository = process.env.GITHUB_REPOSITORY; + if (!repository || !result.runId) return ''; + return `https://github.com/${repository}/actions/runs/${result.runId}`; +} + +export function renderReport(current, baseline, baselineKind = '') { + const compatible = + baseline && + current.platform === baseline.platform && + current.architecture === baseline.architecture; + const lines = [`### ${current.platform[0].toUpperCase()}${current.platform.slice(1)}`, '']; + + if (compatible) { + const link = resultLink(baseline); + const description = link + ? `[${baselineKind || 'baseline'}](${link})` + : baselineKind || 'baseline'; + lines.push(`Compared with ${description} at \`${baseline.commit.slice(0, 8)}\`.`, ''); + } else if (baseline) { + lines.push( + `No comparison: the baseline architecture is \`${baseline.architecture}\`, current is \`${current.architecture}\`.`, + '', + ); + } else { + lines.push('No previous result was available for this runner.', ''); + } + + lines.push( + '| Target | Untracked | Tracked | fspy overhead | Normalized change |', + '| --- | ---: | ---: | ---: | ---: |', + ); + + const targets = Object.hasOwn(current.benchmarks, 'static/tracked') + ? ['dynamic', 'static'] + : ['dynamic']; + for (const target of targets) { + const untracked = current.benchmarks[`${target}/untracked`].meanNs; + const tracked = current.benchmarks[`${target}/tracked`].meanNs; + const overhead = ratio(current, target) - 1; + const normalizedChange = + compatible && Object.hasOwn(baseline.benchmarks, `${target}/tracked`) + ? ratio(current, target) / ratio(baseline, target) - 1 + : undefined; + lines.push( + `| ${target} | ${duration(untracked)} | ${duration(tracked)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, + ); + } + + lines.push( + '', + `Workload: ${current.workload.threadCount} threads, ${current.workload.filesPerThread.toLocaleString('en-US')} files per thread, ${current.workload.totalOpens.toLocaleString('en-US')} total open-and-close operations.`, + '', + `\`${current.architecture}\` · ${current.os.cpu} · run [${current.runId}](${resultLink(current)})`, + '', + ); + return `${lines.join('\n')}\n`; +} + +async function compare(args) { + const current = JSON.parse(await readFile(required(args, 'current'), 'utf8')); + const baselinePath = args.get('baseline'); + let baseline; + if (baselinePath) { + try { + baseline = JSON.parse(await readFile(baselinePath, 'utf8')); + } catch (error) { + if (error.code !== 'ENOENT') throw error; + } + } + const report = renderReport(current, baseline, args.get('baseline-kind') ?? ''); + await writeFile(required(args, 'output'), report); + process.stdout.write(report); +} + +async function main() { + const command = process.argv[2]; + const args = parseArgs(process.argv.slice(3)); + if (command === 'resolve-baseline') { + await resolveBaseline(args); + } else if (command === 'collect') { + await collect(args); + } else if (command === 'compare') { + await compare(args); + } else { + throw new Error(`unknown command: ${command ?? ''}`); + } +} + +const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : ''; +if (import.meta.url === invokedPath) { + main().catch((error) => { + process.stderr.write(`${error.stack ?? error}\n`); + process.exitCode = 1; + }); +} diff --git a/.github/scripts/fspy-benchmark.test.mjs b/.github/scripts/fspy-benchmark.test.mjs new file mode 100644 index 000000000..d219767dc --- /dev/null +++ b/.github/scripts/fspy-benchmark.test.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { renderReport } from './fspy-benchmark.mjs'; + +function result(platform, architecture, runId, untracked, tracked) { + return { + platform, + architecture, + os: { cpu: 'Test CPU' }, + commit: '1234567890abcdef', + runId, + workload: { threadCount: 4, filesPerThread: 2048, totalOpens: 8192 }, + benchmarks: { + 'dynamic/untracked': { meanNs: untracked }, + 'dynamic/tracked': { meanNs: tracked }, + }, + }; +} + +test('renders normalized change against a compatible baseline', () => { + const baseline = result('linux', 'X64', '10', 100, 120); + const current = result('linux', 'X64', '20', 100, 132); + const report = renderReport(current, baseline, 'previous PR run'); + + assert.match(report, /fspy overhead \| Normalized change/); + assert.match(report, /\+32\.00% \| \+10\.00%/); + assert.match(report, /previous PR run/); +}); + +test('does not compare different architectures', () => { + const baseline = result('macos', 'X64', '10', 100, 120); + const current = result('macos', 'ARM64', '20', 100, 132); + const report = renderReport(current, baseline, 'main'); + + assert.match(report, /No comparison/); + assert.match(report, /\+32\.00% \| —/); +}); diff --git a/.github/workflows/fspy-benchmark-fork-report.yml b/.github/workflows/fspy-benchmark-fork-report.yml new file mode 100644 index 000000000..4127d262b --- /dev/null +++ b/.github/workflows/fspy-benchmark-fork-report.yml @@ -0,0 +1,88 @@ +name: Fspy Benchmark Fork Report + +permissions: {} + +on: + workflow_run: + workflows: [Fspy Benchmark] + types: [completed] + +jobs: + comment: + name: Report fork benchmark + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.head_repository.full_name != github.repository && + github.event.workflow_run.pull_requests[0].number != null + runs-on: namespace-profile-linux-x64-default + permissions: + actions: read + contents: read + issues: write + steps: + - name: Download benchmark results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: fspy-benchmark-*-pr-${{ github.event.workflow_run.pull_requests[0].number }} + path: .benchmark-results + merge-multiple: true + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Build PR comment + env: + BENCHMARK_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + { + echo '' + echo '## fspy benchmark' + echo + echo "Results for commit \`$BENCHMARK_SHA\`. Each platform uses its latest result from this PR, falling back to \`main\`." + echo + for platform in linux macos windows; do + cat ".benchmark-results/$platform.md" + done + echo + echo 'This comment is updated by the Fspy Benchmark workflow.' + } > benchmark-comment.md + + - name: Update PR comment + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + run: | + node --input-type=module <<'EOF' + import { readFile } from "node:fs/promises"; + + const marker = ""; + const repository = process.env.GITHUB_REPOSITORY; + const token = process.env.GITHUB_TOKEN; + const pullRequest = process.env.PR_NUMBER; + const headers = { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }; + const commentsResponse = await fetch( + `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, + { headers }, + ); + if (!commentsResponse.ok) throw new Error(await commentsResponse.text()); + const comments = await commentsResponse.json(); + const existing = comments.find( + (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker), + ); + const body = await readFile("benchmark-comment.md", "utf8"); + const url = existing + ? `https://api.github.com/repos/${repository}/issues/comments/${existing.id}` + : `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments`; + const response = await fetch(url, { + method: existing ? "PATCH" : "POST", + headers, + body: JSON.stringify({ body }), + }); + if (!response.ok) throw new Error(await response.text()); + EOF diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml new file mode 100644 index 000000000..b475bd886 --- /dev/null +++ b/.github/workflows/fspy-benchmark.yml @@ -0,0 +1,203 @@ +name: Fspy Benchmark + +permissions: {} + +on: + workflow_dispatch: + pull_request: + types: [opened, reopened, synchronize] + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref_name != 'main' }} + +defaults: + run: + shell: bash + +jobs: + benchmark: + name: Benchmark (${{ matrix.platform }}) + permissions: + actions: read + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux + os: namespace-profile-linux-x64-default + static: true + - platform: macos + os: namespace-profile-mac-default + static: false + - platform: windows + os: namespace-profile-windows-4c-8g + static: false + runs-on: ${{ matrix.os }} + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + + - name: Update Detours submodule + if: matrix.platform == 'windows' + run: git submodule update --init --recursive + + - uses: oxc-project/setup-rust@3d6fb132fbe7cdcb66bf8ec193911c2945369d12 # v1.0.17 + with: + save-cache: ${{ github.ref_name == 'main' }} + cache-key: fspy-benchmark-${{ matrix.platform }} + + - name: Install static target + if: matrix.static + run: rustup target add x86_64-unknown-linux-musl + + - name: Set benchmark context + id: context + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + context="pr-$PR_NUMBER" + pull_request="$PR_NUMBER" + else + context="main" + pull_request="none" + fi + echo "name=$context" >> "$GITHUB_OUTPUT" + echo "pull-request=$pull_request" >> "$GITHUB_OUTPUT" + + - name: Find comparison baseline + id: baseline + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node .github/scripts/fspy-benchmark.mjs resolve-baseline + --platform "${{ matrix.platform }}" + --context "${{ steps.context.outputs.name }}" + + - name: Download comparison baseline + if: steps.baseline.outputs.found == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ steps.baseline.outputs.artifact-name }} + path: .benchmark-baseline + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.baseline.outputs.run-id }} + + - name: Run benchmark + run: cargo bench --locked -p fspy_benchmark --bench fspy + + - name: Collect benchmark results + run: >- + node .github/scripts/fspy-benchmark.mjs collect + --criterion-root "$CARGO_TARGET_DIR/criterion/fspy" + --output ".benchmark-results/${{ matrix.platform }}.json" + --platform "${{ matrix.platform }}" + --expect-static "${{ matrix.static }}" + --runner "${{ matrix.os }}" + --commit "$GITHUB_SHA" + --run-id "$GITHUB_RUN_ID" + --run-attempt "$GITHUB_RUN_ATTEMPT" + --event "$GITHUB_EVENT_NAME" + --pull-request "${{ steps.context.outputs.pull-request }}" + + - name: Compare benchmark results + run: >- + node .github/scripts/fspy-benchmark.mjs compare + --current ".benchmark-results/${{ matrix.platform }}.json" + --baseline ".benchmark-baseline/${{ matrix.platform }}.json" + --baseline-kind "${{ steps.baseline.outputs.kind }}" + --output ".benchmark-results/${{ matrix.platform }}.md" + + - name: Add job summary + run: cat ".benchmark-results/${{ matrix.platform }}.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.baseline.outputs.current-artifact-name }} + path: .benchmark-results/${{ matrix.platform }}.* + if-no-files-found: error + overwrite: true + retention-days: 90 + + comment: + name: Report benchmark + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + needs: benchmark + runs-on: namespace-profile-linux-x64-default + permissions: + actions: read + contents: read + issues: write + steps: + - name: Download current results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: fspy-benchmark-*-pr-${{ github.event.pull_request.number }} + path: .benchmark-results + merge-multiple: true + + - name: Build PR comment + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + { + echo '' + echo '## fspy benchmark' + echo + echo "Results for commit \`$GITHUB_SHA\`. Each platform uses its latest result from this PR, falling back to \`main\`." + echo + for platform in linux macos windows; do + cat ".benchmark-results/$platform.md" + done + echo + echo 'This comment is updated by the Fspy Benchmark workflow.' + } > benchmark-comment.md + + - name: Update PR comment + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + node --input-type=module <<'EOF' + import { readFile } from "node:fs/promises"; + + const marker = ""; + const repository = process.env.GITHUB_REPOSITORY; + const token = process.env.GITHUB_TOKEN; + const pullRequest = process.env.PR_NUMBER; + const headers = { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }; + const commentsResponse = await fetch( + `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, + { headers }, + ); + if (!commentsResponse.ok) throw new Error(await commentsResponse.text()); + const comments = await commentsResponse.json(); + const existing = comments.find( + (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker), + ); + const body = await readFile("benchmark-comment.md", "utf8"); + const url = existing + ? `https://api.github.com/repos/${repository}/issues/comments/${existing.id}` + : `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments`; + const response = await fetch(url, { + method: existing ? "PATCH" : "POST", + headers, + body: JSON.stringify({ body }), + }); + if (!response.ok) throw new Error(await response.text()); + EOF diff --git a/Cargo.lock b/Cargo.lock index fd72b2ce0..f5a4464cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,6 +317,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "bpaf" +version = "0.9.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b86829876e7e200161a5aa6ea688d46c32d64d70ee3100127790dde84688d6e" + [[package]] name = "brush-parser" version = "0.4.0" @@ -411,6 +417,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "castaway" version = "0.2.4" @@ -459,6 +471,33 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -670,6 +709,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion2" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b961b72d7eaf7444d1eb98d353d5c2aa08e2443d77fef6bff0e6e97064162482" +dependencies = [ + "bpaf", + "cast", + "ciborium", + "num-traits", + "oorandom", + "serde", + "serde_json", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -723,6 +777,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -1239,6 +1299,37 @@ dependencies = [ "winsafe 0.0.27", ] +[[package]] +name = "fspy_benchmark" +version = "0.0.0" +dependencies = [ + "criterion2", + "fspy", + "fspy_benchmark_static_target", + "fspy_benchmark_target", + "tempfile", + "tokio", + "tokio-util", + "vite_path", + "vite_str", +] + +[[package]] +name = "fspy_benchmark_static_target" +version = "0.0.0" +dependencies = [ + "vite_path", + "vite_str", +] + +[[package]] +name = "fspy_benchmark_target" +version = "0.0.0" +dependencies = [ + "vite_path", + "vite_str", +] + [[package]] name = "fspy_detours_sys" version = "0.0.0" @@ -1537,6 +1628,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -2394,6 +2496,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "option-ext" version = "0.2.0" @@ -3251,9 +3359,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", diff --git a/Cargo.toml b/Cargo.toml index 804541230..818b45454 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ crossterm = { version = "0.29.0", features = ["event-stream"] } csv-async = { version = "1.3.1", features = ["tokio"] } ctor = "1.0" ctrlc = "3.5.2" +criterion2 = { version = "3.0.4", default-features = false } derive_more = "2.0.1" diff-struct = "0.5.3" directories = "6.0.0" @@ -72,6 +73,7 @@ materialized_artifact = { path = "crates/materialized_artifact" } materialized_artifact_build = { path = "crates/materialized_artifact_build" } flate2 = "1.0.35" fspy = { path = "crates/fspy" } +fspy_benchmark_target = { path = "crates/fspy_benchmark_target", artifact = "bin" } fspy_detours_sys = { path = "crates/fspy_detours_sys" } fspy_preload_unix = { path = "crates/fspy_preload_unix", artifact = "cdylib", target = "target" } fspy_preload_windows = { path = "crates/fspy_preload_windows", artifact = "cdylib", target = "target" } @@ -177,6 +179,7 @@ zstd = "0.13" [workspace.metadata.cargo-shear] ignored = [ # These are artifact dependencies. They are not directly `use`d in Rust code. + "fspy_benchmark_target", "fspy_preload_unix", "fspy_preload_windows", "vite_task_client_napi", diff --git a/crates/fspy_benchmark/Cargo.toml b/crates/fspy_benchmark/Cargo.toml new file mode 100644 index 000000000..18330d1b1 --- /dev/null +++ b/crates/fspy_benchmark/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "fspy_benchmark" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[lib] +test = false +bench = false +doctest = false + +[[bench]] +name = "fspy" +harness = false + +[dev-dependencies] +criterion2 = { workspace = true } +fspy = { workspace = true } +fspy_benchmark_target = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "process", "rt-multi-thread"] } +tokio-util = { workspace = true } +vite_path = { workspace = true } +vite_str = { workspace = true } + +[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dev-dependencies] +fspy_benchmark_static_target = { path = "../fspy_benchmark_static_target", artifact = "bin", target = "x86_64-unknown-linux-musl" } + +[package.metadata.cargo-shear] +ignored = ["fspy_benchmark_target", "fspy_benchmark_static_target"] diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md new file mode 100644 index 000000000..fcda70746 --- /dev/null +++ b/crates/fspy_benchmark/README.md @@ -0,0 +1,26 @@ +# fspy benchmark + +This benchmark measures the wall-clock overhead fspy adds to a filesystem-heavy child process. +The target starts four threads. Each thread opens 2,048 distinct, pre-created files for reading +and drops every handle immediately. + +The fixture is created before Criterion starts measuring. Tracked and untracked runs execute the +same target with the same arguments, empty environment, working directory, and standard streams. +The timer covers spawning the target through process termination and fspy finalization. A tracked +warmup verifies that all fixture accesses were captured outside the timed section. + +On Linux, the suite measures two targets: + +- `dynamic` is dynamically linked and exercises `LD_PRELOAD`. +- `static` is linked for `x86_64-unknown-linux-musl` and exercises seccomp user notification. + +macOS measures `DYLD_INSERT_LIBRARIES`, and Windows measures Detours injection. + +Run the benchmark with: + +```sh +just benchmark-fspy +``` + +CI runs it on the Linux, macOS, and Windows Namespace runner profiles. Each result is compared with +the latest result from the same pull request, falling back to the latest `main` result. diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs new file mode 100644 index 000000000..91dbc11b3 --- /dev/null +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -0,0 +1,144 @@ +use std::{ + ffi::OsStr, + fs::File, + process::{ExitStatus, Stdio}, + time::Duration, +}; + +use criterion::{ + BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, criterion_group, criterion_main, + measurement::WallTime, +}; +use fspy::{ChildTermination, Command}; +use tempfile::TempDir; +use tokio::runtime::{Builder, Runtime}; +use tokio_util::sync::CancellationToken; +use vite_path::{AbsolutePath, AbsolutePathBuf}; + +const THREAD_COUNT: usize = 4; +const FILES_PER_THREAD: usize = 2048; +const TOTAL_FILE_COUNT: usize = THREAD_COUNT * FILES_PER_THREAD; +const THREAD_COUNT_ARG: &str = "4"; +const FILES_PER_THREAD_ARG: &str = "2048"; +const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); + +fn benchmark(criterion: &mut Criterion) { + let (_fixture, fixture_root) = create_fixture(); + let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); + let mut group = criterion.benchmark_group("fspy"); + group.sampling_mode(SamplingMode::Flat); + + benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET, &fixture_root); + + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + benchmark_target(&mut group, &runtime, "static", STATIC_TARGET, &fixture_root); + + group.finish(); +} + +fn benchmark_target( + group: &mut BenchmarkGroup<'_, WallTime>, + runtime: &Runtime, + target_name: &str, + target: &str, + fixture: &AbsolutePath, +) { + validate_tracked_run(runtime, target, fixture); + + group.bench_with_input( + BenchmarkId::new(target_name, "untracked"), + &target, + |bencher, target| { + bencher.iter_with_setup_wrapper(|runner| { + let status = runner.run(|| runtime.block_on(run_untracked(target, fixture))); + assert!(status.success(), "untracked benchmark target failed: {status}"); + }); + }, + ); + + group.bench_with_input(BenchmarkId::new(target_name, "tracked"), &target, |bencher, target| { + bencher.iter_with_setup_wrapper(|runner| { + let termination = runner.run(|| runtime.block_on(run_tracked(target, fixture))); + assert!( + termination.status.success(), + "tracked benchmark target failed: {}", + termination.status + ); + }); + }); +} + +fn create_fixture() -> (TempDir, AbsolutePathBuf) { + let fixture = tempfile::tempdir().expect("failed to create benchmark fixture"); + let fixture_root = AbsolutePathBuf::new(fixture.path().to_path_buf()) + .expect("temporary path must be absolute"); + for index in 0..TOTAL_FILE_COUNT { + let path = fixture_root.join(vite_str::format!("file-{index:05}.txt")); + File::create(&path).unwrap_or_else(|error| { + panic!("failed to create {}: {error}", path.as_absolute_path()) + }); + } + (fixture, fixture_root) +} + +fn validate_tracked_run(runtime: &Runtime, target: &str, fixture: &AbsolutePath) { + let termination = runtime.block_on(run_tracked(target, fixture)); + assert!(termination.status.success(), "benchmark target failed: {}", termination.status); + let fixture_access_count = termination + .path_accesses + .iter() + .filter(|access| access.path.strip_path_prefix(fixture, |result| result.is_ok())) + .count(); + assert!( + fixture_access_count >= TOTAL_FILE_COUNT, + "expected at least {TOTAL_FILE_COUNT} fixture accesses, captured {fixture_access_count}" + ); +} + +async fn run_untracked(target: &str, fixture: &AbsolutePath) -> ExitStatus { + let mut command = tokio::process::Command::new(target); + command + .env_clear() + .args(benchmark_args(fixture)) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + command.status().await.expect("failed to run untracked benchmark target") +} + +async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { + let mut command = Command::new(target); + command + .args(benchmark_args(fixture)) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + command + .spawn(CancellationToken::new()) + .await + .expect("failed to spawn tracked benchmark target") + .wait_handle + .await + .expect("failed to wait for tracked benchmark target") +} + +fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 3] { + [fixture.as_path().as_os_str(), OsStr::new(THREAD_COUNT_ARG), OsStr::new(FILES_PER_THREAD_ARG)] +} + +fn criterion_config() -> Criterion { + Criterion::default() + .sample_size(20) + .warm_up_time(Duration::from_secs(2)) + .measurement_time(Duration::from_secs(10)) +} + +criterion_group! { + name = benches; + config = criterion_config(); + targets = benchmark +} +criterion_main!(benches); diff --git a/crates/fspy_benchmark/src/lib.rs b/crates/fspy_benchmark/src/lib.rs new file mode 100644 index 000000000..6d118a163 --- /dev/null +++ b/crates/fspy_benchmark/src/lib.rs @@ -0,0 +1 @@ +//! Benchmarks for the overhead fspy adds to tracked processes. diff --git a/crates/fspy_benchmark_static_target/Cargo.toml b/crates/fspy_benchmark_static_target/Cargo.toml new file mode 100644 index 000000000..b296739b1 --- /dev/null +++ b/crates/fspy_benchmark_static_target/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "fspy_benchmark_static_target" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +vite_path = { workspace = true } +vite_str = { workspace = true } + +[package.metadata.cargo-shear] +ignored = ["vite_path", "vite_str"] + +[[bin]] +name = "fspy_benchmark_static_target" +test = false +doctest = false diff --git a/crates/fspy_benchmark_static_target/src/main.rs b/crates/fspy_benchmark_static_target/src/main.rs new file mode 100644 index 000000000..6e96ceda0 --- /dev/null +++ b/crates/fspy_benchmark_static_target/src/main.rs @@ -0,0 +1 @@ +include!("../../fspy_benchmark_target/src/main.rs"); diff --git a/crates/fspy_benchmark_target/Cargo.toml b/crates/fspy_benchmark_target/Cargo.toml new file mode 100644 index 000000000..c5d917b47 --- /dev/null +++ b/crates/fspy_benchmark_target/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "fspy_benchmark_target" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +vite_path = { workspace = true } +vite_str = { workspace = true } + +[[bin]] +name = "fspy_benchmark_target" +test = false +doctest = false diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs new file mode 100644 index 000000000..7bc6b50a7 --- /dev/null +++ b/crates/fspy_benchmark_target/src/main.rs @@ -0,0 +1,53 @@ +use std::{ + env, + fs::File, + sync::{Arc, Barrier}, + thread, +}; + +use vite_path::{AbsolutePath, AbsolutePathBuf}; + +fn main() { + let mut args = env::args_os(); + let _program = args.next(); + let root = AbsolutePathBuf::new(args.next().expect("missing fixture root").into()) + .expect("fixture root must be absolute"); + let thread_count = parse_usize(args.next(), "thread count"); + let files_per_thread = parse_usize(args.next(), "files per thread"); + assert!(args.next().is_none(), "unexpected extra arguments"); + + run(&root, thread_count, files_per_thread); +} + +fn parse_usize(value: Option, name: &str) -> usize { + value + .unwrap_or_else(|| panic!("missing {name}")) + .into_string() + .unwrap_or_else(|_| panic!("{name} is not UTF-8")) + .parse() + .unwrap_or_else(|_| panic!("invalid {name}")) +} + +fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize) { + assert!(thread_count > 1, "benchmark requires multiple threads"); + assert!(files_per_thread > 0, "benchmark requires at least one file per thread"); + + let paths = (0..thread_count * files_per_thread) + .map(|index| root.join(vite_str::format!("file-{index:05}.txt"))) + .collect::>(); + let barrier = Arc::new(Barrier::new(thread_count)); + + thread::scope(|scope| { + for thread_paths in paths.chunks_exact(files_per_thread) { + let barrier = Arc::clone(&barrier); + scope.spawn(move || { + barrier.wait(); + for path in thread_paths { + drop(File::open(path).unwrap_or_else(|error| { + panic!("failed to open {}: {error}", path.as_absolute_path()) + })); + } + }); + } + }); +} diff --git a/justfile b/justfile index f4b8b865d..bd2d114f2 100644 --- a/justfile +++ b/justfile @@ -47,6 +47,9 @@ lint-linux: lint-windows: cargo-xwin clippy --workspace --all-targets --all-features --target x86_64-pc-windows-msvc -- --deny warnings +benchmark-fspy: + cargo bench -p fspy_benchmark --bench fspy + [unix] doc: RUSTDOCFLAGS='-D warnings' cargo doc --no-deps --document-private-items From 5048d18b02e05c5b95f0e4b2f1a5d6343a674551 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 00:57:21 +0800 Subject: [PATCH 02/23] ci: document trusted benchmark reporter Make the reporter shell explicit and document why its text-only workflow_run boundary is intentionally safe. Co-authored-by: GPT-5.6 Codex --- .github/workflows/fspy-benchmark-fork-report.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fspy-benchmark-fork-report.yml b/.github/workflows/fspy-benchmark-fork-report.yml index 4127d262b..2f2b5e5de 100644 --- a/.github/workflows/fspy-benchmark-fork-report.yml +++ b/.github/workflows/fspy-benchmark-fork-report.yml @@ -2,11 +2,15 @@ name: Fspy Benchmark Fork Report permissions: {} -on: +on: # zizmor: ignore[dangerous-triggers] PR artifacts are read as text and never executed. workflow_run: workflows: [Fspy Benchmark] types: [completed] +defaults: + run: + shell: bash + jobs: comment: name: Report fork benchmark From 9c24b21166869b784214c7c7c814f0af18f23dde Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 00:58:32 +0800 Subject: [PATCH 03/23] ci: upload hidden benchmark results Allow upload-artifact to include the dot-prefixed benchmark results directory. Co-authored-by: GPT-5.6 Codex --- .github/workflows/fspy-benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index b475bd886..e9dc47ef3 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -124,6 +124,7 @@ jobs: name: ${{ steps.baseline.outputs.current-artifact-name }} path: .benchmark-results/${{ matrix.platform }}.* if-no-files-found: error + include-hidden-files: true overwrite: true retention-days: 90 From 6cbe483a0a2e5dbca268bdd0433fe7d7898c6cee Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 01:03:23 +0800 Subject: [PATCH 04/23] ci: bound fspy benchmark process launches Use a shorter process-level sampling window and inherit child stderr so runtime failures remain diagnosable. Co-authored-by: GPT-5.6 Codex --- crates/fspy_benchmark/benches/fspy.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 91dbc11b3..53e8d518c 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -105,7 +105,7 @@ async fn run_untracked(target: &str, fixture: &AbsolutePath) -> ExitStatus { .args(benchmark_args(fixture)) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()); + .stderr(Stdio::inherit()); command.status().await.expect("failed to run untracked benchmark target") } @@ -115,7 +115,7 @@ async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { .args(benchmark_args(fixture)) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()); + .stderr(Stdio::inherit()); command .spawn(CancellationToken::new()) .await @@ -131,9 +131,9 @@ fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 3] { fn criterion_config() -> Criterion { Criterion::default() - .sample_size(20) - .warm_up_time(Duration::from_secs(2)) - .measurement_time(Duration::from_secs(10)) + .sample_size(10) + .warm_up_time(Duration::from_millis(500)) + .measurement_time(Duration::from_secs(1)) } criterion_group! { From 6c3807be9605c0151879e3dcb1932c9020d0144a Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 01:06:46 +0800 Subject: [PATCH 05/23] ci: grant benchmark PR comment access Use pull-request write permission for the sticky benchmark report endpoint. Co-authored-by: GPT-5.6 Codex --- .github/workflows/fspy-benchmark-fork-report.yml | 2 +- .github/workflows/fspy-benchmark.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fspy-benchmark-fork-report.yml b/.github/workflows/fspy-benchmark-fork-report.yml index 2f2b5e5de..35ff6d7da 100644 --- a/.github/workflows/fspy-benchmark-fork-report.yml +++ b/.github/workflows/fspy-benchmark-fork-report.yml @@ -23,7 +23,7 @@ jobs: permissions: actions: read contents: read - issues: write + pull-requests: write steps: - name: Download benchmark results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index e9dc47ef3..d86162339 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -138,7 +138,7 @@ jobs: permissions: actions: read contents: read - issues: write + pull-requests: write steps: - name: Download current results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 From 8541d088c23427d434fe02d846e2e5605be3598b Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 10:29:03 +0800 Subject: [PATCH 06/23] ci: compare fspy benchmarks against main only Preferring an earlier run of the same PR as the baseline masked cumulative regressions, so the baseline is now always the latest main artifact. Gate the main baseline artifact name on refs/heads/main so a workflow_dispatch run from another branch cannot pollute it. Remove the fork report workflow: workflow_run events carry an empty pull_requests array for PRs from forks, so its trigger condition could never match. Fork PRs keep job summaries and artifacts. Fold the PR comment assembly and upsert into fspy-benchmark.mjs. Co-Authored-By: Claude Fable 5 --- .github/scripts/fspy-benchmark.mjs | 79 +++++++++++----- .github/scripts/fspy-benchmark.test.mjs | 6 +- .../workflows/fspy-benchmark-fork-report.yml | 92 ------------------- .github/workflows/fspy-benchmark.yml | 67 +++----------- crates/fspy_benchmark/README.md | 2 +- 5 files changed, 72 insertions(+), 174 deletions(-) delete mode 100644 .github/workflows/fspy-benchmark-fork-report.yml diff --git a/.github/scripts/fspy-benchmark.mjs b/.github/scripts/fspy-benchmark.mjs index 67cf184e2..3d68f97db 100644 --- a/.github/scripts/fspy-benchmark.mjs +++ b/.github/scripts/fspy-benchmark.mjs @@ -29,14 +29,16 @@ function required(args, name) { return value; } -async function githubJson(path) { +async function githubJson(path, init = {}) { const token = requiredEnv('GITHUB_TOKEN'); const response = await fetch(`https://api.github.com${path}`, { + ...init, headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28', 'User-Agent': 'vite-task-fspy-benchmark', + ...(init.body === undefined ? {} : { 'Content-Type': 'application/json' }), }, }); if (!response.ok) { @@ -58,7 +60,10 @@ async function findLatestArtifact(name, currentRunId) { ); return response.artifacts .filter( - (artifact) => !artifact.expired && String(artifact.workflow_run?.id) !== String(currentRunId), + (artifact) => + !artifact.expired && + artifact.workflow_run?.id != null && + String(artifact.workflow_run.id) !== String(currentRunId), ) .sort((left, right) => Date.parse(right.created_at) - Date.parse(left.created_at))[0]; } @@ -69,29 +74,14 @@ async function resolveBaseline(args) { const context = required(args, 'context'); const currentRunId = requiredEnv('GITHUB_RUN_ID'); const currentArtifactName = `fspy-benchmark-${platform}-${context}`; - const candidates = [currentArtifactName]; - if (context !== 'main') candidates.push(`fspy-benchmark-${platform}-main`); - - let artifact; - let kind; - for (const candidate of candidates) { - artifact = await findLatestArtifact(candidate, currentRunId); - if (artifact) { - kind = candidate === currentArtifactName ? 'previous PR run' : 'main'; - break; - } - } + const artifact = await findLatestArtifact(`fspy-benchmark-${platform}-main`, currentRunId); const outputs = [ `current-artifact-name=${currentArtifactName}`, `found=${artifact ? 'true' : 'false'}`, ]; if (artifact) { - outputs.push( - `artifact-name=${artifact.name}`, - `run-id=${artifact.workflow_run.id}`, - `kind=${kind}`, - ); + outputs.push(`artifact-name=${artifact.name}`, `run-id=${artifact.workflow_run.id}`); } await appendFile(outputPath, `${outputs.join('\n')}\n`); } @@ -172,7 +162,7 @@ function resultLink(result) { return `https://github.com/${repository}/actions/runs/${result.runId}`; } -export function renderReport(current, baseline, baselineKind = '') { +export function renderReport(current, baseline) { const compatible = baseline && current.platform === baseline.platform && @@ -181,9 +171,7 @@ export function renderReport(current, baseline, baselineKind = '') { if (compatible) { const link = resultLink(baseline); - const description = link - ? `[${baselineKind || 'baseline'}](${link})` - : baselineKind || 'baseline'; + const description = link ? `[main](${link})` : 'main'; lines.push(`Compared with ${description} at \`${baseline.commit.slice(0, 8)}\`.`, ''); } else if (baseline) { lines.push( @@ -191,7 +179,7 @@ export function renderReport(current, baseline, baselineKind = '') { '', ); } else { - lines.push('No previous result was available for this runner.', ''); + lines.push('No `main` result was available for this runner.', ''); } lines.push( @@ -236,11 +224,50 @@ async function compare(args) { if (error.code !== 'ENOENT') throw error; } } - const report = renderReport(current, baseline, args.get('baseline-kind') ?? ''); + const report = renderReport(current, baseline); await writeFile(required(args, 'output'), report); process.stdout.write(report); } +const COMMENT_MARKER = ''; + +async function comment(args) { + const repository = requiredEnv('GITHUB_REPOSITORY'); + const pullRequest = required(args, 'pull-request'); + const commit = required(args, 'commit'); + const resultsDir = required(args, 'results-dir'); + + const sections = []; + for (const platform of ['linux', 'macos', 'windows']) { + sections.push(await readFile(join(resultsDir, `${platform}.md`), 'utf8')); + } + const body = [ + COMMENT_MARKER, + '## fspy benchmark', + '', + `Results for commit \`${commit}\`. Each platform is compared with the latest \`main\` result.`, + '', + ...sections, + 'This comment is updated by the Fspy Benchmark workflow.', + '', + ].join('\n'); + + const comments = await githubJson( + `/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, + ); + const existing = comments.find( + (existingComment) => + existingComment.user?.type === 'Bot' && existingComment.body?.includes(COMMENT_MARKER), + ); + const path = existing + ? `/repos/${repository}/issues/comments/${existing.id}` + : `/repos/${repository}/issues/${pullRequest}/comments`; + await githubJson(path, { + method: existing ? 'PATCH' : 'POST', + body: JSON.stringify({ body }), + }); +} + async function main() { const command = process.argv[2]; const args = parseArgs(process.argv.slice(3)); @@ -250,6 +277,8 @@ async function main() { await collect(args); } else if (command === 'compare') { await compare(args); + } else if (command === 'comment') { + await comment(args); } else { throw new Error(`unknown command: ${command ?? ''}`); } diff --git a/.github/scripts/fspy-benchmark.test.mjs b/.github/scripts/fspy-benchmark.test.mjs index d219767dc..486fed4a4 100644 --- a/.github/scripts/fspy-benchmark.test.mjs +++ b/.github/scripts/fspy-benchmark.test.mjs @@ -21,17 +21,17 @@ function result(platform, architecture, runId, untracked, tracked) { test('renders normalized change against a compatible baseline', () => { const baseline = result('linux', 'X64', '10', 100, 120); const current = result('linux', 'X64', '20', 100, 132); - const report = renderReport(current, baseline, 'previous PR run'); + const report = renderReport(current, baseline); assert.match(report, /fspy overhead \| Normalized change/); assert.match(report, /\+32\.00% \| \+10\.00%/); - assert.match(report, /previous PR run/); + assert.match(report, /Compared with main at `12345678`/); }); test('does not compare different architectures', () => { const baseline = result('macos', 'X64', '10', 100, 120); const current = result('macos', 'ARM64', '20', 100, 132); - const report = renderReport(current, baseline, 'main'); + const report = renderReport(current, baseline); assert.match(report, /No comparison/); assert.match(report, /\+32\.00% \| —/); diff --git a/.github/workflows/fspy-benchmark-fork-report.yml b/.github/workflows/fspy-benchmark-fork-report.yml deleted file mode 100644 index 35ff6d7da..000000000 --- a/.github/workflows/fspy-benchmark-fork-report.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Fspy Benchmark Fork Report - -permissions: {} - -on: # zizmor: ignore[dangerous-triggers] PR artifacts are read as text and never executed. - workflow_run: - workflows: [Fspy Benchmark] - types: [completed] - -defaults: - run: - shell: bash - -jobs: - comment: - name: Report fork benchmark - if: >- - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.head_repository.full_name != github.repository && - github.event.workflow_run.pull_requests[0].number != null - runs-on: namespace-profile-linux-x64-default - permissions: - actions: read - contents: read - pull-requests: write - steps: - - name: Download benchmark results - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: fspy-benchmark-*-pr-${{ github.event.workflow_run.pull_requests[0].number }} - path: .benchmark-results - merge-multiple: true - github-token: ${{ github.token }} - repository: ${{ github.repository }} - run-id: ${{ github.event.workflow_run.id }} - - - name: Build PR comment - env: - BENCHMARK_SHA: ${{ github.event.workflow_run.head_sha }} - run: | - { - echo '' - echo '## fspy benchmark' - echo - echo "Results for commit \`$BENCHMARK_SHA\`. Each platform uses its latest result from this PR, falling back to \`main\`." - echo - for platform in linux macos windows; do - cat ".benchmark-results/$platform.md" - done - echo - echo 'This comment is updated by the Fspy Benchmark workflow.' - } > benchmark-comment.md - - - name: Update PR comment - env: - GITHUB_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} - run: | - node --input-type=module <<'EOF' - import { readFile } from "node:fs/promises"; - - const marker = ""; - const repository = process.env.GITHUB_REPOSITORY; - const token = process.env.GITHUB_TOKEN; - const pullRequest = process.env.PR_NUMBER; - const headers = { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28", - }; - const commentsResponse = await fetch( - `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, - { headers }, - ); - if (!commentsResponse.ok) throw new Error(await commentsResponse.text()); - const comments = await commentsResponse.json(); - const existing = comments.find( - (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker), - ); - const body = await readFile("benchmark-comment.md", "utf8"); - const url = existing - ? `https://api.github.com/repos/${repository}/issues/comments/${existing.id}` - : `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments`; - const response = await fetch(url, { - method: existing ? "PATCH" : "POST", - headers, - body: JSON.stringify({ body }), - }); - if (!response.ok) throw new Error(await response.text()); - EOF diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index d86162339..f981223e3 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -61,12 +61,17 @@ jobs: env: PR_NUMBER: ${{ github.event.pull_request.number }} run: | + # Only real main-branch runs may publish the main baseline artifact; + # a workflow_dispatch run from another branch must not pollute it. if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then context="pr-$PR_NUMBER" pull_request="$PR_NUMBER" - else + elif [[ "$GITHUB_REF" == "refs/heads/main" ]]; then context="main" pull_request="none" + else + context="dispatch" + pull_request="none" fi echo "name=$context" >> "$GITHUB_OUTPUT" echo "pull-request=$pull_request" >> "$GITHUB_OUTPUT" @@ -112,7 +117,6 @@ jobs: node .github/scripts/fspy-benchmark.mjs compare --current ".benchmark-results/${{ matrix.platform }}.json" --baseline ".benchmark-baseline/${{ matrix.platform }}.json" - --baseline-kind "${{ steps.baseline.outputs.kind }}" --output ".benchmark-results/${{ matrix.platform }}.md" - name: Add job summary @@ -140,6 +144,8 @@ jobs: contents: read pull-requests: write steps: + - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + - name: Download current results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -147,58 +153,13 @@ jobs: path: .benchmark-results merge-multiple: true - - name: Build PR comment - env: - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - { - echo '' - echo '## fspy benchmark' - echo - echo "Results for commit \`$GITHUB_SHA\`. Each platform uses its latest result from this PR, falling back to \`main\`." - echo - for platform in linux macos windows; do - cat ".benchmark-results/$platform.md" - done - echo - echo 'This comment is updated by the Fspy Benchmark workflow.' - } > benchmark-comment.md - - name: Update PR comment env: GITHUB_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - node --input-type=module <<'EOF' - import { readFile } from "node:fs/promises"; - - const marker = ""; - const repository = process.env.GITHUB_REPOSITORY; - const token = process.env.GITHUB_TOKEN; - const pullRequest = process.env.PR_NUMBER; - const headers = { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", - "X-GitHub-Api-Version": "2022-11-28", - }; - const commentsResponse = await fetch( - `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, - { headers }, - ); - if (!commentsResponse.ok) throw new Error(await commentsResponse.text()); - const comments = await commentsResponse.json(); - const existing = comments.find( - (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker), - ); - const body = await readFile("benchmark-comment.md", "utf8"); - const url = existing - ? `https://api.github.com/repos/${repository}/issues/comments/${existing.id}` - : `https://api.github.com/repos/${repository}/issues/${pullRequest}/comments`; - const response = await fetch(url, { - method: existing ? "PATCH" : "POST", - headers, - body: JSON.stringify({ body }), - }); - if (!response.ok) throw new Error(await response.text()); - EOF + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: >- + node .github/scripts/fspy-benchmark.mjs comment + --results-dir .benchmark-results + --pull-request "$PR_NUMBER" + --commit "$HEAD_SHA" diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index fcda70746..137393eb4 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -23,4 +23,4 @@ just benchmark-fspy ``` CI runs it on the Linux, macOS, and Windows Namespace runner profiles. Each result is compared with -the latest result from the same pull request, falling back to the latest `main` result. +the latest `main` result. From 406d88d6dbb40d87eeb6650614a849c8408c7a18 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 11:47:42 +0800 Subject: [PATCH 07/23] ci: convert fspy benchmark script to TypeScript Node runs .mts files directly by stripping types, so the script stays build-free while gaining typed structures for the result JSON, Criterion estimates, and GitHub API payloads. The new tsconfig gives the scripts directory Node types and enforces erasable-only syntax. Comment the non-obvious constraints: baseline selection, re-run exclusion, ratio normalization, and the workload constants mirrored from the Criterion bench. Co-Authored-By: Claude Fable 5 --- ...{fspy-benchmark.mjs => fspy-benchmark.mts} | 211 +++++++++++++----- ...hmark.test.mjs => fspy-benchmark.test.mts} | 14 +- .github/scripts/tsconfig.json | 14 ++ .github/workflows/fspy-benchmark.yml | 8 +- 4 files changed, 184 insertions(+), 63 deletions(-) rename .github/scripts/{fspy-benchmark.mjs => fspy-benchmark.mts} (55%) rename .github/scripts/{fspy-benchmark.test.mjs => fspy-benchmark.test.mts} (76%) create mode 100644 .github/scripts/tsconfig.json diff --git a/.github/scripts/fspy-benchmark.mjs b/.github/scripts/fspy-benchmark.mts similarity index 55% rename from .github/scripts/fspy-benchmark.mjs rename to .github/scripts/fspy-benchmark.mts index 3d68f97db..c59a5f89f 100644 --- a/.github/scripts/fspy-benchmark.mjs +++ b/.github/scripts/fspy-benchmark.mts @@ -1,15 +1,65 @@ #!/usr/bin/env node +// Support script for the Fspy Benchmark workflow (.github/workflows/fspy-benchmark.yml). +// Executed directly with `node`, which strips the type annotations at load time, +// so only erasable TypeScript syntax is allowed (no enums, namespaces, etc.). + import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; import { arch, cpus, platform as osPlatform, release } from 'node:os'; import { dirname, join } from 'node:path'; import { pathToFileURL } from 'node:url'; +// Mirrors THREAD_COUNT and FILES_PER_THREAD in crates/fspy_benchmark/benches/fspy.rs. +// Only used to describe the workload in reports. const THREAD_COUNT = 4; const FILES_PER_THREAD = 2048; -function parseArgs(args) { - const parsed = new Map(); +type Args = Map; + +/** One Criterion estimate, in nanoseconds. */ +interface Estimate { + meanNs: number; + meanLowerNs: number; + meanUpperNs: number; + medianNs: number; +} + +/** The subset of a stored benchmark result that reports are rendered from. */ +interface ReportInput { + platform: string; + architecture: string; + os: { cpu: string }; + commit: string; + runId: string; + workload: { threadCount: number; filesPerThread: number; totalOpens: number }; + benchmarks: Record; +} + +/** The full result JSON produced by `collect` and uploaded as an artifact. */ +interface BenchmarkResult extends ReportInput { + schemaVersion: number; + os: { platform: string; release: string; cpu: string }; + runner: string; + runAttempt: string; + event: string; + pullRequest: string; + benchmarks: Record; +} + +interface BaselineArtifact { + name: string; + runId: string; + createdAt: string; +} + +interface IssueComment { + id: number; + body?: string | null; + user?: { type?: string } | null; +} + +function parseArgs(args: string[]): Args { + const parsed = new Map(); for (let index = 0; index < args.length; index += 2) { const flag = args[index]; const value = args[index + 1]; @@ -21,7 +71,7 @@ function parseArgs(args) { return parsed; } -function required(args, name) { +function required(args: Args, name: string): string { const value = args.get(name); if (value === undefined || value === '') { throw new Error(`missing --${name}`); @@ -29,7 +79,13 @@ function required(args, name) { return value; } -async function githubJson(path, init = {}) { +function requiredEnv(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`missing ${name}`); + return value; +} + +async function githubJson(path: string, init: RequestInit = {}): Promise { const token = requiredEnv('GITHUB_TOKEN'); const response = await fetch(`https://api.github.com${path}`, { ...init, @@ -44,31 +100,48 @@ async function githubJson(path, init = {}) { if (!response.ok) { throw new Error(`GitHub API ${response.status}: ${await response.text()}`); } - return response.json(); + return response.json() as Promise; } -function requiredEnv(name) { - const value = process.env[name]; - if (!value) throw new Error(`missing ${name}`); - return value; -} - -async function findLatestArtifact(name, currentRunId) { +async function findLatestArtifact( + name: string, + currentRunId: string, +): Promise { const repository = requiredEnv('GITHUB_REPOSITORY'); - const response = await githubJson( - `/repos/${repository}/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`, + // The name filter is applied server-side, so one page covers the recent history. + const response = await githubJson<{ + artifacts: { + name: string; + expired: boolean; + created_at: string; + workflow_run?: { id: number } | null; + }[]; + }>(`/repos/${repository}/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`); + return ( + response.artifacts + .flatMap((artifact) => + !artifact.expired && artifact.workflow_run?.id != null + ? [ + { + name: artifact.name, + runId: String(artifact.workflow_run.id), + createdAt: artifact.created_at, + }, + ] + : [], + ) + // Skip the current run so a re-run attempt does not compare against the + // artifact its own previous attempt uploaded. + .filter((artifact) => artifact.runId !== currentRunId) + .sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt))[0] ); - return response.artifacts - .filter( - (artifact) => - !artifact.expired && - artifact.workflow_run?.id != null && - String(artifact.workflow_run.id) !== String(currentRunId), - ) - .sort((left, right) => Date.parse(right.created_at) - Date.parse(left.created_at))[0]; } -async function resolveBaseline(args) { +// Emits step outputs consumed by the workflow's baseline download and result +// upload steps. The baseline is always the latest `main` artifact: comparing +// against earlier runs of the same PR would mask regressions that accumulate +// over the PR's lifetime. +async function resolveBaseline(args: Args): Promise { const outputPath = requiredEnv('GITHUB_OUTPUT'); const platform = required(args, 'platform'); const context = required(args, 'context'); @@ -81,14 +154,26 @@ async function resolveBaseline(args) { `found=${artifact ? 'true' : 'false'}`, ]; if (artifact) { - outputs.push(`artifact-name=${artifact.name}`, `run-id=${artifact.workflow_run.id}`); + outputs.push(`artifact-name=${artifact.name}`, `run-id=${artifact.runId}`); } await appendFile(outputPath, `${outputs.join('\n')}\n`); } -async function readEstimate(criterionRoot, target, mode) { +// Criterion writes the estimates of its most recent run to +// ///new/estimates.json. +async function readEstimate( + criterionRoot: string, + target: string, + mode: string, +): Promise { const path = join(criterionRoot, target, mode, 'new', 'estimates.json'); - const estimates = JSON.parse(await readFile(path, 'utf8')); + const estimates = JSON.parse(await readFile(path, 'utf8')) as { + mean: { + point_estimate: number; + confidence_interval: { lower_bound: number; upper_bound: number }; + }; + median: { point_estimate: number }; + }; return { meanNs: estimates.mean.point_estimate, meanLowerNs: estimates.mean.confidence_interval.lower_bound, @@ -97,12 +182,14 @@ async function readEstimate(criterionRoot, target, mode) { }; } -async function collect(args) { +// Combines Criterion's output with run metadata into the JSON document that is +// uploaded as an artifact and later consumed as a comparison baseline. +async function collect(args: Args): Promise { const criterionRoot = required(args, 'criterion-root'); const output = required(args, 'output'); const platform = required(args, 'platform'); const expectStatic = args.get('expect-static') === 'true'; - const benchmarks = { + const benchmarks: Record = { 'dynamic/untracked': await readEstimate(criterionRoot, 'dynamic', 'untracked'), 'dynamic/tracked': await readEstimate(criterionRoot, 'dynamic', 'tracked'), }; @@ -111,10 +198,10 @@ async function collect(args) { benchmarks['static/tracked'] = await readEstimate(criterionRoot, 'static', 'tracked'); } - const result = { + const result: BenchmarkResult = { schemaVersion: 1, platform, - architecture: process.env.RUNNER_ARCH ?? arch(), + architecture: process.env['RUNNER_ARCH'] ?? arch(), os: { platform: osPlatform(), release: release(), @@ -138,36 +225,40 @@ async function collect(args) { await writeFile(output, `${JSON.stringify(result, null, 2)}\n`); } -function ratio(result, target) { - return ( - result.benchmarks[`${target}/tracked`].meanNs / result.benchmarks[`${target}/untracked`].meanNs - ); +/** Wall-clock cost of tracking, as tracked time over untracked time. */ +function ratio(result: ReportInput, target: string): number { + const tracked = result.benchmarks[`${target}/tracked`]; + const untracked = result.benchmarks[`${target}/untracked`]; + if (!tracked || !untracked) throw new Error(`missing benchmark data for ${target}`); + return tracked.meanNs / untracked.meanNs; } -function percentage(value) { +function percentage(value: number): string { const sign = value > 0 ? '+' : ''; return `${sign}${(value * 100).toFixed(2)}%`; } -function duration(nanoseconds) { +function duration(nanoseconds: number): string { if (nanoseconds >= 1e9) return `${(nanoseconds / 1e9).toFixed(3)} s`; if (nanoseconds >= 1e6) return `${(nanoseconds / 1e6).toFixed(3)} ms`; if (nanoseconds >= 1e3) return `${(nanoseconds / 1e3).toFixed(3)} µs`; return `${nanoseconds.toFixed(1)} ns`; } -function resultLink(result) { - const repository = process.env.GITHUB_REPOSITORY; +function resultLink(result: ReportInput): string { + const repository = process.env['GITHUB_REPOSITORY']; if (!repository || !result.runId) return ''; return `https://github.com/${repository}/actions/runs/${result.runId}`; } -export function renderReport(current, baseline) { +export function renderReport(current: ReportInput, baseline?: ReportInput): string { + // Overhead ratios are only comparable within one architecture; a runner + // profile can move to different hardware over time. const compatible = - baseline && + baseline !== undefined && current.platform === baseline.platform && current.architecture === baseline.architecture; - const lines = [`### ${current.platform[0].toUpperCase()}${current.platform.slice(1)}`, '']; + const lines = [`### ${current.platform.charAt(0).toUpperCase()}${current.platform.slice(1)}`, '']; if (compatible) { const link = resultLink(baseline); @@ -191,15 +282,19 @@ export function renderReport(current, baseline) { ? ['dynamic', 'static'] : ['dynamic']; for (const target of targets) { - const untracked = current.benchmarks[`${target}/untracked`].meanNs; - const tracked = current.benchmarks[`${target}/tracked`].meanNs; + const untracked = current.benchmarks[`${target}/untracked`]; + const tracked = current.benchmarks[`${target}/tracked`]; + if (!untracked || !tracked) throw new Error(`missing benchmark data for ${target}`); const overhead = ratio(current, target) - 1; + // Change of the overhead ratio relative to the baseline. Comparing ratios + // instead of absolute times cancels machine-speed differences between + // runner instances. const normalizedChange = compatible && Object.hasOwn(baseline.benchmarks, `${target}/tracked`) ? ratio(current, target) / ratio(baseline, target) - 1 : undefined; lines.push( - `| ${target} | ${duration(untracked)} | ${duration(tracked)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, + `| ${target} | ${duration(untracked.meanNs)} | ${duration(tracked.meanNs)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, ); } @@ -213,15 +308,17 @@ export function renderReport(current, baseline) { return `${lines.join('\n')}\n`; } -async function compare(args) { - const current = JSON.parse(await readFile(required(args, 'current'), 'utf8')); +async function compare(args: Args): Promise { + const current = JSON.parse(await readFile(required(args, 'current'), 'utf8')) as BenchmarkResult; const baselinePath = args.get('baseline'); - let baseline; + let baseline: BenchmarkResult | undefined; if (baselinePath) { try { - baseline = JSON.parse(await readFile(baselinePath, 'utf8')); + baseline = JSON.parse(await readFile(baselinePath, 'utf8')) as BenchmarkResult; } catch (error) { - if (error.code !== 'ENOENT') throw error; + // The workflow always passes --baseline; the file is absent when no + // baseline artifact was found. + if ((error as { code?: string }).code !== 'ENOENT') throw error; } } const report = renderReport(current, baseline); @@ -231,13 +328,15 @@ async function compare(args) { const COMMENT_MARKER = ''; -async function comment(args) { +// Creates or updates the sticky PR comment. The invisible marker identifies +// the comment, so later runs edit it in place instead of posting a new one. +async function comment(args: Args): Promise { const repository = requiredEnv('GITHUB_REPOSITORY'); const pullRequest = required(args, 'pull-request'); const commit = required(args, 'commit'); const resultsDir = required(args, 'results-dir'); - const sections = []; + const sections: string[] = []; for (const platform of ['linux', 'macos', 'windows']) { sections.push(await readFile(join(resultsDir, `${platform}.md`), 'utf8')); } @@ -252,7 +351,7 @@ async function comment(args) { '', ].join('\n'); - const comments = await githubJson( + const comments = await githubJson( `/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, ); const existing = comments.find( @@ -268,7 +367,7 @@ async function comment(args) { }); } -async function main() { +async function main(): Promise { const command = process.argv[2]; const args = parseArgs(process.argv.slice(3)); if (command === 'resolve-baseline') { @@ -284,10 +383,12 @@ async function main() { } } +// Run only when executed directly, so the test file can import renderReport. const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : ''; if (import.meta.url === invokedPath) { - main().catch((error) => { - process.stderr.write(`${error.stack ?? error}\n`); + main().catch((error: unknown) => { + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + process.stderr.write(`${message}\n`); process.exitCode = 1; }); } diff --git a/.github/scripts/fspy-benchmark.test.mjs b/.github/scripts/fspy-benchmark.test.mts similarity index 76% rename from .github/scripts/fspy-benchmark.test.mjs rename to .github/scripts/fspy-benchmark.test.mts index 486fed4a4..33d24c4ee 100644 --- a/.github/scripts/fspy-benchmark.test.mjs +++ b/.github/scripts/fspy-benchmark.test.mts @@ -1,9 +1,15 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { renderReport } from './fspy-benchmark.mjs'; +import { renderReport } from './fspy-benchmark.mts'; -function result(platform, architecture, runId, untracked, tracked) { +function result( + platform: string, + architecture: string, + runId: string, + untracked: number, + tracked: number, +) { return { platform, architecture, @@ -18,7 +24,7 @@ function result(platform, architecture, runId, untracked, tracked) { }; } -test('renders normalized change against a compatible baseline', () => { +void test('renders normalized change against a compatible baseline', () => { const baseline = result('linux', 'X64', '10', 100, 120); const current = result('linux', 'X64', '20', 100, 132); const report = renderReport(current, baseline); @@ -28,7 +34,7 @@ test('renders normalized change against a compatible baseline', () => { assert.match(report, /Compared with main at `12345678`/); }); -test('does not compare different architectures', () => { +void test('does not compare different architectures', () => { const baseline = result('macos', 'X64', '10', 100, 120); const current = result('macos', 'ARM64', '20', 100, 132); const report = renderReport(current, baseline); diff --git a/.github/scripts/tsconfig.json b/.github/scripts/tsconfig.json new file mode 100644 index 000000000..07f8c9b9b --- /dev/null +++ b/.github/scripts/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "@tsconfig/strictest/tsconfig.json", + "compilerOptions": { + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "module": "NodeNext", + "moduleResolution": "NodeNext", + "noEmit": true, + "target": "ES2022", + "lib": ["ES2022"], + "types": ["node"] + }, + "include": ["*.mts"] +} diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index f981223e3..a27ba9762 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -81,7 +81,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: >- - node .github/scripts/fspy-benchmark.mjs resolve-baseline + node .github/scripts/fspy-benchmark.mts resolve-baseline --platform "${{ matrix.platform }}" --context "${{ steps.context.outputs.name }}" @@ -100,7 +100,7 @@ jobs: - name: Collect benchmark results run: >- - node .github/scripts/fspy-benchmark.mjs collect + node .github/scripts/fspy-benchmark.mts collect --criterion-root "$CARGO_TARGET_DIR/criterion/fspy" --output ".benchmark-results/${{ matrix.platform }}.json" --platform "${{ matrix.platform }}" @@ -114,7 +114,7 @@ jobs: - name: Compare benchmark results run: >- - node .github/scripts/fspy-benchmark.mjs compare + node .github/scripts/fspy-benchmark.mts compare --current ".benchmark-results/${{ matrix.platform }}.json" --baseline ".benchmark-baseline/${{ matrix.platform }}.json" --output ".benchmark-results/${{ matrix.platform }}.md" @@ -159,7 +159,7 @@ jobs: PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: >- - node .github/scripts/fspy-benchmark.mjs comment + node .github/scripts/fspy-benchmark.mts comment --results-dir .benchmark-results --pull-request "$PR_NUMBER" --commit "$HEAD_SHA" From d029f067db1979680bb374bd5faee4fb24f4b3ed Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 11:50:06 +0800 Subject: [PATCH 08/23] ci: set up pinned Node for the fspy benchmark script The Linux and Windows runner images preinstall Node 20, which cannot run .mts files; type stripping needs the pinned 22.19. Co-Authored-By: Claude Fable 5 --- .github/workflows/fspy-benchmark.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index a27ba9762..f7a90af24 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -52,6 +52,11 @@ jobs: save-cache: ${{ github.ref_name == 'main' }} cache-key: fspy-benchmark-${{ matrix.platform }} + # The support script is TypeScript; running it needs the .node-version + # Node, which strips types natively. Runner-preinstalled Node may be + # older. + - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 + - name: Install static target if: matrix.static run: rustup target add x86_64-unknown-linux-musl @@ -146,6 +151,8 @@ jobs: steps: - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 + - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 + - name: Download current results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: From d4c63ac6bced2da0b90adb5289ae9ec9e87e355f Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 13:39:47 +0800 Subject: [PATCH 09/23] test(fspy): amplify benchmark workload over process startup With a single pass over the fixture, the tracked/untracked delta on the preload paths (2-8 ms) is the same order as spawn-time variance on CI runners: overhead swung 11 points between identical runs, and on Windows the confidence interval exceeded the delta. Repeat the open loop 16 times so per-open interception cost dominates, extend the measurement window to five seconds, and render each mean's confidence interval in the report. Bump the result schema and skip comparisons across schema versions, since ratios from different workloads are not comparable. Co-Authored-By: Claude Fable 5 --- .github/scripts/fspy-benchmark.mts | 49 +++++++++++++++++------- .github/scripts/fspy-benchmark.test.mts | 22 +++++++++-- crates/fspy_benchmark/README.md | 6 ++- crates/fspy_benchmark/benches/fspy.rs | 19 +++++++-- crates/fspy_benchmark_target/src/main.rs | 16 +++++--- 5 files changed, 83 insertions(+), 29 deletions(-) diff --git a/.github/scripts/fspy-benchmark.mts b/.github/scripts/fspy-benchmark.mts index c59a5f89f..ca8493b23 100644 --- a/.github/scripts/fspy-benchmark.mts +++ b/.github/scripts/fspy-benchmark.mts @@ -9,35 +9,45 @@ import { arch, cpus, platform as osPlatform, release } from 'node:os'; import { dirname, join } from 'node:path'; import { pathToFileURL } from 'node:url'; -// Mirrors THREAD_COUNT and FILES_PER_THREAD in crates/fspy_benchmark/benches/fspy.rs. -// Only used to describe the workload in reports. +// Mirrors THREAD_COUNT, FILES_PER_THREAD, and PASSES in +// crates/fspy_benchmark/benches/fspy.rs. Only used to describe the workload in +// reports. const THREAD_COUNT = 4; const FILES_PER_THREAD = 2048; +const PASSES = 16; + +// Bump when the workload or result shape changes; results from different +// schema versions are not comparable. +const SCHEMA_VERSION = 2; type Args = Map; -/** One Criterion estimate, in nanoseconds. */ -interface Estimate { +/** Mean with its confidence interval, in nanoseconds. */ +interface EstimateSummary { meanNs: number; meanLowerNs: number; meanUpperNs: number; +} + +/** One Criterion estimate, in nanoseconds. */ +interface Estimate extends EstimateSummary { medianNs: number; } /** The subset of a stored benchmark result that reports are rendered from. */ interface ReportInput { + schemaVersion: number; platform: string; architecture: string; os: { cpu: string }; commit: string; runId: string; - workload: { threadCount: number; filesPerThread: number; totalOpens: number }; - benchmarks: Record; + workload: { threadCount: number; filesPerThread: number; passes: number; totalOpens: number }; + benchmarks: Record; } /** The full result JSON produced by `collect` and uploaded as an artifact. */ interface BenchmarkResult extends ReportInput { - schemaVersion: number; os: { platform: string; release: string; cpu: string }; runner: string; runAttempt: string; @@ -199,7 +209,7 @@ async function collect(args: Args): Promise { } const result: BenchmarkResult = { - schemaVersion: 1, + schemaVersion: SCHEMA_VERSION, platform, architecture: process.env['RUNNER_ARCH'] ?? arch(), os: { @@ -216,7 +226,8 @@ async function collect(args: Args): Promise { workload: { threadCount: THREAD_COUNT, filesPerThread: FILES_PER_THREAD, - totalOpens: THREAD_COUNT * FILES_PER_THREAD, + passes: PASSES, + totalOpens: THREAD_COUNT * FILES_PER_THREAD * PASSES, }, benchmarks, }; @@ -245,6 +256,12 @@ function duration(nanoseconds: number): string { return `${nanoseconds.toFixed(1)} ns`; } +/** Mean with the half-width of its confidence interval, e.g. `12.3 ms ±1.4%`. */ +function durationWithUncertainty(estimate: EstimateSummary): string { + const halfWidth = (estimate.meanUpperNs - estimate.meanLowerNs) / 2 / estimate.meanNs; + return `${duration(estimate.meanNs)} ±${(halfWidth * 100).toFixed(1)}%`; +} + function resultLink(result: ReportInput): string { const repository = process.env['GITHUB_REPOSITORY']; if (!repository || !result.runId) return ''; @@ -252,10 +269,12 @@ function resultLink(result: ReportInput): string { } export function renderReport(current: ReportInput, baseline?: ReportInput): string { - // Overhead ratios are only comparable within one architecture; a runner - // profile can move to different hardware over time. + // Overhead ratios are only comparable within one architecture and one + // workload: a runner profile can move to different hardware over time, and + // schema bumps change what is measured. const compatible = baseline !== undefined && + current.schemaVersion === baseline.schemaVersion && current.platform === baseline.platform && current.architecture === baseline.architecture; const lines = [`### ${current.platform.charAt(0).toUpperCase()}${current.platform.slice(1)}`, '']; @@ -266,7 +285,9 @@ export function renderReport(current: ReportInput, baseline?: ReportInput): stri lines.push(`Compared with ${description} at \`${baseline.commit.slice(0, 8)}\`.`, ''); } else if (baseline) { lines.push( - `No comparison: the baseline architecture is \`${baseline.architecture}\`, current is \`${current.architecture}\`.`, + baseline.schemaVersion === current.schemaVersion + ? `No comparison: the baseline architecture is \`${baseline.architecture}\`, current is \`${current.architecture}\`.` + : `No comparison: the baseline uses result schema ${baseline.schemaVersion}, current is ${current.schemaVersion}.`, '', ); } else { @@ -294,13 +315,13 @@ export function renderReport(current: ReportInput, baseline?: ReportInput): stri ? ratio(current, target) / ratio(baseline, target) - 1 : undefined; lines.push( - `| ${target} | ${duration(untracked.meanNs)} | ${duration(tracked.meanNs)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, + `| ${target} | ${durationWithUncertainty(untracked)} | ${durationWithUncertainty(tracked)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, ); } lines.push( '', - `Workload: ${current.workload.threadCount} threads, ${current.workload.filesPerThread.toLocaleString('en-US')} files per thread, ${current.workload.totalOpens.toLocaleString('en-US')} total open-and-close operations.`, + `Workload: ${current.workload.threadCount} threads × ${current.workload.filesPerThread.toLocaleString('en-US')} files × ${current.workload.passes.toLocaleString('en-US')} passes, ${current.workload.totalOpens.toLocaleString('en-US')} total open-and-close operations.`, '', `\`${current.architecture}\` · ${current.os.cpu} · run [${current.runId}](${resultLink(current)})`, '', diff --git a/.github/scripts/fspy-benchmark.test.mts b/.github/scripts/fspy-benchmark.test.mts index 33d24c4ee..4b527790f 100644 --- a/.github/scripts/fspy-benchmark.test.mts +++ b/.github/scripts/fspy-benchmark.test.mts @@ -3,6 +3,10 @@ import test from 'node:test'; import { renderReport } from './fspy-benchmark.mts'; +function estimate(meanNs: number) { + return { meanNs, meanLowerNs: meanNs * 0.98, meanUpperNs: meanNs * 1.02 }; +} + function result( platform: string, architecture: string, @@ -11,15 +15,16 @@ function result( tracked: number, ) { return { + schemaVersion: 2, platform, architecture, os: { cpu: 'Test CPU' }, commit: '1234567890abcdef', runId, - workload: { threadCount: 4, filesPerThread: 2048, totalOpens: 8192 }, + workload: { threadCount: 4, filesPerThread: 2048, passes: 16, totalOpens: 131072 }, benchmarks: { - 'dynamic/untracked': { meanNs: untracked }, - 'dynamic/tracked': { meanNs: tracked }, + 'dynamic/untracked': estimate(untracked), + 'dynamic/tracked': estimate(tracked), }, }; } @@ -30,7 +35,7 @@ void test('renders normalized change against a compatible baseline', () => { const report = renderReport(current, baseline); assert.match(report, /fspy overhead \| Normalized change/); - assert.match(report, /\+32\.00% \| \+10\.00%/); + assert.match(report, /100\.0 ns ±2\.0% \| 132\.0 ns ±2\.0% \| \+32\.00% \| \+10\.00%/); assert.match(report, /Compared with main at `12345678`/); }); @@ -42,3 +47,12 @@ void test('does not compare different architectures', () => { assert.match(report, /No comparison/); assert.match(report, /\+32\.00% \| —/); }); + +void test('does not compare different schema versions', () => { + const baseline = { ...result('linux', 'X64', '10', 100, 120), schemaVersion: 1 }; + const current = result('linux', 'X64', '20', 100, 132); + const report = renderReport(current, baseline); + + assert.match(report, /No comparison: the baseline uses result schema 1, current is 2\./); + assert.match(report, /\+32\.00% \| —/); +}); diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index 137393eb4..bb9d50758 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -1,8 +1,10 @@ # fspy benchmark This benchmark measures the wall-clock overhead fspy adds to a filesystem-heavy child process. -The target starts four threads. Each thread opens 2,048 distinct, pre-created files for reading -and drops every handle immediately. +The target starts four threads. Each thread opens 2,048 distinct, pre-created files for reading, +dropping every handle immediately, and repeats this for 16 passes — 131,072 tracked opens per +run. The passes keep per-open interception cost dominant over process startup, whose variance on +CI runners would otherwise drown the tracked/untracked delta. The fixture is created before Criterion starts measuring. Tracked and untracked runs execute the same target with the same arguments, empty environment, working directory, and standard streams. diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 53e8d518c..ef709347b 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -15,11 +15,17 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; use vite_path::{AbsolutePath, AbsolutePathBuf}; +// Mirrored by .github/scripts/fspy-benchmark.mts for report rendering. +// The passes multiplier makes per-open interception cost dominate process +// startup in the measurement; with a single pass the tracked/untracked delta +// is the same order as spawn-time variance on CI runners. const THREAD_COUNT: usize = 4; const FILES_PER_THREAD: usize = 2048; +const PASSES: usize = 16; const TOTAL_FILE_COUNT: usize = THREAD_COUNT * FILES_PER_THREAD; const THREAD_COUNT_ARG: &str = "4"; const FILES_PER_THREAD_ARG: &str = "2048"; +const PASSES_ARG: &str = "16"; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); #[cfg(all(target_os = "linux", target_arch = "x86_64"))] @@ -92,6 +98,8 @@ fn validate_tracked_run(runtime: &Runtime, target: &str, fixture: &AbsolutePath) .iter() .filter(|access| access.path.strip_path_prefix(fixture, |result| result.is_ok())) .count(); + // Requires every distinct fixture file to be observed, independent of + // whether repeated accesses to the same path are deduplicated. assert!( fixture_access_count >= TOTAL_FILE_COUNT, "expected at least {TOTAL_FILE_COUNT} fixture accesses, captured {fixture_access_count}" @@ -125,15 +133,20 @@ async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { .expect("failed to wait for tracked benchmark target") } -fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 3] { - [fixture.as_path().as_os_str(), OsStr::new(THREAD_COUNT_ARG), OsStr::new(FILES_PER_THREAD_ARG)] +fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 4] { + [ + fixture.as_path().as_os_str(), + OsStr::new(THREAD_COUNT_ARG), + OsStr::new(FILES_PER_THREAD_ARG), + OsStr::new(PASSES_ARG), + ] } fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) .warm_up_time(Duration::from_millis(500)) - .measurement_time(Duration::from_secs(1)) + .measurement_time(Duration::from_secs(5)) } criterion_group! { diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index 7bc6b50a7..786fa5fe0 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -14,9 +14,10 @@ fn main() { .expect("fixture root must be absolute"); let thread_count = parse_usize(args.next(), "thread count"); let files_per_thread = parse_usize(args.next(), "files per thread"); + let passes = parse_usize(args.next(), "passes"); assert!(args.next().is_none(), "unexpected extra arguments"); - run(&root, thread_count, files_per_thread); + run(&root, thread_count, files_per_thread, passes); } fn parse_usize(value: Option, name: &str) -> usize { @@ -28,9 +29,10 @@ fn parse_usize(value: Option, name: &str) -> usize { .unwrap_or_else(|_| panic!("invalid {name}")) } -fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize) { +fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize, passes: usize) { assert!(thread_count > 1, "benchmark requires multiple threads"); assert!(files_per_thread > 0, "benchmark requires at least one file per thread"); + assert!(passes > 0, "benchmark requires at least one pass"); let paths = (0..thread_count * files_per_thread) .map(|index| root.join(vite_str::format!("file-{index:05}.txt"))) @@ -42,10 +44,12 @@ fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize) { let barrier = Arc::clone(&barrier); scope.spawn(move || { barrier.wait(); - for path in thread_paths { - drop(File::open(path).unwrap_or_else(|error| { - panic!("failed to open {}: {error}", path.as_absolute_path()) - })); + for _ in 0..passes { + for path in thread_paths { + drop(File::open(path).unwrap_or_else(|error| { + panic!("failed to open {}: {error}", path.as_absolute_path()) + })); + } } }); } From df091fbed0fd734dc0804466404b73f092c307e8 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 13:41:17 +0800 Subject: [PATCH 10/23] test(fspy): report per-open throughput via criterion Co-Authored-By: Claude Fable 5 --- crates/fspy_benchmark/benches/fspy.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index ef709347b..1fd01ef37 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -6,8 +6,8 @@ use std::{ }; use criterion::{ - BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, criterion_group, criterion_main, - measurement::WallTime, + BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, + criterion_main, measurement::WallTime, }; use fspy::{ChildTermination, Command}; use tempfile::TempDir; @@ -36,6 +36,8 @@ fn benchmark(criterion: &mut Criterion) { let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); let mut group = criterion.benchmark_group("fspy"); group.sampling_mode(SamplingMode::Flat); + // Criterion reports per-open time alongside wall time. + group.throughput(Throughput::Elements((TOTAL_FILE_COUNT * PASSES) as u64)); benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET, &fixture_root); From 8f12d36db8014f4014078922060f2d09c5c653bc Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:14:31 +0800 Subject: [PATCH 11/23] test(fspy): simplify overhead benchmark Co-authored-by: GPT-5 Codex --- .github/scripts/fspy-benchmark.mts | 415 ------------------ .github/scripts/fspy-benchmark.test.mts | 58 --- .github/scripts/tsconfig.json | 14 - .github/workflows/fspy-benchmark.yml | 153 +++---- Cargo.lock | 11 - crates/fspy_benchmark/Cargo.toml | 3 - crates/fspy_benchmark/README.md | 22 +- crates/fspy_benchmark/benches/fspy.rs | 145 +++--- .../fspy_benchmark_static_target/Cargo.toml | 7 - crates/fspy_benchmark_target/Cargo.toml | 4 - crates/fspy_benchmark_target/src/main.rs | 50 +-- 11 files changed, 136 insertions(+), 746 deletions(-) delete mode 100644 .github/scripts/fspy-benchmark.mts delete mode 100644 .github/scripts/fspy-benchmark.test.mts delete mode 100644 .github/scripts/tsconfig.json diff --git a/.github/scripts/fspy-benchmark.mts b/.github/scripts/fspy-benchmark.mts deleted file mode 100644 index ca8493b23..000000000 --- a/.github/scripts/fspy-benchmark.mts +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env node - -// Support script for the Fspy Benchmark workflow (.github/workflows/fspy-benchmark.yml). -// Executed directly with `node`, which strips the type annotations at load time, -// so only erasable TypeScript syntax is allowed (no enums, namespaces, etc.). - -import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; -import { arch, cpus, platform as osPlatform, release } from 'node:os'; -import { dirname, join } from 'node:path'; -import { pathToFileURL } from 'node:url'; - -// Mirrors THREAD_COUNT, FILES_PER_THREAD, and PASSES in -// crates/fspy_benchmark/benches/fspy.rs. Only used to describe the workload in -// reports. -const THREAD_COUNT = 4; -const FILES_PER_THREAD = 2048; -const PASSES = 16; - -// Bump when the workload or result shape changes; results from different -// schema versions are not comparable. -const SCHEMA_VERSION = 2; - -type Args = Map; - -/** Mean with its confidence interval, in nanoseconds. */ -interface EstimateSummary { - meanNs: number; - meanLowerNs: number; - meanUpperNs: number; -} - -/** One Criterion estimate, in nanoseconds. */ -interface Estimate extends EstimateSummary { - medianNs: number; -} - -/** The subset of a stored benchmark result that reports are rendered from. */ -interface ReportInput { - schemaVersion: number; - platform: string; - architecture: string; - os: { cpu: string }; - commit: string; - runId: string; - workload: { threadCount: number; filesPerThread: number; passes: number; totalOpens: number }; - benchmarks: Record; -} - -/** The full result JSON produced by `collect` and uploaded as an artifact. */ -interface BenchmarkResult extends ReportInput { - os: { platform: string; release: string; cpu: string }; - runner: string; - runAttempt: string; - event: string; - pullRequest: string; - benchmarks: Record; -} - -interface BaselineArtifact { - name: string; - runId: string; - createdAt: string; -} - -interface IssueComment { - id: number; - body?: string | null; - user?: { type?: string } | null; -} - -function parseArgs(args: string[]): Args { - const parsed = new Map(); - for (let index = 0; index < args.length; index += 2) { - const flag = args[index]; - const value = args[index + 1]; - if (!flag?.startsWith('--') || value === undefined) { - throw new Error(`invalid arguments near ${flag ?? ''}`); - } - parsed.set(flag.slice(2), value); - } - return parsed; -} - -function required(args: Args, name: string): string { - const value = args.get(name); - if (value === undefined || value === '') { - throw new Error(`missing --${name}`); - } - return value; -} - -function requiredEnv(name: string): string { - const value = process.env[name]; - if (!value) throw new Error(`missing ${name}`); - return value; -} - -async function githubJson(path: string, init: RequestInit = {}): Promise { - const token = requiredEnv('GITHUB_TOKEN'); - const response = await fetch(`https://api.github.com${path}`, { - ...init, - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': '2022-11-28', - 'User-Agent': 'vite-task-fspy-benchmark', - ...(init.body === undefined ? {} : { 'Content-Type': 'application/json' }), - }, - }); - if (!response.ok) { - throw new Error(`GitHub API ${response.status}: ${await response.text()}`); - } - return response.json() as Promise; -} - -async function findLatestArtifact( - name: string, - currentRunId: string, -): Promise { - const repository = requiredEnv('GITHUB_REPOSITORY'); - // The name filter is applied server-side, so one page covers the recent history. - const response = await githubJson<{ - artifacts: { - name: string; - expired: boolean; - created_at: string; - workflow_run?: { id: number } | null; - }[]; - }>(`/repos/${repository}/actions/artifacts?name=${encodeURIComponent(name)}&per_page=100`); - return ( - response.artifacts - .flatMap((artifact) => - !artifact.expired && artifact.workflow_run?.id != null - ? [ - { - name: artifact.name, - runId: String(artifact.workflow_run.id), - createdAt: artifact.created_at, - }, - ] - : [], - ) - // Skip the current run so a re-run attempt does not compare against the - // artifact its own previous attempt uploaded. - .filter((artifact) => artifact.runId !== currentRunId) - .sort((left, right) => Date.parse(right.createdAt) - Date.parse(left.createdAt))[0] - ); -} - -// Emits step outputs consumed by the workflow's baseline download and result -// upload steps. The baseline is always the latest `main` artifact: comparing -// against earlier runs of the same PR would mask regressions that accumulate -// over the PR's lifetime. -async function resolveBaseline(args: Args): Promise { - const outputPath = requiredEnv('GITHUB_OUTPUT'); - const platform = required(args, 'platform'); - const context = required(args, 'context'); - const currentRunId = requiredEnv('GITHUB_RUN_ID'); - const currentArtifactName = `fspy-benchmark-${platform}-${context}`; - const artifact = await findLatestArtifact(`fspy-benchmark-${platform}-main`, currentRunId); - - const outputs = [ - `current-artifact-name=${currentArtifactName}`, - `found=${artifact ? 'true' : 'false'}`, - ]; - if (artifact) { - outputs.push(`artifact-name=${artifact.name}`, `run-id=${artifact.runId}`); - } - await appendFile(outputPath, `${outputs.join('\n')}\n`); -} - -// Criterion writes the estimates of its most recent run to -// ///new/estimates.json. -async function readEstimate( - criterionRoot: string, - target: string, - mode: string, -): Promise { - const path = join(criterionRoot, target, mode, 'new', 'estimates.json'); - const estimates = JSON.parse(await readFile(path, 'utf8')) as { - mean: { - point_estimate: number; - confidence_interval: { lower_bound: number; upper_bound: number }; - }; - median: { point_estimate: number }; - }; - return { - meanNs: estimates.mean.point_estimate, - meanLowerNs: estimates.mean.confidence_interval.lower_bound, - meanUpperNs: estimates.mean.confidence_interval.upper_bound, - medianNs: estimates.median.point_estimate, - }; -} - -// Combines Criterion's output with run metadata into the JSON document that is -// uploaded as an artifact and later consumed as a comparison baseline. -async function collect(args: Args): Promise { - const criterionRoot = required(args, 'criterion-root'); - const output = required(args, 'output'); - const platform = required(args, 'platform'); - const expectStatic = args.get('expect-static') === 'true'; - const benchmarks: Record = { - 'dynamic/untracked': await readEstimate(criterionRoot, 'dynamic', 'untracked'), - 'dynamic/tracked': await readEstimate(criterionRoot, 'dynamic', 'tracked'), - }; - if (expectStatic) { - benchmarks['static/untracked'] = await readEstimate(criterionRoot, 'static', 'untracked'); - benchmarks['static/tracked'] = await readEstimate(criterionRoot, 'static', 'tracked'); - } - - const result: BenchmarkResult = { - schemaVersion: SCHEMA_VERSION, - platform, - architecture: process.env['RUNNER_ARCH'] ?? arch(), - os: { - platform: osPlatform(), - release: release(), - cpu: cpus()[0]?.model ?? 'unknown', - }, - runner: args.get('runner') ?? '', - commit: required(args, 'commit'), - runId: required(args, 'run-id'), - runAttempt: args.get('run-attempt') ?? '1', - event: required(args, 'event'), - pullRequest: args.get('pull-request') ?? '', - workload: { - threadCount: THREAD_COUNT, - filesPerThread: FILES_PER_THREAD, - passes: PASSES, - totalOpens: THREAD_COUNT * FILES_PER_THREAD * PASSES, - }, - benchmarks, - }; - - await mkdir(dirname(output), { recursive: true }); - await writeFile(output, `${JSON.stringify(result, null, 2)}\n`); -} - -/** Wall-clock cost of tracking, as tracked time over untracked time. */ -function ratio(result: ReportInput, target: string): number { - const tracked = result.benchmarks[`${target}/tracked`]; - const untracked = result.benchmarks[`${target}/untracked`]; - if (!tracked || !untracked) throw new Error(`missing benchmark data for ${target}`); - return tracked.meanNs / untracked.meanNs; -} - -function percentage(value: number): string { - const sign = value > 0 ? '+' : ''; - return `${sign}${(value * 100).toFixed(2)}%`; -} - -function duration(nanoseconds: number): string { - if (nanoseconds >= 1e9) return `${(nanoseconds / 1e9).toFixed(3)} s`; - if (nanoseconds >= 1e6) return `${(nanoseconds / 1e6).toFixed(3)} ms`; - if (nanoseconds >= 1e3) return `${(nanoseconds / 1e3).toFixed(3)} µs`; - return `${nanoseconds.toFixed(1)} ns`; -} - -/** Mean with the half-width of its confidence interval, e.g. `12.3 ms ±1.4%`. */ -function durationWithUncertainty(estimate: EstimateSummary): string { - const halfWidth = (estimate.meanUpperNs - estimate.meanLowerNs) / 2 / estimate.meanNs; - return `${duration(estimate.meanNs)} ±${(halfWidth * 100).toFixed(1)}%`; -} - -function resultLink(result: ReportInput): string { - const repository = process.env['GITHUB_REPOSITORY']; - if (!repository || !result.runId) return ''; - return `https://github.com/${repository}/actions/runs/${result.runId}`; -} - -export function renderReport(current: ReportInput, baseline?: ReportInput): string { - // Overhead ratios are only comparable within one architecture and one - // workload: a runner profile can move to different hardware over time, and - // schema bumps change what is measured. - const compatible = - baseline !== undefined && - current.schemaVersion === baseline.schemaVersion && - current.platform === baseline.platform && - current.architecture === baseline.architecture; - const lines = [`### ${current.platform.charAt(0).toUpperCase()}${current.platform.slice(1)}`, '']; - - if (compatible) { - const link = resultLink(baseline); - const description = link ? `[main](${link})` : 'main'; - lines.push(`Compared with ${description} at \`${baseline.commit.slice(0, 8)}\`.`, ''); - } else if (baseline) { - lines.push( - baseline.schemaVersion === current.schemaVersion - ? `No comparison: the baseline architecture is \`${baseline.architecture}\`, current is \`${current.architecture}\`.` - : `No comparison: the baseline uses result schema ${baseline.schemaVersion}, current is ${current.schemaVersion}.`, - '', - ); - } else { - lines.push('No `main` result was available for this runner.', ''); - } - - lines.push( - '| Target | Untracked | Tracked | fspy overhead | Normalized change |', - '| --- | ---: | ---: | ---: | ---: |', - ); - - const targets = Object.hasOwn(current.benchmarks, 'static/tracked') - ? ['dynamic', 'static'] - : ['dynamic']; - for (const target of targets) { - const untracked = current.benchmarks[`${target}/untracked`]; - const tracked = current.benchmarks[`${target}/tracked`]; - if (!untracked || !tracked) throw new Error(`missing benchmark data for ${target}`); - const overhead = ratio(current, target) - 1; - // Change of the overhead ratio relative to the baseline. Comparing ratios - // instead of absolute times cancels machine-speed differences between - // runner instances. - const normalizedChange = - compatible && Object.hasOwn(baseline.benchmarks, `${target}/tracked`) - ? ratio(current, target) / ratio(baseline, target) - 1 - : undefined; - lines.push( - `| ${target} | ${durationWithUncertainty(untracked)} | ${durationWithUncertainty(tracked)} | ${percentage(overhead)} | ${normalizedChange === undefined ? '—' : percentage(normalizedChange)} |`, - ); - } - - lines.push( - '', - `Workload: ${current.workload.threadCount} threads × ${current.workload.filesPerThread.toLocaleString('en-US')} files × ${current.workload.passes.toLocaleString('en-US')} passes, ${current.workload.totalOpens.toLocaleString('en-US')} total open-and-close operations.`, - '', - `\`${current.architecture}\` · ${current.os.cpu} · run [${current.runId}](${resultLink(current)})`, - '', - ); - return `${lines.join('\n')}\n`; -} - -async function compare(args: Args): Promise { - const current = JSON.parse(await readFile(required(args, 'current'), 'utf8')) as BenchmarkResult; - const baselinePath = args.get('baseline'); - let baseline: BenchmarkResult | undefined; - if (baselinePath) { - try { - baseline = JSON.parse(await readFile(baselinePath, 'utf8')) as BenchmarkResult; - } catch (error) { - // The workflow always passes --baseline; the file is absent when no - // baseline artifact was found. - if ((error as { code?: string }).code !== 'ENOENT') throw error; - } - } - const report = renderReport(current, baseline); - await writeFile(required(args, 'output'), report); - process.stdout.write(report); -} - -const COMMENT_MARKER = ''; - -// Creates or updates the sticky PR comment. The invisible marker identifies -// the comment, so later runs edit it in place instead of posting a new one. -async function comment(args: Args): Promise { - const repository = requiredEnv('GITHUB_REPOSITORY'); - const pullRequest = required(args, 'pull-request'); - const commit = required(args, 'commit'); - const resultsDir = required(args, 'results-dir'); - - const sections: string[] = []; - for (const platform of ['linux', 'macos', 'windows']) { - sections.push(await readFile(join(resultsDir, `${platform}.md`), 'utf8')); - } - const body = [ - COMMENT_MARKER, - '## fspy benchmark', - '', - `Results for commit \`${commit}\`. Each platform is compared with the latest \`main\` result.`, - '', - ...sections, - 'This comment is updated by the Fspy Benchmark workflow.', - '', - ].join('\n'); - - const comments = await githubJson( - `/repos/${repository}/issues/${pullRequest}/comments?per_page=100`, - ); - const existing = comments.find( - (existingComment) => - existingComment.user?.type === 'Bot' && existingComment.body?.includes(COMMENT_MARKER), - ); - const path = existing - ? `/repos/${repository}/issues/comments/${existing.id}` - : `/repos/${repository}/issues/${pullRequest}/comments`; - await githubJson(path, { - method: existing ? 'PATCH' : 'POST', - body: JSON.stringify({ body }), - }); -} - -async function main(): Promise { - const command = process.argv[2]; - const args = parseArgs(process.argv.slice(3)); - if (command === 'resolve-baseline') { - await resolveBaseline(args); - } else if (command === 'collect') { - await collect(args); - } else if (command === 'compare') { - await compare(args); - } else if (command === 'comment') { - await comment(args); - } else { - throw new Error(`unknown command: ${command ?? ''}`); - } -} - -// Run only when executed directly, so the test file can import renderReport. -const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : ''; -if (import.meta.url === invokedPath) { - main().catch((error: unknown) => { - const message = error instanceof Error ? (error.stack ?? error.message) : String(error); - process.stderr.write(`${message}\n`); - process.exitCode = 1; - }); -} diff --git a/.github/scripts/fspy-benchmark.test.mts b/.github/scripts/fspy-benchmark.test.mts deleted file mode 100644 index 4b527790f..000000000 --- a/.github/scripts/fspy-benchmark.test.mts +++ /dev/null @@ -1,58 +0,0 @@ -import assert from 'node:assert/strict'; -import test from 'node:test'; - -import { renderReport } from './fspy-benchmark.mts'; - -function estimate(meanNs: number) { - return { meanNs, meanLowerNs: meanNs * 0.98, meanUpperNs: meanNs * 1.02 }; -} - -function result( - platform: string, - architecture: string, - runId: string, - untracked: number, - tracked: number, -) { - return { - schemaVersion: 2, - platform, - architecture, - os: { cpu: 'Test CPU' }, - commit: '1234567890abcdef', - runId, - workload: { threadCount: 4, filesPerThread: 2048, passes: 16, totalOpens: 131072 }, - benchmarks: { - 'dynamic/untracked': estimate(untracked), - 'dynamic/tracked': estimate(tracked), - }, - }; -} - -void test('renders normalized change against a compatible baseline', () => { - const baseline = result('linux', 'X64', '10', 100, 120); - const current = result('linux', 'X64', '20', 100, 132); - const report = renderReport(current, baseline); - - assert.match(report, /fspy overhead \| Normalized change/); - assert.match(report, /100\.0 ns ±2\.0% \| 132\.0 ns ±2\.0% \| \+32\.00% \| \+10\.00%/); - assert.match(report, /Compared with main at `12345678`/); -}); - -void test('does not compare different architectures', () => { - const baseline = result('macos', 'X64', '10', 100, 120); - const current = result('macos', 'ARM64', '20', 100, 132); - const report = renderReport(current, baseline); - - assert.match(report, /No comparison/); - assert.match(report, /\+32\.00% \| —/); -}); - -void test('does not compare different schema versions', () => { - const baseline = { ...result('linux', 'X64', '10', 100, 120), schemaVersion: 1 }; - const current = result('linux', 'X64', '20', 100, 132); - const report = renderReport(current, baseline); - - assert.match(report, /No comparison: the baseline uses result schema 1, current is 2\./); - assert.match(report, /\+32\.00% \| —/); -}); diff --git a/.github/scripts/tsconfig.json b/.github/scripts/tsconfig.json deleted file mode 100644 index 07f8c9b9b..000000000 --- a/.github/scripts/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "extends": "@tsconfig/strictest/tsconfig.json", - "compilerOptions": { - "allowImportingTsExtensions": true, - "erasableSyntaxOnly": true, - "module": "NodeNext", - "moduleResolution": "NodeNext", - "noEmit": true, - "target": "ES2022", - "lib": ["ES2022"], - "types": ["node"] - }, - "include": ["*.mts"] -} diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index f7a90af24..29707b726 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -40,6 +40,7 @@ jobs: runs-on: ${{ matrix.os }} env: CARGO_TARGET_DIR: ${{ github.workspace }}/target + CRITERION_HOME: ${{ github.workspace }}/target/criterion steps: - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 @@ -52,88 +53,67 @@ jobs: save-cache: ${{ github.ref_name == 'main' }} cache-key: fspy-benchmark-${{ matrix.platform }} - # The support script is TypeScript; running it needs the .node-version - # Node, which strips types natively. Runner-preinstalled Node may be - # older. - - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 - - name: Install static target if: matrix.static run: rustup target add x86_64-unknown-linux-musl - - name: Set benchmark context - id: context + - name: Download main baseline + id: main-baseline + if: github.event_name != 'push' + uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 + with: + github_token: ${{ github.token }} + workflow: fspy-benchmark.yml + workflow_conclusion: success + branch: main + event: push + name: fspy-benchmark-baseline-v1-${{ matrix.platform }} + path: ${{ env.CRITERION_HOME }}/fspy + search_artifacts: true + if_no_artifact_found: warn + + - name: Run benchmark env: - PR_NUMBER: ${{ github.event.pull_request.number }} + BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }} run: | - # Only real main-branch runs may publish the main baseline artifact; - # a workflow_dispatch run from another branch must not pollute it. - if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - context="pr-$PR_NUMBER" - pull_request="$PR_NUMBER" - elif [[ "$GITHUB_REF" == "refs/heads/main" ]]; then - context="main" - pull_request="none" + mkdir -p .benchmark-results + if [[ "$GITHUB_EVENT_NAME" == "push" || "$BASELINE_FOUND" != "true" ]]; then + baseline=(--save-baseline main) else - context="dispatch" - pull_request="none" + baseline=(--baseline main) fi - echo "name=$context" >> "$GITHUB_OUTPUT" - echo "pull-request=$pull_request" >> "$GITHUB_OUTPUT" - - - name: Find comparison baseline - id: baseline - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node .github/scripts/fspy-benchmark.mts resolve-baseline - --platform "${{ matrix.platform }}" - --context "${{ steps.context.outputs.name }}" - - - name: Download comparison baseline - if: steps.baseline.outputs.found == 'true' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ steps.baseline.outputs.artifact-name }} - path: .benchmark-baseline - github-token: ${{ github.token }} - repository: ${{ github.repository }} - run-id: ${{ steps.baseline.outputs.run-id }} - - - name: Run benchmark - run: cargo bench --locked -p fspy_benchmark --bench fspy - - - name: Collect benchmark results - run: >- - node .github/scripts/fspy-benchmark.mts collect - --criterion-root "$CARGO_TARGET_DIR/criterion/fspy" - --output ".benchmark-results/${{ matrix.platform }}.json" - --platform "${{ matrix.platform }}" - --expect-static "${{ matrix.static }}" - --runner "${{ matrix.os }}" - --commit "$GITHUB_SHA" - --run-id "$GITHUB_RUN_ID" - --run-attempt "$GITHUB_RUN_ATTEMPT" - --event "$GITHUB_EVENT_NAME" - --pull-request "${{ steps.context.outputs.pull-request }}" - - - name: Compare benchmark results - run: >- - node .github/scripts/fspy-benchmark.mts compare - --current ".benchmark-results/${{ matrix.platform }}.json" - --baseline ".benchmark-baseline/${{ matrix.platform }}.json" - --output ".benchmark-results/${{ matrix.platform }}.md" + set -o pipefail + cargo bench --locked -p fspy_benchmark --bench fspy -- \ + --color never -n "${baseline[@]}" | + tee ".benchmark-results/${{ matrix.platform }}.txt" - name: Add job summary - run: cat ".benchmark-results/${{ matrix.platform }}.md" >> "$GITHUB_STEP_SUMMARY" + run: | + { + echo '```text' + cat ".benchmark-results/${{ matrix.platform }}.txt" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload main baseline + if: github.event_name == 'push' && github.ref_name == 'main' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fspy-benchmark-baseline-v1-${{ matrix.platform }} + path: ${{ env.CRITERION_HOME }}/fspy + if-no-files-found: error + overwrite: true + retention-days: 90 - - name: Upload benchmark result + - name: Upload benchmark report + if: github.event_name == 'pull_request' + # Matrix jobs have isolated filesystems, so persist each report for the + # comment job to combine into one sticky comment after all platforms finish. uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ steps.baseline.outputs.current-artifact-name }} - path: .benchmark-results/${{ matrix.platform }}.* + name: fspy-benchmark-report-${{ matrix.platform }} + path: .benchmark-results/${{ matrix.platform }}.txt if-no-files-found: error - include-hidden-files: true overwrite: true retention-days: 90 @@ -146,27 +126,32 @@ jobs: runs-on: namespace-profile-linux-x64-default permissions: actions: read - contents: read pull-requests: write steps: - - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 - - - uses: oxc-project/setup-node@4c588e9266bd930b6ddc34307df0659ed511d187 # v1.3.1 - - - name: Download current results + - name: Download benchmark reports uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: fspy-benchmark-*-pr-${{ github.event.pull_request.number }} + pattern: fspy-benchmark-report-* path: .benchmark-results merge-multiple: true + - name: Assemble comment + run: | + { + echo '## fspy benchmark' + echo + for platform in linux macos windows; do + echo "### $platform" + echo + echo '```text' + cat ".benchmark-results/$platform.txt" + echo '```' + echo + done + } > .benchmark-results/comment.md + - name: Update PR comment - env: - GITHUB_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: >- - node .github/scripts/fspy-benchmark.mts comment - --results-dir .benchmark-results - --pull-request "$PR_NUMBER" - --commit "$HEAD_SHA" + uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4 + with: + header: fspy-benchmark + path: .benchmark-results/comment.md diff --git a/Cargo.lock b/Cargo.lock index f5a4464cd..1a71e1516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1307,28 +1307,17 @@ dependencies = [ "fspy", "fspy_benchmark_static_target", "fspy_benchmark_target", - "tempfile", "tokio", "tokio-util", - "vite_path", - "vite_str", ] [[package]] name = "fspy_benchmark_static_target" version = "0.0.0" -dependencies = [ - "vite_path", - "vite_str", -] [[package]] name = "fspy_benchmark_target" version = "0.0.0" -dependencies = [ - "vite_path", - "vite_str", -] [[package]] name = "fspy_detours_sys" diff --git a/crates/fspy_benchmark/Cargo.toml b/crates/fspy_benchmark/Cargo.toml index 18330d1b1..c6fbe7b67 100644 --- a/crates/fspy_benchmark/Cargo.toml +++ b/crates/fspy_benchmark/Cargo.toml @@ -23,11 +23,8 @@ harness = false criterion2 = { workspace = true } fspy = { workspace = true } fspy_benchmark_target = { workspace = true } -tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "process", "rt-multi-thread"] } tokio-util = { workspace = true } -vite_path = { workspace = true } -vite_str = { workspace = true } [target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dev-dependencies] fspy_benchmark_static_target = { path = "../fspy_benchmark_static_target", artifact = "bin", target = "x86_64-unknown-linux-musl" } diff --git a/crates/fspy_benchmark/README.md b/crates/fspy_benchmark/README.md index bb9d50758..00d39f68f 100644 --- a/crates/fspy_benchmark/README.md +++ b/crates/fspy_benchmark/README.md @@ -1,28 +1,12 @@ # fspy benchmark -This benchmark measures the wall-clock overhead fspy adds to a filesystem-heavy child process. -The target starts four threads. Each thread opens 2,048 distinct, pre-created files for reading, -dropping every handle immediately, and repeats this for 16 passes — 131,072 tracked opens per -run. The passes keep per-open interception cost dominant over process startup, whose variance on -CI runners would otherwise drown the tracked/untracked delta. +Measures fspy's wall-clock overhead while multiple threads repeatedly open a nonexistent path. +Criterion reports the nonnegative difference between tracked and untracked runs. -The fixture is created before Criterion starts measuring. Tracked and untracked runs execute the -same target with the same arguments, empty environment, working directory, and standard streams. -The timer covers spawning the target through process termination and fspy finalization. A tracked -warmup verifies that all fixture accesses were captured outside the timed section. - -On Linux, the suite measures two targets: - -- `dynamic` is dynamically linked and exercises `LD_PRELOAD`. -- `static` is linked for `x86_64-unknown-linux-musl` and exercises seccomp user notification. - -macOS measures `DYLD_INSERT_LIBRARIES`, and Windows measures Detours injection. +Linux measures dynamic and static targets. macOS and Windows measure their preload implementations. Run the benchmark with: ```sh just benchmark-fspy ``` - -CI runs it on the Linux, macOS, and Windows Namespace runner profiles. Each result is compared with -the latest `main` result. diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 1fd01ef37..7929f917a 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -1,48 +1,35 @@ use std::{ - ffi::OsStr, - fs::File, process::{ExitStatus, Stdio}, - time::Duration, + time::{Duration, Instant}, }; use criterion::{ - BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group, - criterion_main, measurement::WallTime, + BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, criterion_group, criterion_main, + measurement::WallTime, }; use fspy::{ChildTermination, Command}; -use tempfile::TempDir; use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; -use vite_path::{AbsolutePath, AbsolutePathBuf}; - -// Mirrored by .github/scripts/fspy-benchmark.mts for report rendering. -// The passes multiplier makes per-open interception cost dominate process -// startup in the measurement; with a single pass the tracked/untracked delta -// is the same order as spawn-time variance on CI runners. -const THREAD_COUNT: usize = 4; -const FILES_PER_THREAD: usize = 2048; -const PASSES: usize = 16; -const TOTAL_FILE_COUNT: usize = THREAD_COUNT * FILES_PER_THREAD; -const THREAD_COUNT_ARG: &str = "4"; -const FILES_PER_THREAD_ARG: &str = "2048"; -const PASSES_ARG: &str = "16"; + const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); +#[cfg(unix)] +const MISSING_PATH: &str = "/.fspy-benchmark-missing"; +#[cfg(windows)] +const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; + fn benchmark(criterion: &mut Criterion) { - let (_fixture, fixture_root) = create_fixture(); let runtime = Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap(); let mut group = criterion.benchmark_group("fspy"); group.sampling_mode(SamplingMode::Flat); - // Criterion reports per-open time alongside wall time. - group.throughput(Throughput::Elements((TOTAL_FILE_COUNT * PASSES) as u64)); - benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET, &fixture_root); + benchmark_target(&mut group, &runtime, "dynamic", DYNAMIC_TARGET); #[cfg(all(target_os = "linux", target_arch = "x86_64"))] - benchmark_target(&mut group, &runtime, "static", STATIC_TARGET, &fixture_root); + benchmark_target(&mut group, &runtime, "static", STATIC_TARGET); group.finish(); } @@ -52,80 +39,63 @@ fn benchmark_target( runtime: &Runtime, target_name: &str, target: &str, - fixture: &AbsolutePath, ) { - validate_tracked_run(runtime, target, fixture); - - group.bench_with_input( - BenchmarkId::new(target_name, "untracked"), - &target, - |bencher, target| { - bencher.iter_with_setup_wrapper(|runner| { - let status = runner.run(|| runtime.block_on(run_untracked(target, fixture))); - assert!(status.success(), "untracked benchmark target failed: {status}"); - }); - }, - ); - - group.bench_with_input(BenchmarkId::new(target_name, "tracked"), &target, |bencher, target| { - bencher.iter_with_setup_wrapper(|runner| { - let termination = runner.run(|| runtime.block_on(run_tracked(target, fixture))); - assert!( - termination.status.success(), - "tracked benchmark target failed: {}", - termination.status - ); + validate_tracked_run(runtime, target); + + group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { + bencher.iter_custom(|iterations| { + let mut tracked = Duration::ZERO; + let mut untracked = Duration::ZERO; + for _ in 0..iterations { + tracked += measure_tracked(runtime, target); + untracked += measure_untracked(runtime, target); + } + // Keep clamped samples reportable at a stable 1 ns per-iteration floor. + tracked.saturating_sub(untracked).max(Duration::from_nanos(iterations)) }); }); } -fn create_fixture() -> (TempDir, AbsolutePathBuf) { - let fixture = tempfile::tempdir().expect("failed to create benchmark fixture"); - let fixture_root = AbsolutePathBuf::new(fixture.path().to_path_buf()) - .expect("temporary path must be absolute"); - for index in 0..TOTAL_FILE_COUNT { - let path = fixture_root.join(vite_str::format!("file-{index:05}.txt")); - File::create(&path).unwrap_or_else(|error| { - panic!("failed to create {}: {error}", path.as_absolute_path()) - }); - } - (fixture, fixture_root) +fn validate_tracked_run(runtime: &Runtime, target: &str) { + let termination = runtime.block_on(run_tracked(target)); + assert!(termination.status.success(), "benchmark target failed: {}", termination.status); + let captured_missing_access = termination.path_accesses.iter().any(|access| { + access.path.strip_path_prefix(MISSING_PATH, |result| { + result.is_ok_and(|path| path.as_os_str().is_empty()) + }) + }); + assert!(captured_missing_access, "failed to capture access to {MISSING_PATH}"); } -fn validate_tracked_run(runtime: &Runtime, target: &str, fixture: &AbsolutePath) { - let termination = runtime.block_on(run_tracked(target, fixture)); - assert!(termination.status.success(), "benchmark target failed: {}", termination.status); - let fixture_access_count = termination - .path_accesses - .iter() - .filter(|access| access.path.strip_path_prefix(fixture, |result| result.is_ok())) - .count(); - // Requires every distinct fixture file to be observed, independent of - // whether repeated accesses to the same path are deduplicated. +fn measure_untracked(runtime: &Runtime, target: &str) -> Duration { + let start = Instant::now(); + let status = runtime.block_on(run_untracked(target)); + let elapsed = start.elapsed(); + assert!(status.success(), "untracked benchmark target failed: {status}"); + elapsed +} + +fn measure_tracked(runtime: &Runtime, target: &str) -> Duration { + let start = Instant::now(); + let termination = runtime.block_on(run_tracked(target)); + let elapsed = start.elapsed(); assert!( - fixture_access_count >= TOTAL_FILE_COUNT, - "expected at least {TOTAL_FILE_COUNT} fixture accesses, captured {fixture_access_count}" + termination.status.success(), + "tracked benchmark target failed: {}", + termination.status ); + elapsed } -async fn run_untracked(target: &str, fixture: &AbsolutePath) -> ExitStatus { +async fn run_untracked(target: &str) -> ExitStatus { let mut command = tokio::process::Command::new(target); - command - .env_clear() - .args(benchmark_args(fixture)) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()); + command.env_clear().stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::inherit()); command.status().await.expect("failed to run untracked benchmark target") } -async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { +async fn run_tracked(target: &str) -> ChildTermination { let mut command = Command::new(target); - command - .args(benchmark_args(fixture)) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::inherit()); + command.stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::inherit()); command .spawn(CancellationToken::new()) .await @@ -135,20 +105,11 @@ async fn run_tracked(target: &str, fixture: &AbsolutePath) -> ChildTermination { .expect("failed to wait for tracked benchmark target") } -fn benchmark_args(fixture: &AbsolutePath) -> [&OsStr; 4] { - [ - fixture.as_path().as_os_str(), - OsStr::new(THREAD_COUNT_ARG), - OsStr::new(FILES_PER_THREAD_ARG), - OsStr::new(PASSES_ARG), - ] -} - fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) .warm_up_time(Duration::from_millis(500)) - .measurement_time(Duration::from_secs(5)) + .measurement_time(Duration::from_secs(60)) } criterion_group! { diff --git a/crates/fspy_benchmark_static_target/Cargo.toml b/crates/fspy_benchmark_static_target/Cargo.toml index b296739b1..d8daccfe0 100644 --- a/crates/fspy_benchmark_static_target/Cargo.toml +++ b/crates/fspy_benchmark_static_target/Cargo.toml @@ -10,13 +10,6 @@ rust-version.workspace = true [lints] workspace = true -[dependencies] -vite_path = { workspace = true } -vite_str = { workspace = true } - -[package.metadata.cargo-shear] -ignored = ["vite_path", "vite_str"] - [[bin]] name = "fspy_benchmark_static_target" test = false diff --git a/crates/fspy_benchmark_target/Cargo.toml b/crates/fspy_benchmark_target/Cargo.toml index c5d917b47..ea5536db2 100644 --- a/crates/fspy_benchmark_target/Cargo.toml +++ b/crates/fspy_benchmark_target/Cargo.toml @@ -10,10 +10,6 @@ rust-version.workspace = true [lints] workspace = true -[dependencies] -vite_path = { workspace = true } -vite_str = { workspace = true } - [[bin]] name = "fspy_benchmark_target" test = false diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index 786fa5fe0..9ce595970 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -1,55 +1,27 @@ use std::{ - env, fs::File, sync::{Arc, Barrier}, thread, }; -use vite_path::{AbsolutePath, AbsolutePathBuf}; +const THREAD_COUNT: usize = 4; +const OPEN_COUNT_PER_THREAD: usize = 32_768; -fn main() { - let mut args = env::args_os(); - let _program = args.next(); - let root = AbsolutePathBuf::new(args.next().expect("missing fixture root").into()) - .expect("fixture root must be absolute"); - let thread_count = parse_usize(args.next(), "thread count"); - let files_per_thread = parse_usize(args.next(), "files per thread"); - let passes = parse_usize(args.next(), "passes"); - assert!(args.next().is_none(), "unexpected extra arguments"); - - run(&root, thread_count, files_per_thread, passes); -} +#[cfg(unix)] +const MISSING_PATH: &str = "/.fspy-benchmark-missing"; +#[cfg(windows)] +const MISSING_PATH: &str = r"C:\.fspy-benchmark-missing"; -fn parse_usize(value: Option, name: &str) -> usize { - value - .unwrap_or_else(|| panic!("missing {name}")) - .into_string() - .unwrap_or_else(|_| panic!("{name} is not UTF-8")) - .parse() - .unwrap_or_else(|_| panic!("invalid {name}")) -} - -fn run(root: &AbsolutePath, thread_count: usize, files_per_thread: usize, passes: usize) { - assert!(thread_count > 1, "benchmark requires multiple threads"); - assert!(files_per_thread > 0, "benchmark requires at least one file per thread"); - assert!(passes > 0, "benchmark requires at least one pass"); - - let paths = (0..thread_count * files_per_thread) - .map(|index| root.join(vite_str::format!("file-{index:05}.txt"))) - .collect::>(); - let barrier = Arc::new(Barrier::new(thread_count)); +fn main() { + let barrier = Arc::new(Barrier::new(THREAD_COUNT)); thread::scope(|scope| { - for thread_paths in paths.chunks_exact(files_per_thread) { + for _ in 0..THREAD_COUNT { let barrier = Arc::clone(&barrier); scope.spawn(move || { barrier.wait(); - for _ in 0..passes { - for path in thread_paths { - drop(File::open(path).unwrap_or_else(|error| { - panic!("failed to open {}: {error}", path.as_absolute_path()) - })); - } + for _ in 0..OPEN_COUNT_PER_THREAD { + drop(File::open(MISSING_PATH)); } }); } From 5534b1ae8bb2ff0032eb26c7d7d370d950a3b3c6 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:21:18 +0800 Subject: [PATCH 12/23] test(fspy): stabilize benchmark sampling Co-authored-by: GPT-5 Codex --- .github/workflows/fspy-benchmark.yml | 60 +++++++++++++++++++++++++-- crates/fspy_benchmark/benches/fspy.rs | 4 +- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 29707b726..55a70fff2 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -72,15 +72,40 @@ jobs: search_artifacts: true if_no_artifact_found: warn + - name: Download pull request baseline + id: pr-baseline + if: >- + github.event_name == 'pull_request' && + steps.main-baseline.outputs.found_artifact != 'true' && + github.event.pull_request.head.repo.full_name == github.repository + uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 + with: + github_token: ${{ github.token }} + workflow: fspy-benchmark.yml + workflow_conclusion: success + branch: ${{ github.event.pull_request.head.ref }} + event: pull_request + name: fspy-benchmark-baseline-pr-v1-${{ github.event.pull_request.number }}-${{ matrix.platform }} + path: ${{ env.CRITERION_HOME }}/fspy + search_artifacts: true + if_no_artifact_found: warn + - name: Run benchmark env: - BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }} + MAIN_BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }} + PR_BASELINE_FOUND: ${{ steps.pr-baseline.outputs.found_artifact }} run: | mkdir -p .benchmark-results - if [[ "$GITHUB_EVENT_NAME" == "push" || "$BASELINE_FOUND" != "true" ]]; then + if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then baseline=(--save-baseline main) - else + elif [[ "$MAIN_BASELINE_FOUND" == "true" ]]; then baseline=(--baseline main) + elif [[ "$PR_BASELINE_FOUND" == "true" ]]; then + baseline=(--baseline pr) + elif [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + baseline=(--save-baseline pr) + else + baseline=(--save-baseline main) fi set -o pipefail cargo bench --locked -p fspy_benchmark --bench fspy -- \ @@ -95,6 +120,20 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" + - name: Save pull request baseline + if: >- + github.event_name == 'pull_request' && + steps.main-baseline.outputs.found_artifact != 'true' && + github.event.pull_request.head.repo.full_name == github.repository + run: | + for target in dynamic static; do + source="$CRITERION_HOME/fspy/$target/new" + if [[ -d "$source" ]]; then + mkdir -p "$CRITERION_HOME/fspy/$target/pr" + cp -R "$source/." "$CRITERION_HOME/fspy/$target/pr/" + fi + done + - name: Upload main baseline if: github.event_name == 'push' && github.ref_name == 'main' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -105,6 +144,21 @@ jobs: overwrite: true retention-days: 90 + - name: Upload pull request baseline + if: >- + github.event_name == 'pull_request' && + steps.main-baseline.outputs.found_artifact != 'true' && + github.event.pull_request.head.repo.full_name == github.repository + # Until main has a baseline, keep Criterion's data so the next commit + # in this pull request can compare against its latest run. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fspy-benchmark-baseline-pr-v1-${{ github.event.pull_request.number }}-${{ matrix.platform }} + path: ${{ env.CRITERION_HOME }}/fspy + if-no-files-found: error + overwrite: true + retention-days: 90 + - name: Upload benchmark report if: github.event_name == 'pull_request' # Matrix jobs have isolated filesystems, so persist each report for the diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 7929f917a..30d2d4138 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -108,8 +108,8 @@ async fn run_tracked(target: &str) -> ChildTermination { fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) - .warm_up_time(Duration::from_millis(500)) - .measurement_time(Duration::from_secs(60)) + .warm_up_time(Duration::from_secs(5)) + .measurement_time(Duration::from_secs(180)) } criterion_group! { From 01fec1bebc6ab8e3abefa38ce014342096e230a2 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:30:12 +0800 Subject: [PATCH 13/23] test(fspy): compare benchmarks with main only Co-authored-by: GPT-5 Codex --- .github/workflows/fspy-benchmark.yml | 56 +--------------------------- 1 file changed, 1 insertion(+), 55 deletions(-) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 55a70fff2..8812add2e 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -72,38 +72,13 @@ jobs: search_artifacts: true if_no_artifact_found: warn - - name: Download pull request baseline - id: pr-baseline - if: >- - github.event_name == 'pull_request' && - steps.main-baseline.outputs.found_artifact != 'true' && - github.event.pull_request.head.repo.full_name == github.repository - uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21 - with: - github_token: ${{ github.token }} - workflow: fspy-benchmark.yml - workflow_conclusion: success - branch: ${{ github.event.pull_request.head.ref }} - event: pull_request - name: fspy-benchmark-baseline-pr-v1-${{ github.event.pull_request.number }}-${{ matrix.platform }} - path: ${{ env.CRITERION_HOME }}/fspy - search_artifacts: true - if_no_artifact_found: warn - - name: Run benchmark env: MAIN_BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }} - PR_BASELINE_FOUND: ${{ steps.pr-baseline.outputs.found_artifact }} run: | mkdir -p .benchmark-results - if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then - baseline=(--save-baseline main) - elif [[ "$MAIN_BASELINE_FOUND" == "true" ]]; then + if [[ "$GITHUB_EVENT_NAME" != "push" && "$MAIN_BASELINE_FOUND" == "true" ]]; then baseline=(--baseline main) - elif [[ "$PR_BASELINE_FOUND" == "true" ]]; then - baseline=(--baseline pr) - elif [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - baseline=(--save-baseline pr) else baseline=(--save-baseline main) fi @@ -120,20 +95,6 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" - - name: Save pull request baseline - if: >- - github.event_name == 'pull_request' && - steps.main-baseline.outputs.found_artifact != 'true' && - github.event.pull_request.head.repo.full_name == github.repository - run: | - for target in dynamic static; do - source="$CRITERION_HOME/fspy/$target/new" - if [[ -d "$source" ]]; then - mkdir -p "$CRITERION_HOME/fspy/$target/pr" - cp -R "$source/." "$CRITERION_HOME/fspy/$target/pr/" - fi - done - - name: Upload main baseline if: github.event_name == 'push' && github.ref_name == 'main' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -144,21 +105,6 @@ jobs: overwrite: true retention-days: 90 - - name: Upload pull request baseline - if: >- - github.event_name == 'pull_request' && - steps.main-baseline.outputs.found_artifact != 'true' && - github.event.pull_request.head.repo.full_name == github.repository - # Until main has a baseline, keep Criterion's data so the next commit - # in this pull request can compare against its latest run. - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fspy-benchmark-baseline-pr-v1-${{ github.event.pull_request.number }}-${{ matrix.platform }} - path: ${{ env.CRITERION_HOME }}/fspy - if-no-files-found: error - overwrite: true - retention-days: 90 - - name: Upload benchmark report if: github.event_name == 'pull_request' # Matrix jobs have isolated filesystems, so persist each report for the From a2854d512e4bab1b2bfab20ed2f13121d49f0da0 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:37:56 +0800 Subject: [PATCH 14/23] test(fspy): stabilize detailed benchmark report Co-authored-by: GPT-5 Codex --- .github/workflows/fspy-benchmark.yml | 11 ++++++++--- crates/fspy_benchmark/benches/fspy.rs | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fspy-benchmark.yml b/.github/workflows/fspy-benchmark.yml index 8812add2e..a0761fb16 100644 --- a/.github/workflows/fspy-benchmark.yml +++ b/.github/workflows/fspy-benchmark.yml @@ -1,4 +1,4 @@ -name: Fspy Benchmark +name: fspy benchmark permissions: {} @@ -83,15 +83,19 @@ jobs: baseline=(--save-baseline main) fi set -o pipefail + result_file=".benchmark-results/${{ matrix.platform }}.txt" cargo bench --locked -p fspy_benchmark --bench fspy -- \ - --color never -n "${baseline[@]}" | - tee ".benchmark-results/${{ matrix.platform }}.txt" + --color never --verbose -n "${baseline[@]}" | + tee "$result_file" + result="$(<"$result_file")" + printf '%s' "$result" > "$result_file" - name: Add job summary run: | { echo '```text' cat ".benchmark-results/${{ matrix.platform }}.txt" + echo echo '```' } >> "$GITHUB_STEP_SUMMARY" @@ -145,6 +149,7 @@ jobs: echo echo '```text' cat ".benchmark-results/$platform.txt" + echo echo '```' echo done diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 30d2d4138..eb6934579 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -47,8 +47,8 @@ fn benchmark_target( let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; for _ in 0..iterations { - tracked += measure_tracked(runtime, target); untracked += measure_untracked(runtime, target); + tracked += measure_tracked(runtime, target); } // Keep clamped samples reportable at a stable 1 ns per-iteration floor. tracked.saturating_sub(untracked).max(Duration::from_nanos(iterations)) From aebd5d262265ced7b175fe128bedfb5d331ce46f Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:48:46 +0800 Subject: [PATCH 15/23] test(fspy): reduce benchmark measurement noise Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 2 +- crates/fspy_benchmark_target/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index eb6934579..3c868075e 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -109,7 +109,7 @@ fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) .warm_up_time(Duration::from_secs(5)) - .measurement_time(Duration::from_secs(180)) + .measurement_time(Duration::from_secs(60)) } criterion_group! { diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index 9ce595970..deb04d3e3 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -5,7 +5,7 @@ use std::{ }; const THREAD_COUNT: usize = 4; -const OPEN_COUNT_PER_THREAD: usize = 32_768; +const OPEN_COUNT_PER_THREAD: usize = 8_192; #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing"; From ea60443d5a997c88349d2e8c33c9be66de88906a Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 17:54:09 +0800 Subject: [PATCH 16/23] test(fspy): bound dynamic benchmark sampling Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 3c868075e..7d67753e4 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,6 +12,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); +const MIN_DYNAMIC_ITERATIONS: u64 = 100; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -41,17 +42,22 @@ fn benchmark_target( target: &str, ) { validate_tracked_run(runtime, target); + let minimum_iterations = if target_name == "dynamic" { MIN_DYNAMIC_ITERATIONS } else { 0 }; group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { + let measured_iterations = iterations.max(minimum_iterations); let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; - for _ in 0..iterations { + for _ in 0..measured_iterations { untracked += measure_untracked(runtime, target); tracked += measure_tracked(runtime, target); } - // Keep clamped samples reportable at a stable 1 ns per-iteration floor. - tracked.saturating_sub(untracked).max(Duration::from_nanos(iterations)) + tracked + .saturating_sub(untracked) + .div_f64(measured_iterations as f64) + .max(Duration::from_nanos(1)) + .mul_f64(iterations as f64) }); }); } From 8f976e434439862829dab1c1cd435b6f39752341 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:03:57 +0800 Subject: [PATCH 17/23] test(fspy): increase dynamic process samples Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 5 +++-- crates/fspy_benchmark_target/src/main.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 7d67753e4..711fa1282 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,7 +12,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); -const MIN_DYNAMIC_ITERATIONS: u64 = 100; +const MIN_DYNAMIC_PROCESS_PAIRS_PER_SAMPLE: u64 = 500; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -42,7 +42,8 @@ fn benchmark_target( target: &str, ) { validate_tracked_run(runtime, target); - let minimum_iterations = if target_name == "dynamic" { MIN_DYNAMIC_ITERATIONS } else { 0 }; + let minimum_iterations = + if target_name == "dynamic" { MIN_DYNAMIC_PROCESS_PAIRS_PER_SAMPLE } else { 0 }; group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index deb04d3e3..f9548576f 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -5,7 +5,7 @@ use std::{ }; const THREAD_COUNT: usize = 4; -const OPEN_COUNT_PER_THREAD: usize = 8_192; +const OPEN_COUNT_PER_THREAD: usize = 2_048; #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing"; From d24928d12a6f86c4da64c16ba0c1c825dbeb9fd2 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:11:45 +0800 Subject: [PATCH 18/23] test(fspy): stabilize static benchmark samples Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 711fa1282..d4430d428 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,7 +12,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); -const MIN_DYNAMIC_PROCESS_PAIRS_PER_SAMPLE: u64 = 500; +const MIN_PROCESS_PAIRS_PER_SAMPLE: u64 = 500; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -42,12 +42,10 @@ fn benchmark_target( target: &str, ) { validate_tracked_run(runtime, target); - let minimum_iterations = - if target_name == "dynamic" { MIN_DYNAMIC_PROCESS_PAIRS_PER_SAMPLE } else { 0 }; group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { - let measured_iterations = iterations.max(minimum_iterations); + let measured_iterations = iterations.max(MIN_PROCESS_PAIRS_PER_SAMPLE); let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; for _ in 0..measured_iterations { From e3ca653a76aa2fe3f54064de9956d8e8bdebbd90 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:35:45 +0800 Subject: [PATCH 19/23] test(fspy): reduce runner sensitivity Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark_target/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index f9548576f..b7e62b707 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -5,7 +5,7 @@ use std::{ }; const THREAD_COUNT: usize = 4; -const OPEN_COUNT_PER_THREAD: usize = 2_048; +const OPEN_COUNT_PER_THREAD: usize = 512; #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing"; From c167db2d08527cf80826804b0f190a81c3d316a3 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:43:02 +0800 Subject: [PATCH 20/23] test(fspy): warm benchmark runners before sampling Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index d4430d428..5efb9b65e 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -113,7 +113,7 @@ async fn run_tracked(target: &str) -> ChildTermination { fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) - .warm_up_time(Duration::from_secs(5)) + .warm_up_time(Duration::from_secs(30)) .measurement_time(Duration::from_secs(60)) } From a0ba73e5872af9a201a04cecdc531c407fd4a25a Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 18:47:12 +0800 Subject: [PATCH 21/23] test(fspy): fix process pairs per sample Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 5efb9b65e..dfb9cf521 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,7 +12,7 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); -const MIN_PROCESS_PAIRS_PER_SAMPLE: u64 = 500; +const PROCESS_PAIRS_PER_SAMPLE: u64 = 500; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -45,7 +45,8 @@ fn benchmark_target( group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { - let measured_iterations = iterations.max(MIN_PROCESS_PAIRS_PER_SAMPLE); + // Keep run averages comparable when the delta misleads Criterion's calibration. + let measured_iterations = PROCESS_PAIRS_PER_SAMPLE; let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; for _ in 0..measured_iterations { From 6c4b2365e05348cd196746b44532c16aa91448ad Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 19:04:41 +0800 Subject: [PATCH 22/23] test(fspy): keep benchmark warmup short Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index dfb9cf521..0462354c2 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -114,7 +114,7 @@ async fn run_tracked(target: &str) -> ChildTermination { fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) - .warm_up_time(Duration::from_secs(30)) + .warm_up_time(Duration::from_secs(5)) .measurement_time(Duration::from_secs(60)) } From c1c6d7ac05fbcad177a468d619287aeeac265b4b Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 24 Jul 2026 19:18:05 +0800 Subject: [PATCH 23/23] test(fspy): simplify benchmark sampling Co-authored-by: GPT-5 Codex --- crates/fspy_benchmark/benches/fspy.rs | 18 ++++++------------ crates/fspy_benchmark_target/src/main.rs | 2 +- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/crates/fspy_benchmark/benches/fspy.rs b/crates/fspy_benchmark/benches/fspy.rs index 0462354c2..c5feda70a 100644 --- a/crates/fspy_benchmark/benches/fspy.rs +++ b/crates/fspy_benchmark/benches/fspy.rs @@ -12,7 +12,6 @@ use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; const DYNAMIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_TARGET"); -const PROCESS_PAIRS_PER_SAMPLE: u64 = 500; #[cfg(all(target_os = "linux", target_arch = "x86_64"))] const STATIC_TARGET: &str = env!("CARGO_BIN_FILE_FSPY_BENCHMARK_STATIC_TARGET"); @@ -45,19 +44,14 @@ fn benchmark_target( group.bench_with_input(BenchmarkId::from_parameter(target_name), &target, |bencher, target| { bencher.iter_custom(|iterations| { - // Keep run averages comparable when the delta misleads Criterion's calibration. - let measured_iterations = PROCESS_PAIRS_PER_SAMPLE; let mut tracked = Duration::ZERO; let mut untracked = Duration::ZERO; - for _ in 0..measured_iterations { - untracked += measure_untracked(runtime, target); + for _ in 0..iterations { tracked += measure_tracked(runtime, target); + untracked += measure_untracked(runtime, target); } - tracked - .saturating_sub(untracked) - .div_f64(measured_iterations as f64) - .max(Duration::from_nanos(1)) - .mul_f64(iterations as f64) + // Keep clamped samples reportable at a stable 1 ns per-iteration floor. + tracked.saturating_sub(untracked).max(Duration::from_nanos(iterations)) }); }); } @@ -114,8 +108,8 @@ async fn run_tracked(target: &str) -> ChildTermination { fn criterion_config() -> Criterion { Criterion::default() .sample_size(10) - .warm_up_time(Duration::from_secs(5)) - .measurement_time(Duration::from_secs(60)) + .warm_up_time(Duration::from_millis(500)) + .measurement_time(Duration::from_secs(5)) } criterion_group! { diff --git a/crates/fspy_benchmark_target/src/main.rs b/crates/fspy_benchmark_target/src/main.rs index b7e62b707..9ce595970 100644 --- a/crates/fspy_benchmark_target/src/main.rs +++ b/crates/fspy_benchmark_target/src/main.rs @@ -5,7 +5,7 @@ use std::{ }; const THREAD_COUNT: usize = 4; -const OPEN_COUNT_PER_THREAD: usize = 512; +const OPEN_COUNT_PER_THREAD: usize = 32_768; #[cfg(unix)] const MISSING_PATH: &str = "/.fspy-benchmark-missing";