test(fspy): benchmark tracking overhead

.github/workflows/fspy-benchmark.yml

@@ -0,0 +1,162 @@
+name: fspy benchmark
+
+permissions: {}
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, reopened, synchronize]
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.ref_name != 'main' }}
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  benchmark:
+    name: Benchmark (${{ matrix.platform }})
+    permissions:
+      actions: read
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux
+            os: namespace-profile-linux-x64-default
+            static: true
+          - platform: macos
+            os: namespace-profile-mac-default
+            static: false
+          - platform: windows
+            os: namespace-profile-windows-4c-8g
+            static: false
+    runs-on: ${{ matrix.os }}
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+      CRITERION_HOME: ${{ github.workspace }}/target/criterion
+    steps:
+      - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2
+
+      - name: Update Detours submodule
+        if: matrix.platform == 'windows'
+        run: git submodule update --init --recursive
+
+      - uses: oxc-project/setup-rust@3d6fb132fbe7cdcb66bf8ec193911c2945369d12 # v1.0.17
+        with:
+          save-cache: ${{ github.ref_name == 'main' }}
+          cache-key: fspy-benchmark-${{ matrix.platform }}
+
+      - name: Install static target
+        if: matrix.static
+        run: rustup target add x86_64-unknown-linux-musl
+
+      - name: Download main baseline
+        id: main-baseline
+        if: github.event_name != 'push'
+        uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151 # v21
+        with:
+          github_token: ${{ github.token }}
+          workflow: fspy-benchmark.yml
+          workflow_conclusion: success
+          branch: main
+          event: push
+          name: fspy-benchmark-baseline-v1-${{ matrix.platform }}
+          path: ${{ env.CRITERION_HOME }}/fspy
+          search_artifacts: true
+          if_no_artifact_found: warn
+
+      - name: Run benchmark
+        env:
+          MAIN_BASELINE_FOUND: ${{ steps.main-baseline.outputs.found_artifact }}
+        run: |
+          mkdir -p .benchmark-results
+          if [[ "$GITHUB_EVENT_NAME" != "push" && "$MAIN_BASELINE_FOUND" == "true" ]]; then
+            baseline=(--baseline main)
+          else
+            baseline=(--save-baseline main)
+          fi
+          set -o pipefail
+          result_file=".benchmark-results/${{ matrix.platform }}.txt"
+          cargo bench --locked -p fspy_benchmark --bench fspy -- \
+            --color never --verbose -n "${baseline[@]}" |
+            tee "$result_file"
+          result="$(<"$result_file")"
+          printf '%s' "$result" > "$result_file"
+
+      - name: Add job summary
+        run: |
+          {
+            echo '```text'
+            cat ".benchmark-results/${{ matrix.platform }}.txt"
+            echo
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload main baseline
+        if: github.event_name == 'push' && github.ref_name == 'main'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: fspy-benchmark-baseline-v1-${{ matrix.platform }}
+          path: ${{ env.CRITERION_HOME }}/fspy
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 90
+
+      - name: Upload benchmark report
+        if: github.event_name == 'pull_request'
+        # Matrix jobs have isolated filesystems, so persist each report for the
+        # comment job to combine into one sticky comment after all platforms finish.
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: fspy-benchmark-report-${{ matrix.platform }}
+          path: .benchmark-results/${{ matrix.platform }}.txt
+          if-no-files-found: error
+          overwrite: true
+          retention-days: 90
+
+  comment:
+    name: Report benchmark
+    if: >-
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    needs: benchmark
+    runs-on: namespace-profile-linux-x64-default
+    permissions:
+      actions: read
+      pull-requests: write
+    steps:
+      - name: Download benchmark reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: fspy-benchmark-report-*
+          path: .benchmark-results
+          merge-multiple: true
+
+      - name: Assemble comment
+        run: |
+          {
+            echo '## fspy benchmark'
+            echo
+            for platform in linux macos windows; do
+              echo "### $platform"
+              echo
+              echo '```text'
+              cat ".benchmark-results/$platform.txt"
+              echo
+              echo '```'
+              echo
+            done
+          } > .benchmark-results/comment.md
+
+      - name: Update PR comment
+        uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v3.0.4
+        with:
+          header: fspy-benchmark
+          path: .benchmark-results/comment.md

Cargo.toml

@@ -64,6 +64,7 @@ crossterm = { version = "0.29.0", features = ["event-stream"] }
 csv-async = { version = "1.3.1", features = ["tokio"] }
 ctor = "1.0"
 ctrlc = "3.5.2"
+criterion2 = { version = "3.0.4", default-features = false }
 derive_more = "2.0.1"
 diff-struct = "0.5.3"
 directories = "6.0.0"
@@ -72,6 +73,7 @@ materialized_artifact = { path = "crates/materialized_artifact" }
 materialized_artifact_build = { path = "crates/materialized_artifact_build" }
 flate2 = "1.0.35"
 fspy = { path = "crates/fspy" }
+fspy_benchmark_target = { path = "crates/fspy_benchmark_target", artifact = "bin" }
 fspy_detours_sys = { path = "crates/fspy_detours_sys" }
 fspy_preload_unix = { path = "crates/fspy_preload_unix", artifact = "cdylib", target = "target" }
 fspy_preload_windows = { path = "crates/fspy_preload_windows", artifact = "cdylib", target = "target" }
@@ -177,6 +179,7 @@ zstd = "0.13"
 [workspace.metadata.cargo-shear]
 ignored = [
   # These are artifact dependencies. They are not directly `use`d in Rust code.
+  "fspy_benchmark_target",
   "fspy_preload_unix",
   "fspy_preload_windows",
   "vite_task_client_napi",

crates/fspy_benchmark/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "fspy_benchmark"
+version = "0.0.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish = false
+rust-version.workspace = true
+
+[lints]
+workspace = true
+
+[lib]
+test = false
+bench = false
+doctest = false
+
+[[bench]]
+name = "fspy"
+harness = false
+
+[dev-dependencies]
+criterion2 = { workspace = true }
+fspy = { workspace = true }
+fspy_benchmark_target = { workspace = true }
+tokio = { workspace = true, features = ["macros", "process", "rt-multi-thread"] }
+tokio-util = { workspace = true }
+
+[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dev-dependencies]
+fspy_benchmark_static_target = { path = "../fspy_benchmark_static_target", artifact = "bin", target = "x86_64-unknown-linux-musl" }
+
+[package.metadata.cargo-shear]
+ignored = ["fspy_benchmark_target", "fspy_benchmark_static_target"]

crates/fspy_benchmark/README.md

@@ -0,0 +1,13 @@
+# fspy benchmark
+
+Measures fspy's wall-clock overhead while multiple threads repeatedly open a nonexistent path.
+Each estimate aggregates ten tracked and untracked runs. Criterion reports the nonnegative overhead
+normalized to one second of untracked execution, so 10 ms represents 1% overhead.
+
+Linux measures dynamic and static targets. macOS and Windows measure their preload implementations.
+
+Run the benchmark with:
+
+```sh
+just benchmark-fspy
+```