From b7e8ee14a77786c96647c02a530dd8b79a8fe3d9 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 15 Sep 2026 03:47:01 -0400 Subject: [PATCH 1/3] perf(ci): gate the statusline bench lanes behind BENCH_LANES plugins/rate-limit-guard/bench/bench.test.sh spawned bench-idle.sh, bench-load.sh and trace-probe.sh on every contract-corpus run, which is a benchmark run whatever the parameters and the only part of that suite that spends real wall-clock seconds. BENCH_LANES=1 now runs them; an ordinary run stops after the lib assertions and reports the lanes as deferred, the same shape lib/hook-utils.test.sh uses for its clock comparisons. The clean-checkout runnability guard (#2582) moves from every pull request to the new weekly bench-harness workflow, which sets the variable, plus workflow_dispatch for a deliberate run. Advisory like hook-utils-timing: outside ci-status, blocking no merge. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01XwY46aTKXzw9wrLhVC1Azz --- .github/workflows/bench-harness.yml | 36 ++++++++++++++++++++ plugins/rate-limit-guard/bench/bench.test.sh | 32 +++++++++++++++-- scripts/run-plugin-tests-serial.txt | 5 ++- 3 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/bench-harness.yml diff --git a/.github/workflows/bench-harness.yml b/.github/workflows/bench-harness.yml new file mode 100644 index 000000000..fcf7dfc16 --- /dev/null +++ b/.github/workflows/bench-harness.yml @@ -0,0 +1,36 @@ +name: bench-harness + +# Weekly run of the rate-limit-guard statusline bench lanes. +# plugins/rate-limit-guard/bench/bench.test.sh asserts the harness still RUNS +# from a clean checkout — the defect that made #2521's measurements +# unreproducible (#2582) — by spawning bench-idle.sh, bench-load.sh and +# trace-probe.sh at tiny parameters. Spawning a lane is running a benchmark +# whatever the parameters, so those runs are gated on BENCH_LANES and every +# ordinary run of the suite reports them as deferred; this lane is where they +# actually run. Advisory by construction, like hook-utils-timing: nothing +# aggregates this workflow into ci-status, and a flake here blocks no merge. +# +# Linux only, because the contract corpus that carries this suite runs in +# test-linux; ci.yml's test-windows job runs named scripts/ suites, not +# scripts/run-plugin-tests.sh, so no Windows coverage of the lanes is lost. +on: + schedule: + - cron: "43 6 * * 1" # Mondays 06:43 UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Run the bench harness suite with the lanes enabled + env: + BENCH_LANES: "1" + run: bash plugins/rate-limit-guard/bench/bench.test.sh diff --git a/plugins/rate-limit-guard/bench/bench.test.sh b/plugins/rate-limit-guard/bench/bench.test.sh index f866e57fd..9d36f1ec0 100755 --- a/plugins/rate-limit-guard/bench/bench.test.sh +++ b/plugins/rate-limit-guard/bench/bench.test.sh @@ -8,6 +8,13 @@ # unrunnable harness is exactly the defect that let #2521's measurements go # unreproducible (#2582). # +# The lane runs are GATED on BENCH_LANES: running a lane is running a benchmark +# however small its parameters, so an ordinary CI run stops after the lib +# assertions and reports the lanes as deferred. `BENCH_LANES=1 bash +# plugins/rate-limit-guard/bench/bench.test.sh` runs them locally, and the +# weekly bench-harness workflow runs them in CI, which is where the #2582 +# clean-checkout guard now fires. +# # Self-contained: defines its own assertion helpers — installed plugins are # cache-isolated with no shared test lib. @@ -26,6 +33,13 @@ ok() { echo "ok: $*" PASS=$((PASS + 1)) } +# One summary shape for both exits: the gated stop after the lib assertions and +# the full run. +summary() { + echo + echo "PASS=$PASS FAIL=$FAIL" + [[ $FAIL -eq 0 ]] +} WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT @@ -77,6 +91,20 @@ else fail "lib: EPOCHREALTIME guard rc=$RC out=$OUT" fi +# --- the gate ---------------------------------------------------------------- +# Everything above asserts pure functions and costs milliseconds. Everything +# below SPAWNS the lanes, which is a benchmark run whatever the parameters, and +# is the only part of this suite that spends real wall-clock seconds on a +# shared runner. Deferring it reads like lib/hook-utils.test.sh's clock +# comparisons: an `ok` line that says the coverage did not run, not a SKIP, +# because the runner's skip accounting is for an absent optional TOOL and +# --strict-skips must stay usable on a box that has everything. +if [[ -z "${BENCH_LANES:-}" ]]; then + ok "bench lanes: deferred (BENCH_LANES unset; the weekly bench-harness lane runs them, and BENCH_LANES=1 runs them here)" + summary + exit +fi + # --- bench-idle: smoke run against the repo tee, isolated HOME --------------- HOME1="$WORK/home1" mkdir -p "$HOME1" @@ -132,6 +160,4 @@ else fail "trace-probe: rc=$RC out=${OUT:0:400}" fi -echo -echo "PASS=$PASS FAIL=$FAIL" -[[ $FAIL -eq 0 ]] +summary diff --git a/scripts/run-plugin-tests-serial.txt b/scripts/run-plugin-tests-serial.txt index 8e62de661..b386522cc 100644 --- a/scripts/run-plugin-tests-serial.txt +++ b/scripts/run-plugin-tests-serial.txt @@ -39,7 +39,10 @@ plugins/context-guard/hooks/zone-gate.test.sh # A/B timing harness; its verdicts are elapsed-time comparisons. plugins/performance/scripts/ab.test.sh -# The statusline bench measures render latency. +# The statusline bench measures render latency. Its lane runs are gated on +# BENCH_LANES, so an ordinary run stops before them and needs no seat here; the +# entry is for the runs that DO set it (a local full corpus with the lanes +# enabled), where the lanes measure exactly what contention distorts. plugins/rate-limit-guard/bench/bench.test.sh # Cancellation-window cases kill a parked rename shim on a short fuse. From 37db04a221836320e45e1d81ddf134f5c6299d2b Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:02:58 -0400 Subject: [PATCH 2/3] ci: run the bench lanes from a dispatch input, not a scheduled lane The first commit here added a weekly bench-harness workflow to run the gated lanes. #4173 deleted the two workflows of exactly that shape three commits ago, under the owner direction to delete stale and overengineered automation, and hook-utils-timing was deleted for having two runs nobody read. A new scheduled advisory lane would be the same thing again. So ci.yml gains a workflow_dispatch `bench_lanes` input instead, threaded into the contract-test step as BENCH_LANES. Nothing runs on a schedule, nothing goes unread, and the lanes still have a deliberate CI run. The deferral now prints a SKIP line rather than counting an ok, so the runner's aggregate names the coverage that did not run instead of reading as a full pass. bench/README.md describes the split, and the plugin takes a version bump with its changelog entry. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M98vWnd3jQJ19Pe6Uj7bV9 --- .github/workflows/bench-harness.yml | 36 ------------------- .github/workflows/ci.yml | 14 ++++++++ .../.claude-plugin/plugin.json | 2 +- plugins/rate-limit-guard/CHANGELOG.md | 6 ++++ plugins/rate-limit-guard/bench/README.md | 22 ++++++++---- plugins/rate-limit-guard/bench/bench.test.sh | 22 +++++++----- 6 files changed, 50 insertions(+), 52 deletions(-) delete mode 100644 .github/workflows/bench-harness.yml diff --git a/.github/workflows/bench-harness.yml b/.github/workflows/bench-harness.yml deleted file mode 100644 index fcf7dfc16..000000000 --- a/.github/workflows/bench-harness.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: bench-harness - -# Weekly run of the rate-limit-guard statusline bench lanes. -# plugins/rate-limit-guard/bench/bench.test.sh asserts the harness still RUNS -# from a clean checkout — the defect that made #2521's measurements -# unreproducible (#2582) — by spawning bench-idle.sh, bench-load.sh and -# trace-probe.sh at tiny parameters. Spawning a lane is running a benchmark -# whatever the parameters, so those runs are gated on BENCH_LANES and every -# ordinary run of the suite reports them as deferred; this lane is where they -# actually run. Advisory by construction, like hook-utils-timing: nothing -# aggregates this workflow into ci-status, and a flake here blocks no merge. -# -# Linux only, because the contract corpus that carries this suite runs in -# test-linux; ci.yml's test-windows job runs named scripts/ suites, not -# scripts/run-plugin-tests.sh, so no Windows coverage of the lanes is lost. -on: - schedule: - - cron: "43 6 * * 1" # Mondays 06:43 UTC - workflow_dispatch: - -permissions: - contents: read - -jobs: - linux: - runs-on: ubuntu-24.04 - timeout-minutes: 15 - steps: - - name: Check out - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Run the bench harness suite with the lanes enabled - env: - BENCH_LANES: "1" - run: bash plugins/rate-limit-guard/bench/bench.test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc5584277..79c11606c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,17 @@ name: ci on: + # The only way to run the rate-limit-guard bench lanes in CI. They are gated + # on BENCH_LANES inside their suite because spawning a lane is running a + # benchmark, and a scheduled lane for them would be the unread advisory + # automation #4173 deleted. A dispatch runs the full contract corpus, so this + # input only decides whether the lanes run inside it. + workflow_dispatch: + inputs: + bench_lanes: + description: "Run the rate-limit-guard statusline bench lanes" + type: boolean + default: false push: branches: [main] pull_request: @@ -1492,6 +1503,9 @@ jobs: if: needs.changes.outputs.run_tests == 'true' env: BASE_REF: ${{ github.base_ref }} + # Empty on every event but a dispatch that asked for the lanes, which + # is what the gate in the bench suite reads as "stay deferred". + BENCH_LANES: ${{ inputs.bench_lanes && '1' || '' }} run: | if [ "$GITHUB_EVENT_NAME" != pull_request ]; then scripts/run-plugin-tests.sh --jobs 3 --shard "$LEG/$LEGS" diff --git a/plugins/rate-limit-guard/.claude-plugin/plugin.json b/plugins/rate-limit-guard/.claude-plugin/plugin.json index 96a256eb4..2bf692e1c 100644 --- a/plugins/rate-limit-guard/.claude-plugin/plugin.json +++ b/plugins/rate-limit-guard/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "rate-limit-guard", - "version": "0.8.18", + "version": "0.8.19", "description": "Shared rate-limit guard for loop lanes: a statusline wrapper tees the subscription rate-limit windows to a fixed machine-scope file, a StopFailure hook records rate-limit stops reactively, and a reader contract fixes how consuming sessions pause and resume.", "author": { "name": "Melodic Software", diff --git a/plugins/rate-limit-guard/CHANGELOG.md b/plugins/rate-limit-guard/CHANGELOG.md index 30be65ba9..263ca4015 100644 --- a/plugins/rate-limit-guard/CHANGELOG.md +++ b/plugins/rate-limit-guard/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to the `rate-limit-guard` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.8.19] + +### Changed + +- `bench.test.sh` gates its five lane cases on `BENCH_LANES`. Without it the suite runs only the lib-helper assertions and prints a `SKIP:` line naming the coverage that did not run; `BENCH_LANES=1` runs every case, locally or through `ci.yml`'s `bench_lanes` dispatch input. + ## [0.8.18] ### Changed diff --git a/plugins/rate-limit-guard/bench/README.md b/plugins/rate-limit-guard/bench/README.md index e92e1aea0..ed6c7d071 100644 --- a/plugins/rate-limit-guard/bench/README.md +++ b/plugins/rate-limit-guard/bench/README.md @@ -83,9 +83,19 @@ HOME="$(mktemp -d)" bash plugins/rate-limit-guard/bench/bench-idle.sh The **benchmarks gate nothing**. Wall-clock numbers on shared CI runners are noise, so no lane's timing ever runs in CI. What does run is `bench.test.sh`, a contract smoke suite discovered by -`scripts/run-plugin-tests.sh` like every other `*.test.sh`: it unit-tests the lib helpers and -runs each lane once with tiny parameters against the repo tee under an isolated `HOME`, asserting -behaviour and output shape, never timing. That keeps the harness runnable from a clean checkout -(an unrunnable harness is exactly the defect that made #2521's measurements unreproducible) and -maps these files into `scripts/affected-tests.sh` coverage. The tee's behavioural coverage lives -in `../scripts/statusline-tee.test.sh`. +`scripts/run-plugin-tests.sh` like every other `*.test.sh`, which maps these files into +`scripts/affected-tests.sh` coverage. It splits in two: + +- **Every run** unit-tests the lib helpers: `median`, `pace_sleep_arg`, `now_ms` and the refusal + on a bash without `EPOCHREALTIME`. No lane is spawned, so it costs milliseconds. +- **`BENCH_LANES=1`** adds the lane cases: one tiny-parameter run of each lane against the repo + tee under an isolated `HOME`, plus the two failing-render aborts, asserting behaviour and + output shape, never timing. Spawning a lane is running a benchmark whatever the parameters, so + those five cases are gated; without the variable the suite prints a `SKIP:` line and the + runner's summary names the coverage that did not run. + +Run them locally with `BENCH_LANES=1 bash plugins/rate-limit-guard/bench/bench.test.sh`, or in CI +by dispatching `ci.yml` with its `bench_lanes` input set. That is the deliberate run that keeps +the harness runnable from a clean checkout; an unrunnable harness is exactly the defect behind the +unreproducible measurements in #2521. The tee's behavioural coverage lives in +`../scripts/statusline-tee.test.sh`. diff --git a/plugins/rate-limit-guard/bench/bench.test.sh b/plugins/rate-limit-guard/bench/bench.test.sh index 9d36f1ec0..1b4c6745e 100755 --- a/plugins/rate-limit-guard/bench/bench.test.sh +++ b/plugins/rate-limit-guard/bench/bench.test.sh @@ -10,10 +10,9 @@ # # The lane runs are GATED on BENCH_LANES: running a lane is running a benchmark # however small its parameters, so an ordinary CI run stops after the lib -# assertions and reports the lanes as deferred. `BENCH_LANES=1 bash -# plugins/rate-limit-guard/bench/bench.test.sh` runs them locally, and the -# weekly bench-harness workflow runs them in CI, which is where the #2582 -# clean-checkout guard now fires. +# assertions and reports the lanes as skipped coverage. BENCH_LANES=1 runs them +# — locally, or in CI through ci.yml's `bench_lanes` dispatch input, which is +# the deliberate run the #2582 guard rests on. # # Self-contained: defines its own assertion helpers — installed plugins are # cache-isolated with no shared test lib. @@ -95,12 +94,17 @@ fi # Everything above asserts pure functions and costs milliseconds. Everything # below SPAWNS the lanes, which is a benchmark run whatever the parameters, and # is the only part of this suite that spends real wall-clock seconds on a -# shared runner. Deferring it reads like lib/hook-utils.test.sh's clock -# comparisons: an `ok` line that says the coverage did not run, not a SKIP, -# because the runner's skip accounting is for an absent optional TOOL and -# --strict-skips must stay usable on a box that has everything. +# shared runner. +# +# The deferral prints a SKIP line rather than counting an ok. scripts/ +# run-plugin-tests.sh reads `^SKIP:` and names the suite under "Suites with +# skipped coverage (exit 0 here is NOT evidence those cases ran)", which is +# what this is: five cases that did not run. An ok would make the aggregate +# read as full coverage. --strict-skips therefore fails here, correctly — a +# caller declaring a fully provisioned environment is asking for every case to +# run, and BENCH_LANES=1 is how it gets them. if [[ -z "${BENCH_LANES:-}" ]]; then - ok "bench lanes: deferred (BENCH_LANES unset; the weekly bench-harness lane runs them, and BENCH_LANES=1 runs them here)" + echo "SKIP: bench lanes deferred; set BENCH_LANES=1 to run them" summary exit fi From d8c034398733f83a4bad4f1f485597cdeab710be Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:35:38 -0400 Subject: [PATCH 3/3] docs(ci): drop the tracker back-reference from the dispatch-input comment The reason stands on its own: a scheduled lane of their own would be unread advisory automation outside ci-status. Who deleted the last two, and when, is git history. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M98vWnd3jQJ19Pe6Uj7bV9 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79c11606c..dbca8e5c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,9 +3,9 @@ name: ci on: # The only way to run the rate-limit-guard bench lanes in CI. They are gated # on BENCH_LANES inside their suite because spawning a lane is running a - # benchmark, and a scheduled lane for them would be the unread advisory - # automation #4173 deleted. A dispatch runs the full contract corpus, so this - # input only decides whether the lanes run inside it. + # benchmark, and a scheduled lane of their own would be unread advisory + # automation outside `ci-status`. A dispatch runs the full contract corpus, + # so this input only decides whether the lanes run inside it. workflow_dispatch: inputs: bench_lanes: