diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44ab073f..ca05422e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,14 @@ on: # Theme-2/3/4 change, and since there is no cron it is the ONLY thing that runs the matrix. # Without this the matrix change is never exercised by its own PR. workflow_dispatch: + inputs: + regenerate_retrieval_baseline: + description: >- + Rewrite the golden-question retrieval baseline from this run and upload it as an + artifact. The eval GATE is skipped on such a run (it would compare the run with + itself). Nothing is committed - download the artifact and commit it deliberately. + type: boolean + default: false # TWO RUNS OF THIS REPO MUST NOT OVERLAP ON ONE RUNNER HOST. # @@ -412,7 +420,41 @@ jobs: # step summary so a reviewer of a ranking PR sees the delta on the run's summary page # without opening the raw log and scrolling. PIPESTATUS preserves the gate's exit # code across the tee so a regression still fails the job. + # REGENERATING THE BASELINE, on demand, here rather than on a laptop. + # + # A new or renamed golden question makes the run `:incomparable` — `question_set_changed?/1` + # compares the question-id SET and refuses rather than letting an unmatched question read + # as a non-regression. So the baseline MUST be regenerated to add a question, and adding + # questions is the whole path to a golden set grown from real logged queries. + # + # It has to happen HERE. The eval's vectors are deterministic functions of the committed + # text, so a provider is not the issue — the CORPUS is: this job seeds a fresh database, + # while a developer's dev DB carries whatever else it has accumulated. Measured + # 2026-08-12, same commit and same code: CI scored mrr 0.746 / answered 22 of 26, a local + # run scored 0.192 / 5 of 26. A baseline captured from the second would bake a private + # database's contents into a deploy gate for everyone. + # + # Uploaded as an ARTIFACT, never committed by CI: a gate that can rewrite its own + # threshold unattended is not a gate. Download it, read the diff, commit it deliberately. + - name: Regenerate the retrieval baseline (manual, artifact only) + if: github.event_name == 'workflow_dispatch' && inputs.regenerate_retrieval_baseline + run: | + set -o pipefail + mix loopctl.retrieval.eval --mode both --update-baseline 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload the regenerated baseline + if: github.event_name == 'workflow_dispatch' && inputs.regenerate_retrieval_baseline + uses: actions/upload-artifact@v4 + with: + name: retrieval-baseline + path: priv/retrieval_eval/baseline_v1.json + if-no-files-found: error + - name: Run golden-question retrieval eval gate + # Skipped on a regeneration run: the baseline was just rewritten from this very run, + # so comparing against it would be comparing a run with itself and would pass no + # matter what the numbers are. + if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.regenerate_retrieval_baseline) }} run: | set -o pipefail { diff --git a/docs/runbooks/search-events-analysis.md b/docs/runbooks/search-events-analysis.md index 3d11be9b..31f5cffc 100644 --- a/docs/runbooks/search-events-analysis.md +++ b/docs/runbooks/search-events-analysis.md @@ -144,3 +144,27 @@ published articles have ever been opened. That is a harness/consumption problem, search or corpus one — fixing retrieval further will not move it. Do not spend a month's work on the retrieval side on the strength of a utilization number that was never segmented by origin (step 2). + +## Appendix — regenerating the retrieval baseline + +Adding a golden question (for example, one grown from real logged queries) makes the eval +run `:incomparable`: `question_set_changed?/1` compares the question-id SET and refuses, +rather than letting an unmatched question read as a non-regression. So the baseline has to be +regenerated whenever the question set changes. + +**Do it in CI, not locally**, and not for the reason it first appears. The eval's vectors are +deterministic functions of the committed text, so no embedding provider is involved. The +difference is the CORPUS: the CI job seeds a fresh database, while a developer's dev DB +carries whatever else it has accumulated. Measured 2026-08-12 on the same commit — CI scored +`mrr 0.746 / answered 22 of 26`; a local run of the same command scored `0.192 / 5 of 26`. A +baseline captured locally bakes one machine's private database into a deploy gate for +everyone. + +```bash +gh workflow run CI --ref -f regenerate_retrieval_baseline=true +gh run download -n retrieval-baseline # then commit it deliberately +``` + +The gate step is skipped on a regeneration run — comparing a run against a baseline it just +wrote would pass whatever the numbers are. CI never commits the file: a gate that can rewrite +its own threshold unattended is not a gate.