diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 00000000..ed385aa8 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,41 @@ +CI runs the full Lua suite for every PR targeting `master`. The shared +`actions/validate` action records the checked-out Git tree in a small artifact +named with the tree hash and run attempt. This identifies what was tested; only +a successful `Test.yml` workflow run makes it a passing result. + +On a push to `master`, validation locates the merged PR's latest CI run. It +reuses that run only if its recorded tree matches the new `master` tree exactly. +This works across merge, squash, and rebase methods when the resulting files +are identical. Direct pushes, changed merge results, and missing or expired +artifacts fall back to the full suite. + +On a `v*` tag, release validation looks for the latest `master` CI run for the +exact tagged commit, including annotated tags. It requires the same tree proof +and a successful conclusion. A queued or running CI run is awaited for up to +70 minutes before falling back to tests. Unavailable API results, cancelled +runs, and missing evidence also fall back to tests. A known failed validation +for the same source blocks progression; fix or rerun the failed CI run first. +Older successful runs never override a newer matching failure. + +Successful fallback tests let the workflow continue. Each validation attempt +records its own source, so a master fallback can be reused by a subsequent tag. +Artifacts are retained for 90 days; older releases can still proceed by testing +again. Existing CI runs without validation artifacts also require fresh tests. +Recording an artifact is best effort: if uploading it fails, validation still +proceeds, and later stages fall back to tests when they cannot find the record. + +Every reused result emits a notice, an explicit skipped-suite step, and a +workflow summary explaining the skip, with the matching tree, source run URL, +and attempt number. Fallback and blocked decisions also explain their reason +in the summary. + +Release packages are built on Linux and macOS, then extracted into a temporary +directory and smoke-tested (version, help, obfuscation, and execution using the +bundled runtime). Publishing requires both packages and their smoke tests to +succeed. The normal PR → merge → tag → release path runs the full suite once. + +Run the validation policy tests locally with: + +```sh +node --test scripts/ci/reuse-validation.test.cjs +``` diff --git a/.github/actions/validate/action.yml b/.github/actions/validate/action.yml new file mode 100644 index 00000000..d0f043e0 --- /dev/null +++ b/.github/actions/validate/action.yml @@ -0,0 +1,57 @@ +name: Validate Lua suite +description: Reuse verified CI results for the same files, or run the full suite. +runs: + using: composite + steps: + - name: Check validation policy + shell: bash + run: node --test scripts/ci/reuse-validation.test.cjs + - name: Identify checked-out files + id: source + shell: bash + run: | + echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "tree=$(git rev-parse HEAD^{tree})" >> "$GITHUB_OUTPUT" + mkdir -p "$RUNNER_TEMP/lua-validation" + git rev-parse HEAD HEAD^{tree} > "$RUNNER_TEMP/lua-validation/source.txt" + # Record what this attempt tests, including failed attempts. Consumers must + # also verify the workflow's conclusion before treating this as a pass. + - name: Record validation source + uses: actions/upload-artifact@v4 + # An artifact outage must not prevent testing. Later stages will test + # again if this run has no source record they can verify. + continue-on-error: true + with: + name: lua-validation-${{ steps.source.outputs.tree }}-attempt-${{ github.run_attempt }} + path: ${{ runner.temp }}/lua-validation/source.txt + retention-days: 90 + if-no-files-found: error + - name: Look for existing validation + id: validation + uses: actions/github-script@v7 + env: + VALIDATION_COMMIT: ${{ steps.source.outputs.commit }} + VALIDATION_TREE: ${{ steps.source.outputs.tree }} + with: + script: | + const validate = require(`${process.env.GITHUB_WORKSPACE}/scripts/ci/reuse-validation.cjs`); + const result = await validate({ github, context, core }); + core.info(result.reason); + core.setOutput('reuse', result.decision === 'reuse' ? 'true' : 'false'); + core.setOutput('reason', result.reason); + const outcome = { reuse: 'Full Lua suite skipped', test: 'Full Lua suite required', block: 'Validation blocked' }[result.decision]; + if (result.decision === 'reuse') core.notice(`${outcome}: ${result.reason}`); + await core.summary.addHeading(outcome).addRaw(result.reason).addEOL() + .addRaw(`Checked-out commit: ${process.env.VALIDATION_COMMIT}`).addEOL() + .addRaw(`Git tree: ${process.env.VALIDATION_TREE}`).addEOL().write(); + if (result.decision === 'block') core.setFailed(result.reason); + - name: Full Lua suite skipped — reused passing validation + if: steps.validation.outputs.reuse == 'true' + shell: bash + env: + SKIP_REASON: ${{ steps.validation.outputs.reason }} + run: printf 'Full Lua suite skipped. %s\n' "$SKIP_REASON" + - name: Build and run tests + if: steps.validation.outputs.reuse != 'true' + shell: bash + run: bash scripts/run-tests.sh -b -n 10 --ci diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index ae23e0cd..f2fe1129 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -7,11 +7,18 @@ on: branches: - master +permissions: + contents: read + actions: read + pull-requests: read + jobs: test-docker: runs-on: ubuntu-latest + # Allow waiting for an existing run, followed by the full fallback suite. + timeout-minutes: 140 steps: - name: Checkout repo uses: actions/checkout@v4 - - name: Build and run tests - run: bash scripts/run-tests.sh -b -n 10 --ci + - name: Validate Lua suite + uses: ./.github/actions/validate diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69ce0eea..e37247e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,16 +6,18 @@ on: - "v*" permissions: - contents: write + contents: read + actions: read jobs: test: - name: Lua tests + name: Validate release source runs-on: ubuntu-latest + timeout-minutes: 140 steps: - uses: actions/checkout@v4 - - name: Build and run tests - run: bash scripts/run-tests.sh -b -n 10 --ci + - name: Validate Lua suite + uses: ./.github/actions/validate package: name: Package (${{ matrix.os_name }}) @@ -40,6 +42,10 @@ jobs: run: brew install luajit - name: Build package run: PROMETHEUS_BUNDLED_LUA="$(command -v luajit)" ./scripts/release/package.sh "${{ matrix.os_name }}" "${GITHUB_REF_NAME}" + - name: Smoke-test release archive + env: + RELEASE_OS: ${{ matrix.os_name }} + run: bash scripts/release/smoke-test.sh "$RELEASE_OS" "$GITHUB_REF_NAME" - name: Upload artifact uses: actions/upload-artifact@v4 with: @@ -50,10 +56,13 @@ jobs: name: Publish release runs-on: ubuntu-latest needs: package + permissions: + contents: write steps: - uses: actions/download-artifact@v4 with: path: dist + pattern: prometheus-lua-* - name: Move release assets run: | mkdir -p dist/out diff --git a/scripts/ci/reuse-validation.cjs b/scripts/ci/reuse-validation.cjs new file mode 100644 index 00000000..89746b56 --- /dev/null +++ b/scripts/ci/reuse-validation.cjs @@ -0,0 +1,81 @@ +// A matching artifact identifies the checked-out Git tree, not a passing test. +// Only the latest attempt of a successful Test.yml run can supply a pass. +module.exports = async function reuseValidation({ github, context, core, + commit = process.env.VALIDATION_COMMIT, + tree = process.env.VALIDATION_TREE, + sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), + now = Date.now, + waitMs = 70 * 60 * 1000, +}) { + const result = (decision, reason) => ({ decision, reason }); + if (context.eventName === 'pull_request') { + return result('test', 'Pull requests always run the full Lua suite.'); + } + const repo = context.repo; + const isTag = context.ref.startsWith('refs/tags/'); + try { + let head; + let pr; + if (isTag) { + head = commit; + } else if (context.ref === 'refs/heads/master') { + const pulls = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, + { ...repo, commit_sha: commit, per_page: 100 }); + pr = pulls.find(p => p.merged_at && p.base.ref === 'master' && + p.base.repo.full_name === `${repo.owner}/${repo.repo}` && p.merge_commit_sha === commit); + if (!pr) return result('test', 'No merged PR found for this master commit; running the full suite.'); + head = pr.head.sha; + } else { + return result('test', 'No reusable validation source for this event; running the full suite.'); + } + + const runs = await github.paginate(github.rest.actions.listWorkflowRuns, { + ...repo, workflow_id: 'Test.yml', head_sha: head, + event: isTag ? 'push' : 'pull_request', per_page: 100, + }); + // Do not select only successful runs: that could hide a newer failure. + let run = runs.filter(r => r.id !== context.runId && + (isTag ? r.head_branch === 'master' : + // GitHub can omit pull_requests for fork runs. The head SHA and + // matching tested tree still identify the source in that case. + !r.pull_requests?.length || r.pull_requests.some(p => p.number === pr.number))) + .sort((a, b) => b.id - a.id)[0]; + if (!run) return result('test', 'No matching CI run found; running the full suite.'); + + const deadline = now() + waitMs; + while (run.status !== 'completed') { + if (now() >= deadline) { + return result('test', `CI run ${run.html_url} did not finish within 70 minutes; running the full suite.`); + } + core.info(`Waiting for ${run.html_url} (${run.status}).`); + await sleep(30_000); + ({ data: run } = await github.rest.actions.getWorkflowRun({ ...repo, run_id: run.id })); + } + + const failed = ['failure', 'timed_out', 'action_required'].includes(run.conclusion); + if (isTag && failed) { + return result('block', `Validation failed for this commit: ${run.html_url}. Fix or rerun that CI run before proceeding.`); + } + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, + { ...repo, run_id: run.id, per_page: 100 }); + const proof = artifacts.some(a => !a.expired && + a.name === `lua-validation-${tree}-attempt-${run.run_attempt}`); + // A master run is already tied to this exact commit. A PR run needs a + // matching tree artifact because head_sha identifies its branch head, + // whereas checkout tests GitHub's temporary merge commit. + if (proof && failed) { + return result('block', `Validation failed for this source: ${run.html_url}. Fix or rerun that CI run before proceeding.`); + } + if (run.conclusion === 'success' && proof) { + const source = isTag ? `master CI for tagged commit ${commit}` : `PR #${pr.number} CI`; + return result('reuse', `${source} passed, and its recorded Git tree exactly matches ${tree}. Reusing run ${run.html_url} (attempt ${run.run_attempt}).`); + } + if (!proof) { + return result('test', `CI run ${run.html_url} (attempt ${run.run_attempt}) has no unexpired validation artifact matching Git tree ${tree}; running the full suite.`); + } + return result('test', `CI run ${run.html_url} concluded ${run.conclusion}; running the full suite.`); + } catch (error) { + core.warning(`Could not verify prior validation: ${error.message}`); + return result('test', `Prior validation could not be verified (${error.message}); running the full suite.`); + } +}; diff --git a/scripts/ci/reuse-validation.test.cjs b/scripts/ci/reuse-validation.test.cjs new file mode 100644 index 00000000..6118c2f1 --- /dev/null +++ b/scripts/ci/reuse-validation.test.cjs @@ -0,0 +1,182 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const validate = require('./reuse-validation.cjs'); + +function fixture(options = {}) { + const pr = { + number: 42, merged_at: '2026-11-02', merge_commit_sha: 'merged', + base: { ref: 'master', repo: { full_name: 'owner/repo' } }, + head: { sha: 'pr-head' }, + }; + const run = { + id: 123, status: 'completed', conclusion: 'success', run_attempt: 2, + head_branch: 'master', pull_requests: [{ number: 42 }], + html_url: 'https://github.com/owner/repo/actions/runs/123', + ...options.run, + }; + const calls = []; + let time = 0; + const rest = { + repos: { listPullRequestsAssociatedWithCommit() {} }, + actions: { + listWorkflowRuns() {}, listWorkflowRunArtifacts() {}, + async getWorkflowRun(args) { + calls.push(['poll', args]); + return { data: { ...run, status: 'completed', ...options.completed } }; + }, + }, + }; + const args = { + github: { + rest, + async paginate(method, params) { + if (method === rest.repos.listPullRequestsAssociatedWithCommit) { + calls.push(['pulls', params]); + return options.pulls ?? [pr]; + } + if (method === rest.actions.listWorkflowRuns) { + calls.push(['runs', params]); + if (options.apiError) throw new Error('API unavailable'); + return options.runs ?? [run]; + } + if (method === rest.actions.listWorkflowRunArtifacts) { + calls.push(['artifacts', params]); + if (options.artifactError) throw new Error('Artifact API unavailable'); + return options.artifacts ?? [{ + name: 'lua-validation-tree-attempt-2', expired: false, + }]; + } + assert.fail('Unexpected API call'); + }, + }, + context: { + repo: { owner: 'owner', repo: 'repo' }, runId: 999, + eventName: 'push', ref: 'refs/heads/master', ...options.context, + }, + core: { info() {}, warning(message) { calls.push(['warning', message]); } }, + commit: 'merged', tree: 'tree', + sleep: async ms => { time += ms; }, now: () => time, + waitMs: options.waitMs ?? 60_000, + }; + return { calls, args, resolve: () => validate(args) }; +} + +test('PRs always test without consulting prior runs', async () => { + const f = fixture({ context: { eventName: 'pull_request' } }); + assert.equal((await f.resolve()).decision, 'test'); + assert.deepEqual(f.calls, []); +}); + +test('master reuses the passing PR merge tree and explains the skip', async () => { + const f = fixture(); + const result = await f.resolve(); + assert.equal(result.decision, 'reuse'); + assert.match(result.reason, /PR #42 CI passed.*exactly matches tree/); + assert.match(result.reason, /https:\/\/github.com\/owner\/repo\/actions\/runs\/123/); + assert.match(result.reason, /attempt 2/); + const query = f.calls.find(([type]) => type === 'runs')[1]; + assert.equal(query.head_sha, 'pr-head'); + assert.equal(query.event, 'pull_request'); + assert.equal(query.workflow_id, 'Test.yml'); +}); + +test('fork PR metadata may omit the PR association', async () => { + assert.equal((await fixture({ run: { pull_requests: [] } }).resolve()).decision, 'reuse'); +}); + +test('a different PR association is not reused', async () => { + assert.equal((await fixture({ run: { pull_requests: [{ number: 7 }] } }).resolve()).decision, 'test'); +}); + +test('direct master pushes and intermediate commits in a push test', async () => { + assert.equal((await fixture({ pulls: [] }).resolve()).decision, 'test'); + const f = fixture(); + f.args.commit = 'intermediate'; + assert.equal((await f.resolve()).decision, 'test'); +}); + +for (const [name, artifacts] of [ + ['missing', []], + ['different tree', [{ name: 'lua-validation-other-tree-attempt-2' }]], + ['previous attempt', [{ name: 'lua-validation-tree-attempt-1' }]], + ['expired', [{ name: 'lua-validation-tree-attempt-2', expired: true }]], +]) { + test(`${name} proof falls back to tests`, async () => { + const result = await fixture({ artifacts }).resolve(); + assert.equal(result.decision, 'test'); + assert.match(result.reason, /no unexpired validation artifact matching Git tree tree/); + }); +} + +test('tags reuse master CI for the peeled, checked-out commit', async () => { + const f = fixture({ context: { ref: 'refs/tags/v1.0.0' } }); + const result = await f.resolve(); + assert.equal(result.decision, 'reuse'); + assert.match(result.reason, /master CI for tagged commit merged passed/); + assert.equal(f.calls.some(([type]) => type === 'pulls'), false); + const query = f.calls.find(([type]) => type === 'runs')[1]; + assert.equal(query.head_sha, 'merged'); + assert.equal(query.event, 'push'); +}); + +test('tags do not reuse push runs from another branch', async () => { + const f = fixture({ context: { ref: 'refs/tags/v1' }, run: { head_branch: 'feature' } }); + assert.equal((await f.resolve()).decision, 'test'); +}); + +test('missing CI runs fall back at both merge and tag stages', async () => { + for (const ref of ['refs/heads/master', 'refs/tags/v1']) { + assert.equal((await fixture({ context: { ref }, runs: [] }).resolve()).decision, 'test'); + } +}); + +test('in-progress validation is awaited instead of starting duplicate tests', async () => { + const f = fixture({ run: { status: 'in_progress', conclusion: null }, completed: { conclusion: 'success' } }); + assert.equal((await f.resolve()).decision, 'reuse'); + assert.equal(f.calls.filter(([type]) => type === 'poll').length, 1); +}); + +test('waiting has a bounded fallback', async () => { + const f = fixture({ run: { status: 'queued' }, waitMs: 0 }); + assert.equal((await f.resolve()).decision, 'test'); +}); + +for (const conclusion of ['failure', 'timed_out', 'action_required']) { + test(`${conclusion} for the same source blocks merge and release`, async () => { + for (const ref of ['refs/heads/master', 'refs/tags/v1']) { + assert.equal((await fixture({ context: { ref }, run: { conclusion } }).resolve()).decision, 'block'); + } + }); +} + +test('failed master validation blocks tags even if artifact lookup would fail', async () => { + const f = fixture({ context: { ref: 'refs/tags/v1' }, run: { conclusion: 'failure' }, artifactError: true }); + assert.equal((await f.resolve()).decision, 'block'); + assert.equal(f.calls.some(([type]) => type === 'artifacts'), false); +}); + +test('a failed PR run for different files requires fresh tests', async () => { + assert.equal((await fixture({ run: { conclusion: 'failure' }, artifacts: [] }).resolve()).decision, 'test'); +}); + +test('latest matching run takes precedence over an older pass', async () => { + const f = fixture({ runs: [ + { id: 1, head_branch: 'master', status: 'completed', conclusion: 'success' }, + { id: 2, head_branch: 'master', status: 'completed', conclusion: 'failure' }, + ], context: { ref: 'refs/tags/v1' } }); + assert.equal((await f.resolve()).decision, 'block'); +}); + +test('cancelled or skipped runs fall back to tests', async () => { + for (const conclusion of ['cancelled', 'skipped', 'neutral', 'stale']) { + assert.equal((await fixture({ run: { conclusion } }).resolve()).decision, 'test'); + } +}); + +test('API failures trigger full tests, never an unverified pass', async () => { + for (const options of [{ apiError: true }, { artifactError: true }]) { + const f = fixture(options); + assert.equal((await f.resolve()).decision, 'test'); + assert.equal(f.calls.some(([type]) => type === 'warning'), true); + } +}); diff --git a/scripts/release/smoke-test.sh b/scripts/release/smoke-test.sh new file mode 100644 index 00000000..cf774081 --- /dev/null +++ b/scripts/release/smoke-test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Exercise the extracted archive outside the repository, using its bundled Lua. +set -euo pipefail + +os="${1:?Usage: smoke-test.sh }" +version="${2:?Usage: smoke-test.sh }" +archive="$PWD/dist/prometheus-lua-$version-$os.tar.gz" +smoke_dir="$(mktemp -d)" +trap 'rm -rf "$smoke_dir"' EXIT +tar -xzf "$archive" -C "$smoke_dir" +cd "$smoke_dir" +package="$smoke_dir/prometheus-lua-$version-$os" +test -x "$package/runtime/lua" +test -x "$package/prometheus-lua" +unset PROMETHEUS_LUA_VERSION +test "$("$package/prometheus-lua" --version)" = "$version" +"$package/prometheus-lua" --help > /dev/null +printf 'local sum = 0; for i = 1, 10 do sum = sum + i end; print(sum)\n' > input.lua +"$package/prometheus-lua" --preset Minify --Lua51 --out output.lua input.lua +test -s output.lua +test "$("$package/runtime/lua" output.lua)" = '55' +echo "Release archive smoke test passed ($os, $version)."