From 29a849edadabba921da0141be23ec196122d0e8e Mon Sep 17 00:00:00 2001 From: Sebastiano Merlino Date: Thu, 15 Oct 2026 20:58:26 -0700 Subject: [PATCH 1/4] FEATURE-ci-v3-integration: enforce native CI and selected-archive release acceptance Run TLS-off/on native suites and independent bounded fuzz shards, repair installed platform prerequisites, and route legacy CI by source capability. Bind one selected archive to fresh platform bundles and the existing complete readiness validator before an isolated inline publisher can release an existing tag. Local verification: native 239/270 tests (two documented nonlocal skips each), eleven sanitizer fuzz targets, GCC QUIC replay, real Doxygen/Graphviz, 99 focused contracts, actionlint and workflow policy checks pass. A failed full Linux installed attempt remains unsuccessful at unprivileged TCP443; focused genuine isolated public TLS passes all four 82-check cells. Fresh full 18-gate and three nonlocal receipts plus actual GitHub PR verification remain CI/root-owned. --- .github/workflows/changelog-check.yml | 13 +- .github/workflows/codeql-analysis.yml | 38 +- .github/workflows/release.yml | 91 ++-- .github/workflows/v3-native.yml | 119 +++++ .github/workflows/v3-package-validation.yml | 117 +++-- .github/workflows/v3-release-acceptance.yml | 514 ++++++++++++++++++++ .github/workflows/verify-build.yml | 19 +- .gitignore | 6 + ChangeLog | 7 + Makefile.am | 7 +- docs/task-185-release-gates.md | 12 + docs/v3-ci.md | 131 +++++ doxyconfig.in | 4 +- scripts/lib/release_checks.py | 12 +- scripts/lib/release_ci.py | 107 ++++ scripts/lib/release_execution.py | 43 +- scripts/lib/release_platform.py | 252 ++++++++++ scripts/run-v3-protocol-fuzz.sh | 43 +- scripts/run-v3-release-ci.py | 173 +++++++ scripts/run-v3-release-gates.py | 32 +- scripts/test-v3-ci-workflows.py | 105 ++++ specs/architecture/v3/architecture.md | 17 + specs/tasks/FEATURE-ci-v3-integration.md | 38 ++ test/Makefile.am | 3 + test/fuzz/quic_parser_fuzz.cpp | 2 +- test/integ/protocol_fuzz_runner_test.py | 44 ++ test/integ/release_ci_test.py | 239 +++++++++ test/integ/release_platform_bundle_test.py | 233 +++++++++ 28 files changed, 2299 insertions(+), 122 deletions(-) create mode 100644 .github/workflows/v3-native.yml create mode 100644 .github/workflows/v3-release-acceptance.yml create mode 100644 docs/v3-ci.md create mode 100644 scripts/lib/release_ci.py create mode 100644 scripts/lib/release_platform.py create mode 100755 scripts/run-v3-release-ci.py create mode 100755 scripts/test-v3-ci-workflows.py create mode 100644 specs/tasks/FEATURE-ci-v3-integration.md create mode 100755 test/integ/release_ci_test.py create mode 100755 test/integ/release_platform_bundle_test.py diff --git a/.github/workflows/changelog-check.yml b/.github/workflows/changelog-check.yml index 632cedfb5..d2d66afb8 100644 --- a/.github/workflows/changelog-check.yml +++ b/.github/workflows/changelog-check.yml @@ -2,7 +2,10 @@ name: "ChangeLog Check" on: pull_request: - branches: [master] + branches: [master, v3] + +permissions: + contents: read jobs: changelog: @@ -10,14 +13,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: fetch-depth: 0 - name: Check for ChangeLog update + env: + BASE_REF: ${{ github.base_ref }} run: | # Get list of changed files - changed_files=$(git diff --name-only origin/master...HEAD) + git check-ref-format "refs/heads/$BASE_REF" + base=$(git rev-parse --verify "refs/remotes/origin/$BASE_REF^{commit}") + changed_files=$(git diff "$base...HEAD" --name-only --) if [ -z "$changed_files" ]; then echo "No files changed." diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 14bbdedba..2af674a19 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -18,10 +18,10 @@ name: "CodeQL" on: push: - branches: [master] + branches: [master, v3] pull_request: # The branches below must be a subset of the branches above - branches: [master] + branches: [master, v3] schedule: - cron: '0 4 * * 4' @@ -68,18 +68,32 @@ jobs: with: languages: ${{ matrix.language }} - # CodeQL's C/C++ database extractor traces the actual compile commands, so - # the library is built explicitly here rather than via the CodeQL Autobuild - # action (autobuild guesses the build and is intentionally not used). The - # build must be a clean from-scratch `make` so every TU is extracted; the - # in-source build (--enable-same-directory-build) keeps object files where - # the extractor expects them. + - name: Install build prerequisites + run: sudo apt-get update && sudo apt-get install -y autoconf automake libtool pkg-config perl + - name: Select source configuration + id: source + run: | + if grep -q 'libhttpserver_v3core_la_SOURCES' src/Makefile.am; then + echo 'native=true' >> "$GITHUB_OUTPUT" + fi + - name: Provision private native TLS provider + if: steps.source.outputs.native == 'true' + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/codeql-prerequisites" - name: Build the library (CodeQL manual mode) run: | - ./bootstrap ; - ./configure --enable-same-directory-build ; - make ; - timeout-minutes: 30 + ./bootstrap + mkdir "$RUNNER_TEMP/codeql-build" + cd "$RUNNER_TEMP/codeql-build" + if grep -q 'libhttpserver_v3core_la_SOURCES' "$GITHUB_WORKSPACE/src/Makefile.am"; then + export LD_LIBRARY_PATH="$RUNNER_TEMP/codeql-prerequisites/provider/lib" + export V3_TLS_CFLAGS="-I$RUNNER_TEMP/codeql-prerequisites/provider/include" + export V3_TLS_LIBS="-L$RUNNER_TEMP/codeql-prerequisites/provider/lib -lssl -lcrypto" + "$GITHUB_WORKSPACE/./configure" --enable-v3-tls + else + "$GITHUB_WORKSPACE/./configure" + fi + make -j2 + timeout-minutes: 40 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@411c4c9a36b3fca4d674f06b6396b2c6d23522c6 # v3.36.3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c5bccde7..a7afce090 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,8 @@ on: required: true permissions: - contents: write + contents: read + actions: read jobs: validate: @@ -20,43 +21,43 @@ jobs: outputs: version: ${{ steps.version.outputs.version }} is_prerelease: ${{ steps.version.outputs.is_prerelease }} + commit: ${{ steps.version.outputs.commit }} + native: ${{ steps.version.outputs.native }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + fetch-depth: 0 - - name: Determine version + - name: Resolve existing tag and source version id: version + env: + REQUEST_TAG: ${{ inputs.tag || github.ref_name }} run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - VERSION="${{ github.event.inputs.tag }}" - else - VERSION="${GITHUB_REF#refs/tags/}" - fi - VERSION="${VERSION#v}" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - if echo "$VERSION" | grep -qE '-(rc|alpha|beta)'; then - echo "is_prerelease=true" >> "$GITHUB_OUTPUT" - else - echo "is_prerelease=false" >> "$GITHUB_OUTPUT" - fi - - echo "Version: $VERSION" - - - name: Validate version consistency - run: | - chmod +x scripts/validate-version.sh - scripts/validate-version.sh "${{ steps.version.outputs.version }}" + python3 - <<'PYTHON' + import os,re,subprocess + from pathlib import Path + version=os.environ['REQUEST_TAG'] + if not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:rc|alpha|beta)[0-9]+)?',version): raise SystemExit('invalid version tag') + commit=subprocess.check_output(['git','rev-parse','--verify','refs/tags/'+version+'^{commit}'],text=True).strip() + subprocess.run(['git','checkout','--detach',commit],check=True) + subprocess.run(['bash','scripts/validate-version.sh',version],check=True) + native='true' if 'libhttpserver_v3core_la_SOURCES' in Path('src/Makefile.am').read_text() else 'false' + with open(os.environ['GITHUB_OUTPUT'],'a') as out: + out.write('version='+version+'\ncommit='+commit+'\nnative='+native+'\nis_prerelease='+str('-' in version).lower()+'\n') + PYTHON - name: Extract release notes + env: + VERSION: ${{ steps.version.outputs.version }} run: | chmod +x scripts/extract-release-notes.sh - scripts/extract-release-notes.sh "${{ steps.version.outputs.version }}" > release-notes.md + scripts/extract-release-notes.sh "$VERSION" > release-notes.md echo "--- Release notes ---" cat release-notes.md - name: Upload release notes - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: release-notes path: release-notes.md @@ -65,9 +66,12 @@ jobs: name: Build distribution tarball runs-on: ubuntu-latest needs: validate + if: needs.validate.outputs.native == 'false' steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.validate.outputs.commit }} - name: Install dependencies run: | @@ -89,7 +93,7 @@ jobs: make dist - name: Upload tarball - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: dist-tarball path: build/libhttpserver-*.tar.gz @@ -105,7 +109,7 @@ jobs: sudo apt-get install -y libcurl4-openssl-dev - name: Download tarball - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: dist-tarball @@ -135,7 +139,7 @@ jobs: - name: Fetch curl from cache id: cache-curl - uses: actions/cache@v4 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 with: path: curl-7.75.0 key: macos-latest-CURL-pre-built-v3 @@ -155,7 +159,7 @@ jobs: sudo make install - name: Download tarball - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: dist-tarball @@ -184,7 +188,7 @@ jobs: shell: msys2 {0} steps: - name: Setup MSYS2 - uses: msys2/setup-msys2@v2 + uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c with: msystem: MINGW64 update: true @@ -197,7 +201,7 @@ jobs: pacman --noconfirm -S --needed mingw-w64-x86_64-{toolchain,libtool,make,pkg-config,libsystre,doxygen,graphviz,curl,python} - name: Download tarball - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: dist-tarball @@ -221,14 +225,17 @@ jobs: name: Create GitHub Release runs-on: ubuntu-latest needs: [validate, verify-dist-linux, verify-dist-macos, verify-dist-windows] + if: needs.validate.outputs.native == 'false' + permissions: + contents: write steps: - name: Download tarball - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: dist-tarball - name: Download release notes - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: release-notes @@ -236,9 +243,9 @@ jobs: env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} + VERSION: ${{ needs.validate.outputs.version }} + IS_PRERELEASE: ${{ needs.validate.outputs.is_prerelease }} run: | - VERSION="${{ needs.validate.outputs.version }}" - IS_PRERELEASE="${{ needs.validate.outputs.is_prerelease }}" PRERELEASE_FLAG="" if [ "$IS_PRERELEASE" = "true" ]; then @@ -250,3 +257,15 @@ jobs: --notes-file release-notes.md \ $PRERELEASE_FLAG \ libhttpserver-*.tar.gz + + v3-acceptance: + needs: validate + if: needs.validate.outputs.native == 'true' + permissions: + contents: write + actions: read + uses: ./.github/workflows/v3-release-acceptance.yml + with: + source_commit: ${{ needs.validate.outputs.commit }} + source_version: ${{ needs.validate.outputs.version }} + publication: true diff --git a/.github/workflows/v3-native.yml b/.github/workflows/v3-native.yml new file mode 100644 index 000000000..5144f69d5 --- /dev/null +++ b/.github/workflows/v3-native.yml @@ -0,0 +1,119 @@ +name: v3 native and protocol fuzz +on: + push: + branches: [v3, master] + pull_request: + branches: [v3, master] + workflow_dispatch: + # Activates only when this workflow reaches the repository default branch. + schedule: + - cron: '17 3 * * *' +permissions: + contents: read +jobs: + native: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + tls: ['no', 'yes'] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - name: Prerequisites + run: sudo apt-get update && sudo apt-get install -y autoconf automake libtool pkg-config clang perl doxygen graphviz python3-yaml + - name: Private provider + if: matrix.tls == 'yes' + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + - name: C++20 native suites, all deterministic replays, headers, examples and hygiene + env: + TLS_MODE: ${{ matrix.tls }} + run: | + ./bootstrap + mkdir "$RUNNER_TEMP/native" + cd "$RUNNER_TEMP/native" + if [ "$TLS_MODE" = yes ]; then + export V3_TLS_CFLAGS="-I$RUNNER_TEMP/prerequisites/provider/include" + export V3_TLS_LIBS="-L$RUNNER_TEMP/prerequisites/provider/lib -lssl -lcrypto" + export LD_LIBRARY_PATH="$RUNNER_TEMP/prerequisites/provider/lib" + fi + CXX=clang++ CC=clang "$GITHUB_WORKSPACE/configure" --enable-v3-tls="$TLS_MODE" + make -j2 + make check -j2 + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: native-${{ matrix.tls }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/native/**/*.log + ${{ runner.temp }}/native/**/*.trs + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + retention-days: 14 + if-no-files-found: error + fuzz: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: [http1_head, http1_body, websocket_codec, hpack, http2_engine, quic_parser, quic_state, quic_recovery, http3_engine, qpack, quic_tls_callbacks] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - name: Prerequisites + run: sudo apt-get update && sudo apt-get install -y clang autoconf automake libtool pkg-config perl + - name: Private provider for TLS callback instrumentation + if: matrix.target == 'quic_tls_callbacks' + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + - name: Configure genuine native sources + env: + TARGET: ${{ matrix.target }} + run: | + ./bootstrap + mkdir "$RUNNER_TEMP/native" + cd "$RUNNER_TEMP/native" + mode=no + if [ "$TARGET" = quic_tls_callbacks ]; then + mode=yes + export V3_TLS_CFLAGS="-I$RUNNER_TEMP/prerequisites/provider/include" + export V3_TLS_LIBS="-L$RUNNER_TEMP/prerequisites/provider/lib -lssl -lcrypto" + fi + CXX=clang++ CC=clang "$GITHUB_WORKSPACE/configure" --enable-v3-tls="$mode" --disable-examples + - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 + with: + path: ${{ runner.temp }}/corpus + key: corpus-v1-${{ runner.os }}-${{ matrix.target }}-clang-asan-ubsan-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: corpus-v1-${{ runner.os }}-${{ matrix.target }}-clang-asan-ubsan- + - name: Bounded smoke or campaign + env: + TARGET: ${{ matrix.target }} + EVENT: ${{ github.event_name }} + LD_LIBRARY_PATH: ${{ runner.temp }}/prerequisites/provider/lib + run: | + runs=2000; seconds=30; seed=128 + if [ "$EVENT" = schedule ] || [ "$EVENT" = workflow_dispatch ]; then + runs=1000000000; seconds=600 + seed=$(python3 -c 'import secrets; print(secrets.randbelow(4294967295)+1)') + fi + bash scripts/run-v3-protocol-fuzz.sh --compiler clang++ --build-dir "$RUNNER_TEMP/fuzz" \ + --native-config-dir "$RUNNER_TEMP/native" --targets "$TARGET" \ + --runs "$runs" --seconds "$seconds" --seed "$seed" --corpus-dir "$RUNNER_TEMP/corpus" + - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 + if: success() && github.event_name != 'pull_request' && (github.ref_name == 'v3' || github.ref_name == github.event.repository.default_branch) + with: + path: ${{ runner.temp }}/corpus + key: corpus-v1-${{ runner.os }}-${{ matrix.target }}-clang-asan-ubsan-${{ github.run_id }}-${{ github.run_attempt }} + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: fuzz-${{ matrix.target }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/fuzz + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + retention-days: 30 + if-no-files-found: error diff --git a/.github/workflows/v3-package-validation.yml b/.github/workflows/v3-package-validation.yml index bf425429b..53e702eef 100644 --- a/.github/workflows/v3-package-validation.yml +++ b/.github/workflows/v3-package-validation.yml @@ -1,9 +1,9 @@ name: v3 installed package validation on: push: - branches: [v3] + branches: [v3, master] pull_request: - branches: [v3] + branches: [v3, master] workflow_dispatch: permissions: contents: read @@ -14,22 +14,34 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - name: Prerequisites - run: sudo apt-get update && sudo apt-get install -y clang autoconf automake libtool pkg-config python3 perl libcurl4-openssl-dev binutils + run: sudo apt-get update && sudo apt-get install -y clang autoconf automake libtool pkg-config python3 perl libcurl4-openssl-dev binutils python3-venv + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '24.15.0' + - name: Selected client environment and executable Git provenance + run: | + mkdir -p "$RUNNER_TEMP/package-evidence" + exec > >(tee "$RUNNER_TEMP/package-evidence/clients.log") 2>&1 + python3 -c 'import subprocess; subprocess.run(["git", "--version"], check=True)' + python3 -m venv "$RUNNER_TEMP/package-evidence/clients" + "$RUNNER_TEMP/package-evidence/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt - name: Build selected prerequisites - run: python3 scripts/prepare-v3-package-ci.py --output package-evidence/prerequisites + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites" - name: Bootstrap run: ./bootstrap - name: Installed matrix - run: >- - python3 scripts/check-v3-installed-consumer.py --output package-evidence/matrix - --provider-prefix "$PWD/package-evidence/prerequisites/provider" - --pkg-config-dir /usr/lib/x86_64-linux-gnu/pkgconfig --pkg-config-dir /usr/share/pkgconfig - --compiler clang++ --cc clang --jobs 2 + run: | + prefix=() + if ! python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",443)); s.close()'; then + prefix=(sudo env "PATH=$PATH" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory "GIT_CONFIG_VALUE_0=$PWD") + fi + "${prefix[@]}" python3 scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" --provider-prefix "$RUNNER_TEMP/package-evidence/prerequisites/provider" --pkg-config-dir /usr/lib/x86_64-linux-gnu/pkgconfig --pkg-config-dir /usr/share/pkgconfig --client-python "$RUNNER_TEMP/package-evidence/clients/bin/python" --node node --compiler clang++ --cc clang --jobs 2 - if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: v3-package-linux-${{ github.sha }} - path: package-evidence + path: ${{ runner.temp }}/package-evidence + retention-days: 14 if-no-files-found: error macos: runs-on: macos-14 @@ -38,20 +50,33 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - name: Prerequisites run: brew install autoconf automake libtool pkg-config python curl + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '24.15.0' + - name: Selected client environment and executable Git provenance + run: | + mkdir -p "$RUNNER_TEMP/package-evidence" + exec > >(tee "$RUNNER_TEMP/package-evidence/clients.log") 2>&1 + python3 -c 'import subprocess; subprocess.run(["git", "--version"], check=True)' + python3 -m venv "$RUNNER_TEMP/package-evidence/clients" + "$RUNNER_TEMP/package-evidence/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt - name: Build selected prerequisites - run: python3 scripts/prepare-v3-package-ci.py --output package-evidence/prerequisites + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites" - name: Bootstrap run: ./bootstrap - name: Installed matrix - run: >- - python3 scripts/check-v3-installed-consumer.py --output package-evidence/matrix - --provider-prefix "$PWD/package-evidence/prerequisites/provider" --pkg-config-dir "$(brew --prefix)/share/pkgconfig" - --compiler clang++ --cc clang --jobs 2 + run: | + prefix=() + if ! python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",443)); s.close()'; then + prefix=(sudo env "PATH=$PATH" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory "GIT_CONFIG_VALUE_0=$PWD") + fi + "${prefix[@]}" python3 scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" --provider-prefix "$RUNNER_TEMP/package-evidence/prerequisites/provider" --pkg-config-dir "$(brew --prefix)/share/pkgconfig" --client-python "$RUNNER_TEMP/package-evidence/clients/bin/python" --node node --compiler clang++ --cc clang --jobs 2 - if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: v3-package-macos-${{ github.sha }} - path: package-evidence + path: ${{ runner.temp }}/package-evidence + retention-days: 14 if-no-files-found: error mingw64: runs-on: windows-2022 @@ -60,33 +85,48 @@ jobs: run: shell: msys2 {0} steps: + - name: Keep selected Git source bytes unchanged + shell: pwsh + run: git config --global core.autocrlf false - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c with: msystem: MINGW64 update: true install: >- - autotools base-devel perl + autotools base-devel perl git mingw-w64-x86_64-clang mingw-w64-x86_64-gcc mingw-w64-x86_64-binutils mingw-w64-x86_64-python mingw-w64-x86_64-pkgconf mingw-w64-x86_64-libtool mingw-w64-x86_64-curl mingw-w64-x86_64-libsystre + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '24.15.0' + - name: Selected client environment and executable Git provenance + run: | + mkdir -p "$RUNNER_TEMP/package-evidence" + exec > >(tee "$RUNNER_TEMP/package-evidence/clients.log") 2>&1 + export PATH="/c/Program Files/Git/cmd:$PATH" + /mingw64/bin/python -c 'import subprocess; subprocess.run(["git", "--version"], check=True)' + /mingw64/bin/python -m venv "$RUNNER_TEMP/package-evidence/clients" + "$(cygpath -m "$RUNNER_TEMP/package-evidence/clients/Scripts/python.exe")" -m pip install -r test/integ/http2-client-requirements.txt - name: Build selected prerequisites - run: /mingw64/bin/python scripts/prepare-v3-package-ci.py --output package-evidence/prerequisites + run: /mingw64/bin/python scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites" - name: Bootstrap run: ./bootstrap - name: Installed matrix (native Python; IOCP and WSAPoll) run: >- - /mingw64/bin/python scripts/check-v3-installed-consumer.py --output package-evidence/matrix - --provider-prefix "$(cygpath -m "$PWD/package-evidence/prerequisites/provider")" --toolchain-prefix "$(cygpath -m /mingw64)" - --compiler clang++ --cc clang --jobs 2 + /mingw64/bin/python scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" + --provider-prefix "$(cygpath -m "$RUNNER_TEMP/package-evidence/prerequisites/provider")" --toolchain-prefix "$(cygpath -m /mingw64)" + --client-python "$(cygpath -m "$RUNNER_TEMP/package-evidence/clients/Scripts/python.exe")" --node node --compiler clang++ --cc clang --jobs 2 - if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: v3-package-mingw64-${{ github.sha }} - path: package-evidence + path: ${{ runner.temp }}/package-evidence + retention-days: 14 if-no-files-found: error freebsd: runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 180 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - name: freebsd_instance @@ -95,16 +135,37 @@ jobs: release: '14.3' mem: 8192 cpu: 2 - prepare: pkg install -y bash autoconf automake libtool pkgconf python311 gmake curl binutils perl5 llvm + prepare: pkg install -y bash autoconf automake libtool pkgconf python311 gmake curl binutils perl5 llvm git py311-pip py311-virtualenv node24 run: | - python3.11 scripts/prepare-v3-package-ci.py --output package-evidence/prerequisites --make gmake + export RUNNER_TEMP=/tmp/v3-package-${{ github.run_id }}-${{ github.run_attempt }} + mkdir -p "$RUNNER_TEMP/package-evidence" + trap 'cp -R "$RUNNER_TEMP/package-evidence" ./package-evidence' EXIT + if [ "$(node --version)" != v24.15.0 ]; then + mkdir -p "$RUNNER_TEMP/node-source" + curl --fail --location https://nodejs.org/dist/v24.15.0/node-v24.15.0.tar.gz -o "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" + test "$(sha256 -q "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz")" = 729de494dd2872e5a3a6c32a1cd156a5413d4aca2772b2d873ee86bb5531bcd9 + tar -xzf "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" -C "$RUNNER_TEMP/node-source" + ( + cd "$RUNNER_TEMP/node-source/node-v24.15.0" + ./configure --prefix="$RUNNER_TEMP/node" + gmake -j2 > "$RUNNER_TEMP/node-source/build.log" 2>&1 + gmake install >> "$RUNNER_TEMP/node-source/build.log" 2>&1 + ) + export PATH="$RUNNER_TEMP/node/bin:$PATH" + fi + test "$(node --version)" = v24.15.0 + python3.11 -c 'import subprocess; subprocess.run(["git", "--version"], check=True)' + python3.11 -m venv "$RUNNER_TEMP/package-evidence/clients" + "$RUNNER_TEMP/package-evidence/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt + python3.11 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites" --make gmake ./bootstrap - python3.11 scripts/check-v3-installed-consumer.py --output package-evidence/matrix \ - --provider-prefix "$PWD/package-evidence/prerequisites/provider" \ - --pkg-config-dir /usr/local/share/pkgconfig --compiler clang++ --cc clang --make gmake --jobs 2 + python3.11 scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" \ + --provider-prefix "$RUNNER_TEMP/package-evidence/prerequisites/provider" \ + --client-python "$RUNNER_TEMP/package-evidence/clients/bin/python" --node node --pkg-config-dir /usr/local/share/pkgconfig --compiler clang++ --cc clang --make gmake --jobs 2 - if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: v3-package-freebsd-${{ github.sha }} path: package-evidence + retention-days: 14 if-no-files-found: error diff --git a/.github/workflows/v3-release-acceptance.yml b/.github/workflows/v3-release-acceptance.yml new file mode 100644 index 000000000..a5a928801 --- /dev/null +++ b/.github/workflows/v3-release-acceptance.yml @@ -0,0 +1,514 @@ +name: v3 selected archive acceptance +'on': + workflow_call: + inputs: + source_commit: + type: string + required: true + source_version: + type: string + required: true + publication: + type: boolean + default: false + pull_request: + branches: + - v3 + - master + workflow_dispatch: + inputs: + source_commit: + description: Resolved 40-hex commit; empty selects this workflow ref + type: string + source_version: + description: Exact source version; empty reads configure.ac + type: string +permissions: + contents: read + actions: read +jobs: + prepare: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + outputs: + artifact_id: ${{ steps.archive.outputs.artifact-id }} + artifact_digest: ${{ steps.archive.outputs.artifact-digest }} + sha256: ${{ steps.prepare.outputs.sha256 }} + commit: ${{ steps.request.outputs.commit }} + version: ${{ steps.request.outputs.version }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ inputs.source_commit || github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Resolve source and version as structured data + id: request + env: + REQUEST_COMMIT: ${{ inputs.source_commit }} + REQUEST_VERSION: ${{ inputs.source_version }} + run: |- + python3 - <<'PYTHON' + import os,re,subprocess + from pathlib import Path + commit=os.environ['REQUEST_COMMIT'] or subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip() + version=os.environ['REQUEST_VERSION'] + if not version: + text=Path('configure.ac').read_text() + version='.'.join(re.search(r'm4_define\(\[libhttpserver_'+name+r'\],\[([0-9]+)\]\)',text)[1] for name in ('MAJOR_VERSION','MINOR_VERSION','REVISION')) + if not re.fullmatch(r'[0-9a-f]{40}',commit) or not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:rc|alpha|beta)[0-9]+)?',version): raise SystemExit('invalid resolved request') + with open(os.environ['GITHUB_OUTPUT'],'a') as out: out.write('commit='+commit+'\nversion='+version+'\n') + PYTHON + - run: sudo apt-get update && sudo apt-get install -y autoconf automake libtool pkg-config + - id: prepare + env: + SOURCE_COMMIT: ${{ steps.request.outputs.commit }} + SOURCE_VERSION: ${{ steps.request.outputs.version }} + run: python3 scripts/run-v3-release-ci.py prepare --commit "$SOURCE_COMMIT" --version "$SOURCE_VERSION" --output "$RUNNER_TEMP/selected" + - id: archive + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: selected-archive + path: |- + ${{ runner.temp }}/selected/selected.json + ${{ runner.temp }}/selected/libhttpserver-*.tar.gz + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: selected-preparation-logs + path: ${{ runner.temp }}/selected/*.log + retention-days: 30 + if-no-files-found: warn + darwin: + runs-on: macos-14 + needs: prepare + timeout-minutes: 180 + outputs: + artifact_id: ${{ steps.proof.outputs.artifact-id }} + artifact_digest: ${{ steps.proof.outputs.artifact-digest }} + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.prepare.outputs.commit }} + persist-credentials: false + - run: brew install autoconf automake libtool pkg-config python curl + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: 24.15.0 + - name: Verify native Git and download immutable selected archive + run: |- + python3 -c 'import subprocess; subprocess.run(["git","--version"],check=True)' + python3 scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected" + - name: Selected provider and clients + run: |- + exec > >(tee "$RUNNER_TEMP/prerequisites-ci.log") 2>&1 + python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + python3 -m venv "$RUNNER_TEMP/clients" + "$RUNNER_TEMP/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt + - name: Complete native and installed TLS off/on proof + run: | + prefix=() + if ! python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",443)); s.close()'; then + prefix=(sudo env "PATH=$PATH" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory "GIT_CONFIG_VALUE_0=$PWD") + fi + "${prefix[@]}" python3 scripts/run-v3-release-ci.py platform --selection "$RUNNER_TEMP/selected/selected.json" --archive "$RUNNER_TEMP/selected/"libhttpserver-*.tar.gz --provider-prefix "$RUNNER_TEMP/prerequisites/provider" --client-python "$RUNNER_TEMP/clients/bin/python" --node node --output "$RUNNER_TEMP/platform" --pkg-config-dir "$(brew --prefix)/share/pkgconfig" + - id: proof + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-Darwin + path: ${{ runner.temp }}/platform/Darwin.tar.gz + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-Darwin-failure-evidence + path: |- + ${{ runner.temp }}/platform + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + ${{ runner.temp }}/prerequisites-ci.log + retention-days: 30 + if-no-files-found: warn + compression-level: 0 + windows: + runs-on: windows-2022 + needs: prepare + timeout-minutes: 180 + outputs: + artifact_id: ${{ steps.proof.outputs.artifact-id }} + artifact_digest: ${{ steps.proof.outputs.artifact-digest }} + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + steps: + - name: Keep selected Git source bytes unchanged + shell: pwsh + run: git config --global core.autocrlf false + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.prepare.outputs.commit }} + persist-credentials: false + - uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c + with: + msystem: MINGW64 + update: true + install: autotools base-devel perl git mingw-w64-x86_64-clang mingw-w64-x86_64-gcc mingw-w64-x86_64-binutils mingw-w64-x86_64-python + mingw-w64-x86_64-pkgconf mingw-w64-x86_64-libtool mingw-w64-x86_64-curl mingw-w64-x86_64-libsystre + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: 24.15.0 + - name: Verify native Git and download immutable selected archive + run: |- + export PATH="/c/Program Files/Git/cmd:$PATH" + /mingw64/bin/python -c 'import subprocess; subprocess.run(["git","--version"],check=True)' + /mingw64/bin/python scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected" + - name: Selected provider and clients + run: |- + exec > >(tee "$RUNNER_TEMP/prerequisites-ci.log") 2>&1 + /mingw64/bin/python scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + /mingw64/bin/python -m venv "$RUNNER_TEMP/clients" + "$(cygpath -m "$RUNNER_TEMP/clients/Scripts/python.exe")" -m pip install -r test/integ/http2-client-requirements.txt + - name: Complete native and installed TLS off/on proof + run: /mingw64/bin/python scripts/run-v3-release-ci.py platform --selection "$RUNNER_TEMP/selected/selected.json" --archive "$RUNNER_TEMP/selected/"libhttpserver-*.tar.gz + --provider-prefix "$RUNNER_TEMP/prerequisites/provider" --client-python "$(cygpath -m "$RUNNER_TEMP/clients/Scripts/python.exe")" + --node node --output "$RUNNER_TEMP/platform" --toolchain-prefix "$(cygpath -m /mingw64)" + - id: proof + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-Windows + path: ${{ runner.temp }}/platform/Windows.tar.gz + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-Windows-failure-evidence + path: |- + ${{ runner.temp }}/platform + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + ${{ runner.temp }}/prerequisites-ci.log + retention-days: 30 + if-no-files-found: warn + compression-level: 0 + defaults: + run: + shell: msys2 {0} + freebsd: + runs-on: ubuntu-24.04 + needs: prepare + timeout-minutes: 180 + outputs: + artifact_id: ${{ steps.proof.outputs.artifact-id }} + artifact_digest: ${{ steps.proof.outputs.artifact-digest }} + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.prepare.outputs.commit }} + persist-credentials: false + - name: Download immutable selected archive + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected" + - run: mkdir -p build cp -R "$RUNNER_TEMP/selected" build/ci-selected-input + - uses: vmactions/freebsd-vm@c9f815bc7aa0d34c9fdd0619b034a32d6ca7b57e + with: + release: '14.3' + mem: 8192 + cpu: 2 + prepare: pkg install -y bash autoconf automake libtool pkgconf python311 gmake curl binutils perl5 llvm git py311-pip py311-virtualenv + node24 + run: |- + set -eu + export RUNNER_TEMP=/tmp/v3-release-${{ github.run_id }}-${{ github.run_attempt }} + mkdir -p "$RUNNER_TEMP" + trap 'mkdir -p platform-evidence; cp -R "$RUNNER_TEMP/platform" platform-evidence/ 2>/dev/null || true; cp -R "$RUNNER_TEMP/prerequisites" platform-evidence/ 2>/dev/null || true' EXIT + if [ "$(node --version)" != v24.15.0 ]; then + mkdir -p "$RUNNER_TEMP/node-source" + curl --fail --location https://nodejs.org/dist/v24.15.0/node-v24.15.0.tar.gz -o "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" + test "$(sha256 -q "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz")" = 729de494dd2872e5a3a6c32a1cd156a5413d4aca2772b2d873ee86bb5531bcd9 + tar -xzf "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" -C "$RUNNER_TEMP/node-source" + ( + cd "$RUNNER_TEMP/node-source/node-v24.15.0" + ./configure --prefix="$RUNNER_TEMP/node" + gmake -j2 > "$RUNNER_TEMP/node-source/build.log" 2>&1 + gmake install >> "$RUNNER_TEMP/node-source/build.log" 2>&1 + ) + export PATH="$RUNNER_TEMP/node/bin:$PATH" + fi + test "$(node --version)" = v24.15.0 + python3.11 -c 'import subprocess; subprocess.run(["git","--version"],check=True)' + python3.11 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" --make gmake + python3.11 -m venv "$RUNNER_TEMP/clients" + "$RUNNER_TEMP/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt + python3.11 scripts/run-v3-release-ci.py platform --selection build/ci-selected-input/selected.json --archive build/ci-selected-input/libhttpserver-*.tar.gz --provider-prefix "$RUNNER_TEMP/prerequisites/provider" --client-python "$RUNNER_TEMP/clients/bin/python" --node node --make gmake --pkg-config-dir /usr/local/share/pkgconfig --output "$RUNNER_TEMP/platform" + - id: proof + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-FreeBSD + path: platform-evidence/platform/FreeBSD.tar.gz + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-FreeBSD-failure-evidence + path: platform-evidence + retention-days: 30 + if-no-files-found: warn + compression-level: 0 + full: + runs-on: ubuntu-24.04 + needs: + - prepare + - darwin + - windows + - freebsd + timeout-minutes: 360 + outputs: + authorization_id: ${{ steps.authorization.outputs.artifact-id }} + authorization_digest: ${{ steps.authorization.outputs.artifact-digest }} + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + GOFLAGS: -mod=readonly + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.prepare.outputs.commit }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: 24.15.0 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 + with: + go-version: 1.25.1 + cache: false + - name: Build and capture prerequisites + run: |- + exec > >(tee "$RUNNER_TEMP/prerequisites-ci.log") 2>&1 + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y clang autoconf automake libtool pkg-config python3-venv perl binutils tshark iproute2 libcurl4-openssl-dev + python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + python3 -m venv "$RUNNER_TEMP/clients" + "$RUNNER_TEMP/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt -r test/integ/http3-client-requirements.txt websockets==15.0.1 + python3 -m venv "$RUNNER_TEMP/runner-python" + git clone https://github.com/quic-interop/quic-interop-runner.git "$RUNNER_TEMP/quic-runner" + git -C "$RUNNER_TEMP/quic-runner" checkout --detach 740c05a10b61d65e8abd3ad38d60898004d335d9 + "$RUNNER_TEMP/runner-python/bin/python" -m pip install -r "$RUNNER_TEMP/quic-runner/requirements.txt" + git clone https://github.com/summerwind/h2spec.git "$RUNNER_TEMP/h2spec-src" + git -C "$RUNNER_TEMP/h2spec-src" checkout --detach 70ac2294010887f48b18e2d64f5cccd48421fad1 + make -C "$RUNNER_TEMP/h2spec-src" build + go -C test/integ/http3-client-go mod download + docker pull martenseemann/quic-network-simulator@sha256:c23d82a55caffe681b1bdae65d4d30d23e1283141a414a7f02ee56cf15f9c6b9 + docker pull martenseemann/quic-go-interop@sha256:6743eea41cb8d525686eb8f2437e3ac45835740ec87c62728f645cd89e85ab88 + docker pull martenseemann/quic-network-simulator-endpoint@sha256:3b2b9e6fa317da238c8140a7b6d2bf8d4d45a8464b873eecac3ff2a32520b71a + - name: Download same selected archive + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected" + - name: Import immutable darwin bundle + env: + ARTIFACT_ID: ${{ needs.darwin.outputs.artifact_id }} + ARTIFACT_DIGEST: ${{ needs.darwin.outputs.artifact_digest }} + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$ARTIFACT_ID" --digest "$ARTIFACT_DIGEST" --output "$RUNNER_TEMP/bundle-darwin" + - name: Import immutable windows bundle + env: + ARTIFACT_ID: ${{ needs.windows.outputs.artifact_id }} + ARTIFACT_DIGEST: ${{ needs.windows.outputs.artifact_digest }} + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$ARTIFACT_ID" --digest "$ARTIFACT_DIGEST" --output "$RUNNER_TEMP/bundle-windows" + - name: Import immutable freebsd bundle + env: + ARTIFACT_ID: ${{ needs.freebsd.outputs.artifact_id }} + ARTIFACT_DIGEST: ${{ needs.freebsd.outputs.artifact_digest }} + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$ARTIFACT_ID" --digest "$ARTIFACT_DIGEST" --output "$RUNNER_TEMP/bundle-freebsd" + - name: Execute all 18 gates against the selected archive + env: + ARCHIVE_SHA256: ${{ needs.prepare.outputs.sha256 }} + run: python3 scripts/run-v3-release-gates.py --output "$RUNNER_TEMP/full" --selected-archive "$RUNNER_TEMP/selected/"libhttpserver-*.tar.gz + --selected-sha256 "$ARCHIVE_SHA256" --provider-prefix "$RUNNER_TEMP/prerequisites/provider" --compiler clang++ --cc clang + --make make --jobs 2 --python "$RUNNER_TEMP/clients/bin/python" --node "$(command -v node)" --go "$(command -v go)" --h2spec + "$RUNNER_TEMP/h2spec-src/h2spec" --tshark "$(command -v tshark)" --runner-python "$RUNNER_TEMP/runner-python/bin/python" --runner-dir + "$RUNNER_TEMP/quic-runner" --server-image "libhttpserver-ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" --docker-direct-routing-compat --pkg-config-dir + /usr/lib/x86_64-linux-gnu/pkgconfig --pkg-config-dir /usr/share/pkgconfig --isolated-public-network + - name: Actual tag readiness and content-bound authorization + env: + PUBLICATION: ${{ inputs.publication || false }} + PRODUCER_DARWIN_ID: ${{ needs.darwin.outputs.artifact_id }} + PRODUCER_DARWIN_DIGEST: ${{ needs.darwin.outputs.artifact_digest }} + PRODUCER_WINDOWS_ID: ${{ needs.windows.outputs.artifact_id }} + PRODUCER_WINDOWS_DIGEST: ${{ needs.windows.outputs.artifact_digest }} + PRODUCER_FREEBSD_ID: ${{ needs.freebsd.outputs.artifact_id }} + PRODUCER_FREEBSD_DIGEST: ${{ needs.freebsd.outputs.artifact_digest }} + run: |- + args=() + if [ "$PUBLICATION" = true ]; then args+=(--publish --artifact-id "$SELECTED_ID"); fi + python3 scripts/run-v3-release-ci.py accept --manifest "$RUNNER_TEMP/full/manifest.json" --selection "$RUNNER_TEMP/selected/selected.json" --platform-bundle "$RUNNER_TEMP/bundle-darwin/Darwin.tar.gz" --platform-bundle "$RUNNER_TEMP/bundle-windows/Windows.tar.gz" --platform-bundle "$RUNNER_TEMP/bundle-freebsd/FreeBSD.tar.gz" --output "$RUNNER_TEMP/acceptance" "${args[@]}" + - id: authorization + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: release-readiness + path: |- + ${{ runner.temp }}/acceptance/authorization.json + ${{ runner.temp }}/acceptance/accepted-manifest.json + ${{ runner.temp }}/acceptance/validation.json + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: full-gate-failure-evidence + path: |- + ${{ runner.temp }}/full + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + ${{ runner.temp }}/prerequisites-ci.log + retention-days: 30 + if-no-files-found: warn + compression-level: 0 + publish: + runs-on: ubuntu-24.04 + needs: + - prepare + - darwin + - windows + - freebsd + - full + if: inputs.publication == true && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + permissions: + contents: write + actions: read + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + GOFLAGS: -mod=readonly + SOURCE_COMMIT: ${{ needs.prepare.outputs.commit }} + SOURCE_VERSION: ${{ needs.prepare.outputs.version }} + ARCHIVE_SHA256: ${{ needs.prepare.outputs.sha256 }} + AUTHORIZATION_ID: ${{ needs.full.outputs.authorization_id }} + AUTHORIZATION_DIGEST: ${{ needs.full.outputs.authorization_digest }} + REQUIRED_RESULTS: '{"prepare":"${{ needs.prepare.result }}","darwin":"${{ needs.darwin.result }}","windows":"${{ needs.windows.result + }}","freebsd":"${{ needs.freebsd.result }}","full":"${{ needs.full.result }}"}' + PRODUCER_DARWIN_ID: ${{ needs.darwin.outputs.artifact_id }} + PRODUCER_DARWIN_DIGEST: ${{ needs.darwin.outputs.artifact_digest }} + PRODUCER_WINDOWS_ID: ${{ needs.windows.outputs.artifact_id }} + PRODUCER_WINDOWS_DIGEST: ${{ needs.windows.outputs.artifact_digest }} + PRODUCER_FREEBSD_ID: ${{ needs.freebsd.outputs.artifact_id }} + PRODUCER_FREEBSD_DIGEST: ${{ needs.freebsd.outputs.artifact_digest }} + steps: + - name: Publish accepted archive without candidate execution + run: |- + python3 - <<'PYTHON' + import hashlib + from datetime import datetime + import json + import os + from pathlib import Path, PurePosixPath, PureWindowsPath + import re + import stat + import subprocess + import zipfile + + + def need(condition, message): + if not condition: + raise ValueError(message) + + + def file_digest(path): + value=hashlib.sha256() + with Path(path).open('rb') as stream: + for chunk in iter(lambda:stream.read(1024*1024),b''): + value.update(chunk) + return value.hexdigest() + + + def canonical(document): + return hashlib.sha256(json.dumps(document,sort_keys=True,separators=(',',':')).encode()).hexdigest() + + + def publish_checked(authorization, expected, archive, manifest, tag_commit, publication): + need(expected['event'] in ('push','workflow_dispatch'), 'untrusted publication event') + need(re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:rc|alpha|beta)[0-9]+)?',expected['tag']), 'invalid tag') + need(authorization.get('schema_version') == 1 and authorization.get('binding') == expected, 'authorization identity differs') + need(canonical(manifest) == authorization['manifest_sha256'], 'accepted manifest changed') + need(tag_commit == expected['source_commit'] == manifest['release_artifact']['source']['head'], 'tag/source commit changed') + need(file_digest(archive) == expected['archive_sha256'] + == manifest['release_artifact']['archive']['sha256'], 'selected archive changed') + need(expected['producer_ids'] == ['FreeBSD','Darwin','Windows'] and + set(expected['producer_artifacts']) == {'freebsd','darwin','windows'}, 'producer identities missing') + publication() + + + def api(endpoint): + return json.loads(subprocess.check_output(['gh','api',endpoint],text=True)) + + + def download(artifact_id, expected_digest, directory, expected, run): + need(re.fullmatch(r'[1-9][0-9]*',artifact_id), 'invalid artifact ID') + endpoint='repos/'+expected['repository']+'/actions/artifacts/'+artifact_id + metadata=api(endpoint) + need(str(metadata['id']) == artifact_id and metadata['expired'] is False, 'wrong/expired artifact') + need(str(metadata['workflow_run']['id']) == expected['run_id'] and metadata['workflow_run']['head_sha'] == expected['workflow_source'], 'artifact source/run differs') + need(datetime.fromisoformat(metadata['created_at']) >= datetime.fromisoformat(run['run_started_at']), 'artifact from earlier run attempt') + archive=directory.with_suffix('.zip') + with archive.open('xb') as output: + subprocess.run(['gh','api',endpoint+'/zip'],stdout=output,check=True) + digest='sha256:'+file_digest(archive) + need(digest == metadata['digest'] and digest == 'sha256:'+expected_digest.removeprefix('sha256:'), 'artifact ZIP digest differs') + with zipfile.ZipFile(archive) as stream: + members=stream.infolist(); names=set() + need(len(members)<20004 and sum(m.file_size for m in members)<9*1024**3, 'artifact size bound') + for member in members: + name=PurePosixPath(member.filename) + need(not name.is_absolute() and not PureWindowsPath(member.filename).drive and '..' not in name.parts and '\\' not in member.filename + and member.filename not in names and not stat.S_ISLNK(member.external_attr>>16), 'unsafe artifact member') + names.add(member.filename) + directory.mkdir(); stream.extractall(directory) + + + def main(): + expected=dict(repository=os.environ['GITHUB_REPOSITORY'],run_id=os.environ['GITHUB_RUN_ID'],run_attempt=os.environ['GITHUB_RUN_ATTEMPT'], + workflow_source=os.environ['GITHUB_SHA'],source_commit=os.environ['SOURCE_COMMIT'],tag=os.environ['SOURCE_VERSION'], + archive_sha256=os.environ['ARCHIVE_SHA256'],event=os.environ['GITHUB_EVENT_NAME'],artifact_id=os.environ['SELECTED_ID'], + producer_ids=['FreeBSD','Darwin','Windows'],producer_artifacts={name:dict(id=os.environ['PRODUCER_'+name.upper()+'_ID'],digest=os.environ['PRODUCER_'+name.upper()+'_DIGEST']) for name in ('freebsd','darwin','windows')}) + need(re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+',expected['repository']), 'invalid repository') + need(re.fullmatch(r'[0-9a-f]{40}',expected['source_commit']), 'invalid commit') + need(all(value=='success' for value in json.loads(os.environ['REQUIRED_RESULTS']).values()), 'required producer not successful') + run=api('repos/'+expected['repository']+'/actions/runs/'+expected['run_id']+'/attempts/'+expected['run_attempt']) + need(str(run['run_attempt'])==expected['run_attempt'] and run['repository']['full_name']==expected['repository'] and run['head_sha']==expected['workflow_source'], 'workflow attempt/source changed') + root=Path(os.environ['RUNNER_TEMP']) + download(expected['artifact_id'],os.environ['SELECTED_DIGEST'],root/'selected',expected,run) + download(os.environ['AUTHORIZATION_ID'],os.environ['AUTHORIZATION_DIGEST'],root/'authorized',expected,run) + selected=json.loads((root/'selected/selected.json').read_text()) + need(selected['commit']==expected['source_commit'] and selected['version']==expected['tag'] and selected['sha256']==expected['archive_sha256'], 'selected descriptor differs') + need(Path(selected['archive']).name==selected['archive'], 'unsafe archive filename') + authorization=json.loads((root/'authorized/authorization.json').read_text()) + manifest=json.loads((root/'authorized/accepted-manifest.json').read_text()) + tag=api('repos/'+expected['repository']+'/git/ref/tags/'+expected['tag'])['object'] + for _ in range(5): + if tag['type']=='commit': break + need(tag['type']=='tag', 'unexpected tag target') + tag=api('repos/'+expected['repository']+'/git/tags/'+tag['sha'])['object'] + need(tag['type']=='commit','unresolved tag') + def create_release(): + argv=['gh','release','create',expected['tag'],'--repo',expected['repository'],'--verify-tag', + '--target',expected['source_commit'],'--title','libhttpserver '+expected['tag'],'--generate-notes'] + if '-' in expected['tag']: argv.append('--prerelease') + subprocess.run(argv+[str(root/'selected'/selected['archive'])],check=True) + publish_checked(authorization,expected,root/'selected'/selected['archive'],manifest,tag['sha'],create_release) + + + main() + + PYTHON diff --git a/.github/workflows/verify-build.yml b/.github/workflows/verify-build.yml index 16de17b87..d25526367 100644 --- a/.github/workflows/verify-build.yml +++ b/.github/workflows/verify-build.yml @@ -2,10 +2,10 @@ name: "Verify Build" on: push: - branches: [master] + branches: [master, v3] pull_request: # The branches below must be a subset of the branches above - branches: [master] + branches: [master, v3] schedule: - cron: '0 0 * * 0' @@ -15,7 +15,22 @@ permissions: contents: read jobs: + source: + runs-on: ubuntu-latest + outputs: + native: ${{ steps.source.outputs.native }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - id: source + run: | + if grep -q 'libhttpserver_v3core_la_SOURCES' src/Makefile.am; then + echo 'native=true' >> "$GITHUB_OUTPUT" + else + echo 'native=false' >> "$GITHUB_OUTPUT" + fi verify: + needs: source + if: needs.source.outputs.native == 'false' name: Verify runs-on: ${{ matrix.os }} env: diff --git a/.gitignore b/.gitignore index 73475dff1..9f1b476a0 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,9 @@ __pycache__/ # v3 source-layout compiled implementation contract checker. !scripts/check-v3-compiled-operations.py + +# v3 CI selected-archive publication and portable platform contracts. +!scripts/run-v3-release-ci.py +!scripts/test-v3-ci-workflows.py +!test/integ/release_ci_test.py +!test/integ/release_platform_bundle_test.py diff --git a/ChangeLog b/ChangeLog index 95b7d5393..5cc2febf6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,11 @@ Version 3.0.0 - 2026-10-10 + Connect native TLS-off/on builds, replay and complete installed matrices to CI. + Add bounded sanitizer fuzz smoke, persistent trusted corpora and scheduled + campaign code that activates after default-branch cutover. + Require exact selected-archive readiness and all platform evidence before + publication, with immutable artifact verification and isolated permissions. + Fix an ambiguous QUIC replay cursor construction rejected by GCC 13. + Restore native Doxygen linked-page input and a Graphviz-compatible font. * Ship the C++20 native engine with SOVERSION 3 and native public headers. * Remove MHD and GnuTLS configure, source, build, link and package prerequisites. * Require strict TLS-on/off installed audits and validate versioned ABI-2/3 runtime coexistence. diff --git a/Makefile.am b/Makefile.am index 90a0b7678..0840e5155 100644 --- a/Makefile.am +++ b/Makefile.am @@ -333,7 +333,7 @@ check-release-notes: # exit 0 if doxygen is not installed. check-doxygen: @echo "=== check-doxygen: enforce TASK-043 zero-warning invariant ===" - @BUILD_DIR="$(abs_top_builddir)" $(top_srcdir)/scripts/check-doxygen.sh + +@BUILD_DIR="$(abs_top_builddir)" $(top_srcdir)/scripts/check-doxygen.sh # TASK-044: SOVERSION acceptance gate. Stages a clean DESTDIR install and # asserts that the v3.0 on-disk SONAME layout is correct (libhttpserver.so.3 @@ -620,3 +620,8 @@ check-v3-compiled-operations: $(top_builddir)/src/libhttpserver.la .PHONY: check-v3-compiled-operations EXTRA_DIST += docs/v3-protocol-limits.md scripts/check-v3-protocol-limits.py test/installed/protocol_limits_consumer.cpp + +EXTRA_DIST += scripts/run-v3-release-ci.py scripts/lib/release_ci.py scripts/lib/release_platform.py scripts/test-v3-ci-workflows.py +EXTRA_DIST += .github/workflows/v3-release-acceptance.yml + +EXTRA_DIST += docs/v3-ci.md diff --git a/docs/task-185-release-gates.md b/docs/task-185-release-gates.md index b19fb799b..7d18aaaeb 100644 --- a/docs/task-185-release-gates.md +++ b/docs/task-185-release-gates.md @@ -8,6 +8,18 @@ adapters. Public TLS/H2/H3, credential and throughput/latency stages now execute their installed-package adapters in both TLS modes with independent receipt validation. Private fixtures remain supporting evidence. +## CI selected-archive execution + +The CI integration in [v3-ci.md](v3-ci.md) selects one immutable archive, +produces portable FreeBSD/Darwin/Windows bundles, then executes the existing +18-gate Linux inventory against that archive. Original platform receipts and +referenced bytes are retained; a scoped catalog resolver validates them without +foreign execution. Full readiness authorizes publication only after the actual +validator accepts every required gate and platform. Existing fresh-archive +callers and the separate release-owner pre-tag prerequisite remain supported. +Validation-only PR/dispatch runs exercise the same acceptance path without +creating tags or releases. Historical evidence below remains historical. + ## Complete local implementation inventory (2026-10-14) The fresh `build/task185-final-19` inventory passes all **18 mandatory local diff --git a/docs/v3-ci.md b/docs/v3-ci.md new file mode 100644 index 000000000..004b90629 --- /dev/null +++ b/docs/v3-ci.md @@ -0,0 +1,131 @@ +# v3 CI and publication + +Native push/PR checks target `v3` and `master`. Routing in Verify Build examines +`src/Makefile.am`: historical sources retain the historical matrix, while native +sources use `v3-native.yml`. CodeQL compiles the native core and private TLS +sources with the selected OpenSSL provider. ChangeLog checks compare the actual +PR base, passed as data through `BASE_REF`. + +`v3-native.yml` runs C++20 `make check` with TLS off/on, including deterministic +H1/WebSocket, HPACK, H2, QUIC parser/state/recovery/callback, H3 and QPACK replay, +registered helper tests, public headers, examples and install hygiene. Checkout +README/release-note/Doxygen checks retain their repository/spec prerequisites. +`v3-package-validation.yml` separately runs complete installed shared/static, +managed/external matrices on Linux, macOS, MinGW64 and FreeBSD. Provider, build +and evidence directories live outside the source checkout; VM evidence is +copied back only after auditing, for upload. Native Python verifies executable +Git on Windows and FreeBSD. Selected clients are hyper-h2 4.4.1 and Node 24.15.0; +FreeBSD builds the checksummed Node source if its ports package differs. Linux +and macOS package jobs probe actual loopback TCP443 binding for ACME witnesses; +when privileges require it, they run the producer with sudo and an exact +checkout-scoped Git ownership exception. + +## Fuzz smoke and campaigns + +Eleven independent target jobs prevent one failure from suppressing other +attempts. Ten targets compile TLS-off native sources; `quic_tls_callbacks` +uses genuine TLS-on configure/provider flags. Native project objects are built +with libFuzzer, ASan and UBSan instrumentation. OpenSSL and external clients +remain outside this instrumentation boundary. + +Push/PR smoke requests 2,000 runs, at most 30 seconds per target, seed 128. +Scheduled/manual campaigns request a high run cap, 600 seconds per target and a +varying recorded seed. Existing 5-second input timeout, 512-MiB RSS and target +input-length bounds remain in force. `compile.sh`, `reproduce.sh`, build/run +logs, source lists, environment and crash artifacts are retained on failure. +Fuzz artifacts retain 30 days; native/package artifacts retain 14 days. + +```sh +bash scripts/run-v3-protocol-fuzz.sh --compiler clang++ \ + --build-dir /tmp/fuzz-attempt --native-config-dir /tmp/configured-off \ + --targets 'http3_engine qpack' --seed 128 --runs 2000 --seconds 30 \ + --corpus-dir /tmp/writable-corpus +``` + +Seeds are copied from regression fixtures into a writable content-addressed +corpus. Restored inputs remain alongside these seeds. Corpus paths must be +outside the checkout and cannot contain symlinks. Neither fixtures nor existing +content-addressed inputs are overwritten. Immutable cache keys include corpus +format, target, platform, instrumentation, run ID and attempt; compatible keys +supply restoration. PRs restore only. Successful trusted `v3`/default-branch +push or campaign jobs may save. Deterministic replay has no cache dependency. +Corpus retention uses [GitHub’s cache policy](https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#usage-limits-and-eviction-policy): +entries expire after seven days without access, and storage pressure evicts older +entries within the repository’s configured limit. Caches are best-effort +persistence, not test proof. + +The nightly schedule is intentionally dormant while this workflow exists only +on `v3` and `master` remains the default branch. It activates when the workflow +reaches the default/master branch. Scheduled checkout uses the event/default +ref, so it follows the new default source after cutover. No dispatcher or +repository setting change is introduced. Manual dispatch is defined but its +availability before default-branch adoption must be verified in the real +repository; push/PR execution does not depend on it. + +## Selected-archive acceptance + +`v3-release-acceptance.yml` prepares one distribution archive and source/tooling +snapshot. Each nonlocal producer checks the exact selected archive against its +checkout, runs the complete installed and native TLS-off/on suites, validates +raw results and exports a platform bundle. The Linux job then executes all 18 +existing local gates against that same selected archive. Full H3 interop, +sanitizers, installed public capabilities and full benchmark workloads retain +their existing acceptance rules and exclusions. + +The fixed FreeBSD/Darwin/Windows bundles preserve original receipt bytes and +logical artifact paths. A bounded per-platform catalog retains exact source, +log, suite, audit, compiler, provider and selected-client bytes. Import checks +safe extraction, namespace uniqueness, every catalog hash and referenced +package input; it reruns the installed matrix, raw-suite/count and native-result +validators. Windows path normalization applies only to logical lookup and +comparison; foreign binaries are never executed. Sealed dyld/API-set OS leaves retain the +existing auditor’s explicit whitelist; nonexistent OS files are never +synthesized. Missing/changed bytes, +missing native replays, failed suites and stale counts block readiness. + +Linux evidence stays on its original filesystem. The actual `check_tag_ready` +validator checks all 18 local gates and all three platform receipts before an +authorization is written. That authorization binds the repository, workflow +source/run/attempt, selected commit/tag/archive, accepted manifest and immutable +selected/platform artifact IDs and digests. Downloads compare the actual ZIP +SHA-256 with both producer output and GitHub artifact metadata; a warning from +a download action is insufficient. Artifact creation must belong to the current +run attempt; rerun all jobs to produce fresh release proof rather than reusing +artifacts from successful jobs in an earlier attempt. + +The final publisher alone receives `contents: write`. It checks successful +mandatory dependencies, same-run artifacts, manifest and archive hashes and the +current tag target before `gh release create --verify-tag`. It runs only inline +workflow code and the GitHub CLI, with no checkout or downloaded candidate code. +Historical release sources retain their legacy distribution validation path. +The workflow starts after an existing tag; release owners must still execute +the separate pre-tag readiness command. This is publication enforcement. + +PR and validation-only dispatch use the same archive, platform and full local +path, and cannot publish. For an explicit manual validation request, provide an +already resolved 40-hex `source_commit` and its `source_version`. Empty inputs +select the current workflow ref and the version from `configure.ac`. The actual +GitHub run and dispatch availability are CI/root verification responsibilities. +No tags or releases are created by local verification. + +For local callers, fresh archive creation remains the default: + +```sh +python3 scripts/run-v3-release-gates.py --output /tmp/release-attempt \ + --selected-archive /tmp/selected/libhttpserver-3.0.0.tar.gz \ + --selected-sha256 ACTUAL_SHA256 --provider-prefix /tmp/provider \ + --compiler clang++ --cc clang --make make +# Supply the existing required pinned clients, captures and interop arguments. +python3 scripts/run-v3-release-gates.py --check-tag-ready /tmp/release-attempt/manifest.json \ + --platform-bundle /tmp/FreeBSD.tar.gz --platform-bundle /tmp/Darwin.tar.gz \ + --platform-bundle /tmp/Windows.tar.gz +``` + +Each output/import destination is exclusive. A failed attempt is preserved; +use a fresh directory for another attempt. A missing required infrastructure +capability remains an unsuccessful required dependency. In particular, the +pinned upstream QUIC simulator is provisioned first. No historical clock +compatibility receipt is imported. If a fresh run demonstrates clock +incompatibility, the QUIC simulator infrastructure owner must reprovision the +existing bounded compatibility recipe/probe and produce fresh evidence before +H3 can pass. diff --git a/doxyconfig.in b/doxyconfig.in index ceda5b9f5..c3937ebe3 100644 --- a/doxyconfig.in +++ b/doxyconfig.in @@ -581,7 +581,7 @@ WARN_LOGFILE = # directories like "/usr/src/myproject". Separate the files or directories # with spaces. -INPUT = $(SRCDIR)/src/httpserver $(SRCDIR)/docs/v3-api.md +INPUT = $(SRCDIR)/src/httpserver $(SRCDIR)/docs/v3-api.md $(SRCDIR)/docs/v3-protocol-limits.md # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is @@ -1479,7 +1479,7 @@ DOT_NUM_THREADS = 0 # DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory # containing the font. -DOT_FONTNAME = FreeSans.ttf +DOT_FONTNAME = Helvetica # The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. # The default size is 10pt. diff --git a/scripts/lib/release_checks.py b/scripts/lib/release_checks.py index 3979e9e6b..2a77d88bf 100644 --- a/scripts/lib/release_checks.py +++ b/scripts/lib/release_checks.py @@ -78,7 +78,9 @@ def extract_archive(archive, directory): names = [Path(m.name) for m in members] need(names and len(members) < 20000 and sum(m.size for m in members) < 512 * 1024 * 1024, 'archive exceeds release extraction bounds') - need(all(not p.is_absolute() and '..' not in p.parts for p in names), 'archive path traversal') + need(all(not p.is_absolute() and '..' not in p.parts and '\\' not in m.name + and (m.isdir() or m.isfile()) for p, m in zip(names, members)), 'archive path traversal/link') + need(len({m.name for m in members}) == len(members), 'duplicate archive member') roots = {p.parts[0] for p in names if p.parts} need(len(roots) == 1, 'archive requires one source root') need(not directory.exists(), 'extracted directory already exists') @@ -149,7 +151,7 @@ def rotation_counts(receipt, source, build, prefix): return counts(4, 20) -def validate_package(root, directory, read_json): +def validate_package(root, directory, read_json, source_root=None, provenance_path=Path): summary = read_json(directory / 'summary.json') need(summary.get('status') == 'passed' and set(summary.get('lanes', {})) == {'no', 'yes'}, 'package matrix incomplete') matrix = load_runner('lib/package_matrix.py', root) @@ -161,8 +163,10 @@ def validate_package(root, directory, read_json): need(audit.get('violations') == [] and audit.get('tls_mode') == mode, 'package audit failed/wrong TLS') for linkage in ('shared', 'static'): public = audit['public'][linkage] - matrix.check_linkage(public['dependencies'], linkage, lane / 'prefix') - matrix.check_dependencies(public['compile']['dependencies'], lane / 'prefix', root, lane / 'build') + graph = dict(public['dependencies'], nodes=[dict(node, id=str(provenance_path(node['id']))) for node in public['dependencies']['nodes']]) + matrix.check_linkage(graph, linkage, provenance_path(lane / 'prefix')) + matrix.check_dependencies([str(provenance_path(p)) for p in public['compile']['dependencies']], + provenance_path(lane / 'prefix'), provenance_path(source_root or root), provenance_path(lane / 'build')) cells = public['cells'] need(len(cells) == 2 and all(len(cell['configurations']) == len(inventory['cases']) for cell in cells), 'package configuration coverage missing') suites = read_json(lane / ('integration-' + linkage) / 'suites.json') diff --git a/scripts/lib/release_ci.py b/scripts/lib/release_ci.py new file mode 100644 index 000000000..7f4c13919 --- /dev/null +++ b/scripts/lib/release_ci.py @@ -0,0 +1,107 @@ +"""Content-bound authorization after the fixed release readiness decision.""" +from datetime import datetime +import hashlib +import json +import re +from pathlib import PurePosixPath, PureWindowsPath +import stat +import subprocess +import zipfile + + +VERSION = r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:rc|alpha|beta)[0-9]+)?' + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def canonical_digest(document): + return hashlib.sha256(json.dumps(document, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def validate_binding(binding): + require(re.fullmatch(VERSION, binding['tag']), 'invalid version tag') + require(re.fullmatch(r'[0-9a-f]{40}', binding['source_commit']), 'invalid source commit') + require(binding['event'] in ('push', 'workflow_dispatch'), 'untrusted publication event') + require(binding['producer_ids'] == ['FreeBSD', 'Darwin', 'Windows'], 'required producer identities missing') + require(re.fullmatch(r'[0-9a-f]{40}', binding['workflow_source']), 'invalid workflow source') + require(set(binding['producer_artifacts']) == {'freebsd','darwin','windows'}, 'missing producer artifact identity') + for artifact in binding['producer_artifacts'].values(): + require(re.fullmatch(r'[1-9][0-9]*', artifact['id']) and re.fullmatch(r'[0-9a-f]{64}', artifact['digest']), 'invalid producer artifact identity') + for key in ('run_id', 'run_attempt', 'artifact_id'): + require(re.fullmatch(r'[1-9][0-9]*', binding[key]), 'invalid run/artifact identity') + require(re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', binding['repository']), 'invalid repository') + + +def authorize(gate, manifest, binding): + validate_binding(binding) + require(manifest['release_artifact']['source'].get('working_diff_sha256') == hashlib.sha256(b'').hexdigest(), 'publication source differs from committed checkout') + gate.check_tag_ready(manifest) + require(manifest['release_artifact']['source']['head'] == binding['source_commit'], 'wrong selected source commit') + require(manifest['release_artifact']['archive']['sha256'] == binding['archive_sha256'], 'wrong selected archive') + return dict(schema_version=1, binding=json.loads(json.dumps(binding)), manifest_sha256=canonical_digest(manifest)) + + +def selected_archive(checkout, source, archive, expected_sha256, extraction): + import release_artifact as identity + from release_checks import extract_archive, verify_snapshot_packaging + require(identity.sha256(archive) == expected_sha256, 'selected archive digest mismatch') + for name, digest in source['files_sha256'].items(): + require(identity.sha256(checkout / name) == digest, 'selected source snapshot changed') + extracted = extract_archive(archive, extraction) + packaged = identity.source_files(extracted) + transformations = verify_snapshot_packaging(checkout, source['files_sha256'], extracted) + tooling = {} + candidates = [checkout / '.gitignore'] + list((checkout / 'specs').rglob('*.md')) + list((checkout / 'docs').rglob('*.md')) + for path in candidates: + if not path.is_file(): continue + name = path.relative_to(checkout).as_posix() + destination = extracted / name + if not destination.exists(): + require(source['files_sha256'].get(name) == identity.sha256(path), 'tooling not from selected source') + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(path.read_bytes()) + tooling[name] = identity.sha256(destination) + return dict(schema_version=1, source_root=str(extracted), source=source, + archive=dict(path=str(archive.resolve()), sha256=expected_sha256, bytes=archive.stat().st_size), + files_sha256=identity.source_files(extracted), tooling_bundle=tooling, + packaged_files_sha256=packaged, configured_transformations=transformations) + + +def validate_request(root, commit, version): + require(re.fullmatch(VERSION, version), 'invalid source version') + require(re.fullmatch(r'[0-9a-f]{40}', commit), 'invalid selected commit') + try: + actual = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD^{commit}'], cwd=root, text=True, stderr=subprocess.PIPE).strip() + except (OSError, subprocess.CalledProcessError) as error: + raise ValueError('selected request requires executable Git and a checkout') from error + require(actual == commit, 'selected commit differs from checkout') + status = subprocess.check_output(['git', 'status', '--porcelain=v1', '--untracked-files=all'], cwd=root, text=True) + require(not status, 'selected checkout has modified/untracked source inputs') + subprocess.run(['bash', str(root / 'scripts/validate-version.sh'), version], cwd=root, check=True) + + +def verify_download(archive, metadata, run, expected, expected_digest, destination): + require(str(metadata['id']) == expected['artifact_id'] and metadata['expired'] is False, 'wrong/expired artifact ID') + require(str(metadata['workflow_run']['id']) == expected['run_id'] and metadata['workflow_run']['head_sha'] == expected['source_commit'], 'artifact from another workflow source/run') + require(str(run['id']) == expected['run_id'] and str(run['run_attempt']) == expected['run_attempt'] + and run['repository']['full_name'] == expected['repository'] and run['head_sha'] == expected['source_commit'], 'wrong repository/source/run attempt') + require(datetime.fromisoformat(metadata['created_at']) >= datetime.fromisoformat(run['run_started_at']), 'artifact belongs to an earlier run attempt') + from release_artifact import sha256 + digest = 'sha256:' + sha256(archive) + require(digest == expected_digest == metadata['digest'], 'downloaded artifact digest mismatch') + require(not destination.exists(), 'artifact destination already exists') + with zipfile.ZipFile(archive) as stream: + members = stream.infolist() + require(len(members) < 20004 and sum(m.file_size for m in members) < 9 * 1024**3, 'artifact extraction bounds exceeded') + names = set() + for member in members: + name = PurePosixPath(member.filename) + mode = member.external_attr >> 16 + require(not name.is_absolute() and not PureWindowsPath(member.filename).drive and '..' not in name.parts and '\\' not in member.filename + and member.filename not in names and not stat.S_ISLNK(mode), 'unsafe/duplicate artifact member') + names.add(member.filename) + destination.mkdir(parents=True) + stream.extractall(destination) diff --git a/scripts/lib/release_execution.py b/scripts/lib/release_execution.py index 8e0aee4a4..ea6a00ba9 100644 --- a/scripts/lib/release_execution.py +++ b/scripts/lib/release_execution.py @@ -84,32 +84,23 @@ def prepare(self, cmd): except Exception as error: self.manifest['client_packages'] = {'failure': str(error)} self.manifest['source'] = release_identity.source_identity(self.root) - bootstrap_env = dict(self.env) - bootstrap_env.pop('V3_TLS_CFLAGS', None); bootstrap_env.pop('V3_TLS_LIBS', None) - self.command(cmd, ['bash', self.root / 'bootstrap'], self.root, bootstrap_env) - dist = self.output / 'distribution-build'; dist.mkdir() - self.command(cmd, ['bash', self.root / 'configure', '--enable-v3-tls=no', '--disable-examples'], dist, bootstrap_env) - self.command(cmd, [self.args.make, 'dist'], dist, bootstrap_env) - archives = list(dist.glob('libhttpserver-*.tar.gz')) - self.gate.need(len(archives) == 1, 'missing/ambiguous fresh distribution archive') - self.source = self.gate.extract_archive(archives[0], self.output / 'extracted') - archive_files = release_identity.source_files(self.source) - snapshot = self.manifest['source']['files_sha256'] - transformations = checks.verify_snapshot_packaging(self.root, snapshot, self.source) - bundle = {} - # Repository specifications are declared tooling inputs, never borrowed libraries. - candidates = [self.root / '.gitignore'] + list((self.root / 'specs').rglob('*.md')) + list((self.root / 'docs').rglob('*.md')) - for path in candidates: - name = path.relative_to(self.root).as_posix() - destination = self.source / name - if not destination.exists(): - self.gate.need(snapshot.get(name) == release_identity.sha256(path), 'tooling input not from selected snapshot: ' + name) - destination.parent.mkdir(parents=True, exist_ok=True); shutil.copyfile(path, destination) - bundle[name] = release_identity.sha256(destination) - files = release_identity.source_files(self.source) - identity = dict(schema_version=1, source_root=str(self.source), source=self.manifest['source'], - archive=self.gate.artifact(archives[0]), files_sha256=files, tooling_bundle=bundle, - packaged_files_sha256=archive_files, configured_transformations=transformations) + selected = getattr(self.args, 'selected_archive', None) + if selected: + archives = [Path(selected).resolve()] + expected_sha256 = self.args.selected_sha256 + else: + bootstrap_env = dict(self.env) + bootstrap_env.pop('V3_TLS_CFLAGS', None); bootstrap_env.pop('V3_TLS_LIBS', None) + self.command(cmd, ['bash', self.root / 'bootstrap'], self.root, bootstrap_env) + dist = self.output / 'distribution-build'; dist.mkdir() + self.command(cmd, ['bash', self.root / 'configure', '--enable-v3-tls=no', '--disable-examples'], dist, bootstrap_env) + self.command(cmd, [self.args.make, 'dist'], dist, bootstrap_env) + archives = list(dist.glob('libhttpserver-*.tar.gz')) + self.gate.need(len(archives) == 1, 'missing/ambiguous fresh distribution archive') + expected_sha256 = self.gate.digest(archives[0]) + from release_ci import selected_archive + identity = selected_archive(self.root, self.manifest['source'], archives[0], expected_sha256, self.output / 'extracted') + self.source = Path(identity['source_root']) path = self.output / 'artifact-identity.json'; self.gate.save(path, identity) self.env['V3_RELEASE_IDENTITY'] = str(path) self.manifest['release_artifact'] = identity diff --git a/scripts/lib/release_platform.py b/scripts/lib/release_platform.py new file mode 100644 index 000000000..6b44e1c1b --- /dev/null +++ b/scripts/lib/release_platform.py @@ -0,0 +1,252 @@ +"""Fixed platform receipt bundles: immutable logical paths, never foreign execution.""" +import hashlib +import json +import posixpath +from pathlib import Path, PurePosixPath, PureWindowsPath +import shutil +import tarfile + +from release_ci import require +import release_artifact as identity + +PLATFORMS = ('FreeBSD', 'Darwin', 'Windows') +MAX_FILES = 20000 +MAX_BYTES = 8 * 1024 * 1024 * 1024 + + +def read_document(path): + def unique(pairs): + value = {} + for key, child in pairs: + require(key not in value, 'duplicate platform receipt/catalog key') + value[key] = child + return value + require(path.is_file() and path.stat().st_size <= 64 * 1024 * 1024, 'missing/oversized platform document') + return json.loads(path.read_text(), object_pairs_hook=unique) + + +def record(path): + return dict(sha256=identity.sha256(path), bytes=Path(path).stat().st_size) + + +def logical_key(value): + windows = PureWindowsPath(str(value)) + return posixpath.normpath(windows.as_posix() if windows.drive else str(value)) + + +def provenance_path(value): + windows = PureWindowsPath(str(value)) + if windows.drive: + # Comparison-only namespace; these paths are never opened or executed. + return Path('/__v3_foreign_windows__') / windows.drive.rstrip(':') / Path(*windows.parts[1:]) + return Path(value) + + +def retain(destination, platform, receipt, paths): + require(platform in PLATFORMS, 'unknown platform namespace') + destination.mkdir(parents=True, exist_ok=False) + (destination / 'bytes').mkdir() + catalog = {} + total = 0 + paths = set(map(Path, paths)) + paths |= {path.resolve() for path in paths} + for path in sorted(paths): + logical = str(path) + value = record(path); total += value['bytes'] + require(len(catalog) < MAX_FILES and total <= MAX_BYTES, 'platform evidence exceeds bounds') + name = 'bytes/' + value['sha256'] + target = destination / name + if not target.exists(): shutil.copyfile(path, target) + require(record(target) == value, 'platform artifact changed during retention') + catalog[logical] = dict(value, retained=name) + (destination / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + (destination / 'catalog.json').write_text(json.dumps(dict(schema_version=1, platform=platform, receipt=record(destination / 'receipt.json'), paths=catalog), indent=2) + '\n') + return Catalog(destination) + + +class Catalog: + def __init__(self, directory): + self.directory = Path(directory).resolve() + require(not any(p.is_symlink() for p in self.directory.rglob('*')), 'platform bundle symlink') + require((self.directory / 'catalog.json').is_file() and (self.directory / 'catalog.json').stat().st_size <= 64 * 1024 * 1024, 'oversized platform catalog') + value = read_document(self.directory / 'catalog.json') + require(value.get('schema_version') == 1 and value.get('platform') in PLATFORMS, 'unknown platform namespace') + self.platform, self.paths = value['platform'], value['paths'] + self.receipt_record = value['receipt'] + self.catalog_record = record(self.directory / 'catalog.json') + self.verify_metadata() + require(0 < len(self.paths) < MAX_FILES, 'empty/oversized platform catalog') + require(sum(r['bytes'] for r in self.paths.values()) <= MAX_BYTES, 'oversized platform bytes') + self.logical_paths = {} + for logical in self.paths: + key = logical_key(logical) + previous = self.logical_paths.get(key) + require(previous is None or self.paths[previous] == self.paths[logical], 'conflicting logical path aliases') + self.logical_paths[key] = logical + permitted = {'catalog.json', 'receipt.json'} + for logical, value in self.paths.items(): + name = value['retained'] + require(name == 'bytes/' + value['sha256'] and len(value['sha256']) == 64 + and all(c in '0123456789abcdef' for c in value['sha256']), 'unsafe platform catalog member') + permitted.add(name) + self.path(logical) + observed = {p.relative_to(self.directory).as_posix() for p in self.directory.rglob('*') if p.is_file()} + require(observed == permitted, 'missing/extraneous platform catalog bytes') + + def verify_metadata(self): + require(record(self.directory / 'catalog.json') == self.catalog_record, 'platform catalog changed') + require(record(self.directory / 'receipt.json') == self.receipt_record, 'original platform receipt bytes changed') + + def path(self, logical): + original = self.logical_paths.get(logical_key(logical)) + value = self.paths.get(original) + require(value is not None, 'missing catalog entry: ' + str(logical)) + path = self.directory / value['retained'] + require(path.is_file() and not path.is_symlink() and record(path) == {k: value[k] for k in ('bytes', 'sha256')}, + 'changed platform catalog bytes: ' + str(logical)) + return path + + def read_json(self, logical): + path = self.path(logical) + require(path.stat().st_size <= 64 * 1024 * 1024, 'oversized platform receipt') + return read_document(path) + + def verify_files(self, records): + require(isinstance(records, list) and records, 'empty platform artifact evidence') + for value in records: + require(record(self.path(value['path'])) == {k: value[k] for k in ('bytes', 'sha256')}, 'platform artifact identity differs') + + +def unpack(archive, output): + require(not output.exists(), 'platform namespace collision') + with tarfile.open(archive) as stream: + members = stream.getmembers() + require(0 < len(members) < MAX_FILES + 4 and sum(m.size for m in members) <= MAX_BYTES, 'platform bundle exceeds bounds') + names = set(); roots = set() + for member in members: + path = PurePosixPath(member.name) + require(not path.is_absolute() and '..' not in path.parts and '\\' not in member.name + and (member.isfile() or member.isdir()) and member.name not in names, 'unsafe/duplicate platform member') + names.add(member.name); roots.add(path.parts[0]) + require(len(roots) == 1 and next(iter(roots)) in PLATFORMS, 'unknown platform namespace') + output.mkdir(parents=True) + stream.extractall(output, filter='data') + return Catalog(output / roots.pop()) + + +def package_origin(root, origin, read_json, read_text, expected, platform, read_bytes=None): + import package_matrix as matrix + from release_checks import validate_package + directory = Path(origin['package']) + summary = read_json(directory / 'summary.json') + require(summary['revision'] == origin['source_commit'] == summary['release_artifact']['source']['head'], 'platform source commit differs') + require(summary['host'] == platform and summary['release_artifact']['archive']['sha256'] == expected['archive_sha256'] + and summary['release_artifact']['source']['snapshot_sha256'] == expected['snapshot_sha256'], 'stale platform package source/archive') + require(hashlib.sha256((read_bytes or (lambda p: Path(p).read_bytes()))(origin['provider_header'])).hexdigest() == summary['provider_header_sha256'], + 'platform provider artifact substituted') + result = validate_package(root, directory, read_json, source_root=Path(origin['source_root']), provenance_path=provenance_path) + inventory = matrix.load() + for mode in ('no', 'yes'): + for linkage in ('shared', 'static'): + suites = read_json(directory / ('tls-' + mode) / ('integration-' + linkage) / 'suites.json') + for name, definition in matrix.suites(inventory, platform, mode).items(): + receipt = suites[name] + observed = matrix.suite_result(read_text(receipt['log']), 0) + require(all(receipt.get(k) == v for k, v in observed.items()), 'stale platform suite summary') + require(set(definition.get('required_cases', [])) <= set(observed['case_names']), 'pruned platform suite') + return result + + +def validate_origin(root, row, catalog): + from release_checks import automake_counts, counts + catalog.verify_metadata() + expected = row['identity']; origin = row['origin'] + prefix = logical_key(origin['package']).rstrip('/') + '/' + for logical in catalog.paths: + if logical_key(logical).startswith(prefix) and logical.endswith('.json'): + referenced_files(catalog.read_json(logical), catalog.path) + result = package_origin(root, origin, catalog.read_json, lambda p: catalog.path(p).read_text(), expected, row['id'], lambda p: catalog.path(p).read_bytes()) + require(set(origin['native']) == {'no', 'yes'}, 'missing native TLS configuration') + for mode, native in origin['native'].items(): + names = native['tests'] + require(len(names) == len(set(names)) and names, 'duplicate/empty native suite inventory') + mandatory = {'protocol_fuzz_replay', 'hpack_corpus', 'http2_fuzz_replay', 'quic_fuzz_replay', + 'quic_recovery_fuzz_replay', 'http3_fuzz_replay', 'qpack_fuzz_replay', + 'io_' + {'Windows':'iocp', 'Darwin':'kqueue', 'FreeBSD':'kqueue'}[row['id']] + '_backend'} + if mode == 'yes': mandatory.add('quic_tls_callbacks_fuzz_replay') + require(mandatory <= set(names), 'pruned mandatory native suite') + logs = [] + for name in names: + value = catalog.path(Path(native['build']) / 'test' / (name + '.trs')).read_text() + if ':test-result: SKIP' in value: + require(name in {'io_iocp_backend','io_epoll_backend','io_kqueue_backend'} and name not in mandatory, 'unexpected native skip') + else: + require(':test-result: PASS' in value, 'failed native test result') + logs.append(name) + text = catalog.path(Path(native['build']) / 'test/test-suite.log').read_text() + import re + skipped = len(names) - len(logs) + require(re.findall(r'^# SKIP:[ \t]+(\d+)$', text, re.M) == [str(skipped)], 'native skip summary differs') + require(re.findall(r'^# TOTAL:[ \t]+(\d+)$', text, re.M) == [str(len(names))], 'native registered suite summary differs') + text = re.sub(r'^# TOTAL:[ \t]+\d+$', '# TOTAL: ' + str(len(logs)), text, flags=re.M) + text = re.sub(r'^# SKIP:[ \t]+\d+$', '# SKIP: 0', text, flags=re.M) + native_counts = automake_counts(text) + result = counts(result['tests'] + native_counts['tests'], result['checks'] + native_counts['checks']) + require(row['receipt']['counts'] == result, 'platform envelope counts differ from origin') + catalog.verify_files(row['artifacts']); catalog.verify_files(row['receipt']['artifacts']) + return result + + +def import_bundle(root, manifest, catalog, expected): + row = read_document(catalog.directory / 'receipt.json') + require(row.get('id') == catalog.platform and row.get('identity') == expected and row.get('status') == 'passed' + and row.get('archive_sha256') == expected['archive_sha256'], + 'wrong/failed platform bundle identity') + current = [r for r in manifest['platforms'] if r['id'] == row['id']] + require(len(current) == 1 and current[0]['status'] == 'pending', 'duplicate/unknown platform producer') + require(row['origin']['source_commit'] == manifest['release_artifact']['source']['head'], 'platform from another source commit') + validate_origin(root, row, catalog) + row['portable_bundle'] = str(catalog.directory) + row['portable_catalog'] = dict(path=str(catalog.directory / 'catalog.json'), **catalog.catalog_record) + manifest['platforms'] = [row if r['id'] == row['id'] else r for r in manifest['platforms']] + + +def referenced_files(document, resolve=Path): + """Fixed package provenance records, including external clients/provider inputs.""" + result = set() + def visit(value): + if isinstance(value, list): + for child in value: visit(child) + elif isinstance(value, dict): + if 'path' in value and 'sha256' in value: + path = Path(value['path']); retained = resolve(str(path)) + require(retained.is_file() and identity.sha256(retained) == value['sha256'], 'referenced package artifact changed') + result.add(path) + for key in ('dependency_sha256', 'sources'): + mapping = value.get(key) + if isinstance(mapping, dict): + for name, digest in mapping.items(): + if isinstance(digest, str) and len(digest) == 64: + path = Path(name); retained = resolve(str(path)) + require(retained.is_file() and identity.sha256(retained) == digest, 'referenced package dependency changed') + result.add(path) + # Dependency graph node IDs name the actual provider/runtime libraries. + for node in value.get('nodes', []): + if isinstance(node, dict) and isinstance(node.get('id'), str): + path = Path(node['id']) + if node.get('provenance') == 'OS/API-set platform leaf': + from release_checks import load_runner + audit = load_runner('audit-v3-installed-package.py') + windows = PureWindowsPath(str(path)) + approved = str(path) in audit.DYLD_LEAVES or ( + windows.parent.name.lower() == 'system32' and windows.parent.parent.name.lower() == 'windows' + and audit.windows_system(windows.name)) + require(approved and node.get('sha256') is None, 'undeclared virtual platform leaf') + continue + if path.is_absolute() or PureWindowsPath(str(path)).is_absolute(): + retained = resolve(str(path)) + require(retained.is_file() and identity.sha256(retained) == node.get('sha256'), 'referenced runtime library missing/changed') + result.add(path) + for child in value.values(): visit(child) + visit(document) + return result diff --git a/scripts/run-v3-protocol-fuzz.sh b/scripts/run-v3-protocol-fuzz.sh index 756eb1950..1126dab2a 100755 --- a/scripts/run-v3-protocol-fuzz.sh +++ b/scripts/run-v3-protocol-fuzz.sh @@ -2,13 +2,15 @@ # Native, sequential libFuzzer lane. Explicit requests never silently skip. set -euo pipefail root=$(cd "$(dirname "$0")/.." && pwd) -compiler=clang++ build= runs=2000 seconds=30 targets='http1_head http1_body websocket_codec' native_config= +compiler=clang++ build= runs=2000 seconds=30 targets='http1_head http1_body websocket_codec' native_config= seed=128 corpus= while [ "$#" -gt 0 ]; do case "$1" in --compiler) compiler=$2; shift 2 ;; --build-dir) build=$2; shift 2 ;; --native-config-dir) native_config=$2; shift 2 ;; --targets) targets=$2; shift 2 ;; + --seed) seed=$2; shift 2 ;; + --corpus-dir) corpus=$2; shift 2 ;; --runs) runs=$2; shift 2 ;; --seconds) seconds=$2; shift 2 ;; *) echo "unknown argument: $1" >&2; exit 2 ;; @@ -19,6 +21,21 @@ done for target in $targets; do case "$target" in http1_head|http1_body|websocket_codec|hpack|http2_engine|quic_parser|quic_state|quic_recovery|quic_tls_callbacks|http3_engine|qpack) ;; *) echo "unknown fuzz target: $target" >&2; exit 2 ;; esac done +case "$seed" in ''|*[!0-9]*) echo 'invalid fuzz seed' >&2; exit 2 ;; esac +[ "${#seed}" -le 10 ] && [ "$seed" -gt 0 ] && [ "$seed" -le 4294967295 ] || { echo 'invalid fuzz seed' >&2; exit 2; } +if [ -n "$corpus" ]; then + python3 - "$corpus" "$root" <<'PYTHON' +from pathlib import Path +import sys +corpus, root = Path(sys.argv[1]), Path(sys.argv[2]) +if corpus.is_symlink() or any(p.is_symlink() for p in corpus.parents) or any(p.is_symlink() for p in corpus.rglob('*')): + raise SystemExit('corpus symlink rejected') +if corpus.resolve() == root or root in corpus.resolve().parents: + raise SystemExit('writable corpus must be outside source checkout') +corpus.mkdir(parents=True, exist_ok=True) +PYTHON + corpus=$(cd "$corpus" && pwd) +fi command -v "$compiler" >/dev/null || { echo 'libFuzzer compiler unavailable' >&2; exit 1; } provider_cflags=() provider_libs=() for target in $targets; do @@ -92,6 +109,26 @@ PY ;; esac [ -n "$(find "$build/$target/seeds" -type f -print -quit)" ] || { echo "empty corpus: $target" >&2; exit 1; } + target_corpus="$build/$target/seeds" + if [ -n "$corpus" ]; then + target_corpus="$corpus/$target" + python3 - "$build/$target/seeds" "$target_corpus" <<'PYTHON' +import hashlib +from pathlib import Path +import sys +source, destination = map(Path, sys.argv[1:]) +destination.mkdir(parents=True, exist_ok=True) +for fixture in source.iterdir(): + data = fixture.read_bytes() + path = destination / hashlib.sha256(data).hexdigest() + if path.exists(): + if path.read_bytes() != data: + raise SystemExit('corpus content-address collision') + else: + with path.open('xb') as output: + output.write(data) +PYTHON + fi sources=("$root/test/fuzz/${target}_fuzz.cpp") if [ "$target" = http1_head ] || [ "$target" = http1_body ] || [ "$target" = hpack ]; then sources+=("$root/src/http/fields.cpp" "$root/src/http/method.cpp" @@ -162,8 +199,8 @@ PYTHON printf '%q ' "$compiler" "${flags[@]}" "${target_flags[@]}" "${sources[@]}" "${link_flags[@]}" -o "$build/$target/fuzz" > "$build/$target/compile.sh" printf '\n' >> "$build/$target/compile.sh" "$compiler" "${flags[@]}" "${target_flags[@]}" "${sources[@]}" "${link_flags[@]}" -o "$build/$target/fuzz" > "$build/$target/build.log" 2>&1 - printf '%q ' "$build/$target/fuzz" "$build/$target/seeds" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed=128 -artifact_prefix="$build/$target/artifacts/" > "$build/$target/reproduce.sh" + printf '%q ' "$build/$target/fuzz" "$target_corpus" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed="$seed" -artifact_prefix="$build/$target/artifacts/" > "$build/$target/reproduce.sh" printf '\n' >> "$build/$target/reproduce.sh" - "$build/$target/fuzz" "$build/$target/seeds" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed=128 -artifact_prefix="$build/$target/artifacts/" > "$build/$target/run.log" 2>&1 || { echo "fuzz failure: $target; see $build/$target/run.log and artifacts" >&2; exit 1; } + "$build/$target/fuzz" "$target_corpus" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed="$seed" -artifact_prefix="$build/$target/artifacts/" > "$build/$target/run.log" 2>&1 || { echo "fuzz failure: $target; see $build/$target/run.log and artifacts" >&2; exit 1; } echo "PASS fuzz $target" done diff --git a/scripts/run-v3-release-ci.py b/scripts/run-v3-release-ci.py new file mode 100755 index 000000000..37743248d --- /dev/null +++ b/scripts/run-v3-release-ci.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Prepare one archive, produce native platform proof, or authorize publication.""" +import argparse +import importlib.util +import json +import os +from pathlib import Path +import platform +import shutil +import sys +import tarfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'scripts/lib')) +from package_commands import Commands +import release_artifact as identity +import release_checks as checks +from release_ci import authorize, require, selected_archive, validate_request +from release_platform import PLATFORMS, referenced_files, retain, validate_origin + + +def gate_runner(): + spec = importlib.util.spec_from_file_location('release_gate_ci', ROOT / 'scripts/run-v3-release-gates.py') + gate = importlib.util.module_from_spec(spec); sys.modules[spec.name] = gate; spec.loader.exec_module(gate) + return gate + + +def prepare(args): + validate_request(ROOT, args.commit, args.version) + args.output.mkdir(parents=True, exist_ok=False) + cmd = Commands(args.output) + env = dict(os.environ); env.pop('V3_RELEASE_IDENTITY', None) + cmd(['bash', str(ROOT / 'bootstrap')], ROOT, env) + build = args.output / 'build'; build.mkdir() + cmd(['bash', str(ROOT / 'configure'), '--disable-v3-tls', '--disable-examples'], build, env) + cmd([args.make, 'dist'], build, env) + archives = list(build.glob('libhttpserver-*.tar.gz')) + require(len(archives) == 1, 'ambiguous/missing selected archive') + archive = args.output / archives[0].name; shutil.copyfile(archives[0], archive) + validate_request(ROOT, args.commit, args.version) + source = identity.source_identity(ROOT) + selected = selected_archive(ROOT, source, archive, identity.sha256(archive), args.output / 'verified') + (args.output / 'selected.json').write_text(json.dumps(dict(commit=args.commit, version=args.version, + archive=archive.name, sha256=identity.sha256(archive), source=source, + files_sha256=selected['files_sha256']), indent=2) + '\n') + if os.environ.get('GITHUB_OUTPUT'): + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write('sha256=' + identity.sha256(archive) + '\n') + + +def platform_proof(args): + require(platform.system() in PLATFORMS, 'native nonlocal platform required') + selected = json.loads(args.selection.read_text()) + validate_request(ROOT, selected['commit'], selected['version']) + args.output.mkdir(parents=True, exist_ok=False) + release = selected_archive(ROOT, identity.source_identity(ROOT), args.archive, selected['sha256'], args.output / 'source') + require(release['source'] == selected['source'] and release['files_sha256'] == selected['files_sha256'], 'selected snapshot/tooling differs') + source = Path(release['source_root']) + identity_path = args.output / 'artifact-identity.json'; identity_path.write_text(json.dumps(release)) + env = dict(os.environ, V3_RELEASE_IDENTITY=str(identity_path), + V3_TLS_CFLAGS='-I' + (args.provider_prefix / 'include').as_posix(), + V3_TLS_LIBS='-L' + (args.provider_prefix / 'lib').as_posix() + ' -lssl -lcrypto') + env['LD_LIBRARY_PATH'] = str(args.provider_prefix / 'lib') + env['DYLD_LIBRARY_PATH'] = str(args.provider_prefix / 'lib') + env['PATH'] = str(args.provider_prefix / 'bin') + os.pathsep + env['PATH'] + cmd = Commands(args.output) + package = args.output / 'matrix' + argv = [sys.executable, source / 'scripts/check-v3-installed-consumer.py', '--output', package, + '--provider-prefix', args.provider_prefix, '--compiler', args.compiler, '--cc', args.cc, + '--make', args.make, '--jobs', str(args.jobs), '--client-python', args.client_python, '--node', args.node] + for directory in args.pkg_config_dir: argv += ['--pkg-config-dir', directory] + if args.toolchain_prefix: argv += ['--toolchain-prefix', args.toolchain_prefix] + cmd(argv, source, env, timeout=10800) + gate = gate_runner() + expected = gate.expected_identity({'release_artifact':release}, platform.system()) + origin = dict(package=str(package), source_root=str(source), source_commit=selected['commit'], + provider_header=str(args.provider_prefix / 'include/openssl/opensslv.h'), native={}) + result = checks.validate_package(ROOT, package, gate.read_json) + for mode in ('no', 'yes'): + build = package / ('tls-' + mode) / 'build' + cmd([args.make, '-j' + str(args.jobs)], build, env, timeout=3600) + cmd([args.make, 'check', '-j' + str(args.jobs)], build, env, timeout=7200) + text = cmd([args.make, '--no-print-directory', '-s', '-C', str(build / 'test'), + '--eval', 'ci-list-tests:;@printf "%s\\n" $(TEST_LOGS)', 'ci-list-tests'], build, env) + tests = [name.removesuffix('.log') for name in text.splitlines() if name] + origin['native'][mode] = dict(build=str(build), tests=tests) + native = checks.automake_result(build, platform.system()) + result = checks.counts(result['tests'] + native['tests'], result['checks'] + native['checks']) + files = [p for p in args.output.rglob('*') if p.is_file() and '__pycache__' not in p.parts] + # Preserve explicit tools/provider inputs referenced outside the platform tree. + files += [args.provider_prefix / 'include/openssl/opensslv.h', args.archive] + for receipt_path in package.rglob('*.json'): + files += list(referenced_files(json.loads(receipt_path.read_text()))) + for tool in (args.compiler, args.cc, args.make, args.client_python, args.node, 'git'): + selected_tool = shutil.which(str(tool)); require(selected_tool, 'selected tool unavailable: ' + str(tool)) + files.append(Path(selected_tool)) + files = sorted(set(files)) + artifacts = [gate.artifact(path) for path in files] + row = dict(id=platform.system(), status='passed', archive_sha256=selected['sha256'], identity=expected, + origin=origin, artifacts=artifacts, receipt=dict(status='passed', gate=platform.system(), identity=expected, counts=result, artifacts=artifacts)) + directory = args.output / 'bundle' / platform.system() + catalog = retain(directory, platform.system(), row, files) + validate_origin(ROOT, row, catalog) + with tarfile.open(args.output / (platform.system() + '.tar.gz'), 'w:gz') as stream: + stream.add(directory, arcname=platform.system(), recursive=True) + + +def accept(args): + gate = gate_runner(); manifest = gate.read_json(args.manifest) + gate.import_platform_bundles(manifest, args.platform_bundle, args.output) + gate.check_tag_ready(manifest) + selected = json.loads(args.selection.read_text()) + require(manifest['release_artifact']['source'] == selected['source'] and + manifest['release_artifact']['archive']['sha256'] == selected['sha256'], 'selected archive/source acceptance differs') + args.output.mkdir(parents=True, exist_ok=True) + gate.save(args.output / 'accepted-manifest.json', manifest) + if args.publish: + binding = dict(repository=os.environ['GITHUB_REPOSITORY'], run_id=os.environ['GITHUB_RUN_ID'], + run_attempt=os.environ['GITHUB_RUN_ATTEMPT'], source_commit=selected['commit'], tag=selected['version'], + archive_sha256=selected['sha256'], event=os.environ['GITHUB_EVENT_NAME'], artifact_id=args.artifact_id, + producer_ids=list(PLATFORMS), workflow_source=os.environ['GITHUB_SHA'], + producer_artifacts={name:dict(id=os.environ['PRODUCER_'+name.upper()+'_ID'],digest=os.environ['PRODUCER_'+name.upper()+'_DIGEST']) for name in ('freebsd','darwin','windows')}) + gate.save(args.output / 'authorization.json', authorize(gate, manifest, binding)) + else: + gate.save(args.output / 'validation.json', dict(status='passed', publication=False, source_commit=selected['commit'], archive_sha256=selected['sha256'])) + + +def download(args): + import subprocess + from release_ci import verify_download + expected=dict(repository=os.environ['GITHUB_REPOSITORY'],run_id=os.environ['GITHUB_RUN_ID'], + run_attempt=os.environ['GITHUB_RUN_ATTEMPT'],source_commit=os.environ['GITHUB_SHA'],artifact_id=args.artifact_id) + require(args.artifact_id.isdigit(), 'invalid artifact ID') + endpoint='repos/'+expected['repository']+'/actions/artifacts/'+args.artifact_id + metadata=json.loads(subprocess.check_output(['gh','api',endpoint],text=True)) + run=json.loads(subprocess.check_output(['gh','api','repos/'+expected['repository']+'/actions/runs/'+expected['run_id']+'/attempts/'+expected['run_attempt']],text=True)) + archive=args.output.parent/('artifact-'+args.artifact_id+'.zip') + with archive.open('xb') as output: + subprocess.run(['gh','api',endpoint+'/zip'],stdout=output,check=True) + digest=args.digest if args.digest.startswith('sha256:') else 'sha256:'+args.digest + verify_download(archive,metadata,run,expected,digest,args.output) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='operation', required=True) + prep = sub.add_parser('prepare'); prep.add_argument('--commit', required=True); prep.add_argument('--version', required=True) + platform_parser = sub.add_parser('platform') + platform_parser.add_argument('--archive', type=Path, required=True) + platform_parser.add_argument('--provider-prefix', type=Path, required=True) + platform_parser.add_argument('--compiler', default='clang++'); platform_parser.add_argument('--cc', default='clang') + platform_parser.add_argument('--client-python', required=True); platform_parser.add_argument('--node', required=True) + platform_parser.add_argument('--jobs', type=int, default=2) + platform_parser.add_argument('--pkg-config-dir', type=Path, action='append', default=[]) + platform_parser.add_argument('--toolchain-prefix', type=Path) + fetch = sub.add_parser('download'); fetch.add_argument('--artifact-id', required=True); fetch.add_argument('--digest', required=True) + acceptance = sub.add_parser('accept'); acceptance.add_argument('--manifest', type=Path, required=True) + acceptance.add_argument('--platform-bundle', type=Path, action='append', default=[]) + acceptance.add_argument('--publish', action='store_true'); acceptance.add_argument('--artifact-id') + for command in (prep, platform_parser, acceptance, fetch): command.add_argument('--output', type=Path, required=True) + for command in (platform_parser, acceptance): command.add_argument('--selection', type=Path, required=True) + for command in (prep, platform_parser): command.add_argument('--make', default='make') + args = parser.parse_args() + args.output = args.output.resolve() + if args.operation == 'prepare': prepare(args) + elif args.operation == 'platform': + args.archive = args.archive.resolve(); args.provider_prefix = args.provider_prefix.resolve(); platform_proof(args) + elif args.operation == 'download': download(args) + else: accept(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/run-v3-release-gates.py b/scripts/run-v3-release-gates.py index 059d147fe..a0bce1c2f 100644 --- a/scripts/run-v3-release-gates.py +++ b/scripts/run-v3-release-gates.py @@ -86,7 +86,7 @@ def validate_inventory(document): need(document == inventory_document(), 'missing/duplicated/changed required gate or unknown exclusion') -def verify_receipt(receipt, identity, public=False): +def verify_receipt(receipt, identity, public=False, verify=verify_files): need(isinstance(receipt, dict) and receipt.get('status') == 'passed', 'missing/failed required receipt') need(receipt.get('identity') == identity and identity, 'historical/wrong archive, source, TLS or sanitizer identity') counts = receipt.get('counts', {}) @@ -94,7 +94,7 @@ def verify_receipt(receipt, identity, public=False): 'missing/malformed counts') need(counts['tests'] > 0 and counts['checks'] > 0 and counts['successes'] == counts['checks'] and counts['failures'] == counts['skipped'] == 0, 'empty/failed/skipped required receipts') - verify_files(receipt.get('artifacts', [])) + verify(receipt.get('artifacts', [])) if public: need(receipt.get('kind') == 'installed public capability' and set(receipt.get('witnesses', [])) >= {'tls', 'h2', 'h3', 'credentials'}, @@ -214,11 +214,21 @@ def verify_local_receipts(manifest): def verify_gate_rows(manifest, rows): for row in rows: - verify_files(row['artifacts']) + verify = verify_files + if row.get('portable_bundle'): + need(row['id'] in PLATFORMS, 'portable resolver is restricted to platform records') + from release_platform import Catalog, validate_origin + verify_files([row['portable_catalog']]) + catalog = Catalog(Path(row['portable_bundle'])) + original = json.loads((catalog.directory / 'receipt.json').read_text()) + need({k: v for k, v in row.items() if k not in ('portable_bundle', 'portable_catalog')} == original, 'platform envelope changed') + validate_origin(ROOT, row, catalog) + verify = catalog.verify_files + verify(row['artifacts']) expected = expected_identity(manifest, row['id']) need(row.get('identity') == expected, 'gate does not belong to selected packaged release') need(row.get('receipt', {}).get('gate') == row['id'], 'wrong required gate receipt') - verify_receipt(row.get('receipt'), expected, public=row['id'] == 'public-capabilities') + verify_receipt(row.get('receipt'), expected, public=row['id'] == 'public-capabilities', verify=verify) def check_tag_ready(manifest): @@ -245,10 +255,19 @@ def import_platform_receipts(manifest, paths): manifest['platforms'] = [receipt if r['id'] == name else r for r in manifest['platforms']] +def import_platform_bundles(manifest, paths, output): + from release_platform import unpack, import_bundle + for index, path in enumerate(paths): + catalog = unpack(path, output / 'imported-platforms' / str(index)) + import_bundle(ROOT, manifest, catalog, expected_identity(manifest, catalog.platform)) + + def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--check-tag-ready', type=Path) parser.add_argument('--output', type=Path) + parser.add_argument('--selected-archive', type=Path) + parser.add_argument('--selected-sha256') parser.add_argument('--provider-prefix', type=Path) parser.add_argument('--compiler') parser.add_argument('--cc') @@ -261,14 +280,18 @@ def main(argv=None): parser.add_argument('--prerequisite-prefix', type=Path, action='append', default=[]) parser.add_argument('--pkg-config-dir', type=Path, action='append', default=[]) parser.add_argument('--platform-receipt', type=Path, action='append', default=[]) + parser.add_argument('--platform-bundle', type=Path, action='append', default=[]) parser.add_argument('--tool-artifact', type=Path, action='append', default=[]) parser.add_argument('--docker-direct-routing-compat', action='store_true') parser.add_argument('--isolated-public-network', action='store_true') args = parser.parse_args(argv) + if bool(args.selected_archive) != bool(args.selected_sha256): + parser.error('--selected-archive and --selected-sha256 must be supplied together') if args.check_tag_ready: try: manifest = read_json(args.check_tag_ready) import_platform_receipts(manifest, args.platform_receipt) + import_platform_bundles(manifest, args.platform_bundle, args.check_tag_ready.parent) check_tag_ready(manifest) print('PASS: full release inventory verified; tag prerequisite satisfied') return 0 @@ -293,6 +316,7 @@ def main(argv=None): ReleaseAttempt(args, sys.modules[__name__], manifest).execute() manifest['artifact_identity_path'] = str(args.output / 'artifact-identity.json') import_platform_receipts(manifest, args.platform_receipt) + import_platform_bundles(manifest, args.platform_bundle, args.output) manifest['local_passed'] = local_passed(manifest) if manifest['local_passed']: try: diff --git a/scripts/test-v3-ci-workflows.py b/scripts/test-v3-ci-workflows.py new file mode 100755 index 000000000..7d97f2edf --- /dev/null +++ b/scripts/test-v3-ci-workflows.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Workflow dependencies and trust boundaries; execution contracts live beside runners.""" +from pathlib import Path +import os +import re +import subprocess +import sys +import tempfile +import unittest +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +def workflow(name): + return yaml.safe_load((ROOT / '.github/workflows' / name).read_text()) + + +class CIWorkflows(unittest.TestCase): + def test_release_request_resolver_emits_structured_commit_and_version(self): + steps = workflow('v3-release-acceptance.yml')['jobs']['prepare']['steps'] + run = next(step['run'] for step in steps if step.get('id') == 'request') + source = run.split("<<'PYTHON'\n", 1)[1].rsplit('\nPYTHON', 1)[0] + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'outputs' + commit = 'a' * 40 + env = dict(os.environ, REQUEST_COMMIT=commit, REQUEST_VERSION='3.0.0', GITHUB_OUTPUT=str(output)) + result = subprocess.run([sys.executable, '-c', source], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_text(), 'commit=' + commit + '\nversion=3.0.0\n') + + def test_native_lane_checks_both_configurations_and_replays_independent_of_cache(self): + path = ROOT / '.github/workflows/v3-native.yml' + self.assertTrue(path.exists(), 'native CI lane missing') + lane = workflow(path.name) + self.assertEqual(lane['permissions'], {'contents': 'read'}) + native = lane['jobs']['native'] + self.assertEqual(native['strategy']['matrix']['tls'], ['no', 'yes']) + runs = '\n'.join(step.get('run', '') for step in native['steps']) + self.assertIn('make check', runs) + self.assertIn('RUNNER_TEMP', runs) + fuzz = lane['jobs']['fuzz'] + targets = fuzz['strategy']['matrix']['target'] + self.assertEqual(set(targets), {'http1_head', 'http1_body', 'websocket_codec', 'hpack', + 'http2_engine', 'quic_parser', 'quic_state', 'quic_recovery', 'http3_engine', 'qpack', 'quic_tls_callbacks'}) + save = next(step for step in fuzz['steps'] if '/save@' in step.get('uses', '')) + self.assertIn("github.event_name != 'pull_request'", save['if']) + self.assertIn('github.event.repository.default_branch', save['if']) + self.assertIn('run_attempt', save['with']['key']) + self.assertTrue(any(step.get('if') == 'always()' and '/upload-artifact@' in step.get('uses', '') for step in fuzz['steps'])) + + def test_package_paths_do_not_contaminate_checkout_and_native_git_is_installed(self): + lane = workflow('v3-package-validation.yml') + for name in ('linux', 'macos', 'mingw64', 'freebsd'): + text = str(lane['jobs'][name]['steps']) + self.assertNotIn('$PWD/package-evidence', text) + self.assertIn('client-python', text) + self.assertIn('node', text) + self.assertIn('git', str(lane['jobs']['mingw64']['steps'])) + self.assertIn('git', str(lane['jobs']['freebsd']['steps'])) + + def test_freebsd_client_is_exact_pinned_node_source_when_ports_differ(self): + for filename in ('v3-package-validation.yml','v3-release-acceptance.yml'): + text=(ROOT/'.github/workflows'/filename).read_text() + self.assertIn('729de494dd2872e5a3a6c32a1cd156a5413d4aca2772b2d873ee86bb5531bcd9',text) + self.assertIn('node-v24.15.0.tar.gz',text) + self.assertIn('node --version',text) + + def test_publisher_receives_only_accepted_manifest_and_authorization(self): + lane=workflow('v3-release-acceptance.yml') + step=next(step for step in lane['jobs']['full']['steps'] if step.get('id')=='authorization') + path=step['with']['path'] + self.assertIn('authorization.json',path) + self.assertIn('accepted-manifest.json',path) + self.assertNotIn('imported-platforms',path) + self.assertEqual(lane['jobs']['publish']['permissions']['contents'],'write') + for name,job in lane['jobs'].items(): + if name!='publish': self.assertNotEqual(job.get('permissions',{}).get('contents'),'write') + + def test_macos_provisions_actual_acme_tcp443_binding_capability(self): + for filename,job in (('v3-package-validation.yml','macos'),('v3-release-acceptance.yml','darwin')): + steps=workflow(filename)['jobs'][job]['steps'] + text='\n'.join(step.get('run','') for step in steps) + self.assertIn('GIT_CONFIG_KEY_0=safe.directory',text) + self.assertIn('bind(("127.0.0.1",443))',text) + + def test_changelog_uses_actual_base_as_data(self): + lane = workflow('changelog-check.yml') + step = next(step for step in lane['jobs']['changelog']['steps'] if 'git diff' in step.get('run', '')) + self.assertIn('BASE_REF', step['env']) + self.assertNotIn('origin/master', step['run']) + self.assertIn('git check-ref-format', step['run']) + self.assertIn('--name-only --', step['run']) + + def test_all_modified_workflows_pin_actions(self): + for name in ('v3-native.yml', 'v3-package-validation.yml', 'verify-build.yml', 'codeql-analysis.yml', 'changelog-check.yml', 'release.yml'): + path = ROOT / '.github/workflows' / name + if not path.exists(): continue + for use in re.findall(r'uses:\s*(\S+)', path.read_text()): + if not use.startswith('./.github/workflows/'): + self.assertRegex(use, r'@[0-9a-f]{40}$') + + +if __name__ == '__main__': + unittest.main() diff --git a/specs/architecture/v3/architecture.md b/specs/architecture/v3/architecture.md index 8d8781944..643c25c2e 100644 --- a/specs/architecture/v3/architecture.md +++ b/specs/architecture/v3/architecture.md @@ -460,3 +460,20 @@ necessary detail declarations. An installed header never includes a noninstalled private header. Normal library linkage supplies compiled operations; standalone header compilation does not imply link-free runtime behavior. Extraction preserves API, representation, suspension points, resource accounting and protocol behavior. + +### Selected-archive CI release acceptance + +CI keeps the existing TASK-185 18-gate validator authoritative. One selected +archive and source/tooling snapshot feed native/installed platform producers, +then the Linux full inventory. Linux evidence remains on its original +filesystem; only fixed FreeBSD/Darwin/Windows receipts cross the platform seam. +A bounded namespaced logical-path catalog preserves original receipt bytes and +retains every referenced package input. Import reruns origin validators and +normalizes foreign paths only for lookup/comparison, never binary execution. + +The final job consumes content-bound authorization, same-run immutable artifact +IDs/digests and the exact archive. It rechecks manifest/source/tag bindings and +executes no downloaded candidate code. Only that job receives publication +permission. PR/validation-only execution cannot publish. This design accepts a +serial release critical path to avoid moving the complete Linux evidence tree +or broadening existing protocol validators. See [v3 CI](../../../docs/v3-ci.md). diff --git a/specs/tasks/FEATURE-ci-v3-integration.md b/specs/tasks/FEATURE-ci-v3-integration.md new file mode 100644 index 000000000..4cb4329d6 --- /dev/null +++ b/specs/tasks/FEATURE-ci-v3-integration.md @@ -0,0 +1,38 @@ +# FEATURE-ci-v3-integration + +Status: locally implemented and tested; local verification recorded in the task evidence; +actual GitHub PR verification and final validation remain root-owned. + +Scope: native CI/replay, installed platform prerequisites, bounded fuzz campaigns, +selected-archive release acceptance and publication enforcement. No production +protocol behavior, default-branch setting, tag or release changes. + +- [x] Add source-aware v3 push/PR native builds and CodeQL TLS coverage. +- [x] Repair actual PR-base ChangeLog comparison and pin touched external actions. +- [x] Keep package prerequisites/builds/evidence outside audited source roots; + select pinned clients and executable native Git. +- [x] Shard all ten TLS-off fuzz targets and TLS-on callbacks; preserve core + instrumentation, bounded smoke, varying campaign seeds and corpus trust. +- [x] Add default-branch schedule code inside v3, with deferred activation. +- [x] Prepare/select one archive; retain fixed portable platform evidence and + invoke the existing complete readiness validator on live Linux evidence. +- [x] Bind immutable artifacts and authorization; isolate publication permissions + and recheck the existing tag before the release side effect. +- [x] Exercise publication refusal and success through the actual validator, + with only the final publication side effect mocked. +- [ ] Root: run the actual GitHub PR pipeline; verify manual dispatch availability. +- [ ] CI / v3 PR: obtain fresh full local gate and FreeBSD/Darwin/Windows receipts + for the selected commit/archive. Missing evidence is not a release pass. + +Details and commands: [v3 CI](../../docs/v3-ci.md). Historical TASK-185 receipts +are unchanged and cannot replace fresh evidence. + +Local verification: native TLS-off/on suites (239/270 tests, two documented +nonlocal backend skips in each), all eleven sanitizer fuzz targets, genuine +Doxygen/Graphviz, GCC QUIC replay and 99 focused CI/release/fuzz contracts passed. +The fresh Linux installed attempt passed TLS-off but failed TLS-on public TLS at +unprivileged TCP443 binding. Its unsuccessful receipt remains unchanged. A fresh +focused public TLS run using the existing isolated network path passed all four +shared/static and managed/external cells (82 checks each). This proves the +affected prerequisite correction; it does not turn the failed whole matrix into +a passing receipt. The fresh complete 18-gate release run remains pending CI. diff --git a/test/Makefile.am b/test/Makefile.am index 96fcdb497..8ce10bad4 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -1647,3 +1647,6 @@ protocol_limits_SOURCES = unit/protocol_limits_test.cpp check_PROGRAMS += protocol_compression TESTS += protocol_compression protocol_compression_SOURCES = unit/protocol_compression_test.cpp + +TESTS += integ/release_ci_test.py integ/release_platform_bundle_test.py +EXTRA_DIST += integ/release_ci_test.py integ/release_platform_bundle_test.py diff --git a/test/fuzz/quic_parser_fuzz.cpp b/test/fuzz/quic_parser_fuzz.cpp index fb3cc5e17..c37af4d5f 100644 --- a/test/fuzz/quic_parser_fuzz.cpp +++ b/test/fuzz/quic_parser_fuzz.cpp @@ -168,7 +168,7 @@ std::uint32_t quic_codec_fuzz_input(std::span input) { require(hd::decode_quic_varint(out).value == scalar.value); for (std::size_t width = 1; width <= 4; ++width) { hd::encode_quic_packet_number(scalar.value, width, out); - hd::quic_cursor c(std::span(out).first(width)); + hd::quic_cursor c{std::span(out).first(width)}; require(hd::reconstruct_quic_packet_number(c.fixed(width), width, scalar.value).code == hd::quic_codec_code::ok); } } diff --git a/test/integ/protocol_fuzz_runner_test.py b/test/integ/protocol_fuzz_runner_test.py index 8999f1988..9e30eeb17 100755 --- a/test/integ/protocol_fuzz_runner_test.py +++ b/test/integ/protocol_fuzz_runner_test.py @@ -139,6 +139,50 @@ def test_http3_qpack_build_and_invocation_contract(self): self.assertTrue(list((build / target / 'seeds').glob('*.seed'))) self.assertTrue((build / target / 'reproduce.sh').is_file()) + def test_seed_and_corpus_arguments_reject_invalid_inputs_before_compilation(self): + for seed in ('0', '-1', '4294967296', '$(touch marker)', '12x'): + with self.subTest(seed=seed), tempfile.TemporaryDirectory() as build: + result = subprocess.run(['bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'), + '--build-dir', build, '--seed', seed, '--compiler', '/missing/compiler'], + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 2) + self.assertIn('invalid fuzz seed', result.stderr) + + def test_restored_corpus_is_merged_without_following_links_or_overwriting_seeds(self): + import os + with tempfile.TemporaryDirectory() as directory: + location = pathlib.Path(directory) + compiler = location / 'compiler' + compiler.write_text("#!/usr/bin/env python3\nimport pathlib,sys\n" + "if '--version' in sys.argv: print('recorder'); sys.exit()\n" + "p=pathlib.Path(sys.argv[sys.argv.index('-o')+1]); " + "p.write_text('#!/bin/sh\\nexit 0\\n'); p.chmod(0o755)\n") + compiler.chmod(0o755) + corpus = location / 'corpus' + (corpus / 'http1_head').mkdir(parents=True) + restored = corpus / 'http1_head' / 'extra' + restored.write_bytes(b'unique restored input') + build = location / 'build' + result = subprocess.run(['bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'), + '--build-dir', str(build), '--compiler', str(compiler), '--targets', 'http1_head', + '--seed', '987', '--corpus-dir', str(corpus)], capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + inputs = [p.read_bytes() for p in (corpus / 'http1_head').iterdir()] + self.assertIn(b'unique restored input', inputs) + for fixture in (ROOT / 'test/conformance/http1').glob('*.wire'): + self.assertIn(fixture.read_bytes(), inputs) + reproduction = (build / 'http1_head/reproduce.sh').read_text() + self.assertIn('-seed=987', reproduction) + self.assertIn(str(corpus / 'http1_head'), reproduction) + restored.unlink() + restored.symlink_to(ROOT / 'configure.ac') + bad = subprocess.run(['bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'), + '--build-dir', str(location / 'bad'), '--compiler', str(compiler), + '--targets', 'http1_head', '--corpus-dir', str(corpus)], + capture_output=True, text=True, timeout=10) + self.assertNotEqual(bad.returncode, 0) + self.assertIn('symlink', bad.stderr) + def test_requested_unavailable_compiler_fails(self): with tempfile.TemporaryDirectory() as build: result = subprocess.run(['bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'), diff --git a/test/integ/release_ci_test.py b/test/integ/release_ci_test.py new file mode 100755 index 000000000..edfd0aee2 --- /dev/null +++ b/test/integ/release_ci_test.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Selected archive and publication decisions exercise the real readiness gate.""" +import copy +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts/lib')) +import release_gate_runner_test as release_contracts + + +class PublicationContracts(unittest.TestCase): + def setUp(self): + import hashlib + release_contracts.ReleaseContracts.setUp(self) + self.artifact_identity['source']['working_diff_sha256']=hashlib.sha256(b'').hexdigest() + self.identity_path.write_text(json.dumps(self.artifact_identity)) + @classmethod + def setUpClass(cls): + release_contracts.ReleaseContracts.setUpClass() + cls.gate = release_contracts.ReleaseContracts.gate + cls.ci_path = ROOT / 'scripts/lib/release_ci.py' + + def ci(self): + self.assertTrue(self.ci_path.exists(), 'selected-archive CI acceptance helper missing') + spec = importlib.util.spec_from_file_location('release_ci', self.ci_path) + module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + return module + + def publisher(self): + import ast + import textwrap + text=(ROOT/'.github/workflows/v3-release-acceptance.yml').read_text() + self.assertIn(' publish:\n',text,'isolated publisher job missing') + job=text.split(' publish:\n',1)[1] + self.assertNotIn('checkout@',job) + script=textwrap.dedent(job.split("python3 - <<'PYTHON'\n",1)[1]).rsplit('\nPYTHON',1)[0] + tree=ast.parse(script) + tree.body=[node for node in tree.body if isinstance(node,(ast.Import,ast.ImportFrom,ast.FunctionDef))] + namespace={}; exec(compile(tree,'publisher','exec'),namespace) + return namespace['publish_checked'] + + def publish(self, authorization, binding, archive, publication): + self.publisher()(authorization,binding,archive,self.complete(),'d'*40,publication) + + def complete(self): + manifest = {'schema_version': 1, 'inventory': self.gate.inventory_document(), + 'release_artifact': self.artifact_identity, 'artifact_identity_path': str(self.identity_path), + 'artifacts': self.receipt['artifacts'], 'stages': [], 'platforms': []} + for name in self.gate.LOCAL_GATES + self.gate.PLATFORMS: + identity = self.gate.expected_identity(manifest, name) + row = {'id': name, 'status': 'passed', 'artifacts': self.receipt['artifacts'], + 'identity': identity, 'receipt': dict(self.receipt, identity=identity, gate=name)} + manifest['stages' if name in self.gate.LOCAL_GATES else 'platforms'].append(row) + return manifest + + def binding(self): + return dict(repository='etr/libhttpserver', run_id='12', run_attempt='1', + source_commit='d' * 40, tag='3.0.0', archive_sha256=self.gate.digest(self.binary), + event='push', artifact_id='23', workflow_source='d'*40, producer_ids=['FreeBSD', 'Darwin', 'Windows'], + producer_artifacts={name:dict(id='24',digest='a'*64) for name in ('freebsd','darwin','windows')}) + + def test_valid_publication_requires_actual_readiness_then_calls_side_effect_once(self): + ci = self.ci(); manifest = self.complete(); binding = self.binding() + publish = Mock() + authorization = ci.authorize(self.gate, manifest, binding) + self.publish(authorization, binding, self.binary, publish) + publish.assert_called_once() + + def test_every_missing_or_unsuccessful_gate_and_platform_blocks_publication(self): + ci = self.ci() + for axis in ('stages', 'platforms'): + for index in range(len(self.complete()[axis])): + for damage in ('missing', 'failed', 'duplicate'): + manifest = self.complete(); publish = Mock() + if damage == 'missing': manifest[axis].pop(index) + if damage == 'failed': manifest[axis][index]['status'] = 'failed' + if damage == 'duplicate': manifest[axis].append(copy.deepcopy(manifest[axis][index])) + with self.subTest(axis=axis, index=index, damage=damage), self.assertRaises(ValueError): + self.publish(ci.authorize(self.gate, manifest, self.binding()), self.binding(), self.binary, publish) + publish.assert_not_called() + + def test_wrong_source_archive_run_tag_and_pr_cannot_publish(self): + ci = self.ci(); manifest = self.complete(); binding = self.binding() + authorization = ci.authorize(self.gate, manifest, binding) + for key in ('repository', 'run_id', 'run_attempt', 'source_commit', 'tag', 'artifact_id'): + altered = dict(binding, **{key: 'wrong'}); publish = Mock() + with self.subTest(key=key), self.assertRaises(ValueError): + self.publish(authorization, altered, self.binary, publish) + publish.assert_not_called() + for alteration in ({'event': 'pull_request'}, {'source_commit': 'a' * 40}, + {'archive_sha256': 'a' * 64}, {'tag': '3.0.0; touch marker'}, + {'producer_ids': ['Darwin']}): + with self.subTest(alteration=alteration), self.assertRaises(ValueError): + ci.authorize(self.gate, manifest, dict(binding, **alteration)) + self.binary.write_text('changed after acceptance') + publish = Mock() + with self.assertRaises(ValueError): self.publish(authorization, binding, self.binary, publish) + publish.assert_not_called() + + def test_ci_entrypoint_rejects_unresolved_request_without_side_effects(self): + import subprocess + script = ROOT / 'scripts/run-v3-release-ci.py' + self.assertTrue(script.exists(), 'selected archive CI entrypoint missing') + output=self.directory/'ci' + result=subprocess.run([sys.executable,str(script),'prepare','--output',str(output), + '--commit','$(touch marker)','--version','3.0.0'],cwd=self.directory,capture_output=True,text=True) + self.assertNotEqual(result.returncode,0) + self.assertFalse((self.directory/'marker').exists()) + self.assertFalse(output.exists()) + + def test_authorization_requires_content_bound_platform_producer_artifacts(self): + ci=self.ci() + for damage in ('missing','wrong-id','wrong-digest'): + binding=self.binding() + if damage=='missing': binding['producer_artifacts'].pop('darwin') + if damage=='wrong-id': binding['producer_artifacts']['darwin']['id']='$(touch marker)' + if damage=='wrong-digest': binding['producer_artifacts']['darwin']['digest']='old' + with self.subTest(damage=damage),self.assertRaises(ValueError): ci.authorize(self.gate,self.complete(),binding) + + def test_producer_binding_cannot_change_after_authorization(self): + ci=self.ci(); binding=self.binding(); authorization=ci.authorize(self.gate,self.complete(),binding) + binding['producer_artifacts']['darwin']['id']='25' + with self.assertRaises(ValueError): self.publish(authorization,binding,self.binary,Mock()) + + def test_actual_inline_publisher_calls_once_only_after_real_readiness(self): + publisher=self.publisher() + ci=self.ci(); binding=self.binding(); manifest=self.complete() + authorization=ci.authorize(self.gate,manifest,binding) + release=Mock() + publisher(authorization,binding,self.binary,manifest,'d'*40,release) + release.assert_called_once() + for alteration in ('head','manifest','archive','binding'): + release=Mock(); changed=copy.deepcopy(manifest); expected=copy.deepcopy(binding); tag_commit='d'*40 + if alteration=='head': tag_commit='e'*40 + if alteration=='manifest': changed['platforms'].pop() + if alteration=='archive': self.binary.write_text('different archive') + if alteration=='binding': expected['run_attempt']='2' + with self.subTest(alteration=alteration),self.assertRaises((ValueError,AssertionError)): + publisher(authorization,expected,self.binary,changed,tag_commit,release) + release.assert_not_called() + + def test_immutable_download_checks_source_run_attempt_digest_and_zip_members(self): + import hashlib + import zipfile + ci=self.ci() + archive=self.directory/'artifact.zip' + with zipfile.ZipFile(archive,'w') as stream: stream.writestr('selected.json','selected bytes') + expected=dict(repository='etr/libhttpserver',run_id='12',run_attempt='1',source_commit='d'*40,artifact_id='23') + metadata=dict(id=23,expired=False,created_at='2026-10-15T03:01:00Z',workflow_run={'id':12,'head_sha':'d'*40},digest='sha256:'+hashlib.sha256(archive.read_bytes()).hexdigest()) + run=dict(id=12,run_attempt=1,run_started_at='2026-10-15T03:00:00Z',repository={'full_name':'etr/libhttpserver'},head_sha='d'*40) + ci.verify_download(archive,metadata,run,expected,metadata['digest'],self.directory/'download') + for field,value in [('id',24),('expired',True),('digest','sha256:'+'a'*64),('created_at','2026-10-14T03:01:00Z')]: + altered=dict(metadata,**{field:value}) + with self.subTest(field=field),self.assertRaises(ValueError): + ci.verify_download(archive,altered,run,expected,metadata['digest'],self.directory/field) + for changed in (dict(run,run_attempt=2),dict(run,head_sha='a'*40),dict(run,repository={'full_name':'other/repo'})): + with self.assertRaises(ValueError): ci.verify_download(archive,metadata,changed,expected,metadata['digest'],self.directory/'wrongrun') + for index,name in enumerate(('../outside','/outside','C:/outside','a\\..\\outside')): + bad=self.directory/('escape-%d.zip'%index) + with zipfile.ZipFile(bad,'w') as stream: stream.writestr(name,'escape') + digest='sha256:'+hashlib.sha256(bad.read_bytes()).hexdigest() + with self.subTest(name=name),self.assertRaises(ValueError): + ci.verify_download(bad,dict(metadata,digest=digest),run,expected,digest,self.directory/('badzip-%d'%index)) + + def test_exact_commit_request_rejects_modified_and_untracked_source_inputs(self): + import subprocess + ci=self.ci(); root=self.directory/'checkout'; (root/'scripts').mkdir(parents=True) + for name in ('configure.ac','ChangeLog','scripts/validate-version.sh'): + (root/name).write_bytes((ROOT/name).read_bytes()) + def git(*args): + return subprocess.check_output(['git','-C',str(root),'-c','core.hooksPath=/dev/null',*args],stderr=subprocess.PIPE,text=True) + git('init','-q'); git('add','.') + git('-c','user.name=CI fixture','-c','user.email=ci@example.invalid','commit','-qm','fixture') + commit=git('rev-parse','HEAD').strip() + ci.validate_request(root,commit,'3.0.0') + original=(root/'configure.ac').read_bytes(); (root/'configure.ac').write_bytes(original+b'\n') + with self.assertRaises(ValueError): ci.validate_request(root,commit,'3.0.0') + (root/'configure.ac').write_bytes(original); (root/'injected.cpp').write_text('untracked source') + with self.assertRaises(ValueError): ci.validate_request(root,commit,'3.0.0') + + def test_publication_rejects_self_consistent_dirty_source_snapshot(self): + ci=self.ci(); manifest=self.complete() + manifest['release_artifact']['source']['working_diff_sha256']='a'*64 + self.identity_path.write_text(json.dumps(manifest['release_artifact'])) + self.gate.check_tag_ready(manifest) + with self.assertRaises(ValueError): ci.authorize(self.gate,manifest,self.binding()) + + def test_requested_commit_and_version_are_data_and_must_match_checkout(self): + ci = self.ci() + for commit, version in [('d'*40, '3.0.0; touch marker'), ('$(touch marker)', '3.0.0'), ('e'*40, '3.0.0')]: + with self.subTest(commit=commit,version=version), self.assertRaises(ValueError): + ci.validate_request(ROOT, commit, version) + + def test_request_requires_a_git_checkout_even_when_versions_are_valid(self): + ci=self.ci() + with self.assertRaises(ValueError): ci.validate_request(self.directory,'e'*40,'3.0.0') + + def test_selected_archive_relocation_preserves_source_and_rejects_tampering(self): + import io + import tarfile + ci = self.ci() + checkout = self.directory / 'checkout'; checkout.mkdir() + (checkout / 'configure.ac').write_text('source') + (checkout / 'specs').mkdir(); (checkout / 'specs/product.md').write_text('tooling') + archive = self.directory / 'candidate.tar.gz' + with tarfile.open(archive, 'w:gz') as stream: + for name in ('configure.ac',): + stream.add(checkout / name, arcname='libhttpserver-3.0.0/' + name) + source = dict(head='d' * 40, files_sha256={name: self.gate.digest(checkout / name) + for name in ('configure.ac', 'specs/product.md')}, snapshot_sha256='b' * 64) + identity = ci.selected_archive(checkout, source, archive, self.gate.digest(archive), self.directory / 'relocated') + self.assertEqual(identity['files_sha256'], source['files_sha256']) + self.assertEqual((Path(identity['source_root']) / 'specs/product.md').read_text(), 'tooling') + (checkout / 'configure.ac').write_text('another commit') + with self.assertRaises(ValueError): + ci.selected_archive(checkout, source, archive, self.gate.digest(archive), self.directory / 'wrong-source') + with self.assertRaises(ValueError): + ci.selected_archive(checkout, source, archive, 'a' * 64, self.directory / 'wrong-archive') + for kind in ('symlink', 'hardlink', 'absolute', 'traversal', 'duplicate'): + bad = self.directory / (kind + '.tar.gz') + with tarfile.open(bad, 'w:gz') as stream: + item = tarfile.TarInfo('root/configure.ac'); item.size = 1 + stream.addfile(item, io.BytesIO(b'x')) + item = tarfile.TarInfo({'absolute':'/outside', 'traversal':'../outside'}.get(kind, 'root/configure.ac')) + if kind == 'symlink': item.type=tarfile.SYMTYPE; item.linkname='/outside' + if kind == 'hardlink': item.type=tarfile.LNKTYPE; item.linkname='root/configure.ac' + stream.addfile(item) + with self.subTest(kind=kind), self.assertRaises(ValueError): + ci.selected_archive(checkout, source, bad, self.gate.digest(bad), self.directory / kind) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/integ/release_platform_bundle_test.py b/test/integ/release_platform_bundle_test.py new file mode 100755 index 000000000..908a48aa8 --- /dev/null +++ b/test/integ/release_platform_bundle_test.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +import importlib.util +import io +import json +from pathlib import Path +import sys +import tarfile +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts/lib')) + + +class PlatformBundles(unittest.TestCase): + def module(self): + path = ROOT / 'scripts/lib/release_platform.py' + self.assertTrue(path.exists(), 'portable platform receipt implementation missing') + spec = importlib.util.spec_from_file_location('release_platform', path) + module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + return module + + def test_catalog_relocates_original_bytes_and_rejects_tampering_missing_and_symlinks(self): + module = self.module() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); original = root / 'original.log' + original.write_text('retained immutable native results') + bundle = root / 'bundle' + module.retain(bundle, 'Darwin', {'id': 'Darwin'}, [original]) + moved = root / 'moved'; bundle.rename(moved); original.unlink() + resolver = module.Catalog(moved) + self.assertEqual(resolver.path(str(original)).read_text(), 'retained immutable native results') + with self.assertRaises(ValueError): resolver.path('/missing/record') + retained = resolver.path(str(original)); retained.write_text('tampered') + with self.assertRaises(ValueError): module.Catalog(moved) + retained.unlink(); retained.symlink_to(ROOT / 'configure.ac') + with self.assertRaises(ValueError): module.Catalog(moved) + + def test_original_and_resolved_tool_paths_both_retain_exact_bytes(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); binary=root/'binary'; binary.write_text('native selected compiler') + alias=root/'compiler'; alias.symlink_to(binary) + catalog=module.retain(root/'bundle','Windows',{'id':'Windows'},[alias]) + self.assertEqual(catalog.path(str(alias)).read_bytes(),catalog.path(str(binary)).read_bytes()) + + def test_equivalent_relative_dependency_paths_preserve_both_original_labels(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); (root/'other').mkdir(); file=root/'file'; file.write_text('dependency bytes') + alias=root/'other/../file' + catalog=module.retain(root/'bundle','FreeBSD',{'id':'FreeBSD'},[alias]) + self.assertEqual(catalog.path(str(alias)).read_bytes(),catalog.path(str(file)).read_bytes()) + + def test_windows_logical_paths_resolve_without_rewriting_original_receipts(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); original=root/'log'; original.write_text('native Windows bytes') + retained=root/'bundle'; module.retain(retained,'Windows',{'id':'Windows'},[original]) + catalog=json.loads((retained/'catalog.json').read_text()) + record=catalog['paths'].pop(str(original)); catalog['paths'][r'C:\native\matrix\log']=record + (retained/'catalog.json').write_text(json.dumps(catalog)) + resolver=module.Catalog(retained) + self.assertEqual(resolver.path('C:\\native/matrix/../matrix/log').read_text(),'native Windows bytes') + + def test_provider_client_and_dependency_records_require_retained_original_bytes(self): + module=self.module() + import hashlib + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); client=root/'client.py'; client.write_text('selected client module') + document={'nested':{'path':str(client),'sha256':hashlib.sha256(client.read_bytes()).hexdigest()}, + 'compile':{'dependency_sha256':{str(client):hashlib.sha256(client.read_bytes()).hexdigest()}}} + self.assertEqual(module.referenced_files(document),{client}) + client.write_text('substituted client module') + with self.assertRaises(ValueError): module.referenced_files(document) + client.unlink() + with self.assertRaises(ValueError): module.referenced_files(document) + + def test_duplicate_catalog_keys_are_rejected(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); file=root/'file'; file.write_text('bytes') + module.retain(root/'bundle','FreeBSD',{'id':'FreeBSD'},[file]) + catalog=root/'bundle/catalog.json' + catalog.write_text(catalog.read_text().replace('"schema_version": 1','"schema_version": 1, "schema_version": 1')) + with self.assertRaises(ValueError): module.Catalog(root/'bundle') + + def test_receipt_bytes_are_immutable_even_for_json_equivalent_edits(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); file=root/'file'; file.write_text('retained bytes') + module.retain(root/'bundle','Darwin',{'id':'Darwin'},[file]) + receipt=root/'bundle/receipt.json' + receipt.write_bytes(receipt.read_bytes()+b'\n') + with self.assertRaises(ValueError): module.Catalog(root/'bundle') + + def test_existing_sealed_os_leaves_do_not_require_fabricated_binary_files(self): + module=self.module() + for name in ('/usr/lib/libSystem.B.dylib',r'C:\Windows\System32\api-ms-win-core-synch-l1-2-0.dll'): + graph={'nodes':[dict(id=name,sha256=None,provenance='OS/API-set platform leaf')]} + self.assertEqual(module.referenced_files(graph),set()) + with self.assertRaises(ValueError): + module.referenced_files({'nodes':[dict(id='/private/provider/libssl.dylib',sha256=None,provenance='OS/API-set platform leaf')]}) + + def test_bundle_extraction_rejects_escape_duplicates_links_and_extra_files(self): + module = self.module() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, kind in (('../outside', None), ('/outside', None), ('Darwin/link', tarfile.SYMTYPE), + ('Darwin/link', tarfile.LNKTYPE), ('Darwin/extra', None)): + with self.subTest(name=name, kind=kind): + archive = root / 'bundle.tar.gz' + with tarfile.open(archive, 'w:gz') as stream: + item = tarfile.TarInfo(name); item.size=1 + if kind: item.type=kind; item.linkname='/outside'; item.size=0 + stream.addfile(item, io.BytesIO(b'x') if item.size else None) + with self.assertRaises(ValueError): module.unpack(archive, root / ('extract-' + str(kind) + str(len(name)))) + + def test_valid_archive_import_rejects_duplicate_members_and_namespace_collisions(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); file=root/'file'; file.write_text('retained bytes') + module.retain(root/'Darwin','Darwin',{'id':'Darwin'},[file]) + archive=root/'valid.tar.gz' + with tarfile.open(archive,'w:gz') as stream: stream.add(root/'Darwin',arcname='Darwin') + module.unpack(archive,root/'imported') + with self.assertRaises(ValueError): module.unpack(archive,root/'imported') + duplicate=root/'duplicate.tar.gz' + with tarfile.open(duplicate,'w:gz') as stream: + stream.add(root/'Darwin',arcname='Darwin') + stream.add(root/'Darwin/catalog.json',arcname='Darwin/catalog.json') + with self.assertRaises(ValueError): module.unpack(duplicate,root/'duplicate') + + def test_platform_origin_rejects_stale_summary_pruned_suite_and_provider_substitution(self): + module = self.module() + import package_matrix as matrix + import release_artifact as identity + import release_checks as checks + import hashlib + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); package = root / 'matrix'; package.mkdir() + inventory = matrix.load() + def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + header = root / 'provider.h'; header.write_bytes(b'selected OpenSSL\r\n') + selected = dict(archive_sha256='a'*64, snapshot_sha256='b'*64, extracted_sha256=hashlib.sha256(json.dumps({},sort_keys=True).encode()).hexdigest(), tls='both', sanitizer='none') + write(package/'summary.json', dict(status='passed', host='Darwin', lanes={'no':{},'yes':{}}, + revision='d'*40, provider_header_sha256=identity.sha256(header), + release_artifact={'archive':{'sha256':'a'*64}, 'source':{'snapshot_sha256':'b'*64,'head':'d'*40}})) + for mode in ('no','yes'): + lane=package/('tls-'+mode); prefix=lane/'prefix' + (prefix/'lib').mkdir(parents=True) + (prefix/'lib/libhttpserver.dylib').write_text('selected native library') + public={} + for linkage in ('shared','static'): + graph={'nodes':[{'id':str(prefix/'lib/libhttpserver.dylib'),'sha256':identity.sha256(prefix/'lib/libhttpserver.dylib')}] if linkage=='shared' else []} + public[linkage]=dict(dependencies=graph, compile={'dependencies':[]}, + cells=[{'configurations':inventory['cases']},{'configurations':inventory['cases']}]) + suites={} + for name, definition in matrix.suites(inventory, 'Darwin', mode).items(): + cases=definition.get('required_cases',[]) or ['fixture_case'] + log=lane/('integration-'+linkage)/(name+'.log') + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(''.join('Running test (%d): %s\n'%(i,case) for i,case in enumerate(cases))+ + '%d tests executed\n%d checks\n-> 0 failures\n-> 0 skipped\n'%(len(cases),len(cases))) + suites[name]=dict(matrix.suite_result(log.read_text(),0),status='passed', log=str(log)) + write(lane/('integration-'+linkage)/'suites.json',suites) + write(lane/'audit.json',dict(violations=[],tls_mode=mode,public=public)) + origin=dict(package=str(package), source_root=str(root/'source'), source_commit='d'*40, provider_header=str(header), native={}) + # This test checks installed matrix origin independently; native origin is mandatory in complete bundles. + counts=module.package_origin(ROOT, origin, lambda p: json.loads(Path(p).read_text()), lambda p: Path(p).read_text(), selected, 'Darwin') + self.assertGreater(counts['tests'],0) + wrong_origin=dict(origin,source_commit='e'*40) + with self.assertRaises(ValueError): + module.package_origin(ROOT,wrong_origin,lambda p:json.loads(Path(p).read_text()),lambda p:Path(p).read_text(),selected,'Darwin') + for mode in ('no','yes'): + build=root/('native-'+mode); (build/'test').mkdir(parents=True) + names=['protocol_fuzz_replay','hpack_corpus','http2_fuzz_replay','quic_fuzz_replay', + 'quic_recovery_fuzz_replay','http3_fuzz_replay','qpack_fuzz_replay','io_kqueue_backend'] + if mode=='yes': names.append('quic_tls_callbacks_fuzz_replay') + for name in names: (build/'test'/(name+'.trs')).write_text(':test-result: PASS\n') + (build/'test/test-suite.log').write_text('# TOTAL: %d\n# PASS: %d\n# SKIP: 0\n# XFAIL: 0\n# FAIL: 0\n# XPASS: 0\n# ERROR: 0\n'%(len(names),len(names))) + origin['native'][mode]=dict(build=str(build), tests=names) + counts=checks.counts(counts['tests']+len(names),counts['checks']+len(names)) + artifacts=[dict(path=str(header),sha256=identity.sha256(header),bytes=header.stat().st_size)] + row=dict(id='Darwin',identity=selected,archive_sha256='a'*64,origin=origin,status='passed',artifacts=artifacts, + receipt=dict(status='passed',gate='Darwin',identity=selected,counts=counts,artifacts=artifacts)) + retained=root/'retained' + catalog=module.retain(retained,'Darwin',row,[p for p in root.rglob('*') if p.is_file()]) + module.validate_origin(ROOT,row,catalog) + manifest=dict(release_artifact={'archive':{'sha256':'a'*64},'source':{'snapshot_sha256':'b'*64,'head':'d'*40},'files_sha256':{}}, + platforms=[dict(id=p,status='pending') for p in module.PLATFORMS]) + module.import_bundle(ROOT, manifest, catalog, expected=selected) + self.assertEqual(manifest['platforms'][1]['status'],'passed') + gate=checks.load_runner('run-v3-release-gates.py') + gate.verify_gate_rows(manifest,[manifest['platforms'][1]]) + raw_catalog=(catalog.directory/'catalog.json').read_bytes() + (catalog.directory/'catalog.json').write_bytes(raw_catalog+b'\n') + with self.assertRaisesRegex(ValueError,'changed artifact'): + gate.verify_gate_rows(manifest,[manifest['platforms'][1]]) + (catalog.directory/'catalog.json').write_bytes(raw_catalog) + with self.assertRaises(ValueError): module.import_bundle(ROOT, manifest, catalog, expected=selected) + wrong=json.loads(json.dumps(row)); wrong['archive_sha256']='e'*64 + (catalog.directory/'receipt.json').write_text(json.dumps(wrong)) + fresh=dict(release_artifact={'source':{'head':'d'*40}},platforms=[dict(id=p,status='pending') for p in module.PLATFORMS]) + with self.assertRaises(ValueError): module.import_bundle(ROOT, fresh, catalog, expected=selected) + (catalog.directory/'receipt.json').write_text(json.dumps(row)) + incomplete=json.loads(json.dumps(row)); incomplete['origin']['native']['yes']['tests'].remove('quic_tls_callbacks_fuzz_replay') + with self.assertRaises(ValueError): module.validate_origin(ROOT,incomplete,catalog) + catalog.path(str(header)).write_text('corrupt retained provider') + with self.assertRaises(ValueError): module.validate_origin(ROOT,row,catalog) + def windows(value): + if isinstance(value,dict): return {k:windows(v) for k,v in value.items()} + if isinstance(value,list): return [windows(v) for v in value] + if isinstance(value,str) and value.startswith(str(root)): + return value.replace(str(root),'C:/native').replace('/','\\') + return value + def physical(value): + return Path(str(value).replace('\\','/').replace('C:/native',str(root))) + module.package_origin(ROOT,windows(origin),lambda p:windows(json.loads(physical(p).read_text())), + lambda p:physical(p).read_text(),selected,'Darwin',lambda p:physical(p).read_bytes()) + suites_path=package/'tls-no/integration-shared/suites.json' + suites=json.loads(suites_path.read_text()); first=next(iter(suites)) + altered=json.loads(suites_path.read_text()); altered[first]['tests']+=1; write(suites_path,altered) + with self.assertRaises(ValueError): module.package_origin(ROOT,origin,lambda p:json.loads(Path(p).read_text()),lambda p:Path(p).read_text(),selected,'Darwin') + write(suites_path,suites); removed=suites.pop(first); write(suites_path,suites) + with self.assertRaises(ValueError): module.package_origin(ROOT,origin,lambda p:json.loads(Path(p).read_text()),lambda p:Path(p).read_text(),selected,'Darwin') + suites[first]=removed; write(suites_path,suites); header.write_text('different provider') + with self.assertRaises(ValueError): module.package_origin(ROOT,origin,lambda p:json.loads(Path(p).read_text()),lambda p:Path(p).read_text(),selected,'Darwin') + + +if __name__ == '__main__': unittest.main() From 7a2334c269ee3914adadc0c0ab3b67136cca1eae Mon Sep 17 00:00:00 2001 From: Sebastiano Merlino Date: Thu, 15 Oct 2026 20:59:31 -0700 Subject: [PATCH 2/4] FEATURE-ci-v3-integration: execute VM archive staging as separate commands Exercise the actual FreeBSD staging step with a selected descriptor, preserving newlines so the VM receives the archive inputs. Behavioral RED/GREEN and actionlint pass; focused contract total is 100. --- .github/workflows/v3-release-acceptance.yml | 4 +++- scripts/test-v3-ci-workflows.py | 13 +++++++++++++ specs/tasks/FEATURE-ci-v3-integration.md | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/v3-release-acceptance.yml b/.github/workflows/v3-release-acceptance.yml index a5a928801..0a6385d32 100644 --- a/.github/workflows/v3-release-acceptance.yml +++ b/.github/workflows/v3-release-acceptance.yml @@ -220,7 +220,9 @@ jobs: persist-credentials: false - name: Download immutable selected archive run: python3 scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected" - - run: mkdir -p build cp -R "$RUNNER_TEMP/selected" build/ci-selected-input + - run: | + mkdir -p build + cp -R "$RUNNER_TEMP/selected" build/ci-selected-input - uses: vmactions/freebsd-vm@c9f815bc7aa0d34c9fdd0619b034a32d6ca7b57e with: release: '14.3' diff --git a/scripts/test-v3-ci-workflows.py b/scripts/test-v3-ci-workflows.py index 7d97f2edf..e20c0531b 100755 --- a/scripts/test-v3-ci-workflows.py +++ b/scripts/test-v3-ci-workflows.py @@ -17,6 +17,19 @@ def workflow(name): class CIWorkflows(unittest.TestCase): + def test_freebsd_stages_selected_archive_for_vm_transfer(self): + steps = workflow('v3-release-acceptance.yml')['jobs']['freebsd']['steps'] + run = next(step['run'] for step in steps if 'ci-selected-input' in step.get('run', '')) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + selected = root / 'runner' / 'selected' + selected.mkdir(parents=True) + (selected / 'selected.json').write_text('selected archive identity') + result = subprocess.run(['bash', '-e', '-c', run], cwd=root, + env=dict(os.environ, RUNNER_TEMP=str(root / 'runner')), capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((root / 'build/ci-selected-input/selected.json').read_text(), 'selected archive identity') + def test_release_request_resolver_emits_structured_commit_and_version(self): steps = workflow('v3-release-acceptance.yml')['jobs']['prepare']['steps'] run = next(step['run'] for step in steps if step.get('id') == 'request') diff --git a/specs/tasks/FEATURE-ci-v3-integration.md b/specs/tasks/FEATURE-ci-v3-integration.md index 4cb4329d6..fba529bbc 100644 --- a/specs/tasks/FEATURE-ci-v3-integration.md +++ b/specs/tasks/FEATURE-ci-v3-integration.md @@ -29,7 +29,7 @@ are unchanged and cannot replace fresh evidence. Local verification: native TLS-off/on suites (239/270 tests, two documented nonlocal backend skips in each), all eleven sanitizer fuzz targets, genuine -Doxygen/Graphviz, GCC QUIC replay and 99 focused CI/release/fuzz contracts passed. +Doxygen/Graphviz, GCC QUIC replay and 100 focused CI/release/fuzz contracts passed. The fresh Linux installed attempt passed TLS-off but failed TLS-on public TLS at unprivileged TCP443 binding. Its unsuccessful receipt remains unchanged. A fresh focused public TLS run using the existing isolated network path passed all four From 08c5b7b5eb5e992f5a12d88136064010fc435f9d Mon Sep 17 00:00:00 2001 From: Sebastiano Merlino Date: Thu, 15 Oct 2026 21:01:58 -0700 Subject: [PATCH 3/4] FEATURE-ci-v3-integration: distribute workflow contract inputs A real relocated archive exposed missing native and ChangeLog workflow inputs for the distributed CI contract script. Include all reviewed workflow dependencies; fresh distribution and relocated workflow/package contracts now pass. --- Makefile.am | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.am b/Makefile.am index 0840e5155..2ae1270c2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -623,5 +623,7 @@ EXTRA_DIST += docs/v3-protocol-limits.md scripts/check-v3-protocol-limits.py tes EXTRA_DIST += scripts/run-v3-release-ci.py scripts/lib/release_ci.py scripts/lib/release_platform.py scripts/test-v3-ci-workflows.py EXTRA_DIST += .github/workflows/v3-release-acceptance.yml +EXTRA_DIST += .github/workflows/v3-native.yml .github/workflows/changelog-check.yml +EXTRA_DIST += .github/workflows/codeql-analysis.yml .github/workflows/verify-build.yml .github/workflows/release.yml EXTRA_DIST += docs/v3-ci.md From 63ffaa2a239a3204ff2a386508eb020614620edc Mon Sep 17 00:00:00 2001 From: Sebastiano Merlino Date: Thu, 15 Oct 2026 21:14:09 -0700 Subject: [PATCH 4/4] FEATURE-ci-v3-integration: bind PR artifact origin to API branch head The real PR artifact and run APIs report the branch head while GITHUB_SHA identifies the test merge checkout. Supply the trusted event head explicitly to download origin checks, retaining exact selected merge source, archive, run, attempt, repository and digest validation. Reproduced the actual captured artifact failure twice before repair. Real ZIP replay now validates the branch-head origin and unchanged merge descriptor/archive; 15 release contracts and 10 workflow contracts pass, including negative source/origin/run/attempt/digest controls. Actionlint and Python compilation pass. Native/fuzz/docs/public443 evidence is reused unchanged; fresh actual CI remains root-owned. --- .github/workflows/v3-release-acceptance.yml | 3 ++ docs/v3-ci.md | 6 +++ scripts/lib/release_ci.py | 4 +- scripts/run-v3-release-ci.py | 2 +- scripts/test-v3-ci-workflows.py | 5 ++ specs/tasks/FEATURE-ci-v3-integration.md | 9 +++- test/integ/release_ci_test.py | 55 ++++++++++++++++++++- 7 files changed, 78 insertions(+), 6 deletions(-) diff --git a/.github/workflows/v3-release-acceptance.yml b/.github/workflows/v3-release-acceptance.yml index 0a6385d32..2aea43330 100644 --- a/.github/workflows/v3-release-acceptance.yml +++ b/.github/workflows/v3-release-acceptance.yml @@ -26,6 +26,9 @@ name: v3 selected archive acceptance permissions: contents: read actions: read +env: + # Artifact/run APIs identify the PR branch head; checkout tests its merge SHA. + WORKFLOW_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} jobs: prepare: runs-on: ubuntu-24.04 diff --git a/docs/v3-ci.md b/docs/v3-ci.md index 004b90629..b567f60e5 100644 --- a/docs/v3-ci.md +++ b/docs/v3-ci.md @@ -93,6 +93,12 @@ a download action is insufficient. Artifact creation must belong to the current run attempt; rerun all jobs to produce fresh release proof rather than reusing artifacts from successful jobs in an earlier attempt. +On PR runs, artifact/run API metadata identifies the PR branch head, while +`GITHUB_SHA` and the selected checkout identify the test merge commit. Downloads +bind the API head to the trusted event's PR head. Selected archive preparation +and platform proof still require the exact selected checkout commit, source +snapshot and archive bytes; these two source identities are not interchangeable. + The final publisher alone receives `contents: write`. It checks successful mandatory dependencies, same-run artifacts, manifest and archive hashes and the current tag target before `gh release create --verify-tag`. It runs only inline diff --git a/scripts/lib/release_ci.py b/scripts/lib/release_ci.py index 7f4c13919..8beebf59b 100644 --- a/scripts/lib/release_ci.py +++ b/scripts/lib/release_ci.py @@ -85,9 +85,9 @@ def validate_request(root, commit, version): def verify_download(archive, metadata, run, expected, expected_digest, destination): require(str(metadata['id']) == expected['artifact_id'] and metadata['expired'] is False, 'wrong/expired artifact ID') - require(str(metadata['workflow_run']['id']) == expected['run_id'] and metadata['workflow_run']['head_sha'] == expected['source_commit'], 'artifact from another workflow source/run') + require(str(metadata['workflow_run']['id']) == expected['run_id'] and metadata['workflow_run']['head_sha'] == expected['workflow_head'], 'artifact from another workflow source/run') require(str(run['id']) == expected['run_id'] and str(run['run_attempt']) == expected['run_attempt'] - and run['repository']['full_name'] == expected['repository'] and run['head_sha'] == expected['source_commit'], 'wrong repository/source/run attempt') + and run['repository']['full_name'] == expected['repository'] and run['head_sha'] == expected['workflow_head'], 'wrong repository/source/run attempt') require(datetime.fromisoformat(metadata['created_at']) >= datetime.fromisoformat(run['run_started_at']), 'artifact belongs to an earlier run attempt') from release_artifact import sha256 digest = 'sha256:' + sha256(archive) diff --git a/scripts/run-v3-release-ci.py b/scripts/run-v3-release-ci.py index 37743248d..abd82e15f 100755 --- a/scripts/run-v3-release-ci.py +++ b/scripts/run-v3-release-ci.py @@ -129,7 +129,7 @@ def download(args): import subprocess from release_ci import verify_download expected=dict(repository=os.environ['GITHUB_REPOSITORY'],run_id=os.environ['GITHUB_RUN_ID'], - run_attempt=os.environ['GITHUB_RUN_ATTEMPT'],source_commit=os.environ['GITHUB_SHA'],artifact_id=args.artifact_id) + run_attempt=os.environ['GITHUB_RUN_ATTEMPT'],workflow_head=os.environ['WORKFLOW_HEAD_SHA'],artifact_id=args.artifact_id) require(args.artifact_id.isdigit(), 'invalid artifact ID') endpoint='repos/'+expected['repository']+'/actions/artifacts/'+args.artifact_id metadata=json.loads(subprocess.check_output(['gh','api',endpoint],text=True)) diff --git a/scripts/test-v3-ci-workflows.py b/scripts/test-v3-ci-workflows.py index e20c0531b..a1ebe838f 100755 --- a/scripts/test-v3-ci-workflows.py +++ b/scripts/test-v3-ci-workflows.py @@ -17,6 +17,11 @@ def workflow(name): class CIWorkflows(unittest.TestCase): + def test_artifact_api_head_is_bound_to_trusted_event_head(self): + lane = workflow('v3-release-acceptance.yml') + self.assertEqual(lane.get('env', {}).get('WORKFLOW_HEAD_SHA'), + '${{ github.event.pull_request.head.sha || github.sha }}') + def test_freebsd_stages_selected_archive_for_vm_transfer(self): steps = workflow('v3-release-acceptance.yml')['jobs']['freebsd']['steps'] run = next(step['run'] for step in steps if 'ci-selected-input' in step.get('run', '')) diff --git a/specs/tasks/FEATURE-ci-v3-integration.md b/specs/tasks/FEATURE-ci-v3-integration.md index fba529bbc..43fafc197 100644 --- a/specs/tasks/FEATURE-ci-v3-integration.md +++ b/specs/tasks/FEATURE-ci-v3-integration.md @@ -20,7 +20,11 @@ protocol behavior, default-branch setting, tag or release changes. and recheck the existing tag before the release side effect. - [x] Exercise publication refusal and success through the actual validator, with only the final publication side effect mocked. -- [ ] Root: run the actual GitHub PR pipeline; verify manual dispatch availability. +- [x] Root: verify manual native campaign dispatch invocation (accepted at + 08c5b7b5; campaign execution outcomes remain pending). +- [ ] Root: complete actual GitHub PR pipeline verification. First selected + archive attempt failed at API PR-head versus merge-SHA origin binding; + causal repair and negative controls passed locally; fresh CI is pending. - [ ] CI / v3 PR: obtain fresh full local gate and FreeBSD/Darwin/Windows receipts for the selected commit/archive. Missing evidence is not a release pass. @@ -29,7 +33,8 @@ are unchanged and cannot replace fresh evidence. Local verification: native TLS-off/on suites (239/270 tests, two documented nonlocal backend skips in each), all eleven sanitizer fuzz targets, genuine -Doxygen/Graphviz, GCC QUIC replay and 100 focused CI/release/fuzz contracts passed. +Doxygen/Graphviz, GCC QUIC replay and 100 initial CI/release/fuzz contracts passed. +The subsequent artifact-origin repair adds three focused contracts (103 total). The fresh Linux installed attempt passed TLS-off but failed TLS-on public TLS at unprivileged TCP443 binding. Its unsuccessful receipt remains unchanged. A fresh focused public TLS run using the existing isolated network path passed all four diff --git a/test/integ/release_ci_test.py b/test/integ/release_ci_test.py index edfd0aee2..61134b26a 100755 --- a/test/integ/release_ci_test.py +++ b/test/integ/release_ci_test.py @@ -151,7 +151,7 @@ def test_immutable_download_checks_source_run_attempt_digest_and_zip_members(sel ci=self.ci() archive=self.directory/'artifact.zip' with zipfile.ZipFile(archive,'w') as stream: stream.writestr('selected.json','selected bytes') - expected=dict(repository='etr/libhttpserver',run_id='12',run_attempt='1',source_commit='d'*40,artifact_id='23') + expected=dict(repository='etr/libhttpserver',run_id='12',run_attempt='1',workflow_head='d'*40,artifact_id='23') metadata=dict(id=23,expired=False,created_at='2026-10-15T03:01:00Z',workflow_run={'id':12,'head_sha':'d'*40},digest='sha256:'+hashlib.sha256(archive.read_bytes()).hexdigest()) run=dict(id=12,run_attempt=1,run_started_at='2026-10-15T03:00:00Z',repository={'full_name':'etr/libhttpserver'},head_sha='d'*40) ci.verify_download(archive,metadata,run,expected,metadata['digest'],self.directory/'download') @@ -168,6 +168,59 @@ def test_immutable_download_checks_source_run_attempt_digest_and_zip_members(sel with self.subTest(name=name),self.assertRaises(ValueError): ci.verify_download(bad,dict(metadata,digest=digest),run,expected,digest,self.directory/('badzip-%d'%index)) + def test_pr_download_binds_api_branch_head_and_retains_merge_source_descriptor(self): + import hashlib + import zipfile + ci=self.ci() + head='08c5b7b5eb5e992f5a12d88136064010fc435f9d' + merge='b5f8d51fe8b597fb3c720aa5459c04b9147c615b' + archive=self.directory/'pr-artifact.zip' + with zipfile.ZipFile(archive,'w') as stream: + stream.writestr('selected.json',json.dumps({'commit':merge,'source':{'head':merge}})) + digest='sha256:'+hashlib.sha256(archive.read_bytes()).hexdigest() + expected=dict(repository='etr/libhttpserver',run_id='38848731938',run_attempt='1',workflow_head=head,artifact_id='12024142302') + metadata=dict(id=12024142302,expired=False,created_at='2026-10-16T04:05:15Z', + workflow_run=dict(id=38848731938,head_sha=head),digest=digest) + run=dict(id=38848731938,run_attempt=1,head_sha=head,event='pull_request', + repository={'full_name':'etr/libhttpserver'},run_started_at='2026-10-16T04:04:45Z') + destination=self.directory/'pr-download' + ci.verify_download(archive,metadata,run,expected,digest,destination) + self.assertEqual(json.loads((destination/'selected.json').read_text())['commit'],merge) + for index, altered in enumerate((dict(expected,workflow_head=merge),dict(expected,workflow_head='a'*40), + dict(expected,run_id='38848731939'),dict(expected,run_attempt='2'),dict(expected,repository='other/repo'))): + with self.subTest(expected=altered),self.assertRaises(ValueError): + ci.verify_download(archive,metadata,run,altered,digest,self.directory/('wrong-pr-'+str(index))) + for field,value in (('id',12024142303),('workflow_run',dict(id=38848731939,head_sha=head)), + ('workflow_run',dict(id=38848731938,head_sha=merge)),('digest','sha256:'+'a'*64)): + with self.subTest(field=field,value=value),self.assertRaises(ValueError): + ci.verify_download(archive,dict(metadata,**{field:value}),run,expected,digest,self.directory/'wrong-origin') + + def test_download_entrypoint_uses_trusted_api_head_instead_of_pr_merge_sha(self): + import argparse + import hashlib + import io + import os + import zipfile + from unittest.mock import patch + spec=importlib.util.spec_from_file_location('release_ci_entrypoint',ROOT/'scripts/run-v3-release-ci.py') + entrypoint=importlib.util.module_from_spec(spec); spec.loader.exec_module(entrypoint) + head='08c5b7b5eb5e992f5a12d88136064010fc435f9d'; merge='b5f8d51fe8b597fb3c720aa5459c04b9147c615b' + stream=io.BytesIO() + with zipfile.ZipFile(stream,'w') as archive: archive.writestr('selected.json',json.dumps({'commit':merge})) + contents=stream.getvalue(); digest='sha256:'+hashlib.sha256(contents).hexdigest() + metadata=dict(id=12024142302,expired=False,created_at='2026-10-16T04:05:15Z', + workflow_run=dict(id=38848731938,head_sha=head),digest=digest) + run=dict(id=38848731938,run_attempt=1,head_sha=head,repository={'full_name':'etr/libhttpserver'},run_started_at='2026-10-16T04:04:45Z') + def transport(argv,stdout,check): + self.assertEqual(argv,['gh','api','repos/etr/libhttpserver/actions/artifacts/12024142302/zip']) + stdout.write(contents) + env=dict(GITHUB_REPOSITORY='etr/libhttpserver',GITHUB_RUN_ID='38848731938',GITHUB_RUN_ATTEMPT='1', + GITHUB_SHA=merge,WORKFLOW_HEAD_SHA=head) + args=argparse.Namespace(artifact_id='12024142302',digest=digest,output=self.directory/'cli-download') + with patch.dict(os.environ,env),patch('subprocess.check_output',side_effect=[json.dumps(metadata),json.dumps(run)]),patch('subprocess.run',side_effect=transport): + entrypoint.download(args) + self.assertEqual(json.loads((args.output/'selected.json').read_text())['commit'],merge) + def test_exact_commit_request_rejects_modified_and_untracked_source_inputs(self): import subprocess ci=self.ci(); root=self.directory/'checkout'; (root/'scripts').mkdir(parents=True)