diff --git a/.github/workflows/changelog-check.yml b/.github/workflows/changelog-check.yml index 632cedfb5..d2d66afb8 100644 --- a/.github/workflows/changelog-check.yml +++ b/.github/workflows/changelog-check.yml @@ -2,7 +2,10 @@ name: "ChangeLog Check" on: pull_request: - branches: [master] + branches: [master, v3] + +permissions: + contents: read jobs: changelog: @@ -10,14 +13,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: fetch-depth: 0 - name: Check for ChangeLog update + env: + BASE_REF: ${{ github.base_ref }} run: | # Get list of changed files - changed_files=$(git diff --name-only origin/master...HEAD) + git check-ref-format "refs/heads/$BASE_REF" + base=$(git rev-parse --verify "refs/remotes/origin/$BASE_REF^{commit}") + changed_files=$(git diff "$base...HEAD" --name-only --) if [ -z "$changed_files" ]; then echo "No files changed." diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 14bbdedba..2af674a19 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -18,10 +18,10 @@ name: "CodeQL" on: push: - branches: [master] + branches: [master, v3] pull_request: # The branches below must be a subset of the branches above - branches: [master] + branches: [master, v3] schedule: - cron: '0 4 * * 4' @@ -68,18 +68,32 @@ jobs: with: languages: ${{ matrix.language }} - # CodeQL's C/C++ database extractor traces the actual compile commands, so - # the library is built explicitly here rather than via the CodeQL Autobuild - # action (autobuild guesses the build and is intentionally not used). The - # build must be a clean from-scratch `make` so every TU is extracted; the - # in-source build (--enable-same-directory-build) keeps object files where - # the extractor expects them. + - name: Install build prerequisites + run: sudo apt-get update && sudo apt-get install -y autoconf automake libtool pkg-config perl + - name: Select source configuration + id: source + run: | + if grep -q 'libhttpserver_v3core_la_SOURCES' src/Makefile.am; then + echo 'native=true' >> "$GITHUB_OUTPUT" + fi + - name: Provision private native TLS provider + if: steps.source.outputs.native == 'true' + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/codeql-prerequisites" - name: Build the library (CodeQL manual mode) run: | - ./bootstrap ; - ./configure --enable-same-directory-build ; - make ; - timeout-minutes: 30 + ./bootstrap + mkdir "$RUNNER_TEMP/codeql-build" + cd "$RUNNER_TEMP/codeql-build" + if grep -q 'libhttpserver_v3core_la_SOURCES' "$GITHUB_WORKSPACE/src/Makefile.am"; then + export LD_LIBRARY_PATH="$RUNNER_TEMP/codeql-prerequisites/provider/lib" + export V3_TLS_CFLAGS="-I$RUNNER_TEMP/codeql-prerequisites/provider/include" + export V3_TLS_LIBS="-L$RUNNER_TEMP/codeql-prerequisites/provider/lib -lssl -lcrypto" + "$GITHUB_WORKSPACE/./configure" --enable-v3-tls + else + "$GITHUB_WORKSPACE/./configure" + fi + make -j2 + timeout-minutes: 40 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@411c4c9a36b3fca4d674f06b6396b2c6d23522c6 # v3.36.3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c5bccde7..a7afce090 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,8 @@ on: required: true permissions: - contents: write + contents: read + actions: read jobs: validate: @@ -20,43 +21,43 @@ jobs: outputs: version: ${{ steps.version.outputs.version }} is_prerelease: ${{ steps.version.outputs.is_prerelease }} + commit: ${{ steps.version.outputs.commit }} + native: ${{ steps.version.outputs.native }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + fetch-depth: 0 - - name: Determine version + - name: Resolve existing tag and source version id: version + env: + REQUEST_TAG: ${{ inputs.tag || github.ref_name }} run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - VERSION="${{ github.event.inputs.tag }}" - else - VERSION="${GITHUB_REF#refs/tags/}" - fi - VERSION="${VERSION#v}" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - if echo "$VERSION" | grep -qE '-(rc|alpha|beta)'; then - echo "is_prerelease=true" >> "$GITHUB_OUTPUT" - else - echo "is_prerelease=false" >> "$GITHUB_OUTPUT" - fi - - echo "Version: $VERSION" - - - name: Validate version consistency - run: | - chmod +x scripts/validate-version.sh - scripts/validate-version.sh "${{ steps.version.outputs.version }}" + python3 - <<'PYTHON' + import os,re,subprocess + from pathlib import Path + version=os.environ['REQUEST_TAG'] + if not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:rc|alpha|beta)[0-9]+)?',version): raise SystemExit('invalid version tag') + commit=subprocess.check_output(['git','rev-parse','--verify','refs/tags/'+version+'^{commit}'],text=True).strip() + subprocess.run(['git','checkout','--detach',commit],check=True) + subprocess.run(['bash','scripts/validate-version.sh',version],check=True) + native='true' if 'libhttpserver_v3core_la_SOURCES' in Path('src/Makefile.am').read_text() else 'false' + with open(os.environ['GITHUB_OUTPUT'],'a') as out: + out.write('version='+version+'\ncommit='+commit+'\nnative='+native+'\nis_prerelease='+str('-' in version).lower()+'\n') + PYTHON - name: Extract release notes + env: + VERSION: ${{ steps.version.outputs.version }} run: | chmod +x scripts/extract-release-notes.sh - scripts/extract-release-notes.sh "${{ steps.version.outputs.version }}" > release-notes.md + scripts/extract-release-notes.sh "$VERSION" > release-notes.md echo "--- Release notes ---" cat release-notes.md - name: Upload release notes - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: release-notes path: release-notes.md @@ -65,9 +66,12 @@ jobs: name: Build distribution tarball runs-on: ubuntu-latest needs: validate + if: needs.validate.outputs.native == 'false' steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.validate.outputs.commit }} - name: Install dependencies run: | @@ -89,7 +93,7 @@ jobs: make dist - name: Upload tarball - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: dist-tarball path: build/libhttpserver-*.tar.gz @@ -105,7 +109,7 @@ jobs: sudo apt-get install -y libcurl4-openssl-dev - name: Download tarball - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: dist-tarball @@ -135,7 +139,7 @@ jobs: - name: Fetch curl from cache id: cache-curl - uses: actions/cache@v4 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 with: path: curl-7.75.0 key: macos-latest-CURL-pre-built-v3 @@ -155,7 +159,7 @@ jobs: sudo make install - name: Download tarball - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: dist-tarball @@ -184,7 +188,7 @@ jobs: shell: msys2 {0} steps: - name: Setup MSYS2 - uses: msys2/setup-msys2@v2 + uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c with: msystem: MINGW64 update: true @@ -197,7 +201,7 @@ jobs: pacman --noconfirm -S --needed mingw-w64-x86_64-{toolchain,libtool,make,pkg-config,libsystre,doxygen,graphviz,curl,python} - name: Download tarball - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: dist-tarball @@ -221,14 +225,17 @@ jobs: name: Create GitHub Release runs-on: ubuntu-latest needs: [validate, verify-dist-linux, verify-dist-macos, verify-dist-windows] + if: needs.validate.outputs.native == 'false' + permissions: + contents: write steps: - name: Download tarball - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: dist-tarball - name: Download release notes - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 with: name: release-notes @@ -236,9 +243,9 @@ jobs: env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} + VERSION: ${{ needs.validate.outputs.version }} + IS_PRERELEASE: ${{ needs.validate.outputs.is_prerelease }} run: | - VERSION="${{ needs.validate.outputs.version }}" - IS_PRERELEASE="${{ needs.validate.outputs.is_prerelease }}" PRERELEASE_FLAG="" if [ "$IS_PRERELEASE" = "true" ]; then @@ -250,3 +257,15 @@ jobs: --notes-file release-notes.md \ $PRERELEASE_FLAG \ libhttpserver-*.tar.gz + + v3-acceptance: + needs: validate + if: needs.validate.outputs.native == 'true' + permissions: + contents: write + actions: read + uses: ./.github/workflows/v3-release-acceptance.yml + with: + source_commit: ${{ needs.validate.outputs.commit }} + source_version: ${{ needs.validate.outputs.version }} + publication: true diff --git a/.github/workflows/v3-native.yml b/.github/workflows/v3-native.yml new file mode 100644 index 000000000..5144f69d5 --- /dev/null +++ b/.github/workflows/v3-native.yml @@ -0,0 +1,119 @@ +name: v3 native and protocol fuzz +on: + push: + branches: [v3, master] + pull_request: + branches: [v3, master] + workflow_dispatch: + # Activates only when this workflow reaches the repository default branch. + schedule: + - cron: '17 3 * * *' +permissions: + contents: read +jobs: + native: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + tls: ['no', 'yes'] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - name: Prerequisites + run: sudo apt-get update && sudo apt-get install -y autoconf automake libtool pkg-config clang perl doxygen graphviz python3-yaml + - name: Private provider + if: matrix.tls == 'yes' + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + - name: C++20 native suites, all deterministic replays, headers, examples and hygiene + env: + TLS_MODE: ${{ matrix.tls }} + run: | + ./bootstrap + mkdir "$RUNNER_TEMP/native" + cd "$RUNNER_TEMP/native" + if [ "$TLS_MODE" = yes ]; then + export V3_TLS_CFLAGS="-I$RUNNER_TEMP/prerequisites/provider/include" + export V3_TLS_LIBS="-L$RUNNER_TEMP/prerequisites/provider/lib -lssl -lcrypto" + export LD_LIBRARY_PATH="$RUNNER_TEMP/prerequisites/provider/lib" + fi + CXX=clang++ CC=clang "$GITHUB_WORKSPACE/configure" --enable-v3-tls="$TLS_MODE" + make -j2 + make check -j2 + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: native-${{ matrix.tls }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/native/**/*.log + ${{ runner.temp }}/native/**/*.trs + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + retention-days: 14 + if-no-files-found: error + fuzz: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: [http1_head, http1_body, websocket_codec, hpack, http2_engine, quic_parser, quic_state, quic_recovery, http3_engine, qpack, quic_tls_callbacks] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - name: Prerequisites + run: sudo apt-get update && sudo apt-get install -y clang autoconf automake libtool pkg-config perl + - name: Private provider for TLS callback instrumentation + if: matrix.target == 'quic_tls_callbacks' + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + - name: Configure genuine native sources + env: + TARGET: ${{ matrix.target }} + run: | + ./bootstrap + mkdir "$RUNNER_TEMP/native" + cd "$RUNNER_TEMP/native" + mode=no + if [ "$TARGET" = quic_tls_callbacks ]; then + mode=yes + export V3_TLS_CFLAGS="-I$RUNNER_TEMP/prerequisites/provider/include" + export V3_TLS_LIBS="-L$RUNNER_TEMP/prerequisites/provider/lib -lssl -lcrypto" + fi + CXX=clang++ CC=clang "$GITHUB_WORKSPACE/configure" --enable-v3-tls="$mode" --disable-examples + - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 + with: + path: ${{ runner.temp }}/corpus + key: corpus-v1-${{ runner.os }}-${{ matrix.target }}-clang-asan-ubsan-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: corpus-v1-${{ runner.os }}-${{ matrix.target }}-clang-asan-ubsan- + - name: Bounded smoke or campaign + env: + TARGET: ${{ matrix.target }} + EVENT: ${{ github.event_name }} + LD_LIBRARY_PATH: ${{ runner.temp }}/prerequisites/provider/lib + run: | + runs=2000; seconds=30; seed=128 + if [ "$EVENT" = schedule ] || [ "$EVENT" = workflow_dispatch ]; then + runs=1000000000; seconds=600 + seed=$(python3 -c 'import secrets; print(secrets.randbelow(4294967295)+1)') + fi + bash scripts/run-v3-protocol-fuzz.sh --compiler clang++ --build-dir "$RUNNER_TEMP/fuzz" \ + --native-config-dir "$RUNNER_TEMP/native" --targets "$TARGET" \ + --runs "$runs" --seconds "$seconds" --seed "$seed" --corpus-dir "$RUNNER_TEMP/corpus" + - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 + if: success() && github.event_name != 'pull_request' && (github.ref_name == 'v3' || github.ref_name == github.event.repository.default_branch) + with: + path: ${{ runner.temp }}/corpus + key: corpus-v1-${{ runner.os }}-${{ matrix.target }}-clang-asan-ubsan-${{ github.run_id }}-${{ github.run_attempt }} + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: fuzz-${{ matrix.target }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/fuzz + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + retention-days: 30 + if-no-files-found: error diff --git a/.github/workflows/v3-package-validation.yml b/.github/workflows/v3-package-validation.yml index bf425429b..53e702eef 100644 --- a/.github/workflows/v3-package-validation.yml +++ b/.github/workflows/v3-package-validation.yml @@ -1,9 +1,9 @@ name: v3 installed package validation on: push: - branches: [v3] + branches: [v3, master] pull_request: - branches: [v3] + branches: [v3, master] workflow_dispatch: permissions: contents: read @@ -14,22 +14,34 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - name: Prerequisites - run: sudo apt-get update && sudo apt-get install -y clang autoconf automake libtool pkg-config python3 perl libcurl4-openssl-dev binutils + run: sudo apt-get update && sudo apt-get install -y clang autoconf automake libtool pkg-config python3 perl libcurl4-openssl-dev binutils python3-venv + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '24.15.0' + - name: Selected client environment and executable Git provenance + run: | + mkdir -p "$RUNNER_TEMP/package-evidence" + exec > >(tee "$RUNNER_TEMP/package-evidence/clients.log") 2>&1 + python3 -c 'import subprocess; subprocess.run(["git", "--version"], check=True)' + python3 -m venv "$RUNNER_TEMP/package-evidence/clients" + "$RUNNER_TEMP/package-evidence/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt - name: Build selected prerequisites - run: python3 scripts/prepare-v3-package-ci.py --output package-evidence/prerequisites + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites" - name: Bootstrap run: ./bootstrap - name: Installed matrix - run: >- - python3 scripts/check-v3-installed-consumer.py --output package-evidence/matrix - --provider-prefix "$PWD/package-evidence/prerequisites/provider" - --pkg-config-dir /usr/lib/x86_64-linux-gnu/pkgconfig --pkg-config-dir /usr/share/pkgconfig - --compiler clang++ --cc clang --jobs 2 + run: | + prefix=() + if ! python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",443)); s.close()'; then + prefix=(sudo env "PATH=$PATH" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory "GIT_CONFIG_VALUE_0=$PWD") + fi + "${prefix[@]}" python3 scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" --provider-prefix "$RUNNER_TEMP/package-evidence/prerequisites/provider" --pkg-config-dir /usr/lib/x86_64-linux-gnu/pkgconfig --pkg-config-dir /usr/share/pkgconfig --client-python "$RUNNER_TEMP/package-evidence/clients/bin/python" --node node --compiler clang++ --cc clang --jobs 2 - if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: v3-package-linux-${{ github.sha }} - path: package-evidence + path: ${{ runner.temp }}/package-evidence + retention-days: 14 if-no-files-found: error macos: runs-on: macos-14 @@ -38,20 +50,33 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - name: Prerequisites run: brew install autoconf automake libtool pkg-config python curl + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '24.15.0' + - name: Selected client environment and executable Git provenance + run: | + mkdir -p "$RUNNER_TEMP/package-evidence" + exec > >(tee "$RUNNER_TEMP/package-evidence/clients.log") 2>&1 + python3 -c 'import subprocess; subprocess.run(["git", "--version"], check=True)' + python3 -m venv "$RUNNER_TEMP/package-evidence/clients" + "$RUNNER_TEMP/package-evidence/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt - name: Build selected prerequisites - run: python3 scripts/prepare-v3-package-ci.py --output package-evidence/prerequisites + run: python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites" - name: Bootstrap run: ./bootstrap - name: Installed matrix - run: >- - python3 scripts/check-v3-installed-consumer.py --output package-evidence/matrix - --provider-prefix "$PWD/package-evidence/prerequisites/provider" --pkg-config-dir "$(brew --prefix)/share/pkgconfig" - --compiler clang++ --cc clang --jobs 2 + run: | + prefix=() + if ! python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",443)); s.close()'; then + prefix=(sudo env "PATH=$PATH" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory "GIT_CONFIG_VALUE_0=$PWD") + fi + "${prefix[@]}" python3 scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" --provider-prefix "$RUNNER_TEMP/package-evidence/prerequisites/provider" --pkg-config-dir "$(brew --prefix)/share/pkgconfig" --client-python "$RUNNER_TEMP/package-evidence/clients/bin/python" --node node --compiler clang++ --cc clang --jobs 2 - if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: v3-package-macos-${{ github.sha }} - path: package-evidence + path: ${{ runner.temp }}/package-evidence + retention-days: 14 if-no-files-found: error mingw64: runs-on: windows-2022 @@ -60,33 +85,48 @@ jobs: run: shell: msys2 {0} steps: + - name: Keep selected Git source bytes unchanged + shell: pwsh + run: git config --global core.autocrlf false - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c with: msystem: MINGW64 update: true install: >- - autotools base-devel perl + autotools base-devel perl git mingw-w64-x86_64-clang mingw-w64-x86_64-gcc mingw-w64-x86_64-binutils mingw-w64-x86_64-python mingw-w64-x86_64-pkgconf mingw-w64-x86_64-libtool mingw-w64-x86_64-curl mingw-w64-x86_64-libsystre + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: '24.15.0' + - name: Selected client environment and executable Git provenance + run: | + mkdir -p "$RUNNER_TEMP/package-evidence" + exec > >(tee "$RUNNER_TEMP/package-evidence/clients.log") 2>&1 + export PATH="/c/Program Files/Git/cmd:$PATH" + /mingw64/bin/python -c 'import subprocess; subprocess.run(["git", "--version"], check=True)' + /mingw64/bin/python -m venv "$RUNNER_TEMP/package-evidence/clients" + "$(cygpath -m "$RUNNER_TEMP/package-evidence/clients/Scripts/python.exe")" -m pip install -r test/integ/http2-client-requirements.txt - name: Build selected prerequisites - run: /mingw64/bin/python scripts/prepare-v3-package-ci.py --output package-evidence/prerequisites + run: /mingw64/bin/python scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites" - name: Bootstrap run: ./bootstrap - name: Installed matrix (native Python; IOCP and WSAPoll) run: >- - /mingw64/bin/python scripts/check-v3-installed-consumer.py --output package-evidence/matrix - --provider-prefix "$(cygpath -m "$PWD/package-evidence/prerequisites/provider")" --toolchain-prefix "$(cygpath -m /mingw64)" - --compiler clang++ --cc clang --jobs 2 + /mingw64/bin/python scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" + --provider-prefix "$(cygpath -m "$RUNNER_TEMP/package-evidence/prerequisites/provider")" --toolchain-prefix "$(cygpath -m /mingw64)" + --client-python "$(cygpath -m "$RUNNER_TEMP/package-evidence/clients/Scripts/python.exe")" --node node --compiler clang++ --cc clang --jobs 2 - if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: v3-package-mingw64-${{ github.sha }} - path: package-evidence + path: ${{ runner.temp }}/package-evidence + retention-days: 14 if-no-files-found: error freebsd: runs-on: ubuntu-24.04 - timeout-minutes: 90 + timeout-minutes: 180 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - name: freebsd_instance @@ -95,16 +135,37 @@ jobs: release: '14.3' mem: 8192 cpu: 2 - prepare: pkg install -y bash autoconf automake libtool pkgconf python311 gmake curl binutils perl5 llvm + prepare: pkg install -y bash autoconf automake libtool pkgconf python311 gmake curl binutils perl5 llvm git py311-pip py311-virtualenv node24 run: | - python3.11 scripts/prepare-v3-package-ci.py --output package-evidence/prerequisites --make gmake + export RUNNER_TEMP=/tmp/v3-package-${{ github.run_id }}-${{ github.run_attempt }} + mkdir -p "$RUNNER_TEMP/package-evidence" + trap 'cp -R "$RUNNER_TEMP/package-evidence" ./package-evidence' EXIT + if [ "$(node --version)" != v24.15.0 ]; then + mkdir -p "$RUNNER_TEMP/node-source" + curl --fail --location https://nodejs.org/dist/v24.15.0/node-v24.15.0.tar.gz -o "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" + test "$(sha256 -q "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz")" = 729de494dd2872e5a3a6c32a1cd156a5413d4aca2772b2d873ee86bb5531bcd9 + tar -xzf "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" -C "$RUNNER_TEMP/node-source" + ( + cd "$RUNNER_TEMP/node-source/node-v24.15.0" + ./configure --prefix="$RUNNER_TEMP/node" + gmake -j2 > "$RUNNER_TEMP/node-source/build.log" 2>&1 + gmake install >> "$RUNNER_TEMP/node-source/build.log" 2>&1 + ) + export PATH="$RUNNER_TEMP/node/bin:$PATH" + fi + test "$(node --version)" = v24.15.0 + python3.11 -c 'import subprocess; subprocess.run(["git", "--version"], check=True)' + python3.11 -m venv "$RUNNER_TEMP/package-evidence/clients" + "$RUNNER_TEMP/package-evidence/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt + python3.11 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/package-evidence/prerequisites" --make gmake ./bootstrap - python3.11 scripts/check-v3-installed-consumer.py --output package-evidence/matrix \ - --provider-prefix "$PWD/package-evidence/prerequisites/provider" \ - --pkg-config-dir /usr/local/share/pkgconfig --compiler clang++ --cc clang --make gmake --jobs 2 + python3.11 scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" \ + --provider-prefix "$RUNNER_TEMP/package-evidence/prerequisites/provider" \ + --client-python "$RUNNER_TEMP/package-evidence/clients/bin/python" --node node --pkg-config-dir /usr/local/share/pkgconfig --compiler clang++ --cc clang --make gmake --jobs 2 - if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: v3-package-freebsd-${{ github.sha }} path: package-evidence + retention-days: 14 if-no-files-found: error diff --git a/.github/workflows/v3-release-acceptance.yml b/.github/workflows/v3-release-acceptance.yml new file mode 100644 index 000000000..2aea43330 --- /dev/null +++ b/.github/workflows/v3-release-acceptance.yml @@ -0,0 +1,519 @@ +name: v3 selected archive acceptance +'on': + workflow_call: + inputs: + source_commit: + type: string + required: true + source_version: + type: string + required: true + publication: + type: boolean + default: false + pull_request: + branches: + - v3 + - master + workflow_dispatch: + inputs: + source_commit: + description: Resolved 40-hex commit; empty selects this workflow ref + type: string + source_version: + description: Exact source version; empty reads configure.ac + type: string +permissions: + contents: read + actions: read +env: + # Artifact/run APIs identify the PR branch head; checkout tests its merge SHA. + WORKFLOW_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} +jobs: + prepare: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + outputs: + artifact_id: ${{ steps.archive.outputs.artifact-id }} + artifact_digest: ${{ steps.archive.outputs.artifact-digest }} + sha256: ${{ steps.prepare.outputs.sha256 }} + commit: ${{ steps.request.outputs.commit }} + version: ${{ steps.request.outputs.version }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ inputs.source_commit || github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Resolve source and version as structured data + id: request + env: + REQUEST_COMMIT: ${{ inputs.source_commit }} + REQUEST_VERSION: ${{ inputs.source_version }} + run: |- + python3 - <<'PYTHON' + import os,re,subprocess + from pathlib import Path + commit=os.environ['REQUEST_COMMIT'] or subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip() + version=os.environ['REQUEST_VERSION'] + if not version: + text=Path('configure.ac').read_text() + version='.'.join(re.search(r'm4_define\(\[libhttpserver_'+name+r'\],\[([0-9]+)\]\)',text)[1] for name in ('MAJOR_VERSION','MINOR_VERSION','REVISION')) + if not re.fullmatch(r'[0-9a-f]{40}',commit) or not re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:rc|alpha|beta)[0-9]+)?',version): raise SystemExit('invalid resolved request') + with open(os.environ['GITHUB_OUTPUT'],'a') as out: out.write('commit='+commit+'\nversion='+version+'\n') + PYTHON + - run: sudo apt-get update && sudo apt-get install -y autoconf automake libtool pkg-config + - id: prepare + env: + SOURCE_COMMIT: ${{ steps.request.outputs.commit }} + SOURCE_VERSION: ${{ steps.request.outputs.version }} + run: python3 scripts/run-v3-release-ci.py prepare --commit "$SOURCE_COMMIT" --version "$SOURCE_VERSION" --output "$RUNNER_TEMP/selected" + - id: archive + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: selected-archive + path: |- + ${{ runner.temp }}/selected/selected.json + ${{ runner.temp }}/selected/libhttpserver-*.tar.gz + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: selected-preparation-logs + path: ${{ runner.temp }}/selected/*.log + retention-days: 30 + if-no-files-found: warn + darwin: + runs-on: macos-14 + needs: prepare + timeout-minutes: 180 + outputs: + artifact_id: ${{ steps.proof.outputs.artifact-id }} + artifact_digest: ${{ steps.proof.outputs.artifact-digest }} + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.prepare.outputs.commit }} + persist-credentials: false + - run: brew install autoconf automake libtool pkg-config python curl + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: 24.15.0 + - name: Verify native Git and download immutable selected archive + run: |- + python3 -c 'import subprocess; subprocess.run(["git","--version"],check=True)' + python3 scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected" + - name: Selected provider and clients + run: |- + exec > >(tee "$RUNNER_TEMP/prerequisites-ci.log") 2>&1 + python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + python3 -m venv "$RUNNER_TEMP/clients" + "$RUNNER_TEMP/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt + - name: Complete native and installed TLS off/on proof + run: | + prefix=() + if ! python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",443)); s.close()'; then + prefix=(sudo env "PATH=$PATH" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory "GIT_CONFIG_VALUE_0=$PWD") + fi + "${prefix[@]}" python3 scripts/run-v3-release-ci.py platform --selection "$RUNNER_TEMP/selected/selected.json" --archive "$RUNNER_TEMP/selected/"libhttpserver-*.tar.gz --provider-prefix "$RUNNER_TEMP/prerequisites/provider" --client-python "$RUNNER_TEMP/clients/bin/python" --node node --output "$RUNNER_TEMP/platform" --pkg-config-dir "$(brew --prefix)/share/pkgconfig" + - id: proof + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-Darwin + path: ${{ runner.temp }}/platform/Darwin.tar.gz + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-Darwin-failure-evidence + path: |- + ${{ runner.temp }}/platform + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + ${{ runner.temp }}/prerequisites-ci.log + retention-days: 30 + if-no-files-found: warn + compression-level: 0 + windows: + runs-on: windows-2022 + needs: prepare + timeout-minutes: 180 + outputs: + artifact_id: ${{ steps.proof.outputs.artifact-id }} + artifact_digest: ${{ steps.proof.outputs.artifact-digest }} + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + steps: + - name: Keep selected Git source bytes unchanged + shell: pwsh + run: git config --global core.autocrlf false + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.prepare.outputs.commit }} + persist-credentials: false + - uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c + with: + msystem: MINGW64 + update: true + install: autotools base-devel perl git mingw-w64-x86_64-clang mingw-w64-x86_64-gcc mingw-w64-x86_64-binutils mingw-w64-x86_64-python + mingw-w64-x86_64-pkgconf mingw-w64-x86_64-libtool mingw-w64-x86_64-curl mingw-w64-x86_64-libsystre + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: 24.15.0 + - name: Verify native Git and download immutable selected archive + run: |- + export PATH="/c/Program Files/Git/cmd:$PATH" + /mingw64/bin/python -c 'import subprocess; subprocess.run(["git","--version"],check=True)' + /mingw64/bin/python scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected" + - name: Selected provider and clients + run: |- + exec > >(tee "$RUNNER_TEMP/prerequisites-ci.log") 2>&1 + /mingw64/bin/python scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + /mingw64/bin/python -m venv "$RUNNER_TEMP/clients" + "$(cygpath -m "$RUNNER_TEMP/clients/Scripts/python.exe")" -m pip install -r test/integ/http2-client-requirements.txt + - name: Complete native and installed TLS off/on proof + run: /mingw64/bin/python scripts/run-v3-release-ci.py platform --selection "$RUNNER_TEMP/selected/selected.json" --archive "$RUNNER_TEMP/selected/"libhttpserver-*.tar.gz + --provider-prefix "$RUNNER_TEMP/prerequisites/provider" --client-python "$(cygpath -m "$RUNNER_TEMP/clients/Scripts/python.exe")" + --node node --output "$RUNNER_TEMP/platform" --toolchain-prefix "$(cygpath -m /mingw64)" + - id: proof + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-Windows + path: ${{ runner.temp }}/platform/Windows.tar.gz + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-Windows-failure-evidence + path: |- + ${{ runner.temp }}/platform + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + ${{ runner.temp }}/prerequisites-ci.log + retention-days: 30 + if-no-files-found: warn + compression-level: 0 + defaults: + run: + shell: msys2 {0} + freebsd: + runs-on: ubuntu-24.04 + needs: prepare + timeout-minutes: 180 + outputs: + artifact_id: ${{ steps.proof.outputs.artifact-id }} + artifact_digest: ${{ steps.proof.outputs.artifact-digest }} + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.prepare.outputs.commit }} + persist-credentials: false + - name: Download immutable selected archive + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected" + - run: | + mkdir -p build + cp -R "$RUNNER_TEMP/selected" build/ci-selected-input + - uses: vmactions/freebsd-vm@c9f815bc7aa0d34c9fdd0619b034a32d6ca7b57e + with: + release: '14.3' + mem: 8192 + cpu: 2 + prepare: pkg install -y bash autoconf automake libtool pkgconf python311 gmake curl binutils perl5 llvm git py311-pip py311-virtualenv + node24 + run: |- + set -eu + export RUNNER_TEMP=/tmp/v3-release-${{ github.run_id }}-${{ github.run_attempt }} + mkdir -p "$RUNNER_TEMP" + trap 'mkdir -p platform-evidence; cp -R "$RUNNER_TEMP/platform" platform-evidence/ 2>/dev/null || true; cp -R "$RUNNER_TEMP/prerequisites" platform-evidence/ 2>/dev/null || true' EXIT + if [ "$(node --version)" != v24.15.0 ]; then + mkdir -p "$RUNNER_TEMP/node-source" + curl --fail --location https://nodejs.org/dist/v24.15.0/node-v24.15.0.tar.gz -o "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" + test "$(sha256 -q "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz")" = 729de494dd2872e5a3a6c32a1cd156a5413d4aca2772b2d873ee86bb5531bcd9 + tar -xzf "$RUNNER_TEMP/node-source/node-v24.15.0.tar.gz" -C "$RUNNER_TEMP/node-source" + ( + cd "$RUNNER_TEMP/node-source/node-v24.15.0" + ./configure --prefix="$RUNNER_TEMP/node" + gmake -j2 > "$RUNNER_TEMP/node-source/build.log" 2>&1 + gmake install >> "$RUNNER_TEMP/node-source/build.log" 2>&1 + ) + export PATH="$RUNNER_TEMP/node/bin:$PATH" + fi + test "$(node --version)" = v24.15.0 + python3.11 -c 'import subprocess; subprocess.run(["git","--version"],check=True)' + python3.11 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" --make gmake + python3.11 -m venv "$RUNNER_TEMP/clients" + "$RUNNER_TEMP/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt + python3.11 scripts/run-v3-release-ci.py platform --selection build/ci-selected-input/selected.json --archive build/ci-selected-input/libhttpserver-*.tar.gz --provider-prefix "$RUNNER_TEMP/prerequisites/provider" --client-python "$RUNNER_TEMP/clients/bin/python" --node node --make gmake --pkg-config-dir /usr/local/share/pkgconfig --output "$RUNNER_TEMP/platform" + - id: proof + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-FreeBSD + path: platform-evidence/platform/FreeBSD.tar.gz + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: platform-FreeBSD-failure-evidence + path: platform-evidence + retention-days: 30 + if-no-files-found: warn + compression-level: 0 + full: + runs-on: ubuntu-24.04 + needs: + - prepare + - darwin + - windows + - freebsd + timeout-minutes: 360 + outputs: + authorization_id: ${{ steps.authorization.outputs.artifact-id }} + authorization_digest: ${{ steps.authorization.outputs.artifact-digest }} + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + GOFLAGS: -mod=readonly + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ needs.prepare.outputs.commit }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 + with: + node-version: 24.15.0 + - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 + with: + go-version: 1.25.1 + cache: false + - name: Build and capture prerequisites + run: |- + exec > >(tee "$RUNNER_TEMP/prerequisites-ci.log") 2>&1 + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y clang autoconf automake libtool pkg-config python3-venv perl binutils tshark iproute2 libcurl4-openssl-dev + python3 scripts/prepare-v3-package-ci.py --output "$RUNNER_TEMP/prerequisites" + python3 -m venv "$RUNNER_TEMP/clients" + "$RUNNER_TEMP/clients/bin/python" -m pip install -r test/integ/http2-client-requirements.txt -r test/integ/http3-client-requirements.txt websockets==15.0.1 + python3 -m venv "$RUNNER_TEMP/runner-python" + git clone https://github.com/quic-interop/quic-interop-runner.git "$RUNNER_TEMP/quic-runner" + git -C "$RUNNER_TEMP/quic-runner" checkout --detach 740c05a10b61d65e8abd3ad38d60898004d335d9 + "$RUNNER_TEMP/runner-python/bin/python" -m pip install -r "$RUNNER_TEMP/quic-runner/requirements.txt" + git clone https://github.com/summerwind/h2spec.git "$RUNNER_TEMP/h2spec-src" + git -C "$RUNNER_TEMP/h2spec-src" checkout --detach 70ac2294010887f48b18e2d64f5cccd48421fad1 + make -C "$RUNNER_TEMP/h2spec-src" build + go -C test/integ/http3-client-go mod download + docker pull martenseemann/quic-network-simulator@sha256:c23d82a55caffe681b1bdae65d4d30d23e1283141a414a7f02ee56cf15f9c6b9 + docker pull martenseemann/quic-go-interop@sha256:6743eea41cb8d525686eb8f2437e3ac45835740ec87c62728f645cd89e85ab88 + docker pull martenseemann/quic-network-simulator-endpoint@sha256:3b2b9e6fa317da238c8140a7b6d2bf8d4d45a8464b873eecac3ff2a32520b71a + - name: Download same selected archive + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$SELECTED_ID" --digest "$SELECTED_DIGEST" --output "$RUNNER_TEMP/selected" + - name: Import immutable darwin bundle + env: + ARTIFACT_ID: ${{ needs.darwin.outputs.artifact_id }} + ARTIFACT_DIGEST: ${{ needs.darwin.outputs.artifact_digest }} + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$ARTIFACT_ID" --digest "$ARTIFACT_DIGEST" --output "$RUNNER_TEMP/bundle-darwin" + - name: Import immutable windows bundle + env: + ARTIFACT_ID: ${{ needs.windows.outputs.artifact_id }} + ARTIFACT_DIGEST: ${{ needs.windows.outputs.artifact_digest }} + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$ARTIFACT_ID" --digest "$ARTIFACT_DIGEST" --output "$RUNNER_TEMP/bundle-windows" + - name: Import immutable freebsd bundle + env: + ARTIFACT_ID: ${{ needs.freebsd.outputs.artifact_id }} + ARTIFACT_DIGEST: ${{ needs.freebsd.outputs.artifact_digest }} + run: python3 scripts/run-v3-release-ci.py download --artifact-id "$ARTIFACT_ID" --digest "$ARTIFACT_DIGEST" --output "$RUNNER_TEMP/bundle-freebsd" + - name: Execute all 18 gates against the selected archive + env: + ARCHIVE_SHA256: ${{ needs.prepare.outputs.sha256 }} + run: python3 scripts/run-v3-release-gates.py --output "$RUNNER_TEMP/full" --selected-archive "$RUNNER_TEMP/selected/"libhttpserver-*.tar.gz + --selected-sha256 "$ARCHIVE_SHA256" --provider-prefix "$RUNNER_TEMP/prerequisites/provider" --compiler clang++ --cc clang + --make make --jobs 2 --python "$RUNNER_TEMP/clients/bin/python" --node "$(command -v node)" --go "$(command -v go)" --h2spec + "$RUNNER_TEMP/h2spec-src/h2spec" --tshark "$(command -v tshark)" --runner-python "$RUNNER_TEMP/runner-python/bin/python" --runner-dir + "$RUNNER_TEMP/quic-runner" --server-image "libhttpserver-ci-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" --docker-direct-routing-compat --pkg-config-dir + /usr/lib/x86_64-linux-gnu/pkgconfig --pkg-config-dir /usr/share/pkgconfig --isolated-public-network + - name: Actual tag readiness and content-bound authorization + env: + PUBLICATION: ${{ inputs.publication || false }} + PRODUCER_DARWIN_ID: ${{ needs.darwin.outputs.artifact_id }} + PRODUCER_DARWIN_DIGEST: ${{ needs.darwin.outputs.artifact_digest }} + PRODUCER_WINDOWS_ID: ${{ needs.windows.outputs.artifact_id }} + PRODUCER_WINDOWS_DIGEST: ${{ needs.windows.outputs.artifact_digest }} + PRODUCER_FREEBSD_ID: ${{ needs.freebsd.outputs.artifact_id }} + PRODUCER_FREEBSD_DIGEST: ${{ needs.freebsd.outputs.artifact_digest }} + run: |- + args=() + if [ "$PUBLICATION" = true ]; then args+=(--publish --artifact-id "$SELECTED_ID"); fi + python3 scripts/run-v3-release-ci.py accept --manifest "$RUNNER_TEMP/full/manifest.json" --selection "$RUNNER_TEMP/selected/selected.json" --platform-bundle "$RUNNER_TEMP/bundle-darwin/Darwin.tar.gz" --platform-bundle "$RUNNER_TEMP/bundle-windows/Windows.tar.gz" --platform-bundle "$RUNNER_TEMP/bundle-freebsd/FreeBSD.tar.gz" --output "$RUNNER_TEMP/acceptance" "${args[@]}" + - id: authorization + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: release-readiness + path: |- + ${{ runner.temp }}/acceptance/authorization.json + ${{ runner.temp }}/acceptance/accepted-manifest.json + ${{ runner.temp }}/acceptance/validation.json + retention-days: 30 + if-no-files-found: error + - if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: full-gate-failure-evidence + path: |- + ${{ runner.temp }}/full + ${{ runner.temp }}/prerequisites/preparation.json + ${{ runner.temp }}/prerequisites/*.log + ${{ runner.temp }}/prerequisites-ci.log + retention-days: 30 + if-no-files-found: warn + compression-level: 0 + publish: + runs-on: ubuntu-24.04 + needs: + - prepare + - darwin + - windows + - freebsd + - full + if: inputs.publication == true && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + permissions: + contents: write + actions: read + env: + GH_TOKEN: ${{ github.token }} + SELECTED_ID: ${{ needs.prepare.outputs.artifact_id }} + SELECTED_DIGEST: ${{ needs.prepare.outputs.artifact_digest }} + GOFLAGS: -mod=readonly + SOURCE_COMMIT: ${{ needs.prepare.outputs.commit }} + SOURCE_VERSION: ${{ needs.prepare.outputs.version }} + ARCHIVE_SHA256: ${{ needs.prepare.outputs.sha256 }} + AUTHORIZATION_ID: ${{ needs.full.outputs.authorization_id }} + AUTHORIZATION_DIGEST: ${{ needs.full.outputs.authorization_digest }} + REQUIRED_RESULTS: '{"prepare":"${{ needs.prepare.result }}","darwin":"${{ needs.darwin.result }}","windows":"${{ needs.windows.result + }}","freebsd":"${{ needs.freebsd.result }}","full":"${{ needs.full.result }}"}' + PRODUCER_DARWIN_ID: ${{ needs.darwin.outputs.artifact_id }} + PRODUCER_DARWIN_DIGEST: ${{ needs.darwin.outputs.artifact_digest }} + PRODUCER_WINDOWS_ID: ${{ needs.windows.outputs.artifact_id }} + PRODUCER_WINDOWS_DIGEST: ${{ needs.windows.outputs.artifact_digest }} + PRODUCER_FREEBSD_ID: ${{ needs.freebsd.outputs.artifact_id }} + PRODUCER_FREEBSD_DIGEST: ${{ needs.freebsd.outputs.artifact_digest }} + steps: + - name: Publish accepted archive without candidate execution + run: |- + python3 - <<'PYTHON' + import hashlib + from datetime import datetime + import json + import os + from pathlib import Path, PurePosixPath, PureWindowsPath + import re + import stat + import subprocess + import zipfile + + + def need(condition, message): + if not condition: + raise ValueError(message) + + + def file_digest(path): + value=hashlib.sha256() + with Path(path).open('rb') as stream: + for chunk in iter(lambda:stream.read(1024*1024),b''): + value.update(chunk) + return value.hexdigest() + + + def canonical(document): + return hashlib.sha256(json.dumps(document,sort_keys=True,separators=(',',':')).encode()).hexdigest() + + + def publish_checked(authorization, expected, archive, manifest, tag_commit, publication): + need(expected['event'] in ('push','workflow_dispatch'), 'untrusted publication event') + need(re.fullmatch(r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:rc|alpha|beta)[0-9]+)?',expected['tag']), 'invalid tag') + need(authorization.get('schema_version') == 1 and authorization.get('binding') == expected, 'authorization identity differs') + need(canonical(manifest) == authorization['manifest_sha256'], 'accepted manifest changed') + need(tag_commit == expected['source_commit'] == manifest['release_artifact']['source']['head'], 'tag/source commit changed') + need(file_digest(archive) == expected['archive_sha256'] + == manifest['release_artifact']['archive']['sha256'], 'selected archive changed') + need(expected['producer_ids'] == ['FreeBSD','Darwin','Windows'] and + set(expected['producer_artifacts']) == {'freebsd','darwin','windows'}, 'producer identities missing') + publication() + + + def api(endpoint): + return json.loads(subprocess.check_output(['gh','api',endpoint],text=True)) + + + def download(artifact_id, expected_digest, directory, expected, run): + need(re.fullmatch(r'[1-9][0-9]*',artifact_id), 'invalid artifact ID') + endpoint='repos/'+expected['repository']+'/actions/artifacts/'+artifact_id + metadata=api(endpoint) + need(str(metadata['id']) == artifact_id and metadata['expired'] is False, 'wrong/expired artifact') + need(str(metadata['workflow_run']['id']) == expected['run_id'] and metadata['workflow_run']['head_sha'] == expected['workflow_source'], 'artifact source/run differs') + need(datetime.fromisoformat(metadata['created_at']) >= datetime.fromisoformat(run['run_started_at']), 'artifact from earlier run attempt') + archive=directory.with_suffix('.zip') + with archive.open('xb') as output: + subprocess.run(['gh','api',endpoint+'/zip'],stdout=output,check=True) + digest='sha256:'+file_digest(archive) + need(digest == metadata['digest'] and digest == 'sha256:'+expected_digest.removeprefix('sha256:'), 'artifact ZIP digest differs') + with zipfile.ZipFile(archive) as stream: + members=stream.infolist(); names=set() + need(len(members)<20004 and sum(m.file_size for m in members)<9*1024**3, 'artifact size bound') + for member in members: + name=PurePosixPath(member.filename) + need(not name.is_absolute() and not PureWindowsPath(member.filename).drive and '..' not in name.parts and '\\' not in member.filename + and member.filename not in names and not stat.S_ISLNK(member.external_attr>>16), 'unsafe artifact member') + names.add(member.filename) + directory.mkdir(); stream.extractall(directory) + + + def main(): + expected=dict(repository=os.environ['GITHUB_REPOSITORY'],run_id=os.environ['GITHUB_RUN_ID'],run_attempt=os.environ['GITHUB_RUN_ATTEMPT'], + workflow_source=os.environ['GITHUB_SHA'],source_commit=os.environ['SOURCE_COMMIT'],tag=os.environ['SOURCE_VERSION'], + archive_sha256=os.environ['ARCHIVE_SHA256'],event=os.environ['GITHUB_EVENT_NAME'],artifact_id=os.environ['SELECTED_ID'], + producer_ids=['FreeBSD','Darwin','Windows'],producer_artifacts={name:dict(id=os.environ['PRODUCER_'+name.upper()+'_ID'],digest=os.environ['PRODUCER_'+name.upper()+'_DIGEST']) for name in ('freebsd','darwin','windows')}) + need(re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+',expected['repository']), 'invalid repository') + need(re.fullmatch(r'[0-9a-f]{40}',expected['source_commit']), 'invalid commit') + need(all(value=='success' for value in json.loads(os.environ['REQUIRED_RESULTS']).values()), 'required producer not successful') + run=api('repos/'+expected['repository']+'/actions/runs/'+expected['run_id']+'/attempts/'+expected['run_attempt']) + need(str(run['run_attempt'])==expected['run_attempt'] and run['repository']['full_name']==expected['repository'] and run['head_sha']==expected['workflow_source'], 'workflow attempt/source changed') + root=Path(os.environ['RUNNER_TEMP']) + download(expected['artifact_id'],os.environ['SELECTED_DIGEST'],root/'selected',expected,run) + download(os.environ['AUTHORIZATION_ID'],os.environ['AUTHORIZATION_DIGEST'],root/'authorized',expected,run) + selected=json.loads((root/'selected/selected.json').read_text()) + need(selected['commit']==expected['source_commit'] and selected['version']==expected['tag'] and selected['sha256']==expected['archive_sha256'], 'selected descriptor differs') + need(Path(selected['archive']).name==selected['archive'], 'unsafe archive filename') + authorization=json.loads((root/'authorized/authorization.json').read_text()) + manifest=json.loads((root/'authorized/accepted-manifest.json').read_text()) + tag=api('repos/'+expected['repository']+'/git/ref/tags/'+expected['tag'])['object'] + for _ in range(5): + if tag['type']=='commit': break + need(tag['type']=='tag', 'unexpected tag target') + tag=api('repos/'+expected['repository']+'/git/tags/'+tag['sha'])['object'] + need(tag['type']=='commit','unresolved tag') + def create_release(): + argv=['gh','release','create',expected['tag'],'--repo',expected['repository'],'--verify-tag', + '--target',expected['source_commit'],'--title','libhttpserver '+expected['tag'],'--generate-notes'] + if '-' in expected['tag']: argv.append('--prerelease') + subprocess.run(argv+[str(root/'selected'/selected['archive'])],check=True) + publish_checked(authorization,expected,root/'selected'/selected['archive'],manifest,tag['sha'],create_release) + + + main() + + PYTHON diff --git a/.github/workflows/verify-build.yml b/.github/workflows/verify-build.yml index 16de17b87..d25526367 100644 --- a/.github/workflows/verify-build.yml +++ b/.github/workflows/verify-build.yml @@ -2,10 +2,10 @@ name: "Verify Build" on: push: - branches: [master] + branches: [master, v3] pull_request: # The branches below must be a subset of the branches above - branches: [master] + branches: [master, v3] schedule: - cron: '0 0 * * 0' @@ -15,7 +15,22 @@ permissions: contents: read jobs: + source: + runs-on: ubuntu-latest + outputs: + native: ${{ steps.source.outputs.native }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - id: source + run: | + if grep -q 'libhttpserver_v3core_la_SOURCES' src/Makefile.am; then + echo 'native=true' >> "$GITHUB_OUTPUT" + else + echo 'native=false' >> "$GITHUB_OUTPUT" + fi verify: + needs: source + if: needs.source.outputs.native == 'false' name: Verify runs-on: ${{ matrix.os }} env: diff --git a/.gitignore b/.gitignore index 73475dff1..9f1b476a0 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,9 @@ __pycache__/ # v3 source-layout compiled implementation contract checker. !scripts/check-v3-compiled-operations.py + +# v3 CI selected-archive publication and portable platform contracts. +!scripts/run-v3-release-ci.py +!scripts/test-v3-ci-workflows.py +!test/integ/release_ci_test.py +!test/integ/release_platform_bundle_test.py diff --git a/ChangeLog b/ChangeLog index 95b7d5393..5cc2febf6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,11 @@ Version 3.0.0 - 2026-10-10 + Connect native TLS-off/on builds, replay and complete installed matrices to CI. + Add bounded sanitizer fuzz smoke, persistent trusted corpora and scheduled + campaign code that activates after default-branch cutover. + Require exact selected-archive readiness and all platform evidence before + publication, with immutable artifact verification and isolated permissions. + Fix an ambiguous QUIC replay cursor construction rejected by GCC 13. + Restore native Doxygen linked-page input and a Graphviz-compatible font. * Ship the C++20 native engine with SOVERSION 3 and native public headers. * Remove MHD and GnuTLS configure, source, build, link and package prerequisites. * Require strict TLS-on/off installed audits and validate versioned ABI-2/3 runtime coexistence. diff --git a/Makefile.am b/Makefile.am index 90a0b7678..2ae1270c2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -333,7 +333,7 @@ check-release-notes: # exit 0 if doxygen is not installed. check-doxygen: @echo "=== check-doxygen: enforce TASK-043 zero-warning invariant ===" - @BUILD_DIR="$(abs_top_builddir)" $(top_srcdir)/scripts/check-doxygen.sh + +@BUILD_DIR="$(abs_top_builddir)" $(top_srcdir)/scripts/check-doxygen.sh # TASK-044: SOVERSION acceptance gate. Stages a clean DESTDIR install and # asserts that the v3.0 on-disk SONAME layout is correct (libhttpserver.so.3 @@ -620,3 +620,10 @@ check-v3-compiled-operations: $(top_builddir)/src/libhttpserver.la .PHONY: check-v3-compiled-operations EXTRA_DIST += docs/v3-protocol-limits.md scripts/check-v3-protocol-limits.py test/installed/protocol_limits_consumer.cpp + +EXTRA_DIST += scripts/run-v3-release-ci.py scripts/lib/release_ci.py scripts/lib/release_platform.py scripts/test-v3-ci-workflows.py +EXTRA_DIST += .github/workflows/v3-release-acceptance.yml +EXTRA_DIST += .github/workflows/v3-native.yml .github/workflows/changelog-check.yml +EXTRA_DIST += .github/workflows/codeql-analysis.yml .github/workflows/verify-build.yml .github/workflows/release.yml + +EXTRA_DIST += docs/v3-ci.md diff --git a/docs/task-185-release-gates.md b/docs/task-185-release-gates.md index b19fb799b..7d18aaaeb 100644 --- a/docs/task-185-release-gates.md +++ b/docs/task-185-release-gates.md @@ -8,6 +8,18 @@ adapters. Public TLS/H2/H3, credential and throughput/latency stages now execute their installed-package adapters in both TLS modes with independent receipt validation. Private fixtures remain supporting evidence. +## CI selected-archive execution + +The CI integration in [v3-ci.md](v3-ci.md) selects one immutable archive, +produces portable FreeBSD/Darwin/Windows bundles, then executes the existing +18-gate Linux inventory against that archive. Original platform receipts and +referenced bytes are retained; a scoped catalog resolver validates them without +foreign execution. Full readiness authorizes publication only after the actual +validator accepts every required gate and platform. Existing fresh-archive +callers and the separate release-owner pre-tag prerequisite remain supported. +Validation-only PR/dispatch runs exercise the same acceptance path without +creating tags or releases. Historical evidence below remains historical. + ## Complete local implementation inventory (2026-10-14) The fresh `build/task185-final-19` inventory passes all **18 mandatory local diff --git a/docs/v3-ci.md b/docs/v3-ci.md new file mode 100644 index 000000000..b567f60e5 --- /dev/null +++ b/docs/v3-ci.md @@ -0,0 +1,137 @@ +# v3 CI and publication + +Native push/PR checks target `v3` and `master`. Routing in Verify Build examines +`src/Makefile.am`: historical sources retain the historical matrix, while native +sources use `v3-native.yml`. CodeQL compiles the native core and private TLS +sources with the selected OpenSSL provider. ChangeLog checks compare the actual +PR base, passed as data through `BASE_REF`. + +`v3-native.yml` runs C++20 `make check` with TLS off/on, including deterministic +H1/WebSocket, HPACK, H2, QUIC parser/state/recovery/callback, H3 and QPACK replay, +registered helper tests, public headers, examples and install hygiene. Checkout +README/release-note/Doxygen checks retain their repository/spec prerequisites. +`v3-package-validation.yml` separately runs complete installed shared/static, +managed/external matrices on Linux, macOS, MinGW64 and FreeBSD. Provider, build +and evidence directories live outside the source checkout; VM evidence is +copied back only after auditing, for upload. Native Python verifies executable +Git on Windows and FreeBSD. Selected clients are hyper-h2 4.4.1 and Node 24.15.0; +FreeBSD builds the checksummed Node source if its ports package differs. Linux +and macOS package jobs probe actual loopback TCP443 binding for ACME witnesses; +when privileges require it, they run the producer with sudo and an exact +checkout-scoped Git ownership exception. + +## Fuzz smoke and campaigns + +Eleven independent target jobs prevent one failure from suppressing other +attempts. Ten targets compile TLS-off native sources; `quic_tls_callbacks` +uses genuine TLS-on configure/provider flags. Native project objects are built +with libFuzzer, ASan and UBSan instrumentation. OpenSSL and external clients +remain outside this instrumentation boundary. + +Push/PR smoke requests 2,000 runs, at most 30 seconds per target, seed 128. +Scheduled/manual campaigns request a high run cap, 600 seconds per target and a +varying recorded seed. Existing 5-second input timeout, 512-MiB RSS and target +input-length bounds remain in force. `compile.sh`, `reproduce.sh`, build/run +logs, source lists, environment and crash artifacts are retained on failure. +Fuzz artifacts retain 30 days; native/package artifacts retain 14 days. + +```sh +bash scripts/run-v3-protocol-fuzz.sh --compiler clang++ \ + --build-dir /tmp/fuzz-attempt --native-config-dir /tmp/configured-off \ + --targets 'http3_engine qpack' --seed 128 --runs 2000 --seconds 30 \ + --corpus-dir /tmp/writable-corpus +``` + +Seeds are copied from regression fixtures into a writable content-addressed +corpus. Restored inputs remain alongside these seeds. Corpus paths must be +outside the checkout and cannot contain symlinks. Neither fixtures nor existing +content-addressed inputs are overwritten. Immutable cache keys include corpus +format, target, platform, instrumentation, run ID and attempt; compatible keys +supply restoration. PRs restore only. Successful trusted `v3`/default-branch +push or campaign jobs may save. Deterministic replay has no cache dependency. +Corpus retention uses [GitHub’s cache policy](https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#usage-limits-and-eviction-policy): +entries expire after seven days without access, and storage pressure evicts older +entries within the repository’s configured limit. Caches are best-effort +persistence, not test proof. + +The nightly schedule is intentionally dormant while this workflow exists only +on `v3` and `master` remains the default branch. It activates when the workflow +reaches the default/master branch. Scheduled checkout uses the event/default +ref, so it follows the new default source after cutover. No dispatcher or +repository setting change is introduced. Manual dispatch is defined but its +availability before default-branch adoption must be verified in the real +repository; push/PR execution does not depend on it. + +## Selected-archive acceptance + +`v3-release-acceptance.yml` prepares one distribution archive and source/tooling +snapshot. Each nonlocal producer checks the exact selected archive against its +checkout, runs the complete installed and native TLS-off/on suites, validates +raw results and exports a platform bundle. The Linux job then executes all 18 +existing local gates against that same selected archive. Full H3 interop, +sanitizers, installed public capabilities and full benchmark workloads retain +their existing acceptance rules and exclusions. + +The fixed FreeBSD/Darwin/Windows bundles preserve original receipt bytes and +logical artifact paths. A bounded per-platform catalog retains exact source, +log, suite, audit, compiler, provider and selected-client bytes. Import checks +safe extraction, namespace uniqueness, every catalog hash and referenced +package input; it reruns the installed matrix, raw-suite/count and native-result +validators. Windows path normalization applies only to logical lookup and +comparison; foreign binaries are never executed. Sealed dyld/API-set OS leaves retain the +existing auditor’s explicit whitelist; nonexistent OS files are never +synthesized. Missing/changed bytes, +missing native replays, failed suites and stale counts block readiness. + +Linux evidence stays on its original filesystem. The actual `check_tag_ready` +validator checks all 18 local gates and all three platform receipts before an +authorization is written. That authorization binds the repository, workflow +source/run/attempt, selected commit/tag/archive, accepted manifest and immutable +selected/platform artifact IDs and digests. Downloads compare the actual ZIP +SHA-256 with both producer output and GitHub artifact metadata; a warning from +a download action is insufficient. Artifact creation must belong to the current +run attempt; rerun all jobs to produce fresh release proof rather than reusing +artifacts from successful jobs in an earlier attempt. + +On PR runs, artifact/run API metadata identifies the PR branch head, while +`GITHUB_SHA` and the selected checkout identify the test merge commit. Downloads +bind the API head to the trusted event's PR head. Selected archive preparation +and platform proof still require the exact selected checkout commit, source +snapshot and archive bytes; these two source identities are not interchangeable. + +The final publisher alone receives `contents: write`. It checks successful +mandatory dependencies, same-run artifacts, manifest and archive hashes and the +current tag target before `gh release create --verify-tag`. It runs only inline +workflow code and the GitHub CLI, with no checkout or downloaded candidate code. +Historical release sources retain their legacy distribution validation path. +The workflow starts after an existing tag; release owners must still execute +the separate pre-tag readiness command. This is publication enforcement. + +PR and validation-only dispatch use the same archive, platform and full local +path, and cannot publish. For an explicit manual validation request, provide an +already resolved 40-hex `source_commit` and its `source_version`. Empty inputs +select the current workflow ref and the version from `configure.ac`. The actual +GitHub run and dispatch availability are CI/root verification responsibilities. +No tags or releases are created by local verification. + +For local callers, fresh archive creation remains the default: + +```sh +python3 scripts/run-v3-release-gates.py --output /tmp/release-attempt \ + --selected-archive /tmp/selected/libhttpserver-3.0.0.tar.gz \ + --selected-sha256 ACTUAL_SHA256 --provider-prefix /tmp/provider \ + --compiler clang++ --cc clang --make make +# Supply the existing required pinned clients, captures and interop arguments. +python3 scripts/run-v3-release-gates.py --check-tag-ready /tmp/release-attempt/manifest.json \ + --platform-bundle /tmp/FreeBSD.tar.gz --platform-bundle /tmp/Darwin.tar.gz \ + --platform-bundle /tmp/Windows.tar.gz +``` + +Each output/import destination is exclusive. A failed attempt is preserved; +use a fresh directory for another attempt. A missing required infrastructure +capability remains an unsuccessful required dependency. In particular, the +pinned upstream QUIC simulator is provisioned first. No historical clock +compatibility receipt is imported. If a fresh run demonstrates clock +incompatibility, the QUIC simulator infrastructure owner must reprovision the +existing bounded compatibility recipe/probe and produce fresh evidence before +H3 can pass. diff --git a/doxyconfig.in b/doxyconfig.in index ceda5b9f5..c3937ebe3 100644 --- a/doxyconfig.in +++ b/doxyconfig.in @@ -581,7 +581,7 @@ WARN_LOGFILE = # directories like "/usr/src/myproject". Separate the files or directories # with spaces. -INPUT = $(SRCDIR)/src/httpserver $(SRCDIR)/docs/v3-api.md +INPUT = $(SRCDIR)/src/httpserver $(SRCDIR)/docs/v3-api.md $(SRCDIR)/docs/v3-protocol-limits.md # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is @@ -1479,7 +1479,7 @@ DOT_NUM_THREADS = 0 # DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory # containing the font. -DOT_FONTNAME = FreeSans.ttf +DOT_FONTNAME = Helvetica # The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. # The default size is 10pt. diff --git a/scripts/lib/release_checks.py b/scripts/lib/release_checks.py index 3979e9e6b..2a77d88bf 100644 --- a/scripts/lib/release_checks.py +++ b/scripts/lib/release_checks.py @@ -78,7 +78,9 @@ def extract_archive(archive, directory): names = [Path(m.name) for m in members] need(names and len(members) < 20000 and sum(m.size for m in members) < 512 * 1024 * 1024, 'archive exceeds release extraction bounds') - need(all(not p.is_absolute() and '..' not in p.parts for p in names), 'archive path traversal') + need(all(not p.is_absolute() and '..' not in p.parts and '\\' not in m.name + and (m.isdir() or m.isfile()) for p, m in zip(names, members)), 'archive path traversal/link') + need(len({m.name for m in members}) == len(members), 'duplicate archive member') roots = {p.parts[0] for p in names if p.parts} need(len(roots) == 1, 'archive requires one source root') need(not directory.exists(), 'extracted directory already exists') @@ -149,7 +151,7 @@ def rotation_counts(receipt, source, build, prefix): return counts(4, 20) -def validate_package(root, directory, read_json): +def validate_package(root, directory, read_json, source_root=None, provenance_path=Path): summary = read_json(directory / 'summary.json') need(summary.get('status') == 'passed' and set(summary.get('lanes', {})) == {'no', 'yes'}, 'package matrix incomplete') matrix = load_runner('lib/package_matrix.py', root) @@ -161,8 +163,10 @@ def validate_package(root, directory, read_json): need(audit.get('violations') == [] and audit.get('tls_mode') == mode, 'package audit failed/wrong TLS') for linkage in ('shared', 'static'): public = audit['public'][linkage] - matrix.check_linkage(public['dependencies'], linkage, lane / 'prefix') - matrix.check_dependencies(public['compile']['dependencies'], lane / 'prefix', root, lane / 'build') + graph = dict(public['dependencies'], nodes=[dict(node, id=str(provenance_path(node['id']))) for node in public['dependencies']['nodes']]) + matrix.check_linkage(graph, linkage, provenance_path(lane / 'prefix')) + matrix.check_dependencies([str(provenance_path(p)) for p in public['compile']['dependencies']], + provenance_path(lane / 'prefix'), provenance_path(source_root or root), provenance_path(lane / 'build')) cells = public['cells'] need(len(cells) == 2 and all(len(cell['configurations']) == len(inventory['cases']) for cell in cells), 'package configuration coverage missing') suites = read_json(lane / ('integration-' + linkage) / 'suites.json') diff --git a/scripts/lib/release_ci.py b/scripts/lib/release_ci.py new file mode 100644 index 000000000..8beebf59b --- /dev/null +++ b/scripts/lib/release_ci.py @@ -0,0 +1,107 @@ +"""Content-bound authorization after the fixed release readiness decision.""" +from datetime import datetime +import hashlib +import json +import re +from pathlib import PurePosixPath, PureWindowsPath +import stat +import subprocess +import zipfile + + +VERSION = r'[0-9]+\.[0-9]+\.[0-9]+(?:-(?:rc|alpha|beta)[0-9]+)?' + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def canonical_digest(document): + return hashlib.sha256(json.dumps(document, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def validate_binding(binding): + require(re.fullmatch(VERSION, binding['tag']), 'invalid version tag') + require(re.fullmatch(r'[0-9a-f]{40}', binding['source_commit']), 'invalid source commit') + require(binding['event'] in ('push', 'workflow_dispatch'), 'untrusted publication event') + require(binding['producer_ids'] == ['FreeBSD', 'Darwin', 'Windows'], 'required producer identities missing') + require(re.fullmatch(r'[0-9a-f]{40}', binding['workflow_source']), 'invalid workflow source') + require(set(binding['producer_artifacts']) == {'freebsd','darwin','windows'}, 'missing producer artifact identity') + for artifact in binding['producer_artifacts'].values(): + require(re.fullmatch(r'[1-9][0-9]*', artifact['id']) and re.fullmatch(r'[0-9a-f]{64}', artifact['digest']), 'invalid producer artifact identity') + for key in ('run_id', 'run_attempt', 'artifact_id'): + require(re.fullmatch(r'[1-9][0-9]*', binding[key]), 'invalid run/artifact identity') + require(re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', binding['repository']), 'invalid repository') + + +def authorize(gate, manifest, binding): + validate_binding(binding) + require(manifest['release_artifact']['source'].get('working_diff_sha256') == hashlib.sha256(b'').hexdigest(), 'publication source differs from committed checkout') + gate.check_tag_ready(manifest) + require(manifest['release_artifact']['source']['head'] == binding['source_commit'], 'wrong selected source commit') + require(manifest['release_artifact']['archive']['sha256'] == binding['archive_sha256'], 'wrong selected archive') + return dict(schema_version=1, binding=json.loads(json.dumps(binding)), manifest_sha256=canonical_digest(manifest)) + + +def selected_archive(checkout, source, archive, expected_sha256, extraction): + import release_artifact as identity + from release_checks import extract_archive, verify_snapshot_packaging + require(identity.sha256(archive) == expected_sha256, 'selected archive digest mismatch') + for name, digest in source['files_sha256'].items(): + require(identity.sha256(checkout / name) == digest, 'selected source snapshot changed') + extracted = extract_archive(archive, extraction) + packaged = identity.source_files(extracted) + transformations = verify_snapshot_packaging(checkout, source['files_sha256'], extracted) + tooling = {} + candidates = [checkout / '.gitignore'] + list((checkout / 'specs').rglob('*.md')) + list((checkout / 'docs').rglob('*.md')) + for path in candidates: + if not path.is_file(): continue + name = path.relative_to(checkout).as_posix() + destination = extracted / name + if not destination.exists(): + require(source['files_sha256'].get(name) == identity.sha256(path), 'tooling not from selected source') + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(path.read_bytes()) + tooling[name] = identity.sha256(destination) + return dict(schema_version=1, source_root=str(extracted), source=source, + archive=dict(path=str(archive.resolve()), sha256=expected_sha256, bytes=archive.stat().st_size), + files_sha256=identity.source_files(extracted), tooling_bundle=tooling, + packaged_files_sha256=packaged, configured_transformations=transformations) + + +def validate_request(root, commit, version): + require(re.fullmatch(VERSION, version), 'invalid source version') + require(re.fullmatch(r'[0-9a-f]{40}', commit), 'invalid selected commit') + try: + actual = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD^{commit}'], cwd=root, text=True, stderr=subprocess.PIPE).strip() + except (OSError, subprocess.CalledProcessError) as error: + raise ValueError('selected request requires executable Git and a checkout') from error + require(actual == commit, 'selected commit differs from checkout') + status = subprocess.check_output(['git', 'status', '--porcelain=v1', '--untracked-files=all'], cwd=root, text=True) + require(not status, 'selected checkout has modified/untracked source inputs') + subprocess.run(['bash', str(root / 'scripts/validate-version.sh'), version], cwd=root, check=True) + + +def verify_download(archive, metadata, run, expected, expected_digest, destination): + require(str(metadata['id']) == expected['artifact_id'] and metadata['expired'] is False, 'wrong/expired artifact ID') + require(str(metadata['workflow_run']['id']) == expected['run_id'] and metadata['workflow_run']['head_sha'] == expected['workflow_head'], 'artifact from another workflow source/run') + require(str(run['id']) == expected['run_id'] and str(run['run_attempt']) == expected['run_attempt'] + and run['repository']['full_name'] == expected['repository'] and run['head_sha'] == expected['workflow_head'], 'wrong repository/source/run attempt') + require(datetime.fromisoformat(metadata['created_at']) >= datetime.fromisoformat(run['run_started_at']), 'artifact belongs to an earlier run attempt') + from release_artifact import sha256 + digest = 'sha256:' + sha256(archive) + require(digest == expected_digest == metadata['digest'], 'downloaded artifact digest mismatch') + require(not destination.exists(), 'artifact destination already exists') + with zipfile.ZipFile(archive) as stream: + members = stream.infolist() + require(len(members) < 20004 and sum(m.file_size for m in members) < 9 * 1024**3, 'artifact extraction bounds exceeded') + names = set() + for member in members: + name = PurePosixPath(member.filename) + mode = member.external_attr >> 16 + require(not name.is_absolute() and not PureWindowsPath(member.filename).drive and '..' not in name.parts and '\\' not in member.filename + and member.filename not in names and not stat.S_ISLNK(mode), 'unsafe/duplicate artifact member') + names.add(member.filename) + destination.mkdir(parents=True) + stream.extractall(destination) diff --git a/scripts/lib/release_execution.py b/scripts/lib/release_execution.py index 8e0aee4a4..ea6a00ba9 100644 --- a/scripts/lib/release_execution.py +++ b/scripts/lib/release_execution.py @@ -84,32 +84,23 @@ def prepare(self, cmd): except Exception as error: self.manifest['client_packages'] = {'failure': str(error)} self.manifest['source'] = release_identity.source_identity(self.root) - bootstrap_env = dict(self.env) - bootstrap_env.pop('V3_TLS_CFLAGS', None); bootstrap_env.pop('V3_TLS_LIBS', None) - self.command(cmd, ['bash', self.root / 'bootstrap'], self.root, bootstrap_env) - dist = self.output / 'distribution-build'; dist.mkdir() - self.command(cmd, ['bash', self.root / 'configure', '--enable-v3-tls=no', '--disable-examples'], dist, bootstrap_env) - self.command(cmd, [self.args.make, 'dist'], dist, bootstrap_env) - archives = list(dist.glob('libhttpserver-*.tar.gz')) - self.gate.need(len(archives) == 1, 'missing/ambiguous fresh distribution archive') - self.source = self.gate.extract_archive(archives[0], self.output / 'extracted') - archive_files = release_identity.source_files(self.source) - snapshot = self.manifest['source']['files_sha256'] - transformations = checks.verify_snapshot_packaging(self.root, snapshot, self.source) - bundle = {} - # Repository specifications are declared tooling inputs, never borrowed libraries. - candidates = [self.root / '.gitignore'] + list((self.root / 'specs').rglob('*.md')) + list((self.root / 'docs').rglob('*.md')) - for path in candidates: - name = path.relative_to(self.root).as_posix() - destination = self.source / name - if not destination.exists(): - self.gate.need(snapshot.get(name) == release_identity.sha256(path), 'tooling input not from selected snapshot: ' + name) - destination.parent.mkdir(parents=True, exist_ok=True); shutil.copyfile(path, destination) - bundle[name] = release_identity.sha256(destination) - files = release_identity.source_files(self.source) - identity = dict(schema_version=1, source_root=str(self.source), source=self.manifest['source'], - archive=self.gate.artifact(archives[0]), files_sha256=files, tooling_bundle=bundle, - packaged_files_sha256=archive_files, configured_transformations=transformations) + selected = getattr(self.args, 'selected_archive', None) + if selected: + archives = [Path(selected).resolve()] + expected_sha256 = self.args.selected_sha256 + else: + bootstrap_env = dict(self.env) + bootstrap_env.pop('V3_TLS_CFLAGS', None); bootstrap_env.pop('V3_TLS_LIBS', None) + self.command(cmd, ['bash', self.root / 'bootstrap'], self.root, bootstrap_env) + dist = self.output / 'distribution-build'; dist.mkdir() + self.command(cmd, ['bash', self.root / 'configure', '--enable-v3-tls=no', '--disable-examples'], dist, bootstrap_env) + self.command(cmd, [self.args.make, 'dist'], dist, bootstrap_env) + archives = list(dist.glob('libhttpserver-*.tar.gz')) + self.gate.need(len(archives) == 1, 'missing/ambiguous fresh distribution archive') + expected_sha256 = self.gate.digest(archives[0]) + from release_ci import selected_archive + identity = selected_archive(self.root, self.manifest['source'], archives[0], expected_sha256, self.output / 'extracted') + self.source = Path(identity['source_root']) path = self.output / 'artifact-identity.json'; self.gate.save(path, identity) self.env['V3_RELEASE_IDENTITY'] = str(path) self.manifest['release_artifact'] = identity diff --git a/scripts/lib/release_platform.py b/scripts/lib/release_platform.py new file mode 100644 index 000000000..6b44e1c1b --- /dev/null +++ b/scripts/lib/release_platform.py @@ -0,0 +1,252 @@ +"""Fixed platform receipt bundles: immutable logical paths, never foreign execution.""" +import hashlib +import json +import posixpath +from pathlib import Path, PurePosixPath, PureWindowsPath +import shutil +import tarfile + +from release_ci import require +import release_artifact as identity + +PLATFORMS = ('FreeBSD', 'Darwin', 'Windows') +MAX_FILES = 20000 +MAX_BYTES = 8 * 1024 * 1024 * 1024 + + +def read_document(path): + def unique(pairs): + value = {} + for key, child in pairs: + require(key not in value, 'duplicate platform receipt/catalog key') + value[key] = child + return value + require(path.is_file() and path.stat().st_size <= 64 * 1024 * 1024, 'missing/oversized platform document') + return json.loads(path.read_text(), object_pairs_hook=unique) + + +def record(path): + return dict(sha256=identity.sha256(path), bytes=Path(path).stat().st_size) + + +def logical_key(value): + windows = PureWindowsPath(str(value)) + return posixpath.normpath(windows.as_posix() if windows.drive else str(value)) + + +def provenance_path(value): + windows = PureWindowsPath(str(value)) + if windows.drive: + # Comparison-only namespace; these paths are never opened or executed. + return Path('/__v3_foreign_windows__') / windows.drive.rstrip(':') / Path(*windows.parts[1:]) + return Path(value) + + +def retain(destination, platform, receipt, paths): + require(platform in PLATFORMS, 'unknown platform namespace') + destination.mkdir(parents=True, exist_ok=False) + (destination / 'bytes').mkdir() + catalog = {} + total = 0 + paths = set(map(Path, paths)) + paths |= {path.resolve() for path in paths} + for path in sorted(paths): + logical = str(path) + value = record(path); total += value['bytes'] + require(len(catalog) < MAX_FILES and total <= MAX_BYTES, 'platform evidence exceeds bounds') + name = 'bytes/' + value['sha256'] + target = destination / name + if not target.exists(): shutil.copyfile(path, target) + require(record(target) == value, 'platform artifact changed during retention') + catalog[logical] = dict(value, retained=name) + (destination / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + (destination / 'catalog.json').write_text(json.dumps(dict(schema_version=1, platform=platform, receipt=record(destination / 'receipt.json'), paths=catalog), indent=2) + '\n') + return Catalog(destination) + + +class Catalog: + def __init__(self, directory): + self.directory = Path(directory).resolve() + require(not any(p.is_symlink() for p in self.directory.rglob('*')), 'platform bundle symlink') + require((self.directory / 'catalog.json').is_file() and (self.directory / 'catalog.json').stat().st_size <= 64 * 1024 * 1024, 'oversized platform catalog') + value = read_document(self.directory / 'catalog.json') + require(value.get('schema_version') == 1 and value.get('platform') in PLATFORMS, 'unknown platform namespace') + self.platform, self.paths = value['platform'], value['paths'] + self.receipt_record = value['receipt'] + self.catalog_record = record(self.directory / 'catalog.json') + self.verify_metadata() + require(0 < len(self.paths) < MAX_FILES, 'empty/oversized platform catalog') + require(sum(r['bytes'] for r in self.paths.values()) <= MAX_BYTES, 'oversized platform bytes') + self.logical_paths = {} + for logical in self.paths: + key = logical_key(logical) + previous = self.logical_paths.get(key) + require(previous is None or self.paths[previous] == self.paths[logical], 'conflicting logical path aliases') + self.logical_paths[key] = logical + permitted = {'catalog.json', 'receipt.json'} + for logical, value in self.paths.items(): + name = value['retained'] + require(name == 'bytes/' + value['sha256'] and len(value['sha256']) == 64 + and all(c in '0123456789abcdef' for c in value['sha256']), 'unsafe platform catalog member') + permitted.add(name) + self.path(logical) + observed = {p.relative_to(self.directory).as_posix() for p in self.directory.rglob('*') if p.is_file()} + require(observed == permitted, 'missing/extraneous platform catalog bytes') + + def verify_metadata(self): + require(record(self.directory / 'catalog.json') == self.catalog_record, 'platform catalog changed') + require(record(self.directory / 'receipt.json') == self.receipt_record, 'original platform receipt bytes changed') + + def path(self, logical): + original = self.logical_paths.get(logical_key(logical)) + value = self.paths.get(original) + require(value is not None, 'missing catalog entry: ' + str(logical)) + path = self.directory / value['retained'] + require(path.is_file() and not path.is_symlink() and record(path) == {k: value[k] for k in ('bytes', 'sha256')}, + 'changed platform catalog bytes: ' + str(logical)) + return path + + def read_json(self, logical): + path = self.path(logical) + require(path.stat().st_size <= 64 * 1024 * 1024, 'oversized platform receipt') + return read_document(path) + + def verify_files(self, records): + require(isinstance(records, list) and records, 'empty platform artifact evidence') + for value in records: + require(record(self.path(value['path'])) == {k: value[k] for k in ('bytes', 'sha256')}, 'platform artifact identity differs') + + +def unpack(archive, output): + require(not output.exists(), 'platform namespace collision') + with tarfile.open(archive) as stream: + members = stream.getmembers() + require(0 < len(members) < MAX_FILES + 4 and sum(m.size for m in members) <= MAX_BYTES, 'platform bundle exceeds bounds') + names = set(); roots = set() + for member in members: + path = PurePosixPath(member.name) + require(not path.is_absolute() and '..' not in path.parts and '\\' not in member.name + and (member.isfile() or member.isdir()) and member.name not in names, 'unsafe/duplicate platform member') + names.add(member.name); roots.add(path.parts[0]) + require(len(roots) == 1 and next(iter(roots)) in PLATFORMS, 'unknown platform namespace') + output.mkdir(parents=True) + stream.extractall(output, filter='data') + return Catalog(output / roots.pop()) + + +def package_origin(root, origin, read_json, read_text, expected, platform, read_bytes=None): + import package_matrix as matrix + from release_checks import validate_package + directory = Path(origin['package']) + summary = read_json(directory / 'summary.json') + require(summary['revision'] == origin['source_commit'] == summary['release_artifact']['source']['head'], 'platform source commit differs') + require(summary['host'] == platform and summary['release_artifact']['archive']['sha256'] == expected['archive_sha256'] + and summary['release_artifact']['source']['snapshot_sha256'] == expected['snapshot_sha256'], 'stale platform package source/archive') + require(hashlib.sha256((read_bytes or (lambda p: Path(p).read_bytes()))(origin['provider_header'])).hexdigest() == summary['provider_header_sha256'], + 'platform provider artifact substituted') + result = validate_package(root, directory, read_json, source_root=Path(origin['source_root']), provenance_path=provenance_path) + inventory = matrix.load() + for mode in ('no', 'yes'): + for linkage in ('shared', 'static'): + suites = read_json(directory / ('tls-' + mode) / ('integration-' + linkage) / 'suites.json') + for name, definition in matrix.suites(inventory, platform, mode).items(): + receipt = suites[name] + observed = matrix.suite_result(read_text(receipt['log']), 0) + require(all(receipt.get(k) == v for k, v in observed.items()), 'stale platform suite summary') + require(set(definition.get('required_cases', [])) <= set(observed['case_names']), 'pruned platform suite') + return result + + +def validate_origin(root, row, catalog): + from release_checks import automake_counts, counts + catalog.verify_metadata() + expected = row['identity']; origin = row['origin'] + prefix = logical_key(origin['package']).rstrip('/') + '/' + for logical in catalog.paths: + if logical_key(logical).startswith(prefix) and logical.endswith('.json'): + referenced_files(catalog.read_json(logical), catalog.path) + result = package_origin(root, origin, catalog.read_json, lambda p: catalog.path(p).read_text(), expected, row['id'], lambda p: catalog.path(p).read_bytes()) + require(set(origin['native']) == {'no', 'yes'}, 'missing native TLS configuration') + for mode, native in origin['native'].items(): + names = native['tests'] + require(len(names) == len(set(names)) and names, 'duplicate/empty native suite inventory') + mandatory = {'protocol_fuzz_replay', 'hpack_corpus', 'http2_fuzz_replay', 'quic_fuzz_replay', + 'quic_recovery_fuzz_replay', 'http3_fuzz_replay', 'qpack_fuzz_replay', + 'io_' + {'Windows':'iocp', 'Darwin':'kqueue', 'FreeBSD':'kqueue'}[row['id']] + '_backend'} + if mode == 'yes': mandatory.add('quic_tls_callbacks_fuzz_replay') + require(mandatory <= set(names), 'pruned mandatory native suite') + logs = [] + for name in names: + value = catalog.path(Path(native['build']) / 'test' / (name + '.trs')).read_text() + if ':test-result: SKIP' in value: + require(name in {'io_iocp_backend','io_epoll_backend','io_kqueue_backend'} and name not in mandatory, 'unexpected native skip') + else: + require(':test-result: PASS' in value, 'failed native test result') + logs.append(name) + text = catalog.path(Path(native['build']) / 'test/test-suite.log').read_text() + import re + skipped = len(names) - len(logs) + require(re.findall(r'^# SKIP:[ \t]+(\d+)$', text, re.M) == [str(skipped)], 'native skip summary differs') + require(re.findall(r'^# TOTAL:[ \t]+(\d+)$', text, re.M) == [str(len(names))], 'native registered suite summary differs') + text = re.sub(r'^# TOTAL:[ \t]+\d+$', '# TOTAL: ' + str(len(logs)), text, flags=re.M) + text = re.sub(r'^# SKIP:[ \t]+\d+$', '# SKIP: 0', text, flags=re.M) + native_counts = automake_counts(text) + result = counts(result['tests'] + native_counts['tests'], result['checks'] + native_counts['checks']) + require(row['receipt']['counts'] == result, 'platform envelope counts differ from origin') + catalog.verify_files(row['artifacts']); catalog.verify_files(row['receipt']['artifacts']) + return result + + +def import_bundle(root, manifest, catalog, expected): + row = read_document(catalog.directory / 'receipt.json') + require(row.get('id') == catalog.platform and row.get('identity') == expected and row.get('status') == 'passed' + and row.get('archive_sha256') == expected['archive_sha256'], + 'wrong/failed platform bundle identity') + current = [r for r in manifest['platforms'] if r['id'] == row['id']] + require(len(current) == 1 and current[0]['status'] == 'pending', 'duplicate/unknown platform producer') + require(row['origin']['source_commit'] == manifest['release_artifact']['source']['head'], 'platform from another source commit') + validate_origin(root, row, catalog) + row['portable_bundle'] = str(catalog.directory) + row['portable_catalog'] = dict(path=str(catalog.directory / 'catalog.json'), **catalog.catalog_record) + manifest['platforms'] = [row if r['id'] == row['id'] else r for r in manifest['platforms']] + + +def referenced_files(document, resolve=Path): + """Fixed package provenance records, including external clients/provider inputs.""" + result = set() + def visit(value): + if isinstance(value, list): + for child in value: visit(child) + elif isinstance(value, dict): + if 'path' in value and 'sha256' in value: + path = Path(value['path']); retained = resolve(str(path)) + require(retained.is_file() and identity.sha256(retained) == value['sha256'], 'referenced package artifact changed') + result.add(path) + for key in ('dependency_sha256', 'sources'): + mapping = value.get(key) + if isinstance(mapping, dict): + for name, digest in mapping.items(): + if isinstance(digest, str) and len(digest) == 64: + path = Path(name); retained = resolve(str(path)) + require(retained.is_file() and identity.sha256(retained) == digest, 'referenced package dependency changed') + result.add(path) + # Dependency graph node IDs name the actual provider/runtime libraries. + for node in value.get('nodes', []): + if isinstance(node, dict) and isinstance(node.get('id'), str): + path = Path(node['id']) + if node.get('provenance') == 'OS/API-set platform leaf': + from release_checks import load_runner + audit = load_runner('audit-v3-installed-package.py') + windows = PureWindowsPath(str(path)) + approved = str(path) in audit.DYLD_LEAVES or ( + windows.parent.name.lower() == 'system32' and windows.parent.parent.name.lower() == 'windows' + and audit.windows_system(windows.name)) + require(approved and node.get('sha256') is None, 'undeclared virtual platform leaf') + continue + if path.is_absolute() or PureWindowsPath(str(path)).is_absolute(): + retained = resolve(str(path)) + require(retained.is_file() and identity.sha256(retained) == node.get('sha256'), 'referenced runtime library missing/changed') + result.add(path) + for child in value.values(): visit(child) + visit(document) + return result diff --git a/scripts/run-v3-protocol-fuzz.sh b/scripts/run-v3-protocol-fuzz.sh index 756eb1950..1126dab2a 100755 --- a/scripts/run-v3-protocol-fuzz.sh +++ b/scripts/run-v3-protocol-fuzz.sh @@ -2,13 +2,15 @@ # Native, sequential libFuzzer lane. Explicit requests never silently skip. set -euo pipefail root=$(cd "$(dirname "$0")/.." && pwd) -compiler=clang++ build= runs=2000 seconds=30 targets='http1_head http1_body websocket_codec' native_config= +compiler=clang++ build= runs=2000 seconds=30 targets='http1_head http1_body websocket_codec' native_config= seed=128 corpus= while [ "$#" -gt 0 ]; do case "$1" in --compiler) compiler=$2; shift 2 ;; --build-dir) build=$2; shift 2 ;; --native-config-dir) native_config=$2; shift 2 ;; --targets) targets=$2; shift 2 ;; + --seed) seed=$2; shift 2 ;; + --corpus-dir) corpus=$2; shift 2 ;; --runs) runs=$2; shift 2 ;; --seconds) seconds=$2; shift 2 ;; *) echo "unknown argument: $1" >&2; exit 2 ;; @@ -19,6 +21,21 @@ done for target in $targets; do case "$target" in http1_head|http1_body|websocket_codec|hpack|http2_engine|quic_parser|quic_state|quic_recovery|quic_tls_callbacks|http3_engine|qpack) ;; *) echo "unknown fuzz target: $target" >&2; exit 2 ;; esac done +case "$seed" in ''|*[!0-9]*) echo 'invalid fuzz seed' >&2; exit 2 ;; esac +[ "${#seed}" -le 10 ] && [ "$seed" -gt 0 ] && [ "$seed" -le 4294967295 ] || { echo 'invalid fuzz seed' >&2; exit 2; } +if [ -n "$corpus" ]; then + python3 - "$corpus" "$root" <<'PYTHON' +from pathlib import Path +import sys +corpus, root = Path(sys.argv[1]), Path(sys.argv[2]) +if corpus.is_symlink() or any(p.is_symlink() for p in corpus.parents) or any(p.is_symlink() for p in corpus.rglob('*')): + raise SystemExit('corpus symlink rejected') +if corpus.resolve() == root or root in corpus.resolve().parents: + raise SystemExit('writable corpus must be outside source checkout') +corpus.mkdir(parents=True, exist_ok=True) +PYTHON + corpus=$(cd "$corpus" && pwd) +fi command -v "$compiler" >/dev/null || { echo 'libFuzzer compiler unavailable' >&2; exit 1; } provider_cflags=() provider_libs=() for target in $targets; do @@ -92,6 +109,26 @@ PY ;; esac [ -n "$(find "$build/$target/seeds" -type f -print -quit)" ] || { echo "empty corpus: $target" >&2; exit 1; } + target_corpus="$build/$target/seeds" + if [ -n "$corpus" ]; then + target_corpus="$corpus/$target" + python3 - "$build/$target/seeds" "$target_corpus" <<'PYTHON' +import hashlib +from pathlib import Path +import sys +source, destination = map(Path, sys.argv[1:]) +destination.mkdir(parents=True, exist_ok=True) +for fixture in source.iterdir(): + data = fixture.read_bytes() + path = destination / hashlib.sha256(data).hexdigest() + if path.exists(): + if path.read_bytes() != data: + raise SystemExit('corpus content-address collision') + else: + with path.open('xb') as output: + output.write(data) +PYTHON + fi sources=("$root/test/fuzz/${target}_fuzz.cpp") if [ "$target" = http1_head ] || [ "$target" = http1_body ] || [ "$target" = hpack ]; then sources+=("$root/src/http/fields.cpp" "$root/src/http/method.cpp" @@ -162,8 +199,8 @@ PYTHON printf '%q ' "$compiler" "${flags[@]}" "${target_flags[@]}" "${sources[@]}" "${link_flags[@]}" -o "$build/$target/fuzz" > "$build/$target/compile.sh" printf '\n' >> "$build/$target/compile.sh" "$compiler" "${flags[@]}" "${target_flags[@]}" "${sources[@]}" "${link_flags[@]}" -o "$build/$target/fuzz" > "$build/$target/build.log" 2>&1 - printf '%q ' "$build/$target/fuzz" "$build/$target/seeds" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed=128 -artifact_prefix="$build/$target/artifacts/" > "$build/$target/reproduce.sh" + printf '%q ' "$build/$target/fuzz" "$target_corpus" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed="$seed" -artifact_prefix="$build/$target/artifacts/" > "$build/$target/reproduce.sh" printf '\n' >> "$build/$target/reproduce.sh" - "$build/$target/fuzz" "$build/$target/seeds" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed=128 -artifact_prefix="$build/$target/artifacts/" > "$build/$target/run.log" 2>&1 || { echo "fuzz failure: $target; see $build/$target/run.log and artifacts" >&2; exit 1; } + "$build/$target/fuzz" "$target_corpus" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed="$seed" -artifact_prefix="$build/$target/artifacts/" > "$build/$target/run.log" 2>&1 || { echo "fuzz failure: $target; see $build/$target/run.log and artifacts" >&2; exit 1; } echo "PASS fuzz $target" done diff --git a/scripts/run-v3-release-ci.py b/scripts/run-v3-release-ci.py new file mode 100755 index 000000000..abd82e15f --- /dev/null +++ b/scripts/run-v3-release-ci.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Prepare one archive, produce native platform proof, or authorize publication.""" +import argparse +import importlib.util +import json +import os +from pathlib import Path +import platform +import shutil +import sys +import tarfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'scripts/lib')) +from package_commands import Commands +import release_artifact as identity +import release_checks as checks +from release_ci import authorize, require, selected_archive, validate_request +from release_platform import PLATFORMS, referenced_files, retain, validate_origin + + +def gate_runner(): + spec = importlib.util.spec_from_file_location('release_gate_ci', ROOT / 'scripts/run-v3-release-gates.py') + gate = importlib.util.module_from_spec(spec); sys.modules[spec.name] = gate; spec.loader.exec_module(gate) + return gate + + +def prepare(args): + validate_request(ROOT, args.commit, args.version) + args.output.mkdir(parents=True, exist_ok=False) + cmd = Commands(args.output) + env = dict(os.environ); env.pop('V3_RELEASE_IDENTITY', None) + cmd(['bash', str(ROOT / 'bootstrap')], ROOT, env) + build = args.output / 'build'; build.mkdir() + cmd(['bash', str(ROOT / 'configure'), '--disable-v3-tls', '--disable-examples'], build, env) + cmd([args.make, 'dist'], build, env) + archives = list(build.glob('libhttpserver-*.tar.gz')) + require(len(archives) == 1, 'ambiguous/missing selected archive') + archive = args.output / archives[0].name; shutil.copyfile(archives[0], archive) + validate_request(ROOT, args.commit, args.version) + source = identity.source_identity(ROOT) + selected = selected_archive(ROOT, source, archive, identity.sha256(archive), args.output / 'verified') + (args.output / 'selected.json').write_text(json.dumps(dict(commit=args.commit, version=args.version, + archive=archive.name, sha256=identity.sha256(archive), source=source, + files_sha256=selected['files_sha256']), indent=2) + '\n') + if os.environ.get('GITHUB_OUTPUT'): + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write('sha256=' + identity.sha256(archive) + '\n') + + +def platform_proof(args): + require(platform.system() in PLATFORMS, 'native nonlocal platform required') + selected = json.loads(args.selection.read_text()) + validate_request(ROOT, selected['commit'], selected['version']) + args.output.mkdir(parents=True, exist_ok=False) + release = selected_archive(ROOT, identity.source_identity(ROOT), args.archive, selected['sha256'], args.output / 'source') + require(release['source'] == selected['source'] and release['files_sha256'] == selected['files_sha256'], 'selected snapshot/tooling differs') + source = Path(release['source_root']) + identity_path = args.output / 'artifact-identity.json'; identity_path.write_text(json.dumps(release)) + env = dict(os.environ, V3_RELEASE_IDENTITY=str(identity_path), + V3_TLS_CFLAGS='-I' + (args.provider_prefix / 'include').as_posix(), + V3_TLS_LIBS='-L' + (args.provider_prefix / 'lib').as_posix() + ' -lssl -lcrypto') + env['LD_LIBRARY_PATH'] = str(args.provider_prefix / 'lib') + env['DYLD_LIBRARY_PATH'] = str(args.provider_prefix / 'lib') + env['PATH'] = str(args.provider_prefix / 'bin') + os.pathsep + env['PATH'] + cmd = Commands(args.output) + package = args.output / 'matrix' + argv = [sys.executable, source / 'scripts/check-v3-installed-consumer.py', '--output', package, + '--provider-prefix', args.provider_prefix, '--compiler', args.compiler, '--cc', args.cc, + '--make', args.make, '--jobs', str(args.jobs), '--client-python', args.client_python, '--node', args.node] + for directory in args.pkg_config_dir: argv += ['--pkg-config-dir', directory] + if args.toolchain_prefix: argv += ['--toolchain-prefix', args.toolchain_prefix] + cmd(argv, source, env, timeout=10800) + gate = gate_runner() + expected = gate.expected_identity({'release_artifact':release}, platform.system()) + origin = dict(package=str(package), source_root=str(source), source_commit=selected['commit'], + provider_header=str(args.provider_prefix / 'include/openssl/opensslv.h'), native={}) + result = checks.validate_package(ROOT, package, gate.read_json) + for mode in ('no', 'yes'): + build = package / ('tls-' + mode) / 'build' + cmd([args.make, '-j' + str(args.jobs)], build, env, timeout=3600) + cmd([args.make, 'check', '-j' + str(args.jobs)], build, env, timeout=7200) + text = cmd([args.make, '--no-print-directory', '-s', '-C', str(build / 'test'), + '--eval', 'ci-list-tests:;@printf "%s\\n" $(TEST_LOGS)', 'ci-list-tests'], build, env) + tests = [name.removesuffix('.log') for name in text.splitlines() if name] + origin['native'][mode] = dict(build=str(build), tests=tests) + native = checks.automake_result(build, platform.system()) + result = checks.counts(result['tests'] + native['tests'], result['checks'] + native['checks']) + files = [p for p in args.output.rglob('*') if p.is_file() and '__pycache__' not in p.parts] + # Preserve explicit tools/provider inputs referenced outside the platform tree. + files += [args.provider_prefix / 'include/openssl/opensslv.h', args.archive] + for receipt_path in package.rglob('*.json'): + files += list(referenced_files(json.loads(receipt_path.read_text()))) + for tool in (args.compiler, args.cc, args.make, args.client_python, args.node, 'git'): + selected_tool = shutil.which(str(tool)); require(selected_tool, 'selected tool unavailable: ' + str(tool)) + files.append(Path(selected_tool)) + files = sorted(set(files)) + artifacts = [gate.artifact(path) for path in files] + row = dict(id=platform.system(), status='passed', archive_sha256=selected['sha256'], identity=expected, + origin=origin, artifacts=artifacts, receipt=dict(status='passed', gate=platform.system(), identity=expected, counts=result, artifacts=artifacts)) + directory = args.output / 'bundle' / platform.system() + catalog = retain(directory, platform.system(), row, files) + validate_origin(ROOT, row, catalog) + with tarfile.open(args.output / (platform.system() + '.tar.gz'), 'w:gz') as stream: + stream.add(directory, arcname=platform.system(), recursive=True) + + +def accept(args): + gate = gate_runner(); manifest = gate.read_json(args.manifest) + gate.import_platform_bundles(manifest, args.platform_bundle, args.output) + gate.check_tag_ready(manifest) + selected = json.loads(args.selection.read_text()) + require(manifest['release_artifact']['source'] == selected['source'] and + manifest['release_artifact']['archive']['sha256'] == selected['sha256'], 'selected archive/source acceptance differs') + args.output.mkdir(parents=True, exist_ok=True) + gate.save(args.output / 'accepted-manifest.json', manifest) + if args.publish: + binding = dict(repository=os.environ['GITHUB_REPOSITORY'], run_id=os.environ['GITHUB_RUN_ID'], + run_attempt=os.environ['GITHUB_RUN_ATTEMPT'], source_commit=selected['commit'], tag=selected['version'], + archive_sha256=selected['sha256'], event=os.environ['GITHUB_EVENT_NAME'], artifact_id=args.artifact_id, + producer_ids=list(PLATFORMS), workflow_source=os.environ['GITHUB_SHA'], + producer_artifacts={name:dict(id=os.environ['PRODUCER_'+name.upper()+'_ID'],digest=os.environ['PRODUCER_'+name.upper()+'_DIGEST']) for name in ('freebsd','darwin','windows')}) + gate.save(args.output / 'authorization.json', authorize(gate, manifest, binding)) + else: + gate.save(args.output / 'validation.json', dict(status='passed', publication=False, source_commit=selected['commit'], archive_sha256=selected['sha256'])) + + +def download(args): + import subprocess + from release_ci import verify_download + expected=dict(repository=os.environ['GITHUB_REPOSITORY'],run_id=os.environ['GITHUB_RUN_ID'], + run_attempt=os.environ['GITHUB_RUN_ATTEMPT'],workflow_head=os.environ['WORKFLOW_HEAD_SHA'],artifact_id=args.artifact_id) + require(args.artifact_id.isdigit(), 'invalid artifact ID') + endpoint='repos/'+expected['repository']+'/actions/artifacts/'+args.artifact_id + metadata=json.loads(subprocess.check_output(['gh','api',endpoint],text=True)) + run=json.loads(subprocess.check_output(['gh','api','repos/'+expected['repository']+'/actions/runs/'+expected['run_id']+'/attempts/'+expected['run_attempt']],text=True)) + archive=args.output.parent/('artifact-'+args.artifact_id+'.zip') + with archive.open('xb') as output: + subprocess.run(['gh','api',endpoint+'/zip'],stdout=output,check=True) + digest=args.digest if args.digest.startswith('sha256:') else 'sha256:'+args.digest + verify_download(archive,metadata,run,expected,digest,args.output) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='operation', required=True) + prep = sub.add_parser('prepare'); prep.add_argument('--commit', required=True); prep.add_argument('--version', required=True) + platform_parser = sub.add_parser('platform') + platform_parser.add_argument('--archive', type=Path, required=True) + platform_parser.add_argument('--provider-prefix', type=Path, required=True) + platform_parser.add_argument('--compiler', default='clang++'); platform_parser.add_argument('--cc', default='clang') + platform_parser.add_argument('--client-python', required=True); platform_parser.add_argument('--node', required=True) + platform_parser.add_argument('--jobs', type=int, default=2) + platform_parser.add_argument('--pkg-config-dir', type=Path, action='append', default=[]) + platform_parser.add_argument('--toolchain-prefix', type=Path) + fetch = sub.add_parser('download'); fetch.add_argument('--artifact-id', required=True); fetch.add_argument('--digest', required=True) + acceptance = sub.add_parser('accept'); acceptance.add_argument('--manifest', type=Path, required=True) + acceptance.add_argument('--platform-bundle', type=Path, action='append', default=[]) + acceptance.add_argument('--publish', action='store_true'); acceptance.add_argument('--artifact-id') + for command in (prep, platform_parser, acceptance, fetch): command.add_argument('--output', type=Path, required=True) + for command in (platform_parser, acceptance): command.add_argument('--selection', type=Path, required=True) + for command in (prep, platform_parser): command.add_argument('--make', default='make') + args = parser.parse_args() + args.output = args.output.resolve() + if args.operation == 'prepare': prepare(args) + elif args.operation == 'platform': + args.archive = args.archive.resolve(); args.provider_prefix = args.provider_prefix.resolve(); platform_proof(args) + elif args.operation == 'download': download(args) + else: accept(args) + + +if __name__ == '__main__': + main() diff --git a/scripts/run-v3-release-gates.py b/scripts/run-v3-release-gates.py index 059d147fe..a0bce1c2f 100644 --- a/scripts/run-v3-release-gates.py +++ b/scripts/run-v3-release-gates.py @@ -86,7 +86,7 @@ def validate_inventory(document): need(document == inventory_document(), 'missing/duplicated/changed required gate or unknown exclusion') -def verify_receipt(receipt, identity, public=False): +def verify_receipt(receipt, identity, public=False, verify=verify_files): need(isinstance(receipt, dict) and receipt.get('status') == 'passed', 'missing/failed required receipt') need(receipt.get('identity') == identity and identity, 'historical/wrong archive, source, TLS or sanitizer identity') counts = receipt.get('counts', {}) @@ -94,7 +94,7 @@ def verify_receipt(receipt, identity, public=False): 'missing/malformed counts') need(counts['tests'] > 0 and counts['checks'] > 0 and counts['successes'] == counts['checks'] and counts['failures'] == counts['skipped'] == 0, 'empty/failed/skipped required receipts') - verify_files(receipt.get('artifacts', [])) + verify(receipt.get('artifacts', [])) if public: need(receipt.get('kind') == 'installed public capability' and set(receipt.get('witnesses', [])) >= {'tls', 'h2', 'h3', 'credentials'}, @@ -214,11 +214,21 @@ def verify_local_receipts(manifest): def verify_gate_rows(manifest, rows): for row in rows: - verify_files(row['artifacts']) + verify = verify_files + if row.get('portable_bundle'): + need(row['id'] in PLATFORMS, 'portable resolver is restricted to platform records') + from release_platform import Catalog, validate_origin + verify_files([row['portable_catalog']]) + catalog = Catalog(Path(row['portable_bundle'])) + original = json.loads((catalog.directory / 'receipt.json').read_text()) + need({k: v for k, v in row.items() if k not in ('portable_bundle', 'portable_catalog')} == original, 'platform envelope changed') + validate_origin(ROOT, row, catalog) + verify = catalog.verify_files + verify(row['artifacts']) expected = expected_identity(manifest, row['id']) need(row.get('identity') == expected, 'gate does not belong to selected packaged release') need(row.get('receipt', {}).get('gate') == row['id'], 'wrong required gate receipt') - verify_receipt(row.get('receipt'), expected, public=row['id'] == 'public-capabilities') + verify_receipt(row.get('receipt'), expected, public=row['id'] == 'public-capabilities', verify=verify) def check_tag_ready(manifest): @@ -245,10 +255,19 @@ def import_platform_receipts(manifest, paths): manifest['platforms'] = [receipt if r['id'] == name else r for r in manifest['platforms']] +def import_platform_bundles(manifest, paths, output): + from release_platform import unpack, import_bundle + for index, path in enumerate(paths): + catalog = unpack(path, output / 'imported-platforms' / str(index)) + import_bundle(ROOT, manifest, catalog, expected_identity(manifest, catalog.platform)) + + def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--check-tag-ready', type=Path) parser.add_argument('--output', type=Path) + parser.add_argument('--selected-archive', type=Path) + parser.add_argument('--selected-sha256') parser.add_argument('--provider-prefix', type=Path) parser.add_argument('--compiler') parser.add_argument('--cc') @@ -261,14 +280,18 @@ def main(argv=None): parser.add_argument('--prerequisite-prefix', type=Path, action='append', default=[]) parser.add_argument('--pkg-config-dir', type=Path, action='append', default=[]) parser.add_argument('--platform-receipt', type=Path, action='append', default=[]) + parser.add_argument('--platform-bundle', type=Path, action='append', default=[]) parser.add_argument('--tool-artifact', type=Path, action='append', default=[]) parser.add_argument('--docker-direct-routing-compat', action='store_true') parser.add_argument('--isolated-public-network', action='store_true') args = parser.parse_args(argv) + if bool(args.selected_archive) != bool(args.selected_sha256): + parser.error('--selected-archive and --selected-sha256 must be supplied together') if args.check_tag_ready: try: manifest = read_json(args.check_tag_ready) import_platform_receipts(manifest, args.platform_receipt) + import_platform_bundles(manifest, args.platform_bundle, args.check_tag_ready.parent) check_tag_ready(manifest) print('PASS: full release inventory verified; tag prerequisite satisfied') return 0 @@ -293,6 +316,7 @@ def main(argv=None): ReleaseAttempt(args, sys.modules[__name__], manifest).execute() manifest['artifact_identity_path'] = str(args.output / 'artifact-identity.json') import_platform_receipts(manifest, args.platform_receipt) + import_platform_bundles(manifest, args.platform_bundle, args.output) manifest['local_passed'] = local_passed(manifest) if manifest['local_passed']: try: diff --git a/scripts/test-v3-ci-workflows.py b/scripts/test-v3-ci-workflows.py new file mode 100755 index 000000000..a1ebe838f --- /dev/null +++ b/scripts/test-v3-ci-workflows.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Workflow dependencies and trust boundaries; execution contracts live beside runners.""" +from pathlib import Path +import os +import re +import subprocess +import sys +import tempfile +import unittest +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +def workflow(name): + return yaml.safe_load((ROOT / '.github/workflows' / name).read_text()) + + +class CIWorkflows(unittest.TestCase): + def test_artifact_api_head_is_bound_to_trusted_event_head(self): + lane = workflow('v3-release-acceptance.yml') + self.assertEqual(lane.get('env', {}).get('WORKFLOW_HEAD_SHA'), + '${{ github.event.pull_request.head.sha || github.sha }}') + + def test_freebsd_stages_selected_archive_for_vm_transfer(self): + steps = workflow('v3-release-acceptance.yml')['jobs']['freebsd']['steps'] + run = next(step['run'] for step in steps if 'ci-selected-input' in step.get('run', '')) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + selected = root / 'runner' / 'selected' + selected.mkdir(parents=True) + (selected / 'selected.json').write_text('selected archive identity') + result = subprocess.run(['bash', '-e', '-c', run], cwd=root, + env=dict(os.environ, RUNNER_TEMP=str(root / 'runner')), capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((root / 'build/ci-selected-input/selected.json').read_text(), 'selected archive identity') + + def test_release_request_resolver_emits_structured_commit_and_version(self): + steps = workflow('v3-release-acceptance.yml')['jobs']['prepare']['steps'] + run = next(step['run'] for step in steps if step.get('id') == 'request') + source = run.split("<<'PYTHON'\n", 1)[1].rsplit('\nPYTHON', 1)[0] + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'outputs' + commit = 'a' * 40 + env = dict(os.environ, REQUEST_COMMIT=commit, REQUEST_VERSION='3.0.0', GITHUB_OUTPUT=str(output)) + result = subprocess.run([sys.executable, '-c', source], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_text(), 'commit=' + commit + '\nversion=3.0.0\n') + + def test_native_lane_checks_both_configurations_and_replays_independent_of_cache(self): + path = ROOT / '.github/workflows/v3-native.yml' + self.assertTrue(path.exists(), 'native CI lane missing') + lane = workflow(path.name) + self.assertEqual(lane['permissions'], {'contents': 'read'}) + native = lane['jobs']['native'] + self.assertEqual(native['strategy']['matrix']['tls'], ['no', 'yes']) + runs = '\n'.join(step.get('run', '') for step in native['steps']) + self.assertIn('make check', runs) + self.assertIn('RUNNER_TEMP', runs) + fuzz = lane['jobs']['fuzz'] + targets = fuzz['strategy']['matrix']['target'] + self.assertEqual(set(targets), {'http1_head', 'http1_body', 'websocket_codec', 'hpack', + 'http2_engine', 'quic_parser', 'quic_state', 'quic_recovery', 'http3_engine', 'qpack', 'quic_tls_callbacks'}) + save = next(step for step in fuzz['steps'] if '/save@' in step.get('uses', '')) + self.assertIn("github.event_name != 'pull_request'", save['if']) + self.assertIn('github.event.repository.default_branch', save['if']) + self.assertIn('run_attempt', save['with']['key']) + self.assertTrue(any(step.get('if') == 'always()' and '/upload-artifact@' in step.get('uses', '') for step in fuzz['steps'])) + + def test_package_paths_do_not_contaminate_checkout_and_native_git_is_installed(self): + lane = workflow('v3-package-validation.yml') + for name in ('linux', 'macos', 'mingw64', 'freebsd'): + text = str(lane['jobs'][name]['steps']) + self.assertNotIn('$PWD/package-evidence', text) + self.assertIn('client-python', text) + self.assertIn('node', text) + self.assertIn('git', str(lane['jobs']['mingw64']['steps'])) + self.assertIn('git', str(lane['jobs']['freebsd']['steps'])) + + def test_freebsd_client_is_exact_pinned_node_source_when_ports_differ(self): + for filename in ('v3-package-validation.yml','v3-release-acceptance.yml'): + text=(ROOT/'.github/workflows'/filename).read_text() + self.assertIn('729de494dd2872e5a3a6c32a1cd156a5413d4aca2772b2d873ee86bb5531bcd9',text) + self.assertIn('node-v24.15.0.tar.gz',text) + self.assertIn('node --version',text) + + def test_publisher_receives_only_accepted_manifest_and_authorization(self): + lane=workflow('v3-release-acceptance.yml') + step=next(step for step in lane['jobs']['full']['steps'] if step.get('id')=='authorization') + path=step['with']['path'] + self.assertIn('authorization.json',path) + self.assertIn('accepted-manifest.json',path) + self.assertNotIn('imported-platforms',path) + self.assertEqual(lane['jobs']['publish']['permissions']['contents'],'write') + for name,job in lane['jobs'].items(): + if name!='publish': self.assertNotEqual(job.get('permissions',{}).get('contents'),'write') + + def test_macos_provisions_actual_acme_tcp443_binding_capability(self): + for filename,job in (('v3-package-validation.yml','macos'),('v3-release-acceptance.yml','darwin')): + steps=workflow(filename)['jobs'][job]['steps'] + text='\n'.join(step.get('run','') for step in steps) + self.assertIn('GIT_CONFIG_KEY_0=safe.directory',text) + self.assertIn('bind(("127.0.0.1",443))',text) + + def test_changelog_uses_actual_base_as_data(self): + lane = workflow('changelog-check.yml') + step = next(step for step in lane['jobs']['changelog']['steps'] if 'git diff' in step.get('run', '')) + self.assertIn('BASE_REF', step['env']) + self.assertNotIn('origin/master', step['run']) + self.assertIn('git check-ref-format', step['run']) + self.assertIn('--name-only --', step['run']) + + def test_all_modified_workflows_pin_actions(self): + for name in ('v3-native.yml', 'v3-package-validation.yml', 'verify-build.yml', 'codeql-analysis.yml', 'changelog-check.yml', 'release.yml'): + path = ROOT / '.github/workflows' / name + if not path.exists(): continue + for use in re.findall(r'uses:\s*(\S+)', path.read_text()): + if not use.startswith('./.github/workflows/'): + self.assertRegex(use, r'@[0-9a-f]{40}$') + + +if __name__ == '__main__': + unittest.main() diff --git a/specs/architecture/v3/architecture.md b/specs/architecture/v3/architecture.md index 8d8781944..643c25c2e 100644 --- a/specs/architecture/v3/architecture.md +++ b/specs/architecture/v3/architecture.md @@ -460,3 +460,20 @@ necessary detail declarations. An installed header never includes a noninstalled private header. Normal library linkage supplies compiled operations; standalone header compilation does not imply link-free runtime behavior. Extraction preserves API, representation, suspension points, resource accounting and protocol behavior. + +### Selected-archive CI release acceptance + +CI keeps the existing TASK-185 18-gate validator authoritative. One selected +archive and source/tooling snapshot feed native/installed platform producers, +then the Linux full inventory. Linux evidence remains on its original +filesystem; only fixed FreeBSD/Darwin/Windows receipts cross the platform seam. +A bounded namespaced logical-path catalog preserves original receipt bytes and +retains every referenced package input. Import reruns origin validators and +normalizes foreign paths only for lookup/comparison, never binary execution. + +The final job consumes content-bound authorization, same-run immutable artifact +IDs/digests and the exact archive. It rechecks manifest/source/tag bindings and +executes no downloaded candidate code. Only that job receives publication +permission. PR/validation-only execution cannot publish. This design accepts a +serial release critical path to avoid moving the complete Linux evidence tree +or broadening existing protocol validators. See [v3 CI](../../../docs/v3-ci.md). diff --git a/specs/tasks/FEATURE-ci-v3-integration.md b/specs/tasks/FEATURE-ci-v3-integration.md new file mode 100644 index 000000000..43fafc197 --- /dev/null +++ b/specs/tasks/FEATURE-ci-v3-integration.md @@ -0,0 +1,43 @@ +# FEATURE-ci-v3-integration + +Status: locally implemented and tested; local verification recorded in the task evidence; +actual GitHub PR verification and final validation remain root-owned. + +Scope: native CI/replay, installed platform prerequisites, bounded fuzz campaigns, +selected-archive release acceptance and publication enforcement. No production +protocol behavior, default-branch setting, tag or release changes. + +- [x] Add source-aware v3 push/PR native builds and CodeQL TLS coverage. +- [x] Repair actual PR-base ChangeLog comparison and pin touched external actions. +- [x] Keep package prerequisites/builds/evidence outside audited source roots; + select pinned clients and executable native Git. +- [x] Shard all ten TLS-off fuzz targets and TLS-on callbacks; preserve core + instrumentation, bounded smoke, varying campaign seeds and corpus trust. +- [x] Add default-branch schedule code inside v3, with deferred activation. +- [x] Prepare/select one archive; retain fixed portable platform evidence and + invoke the existing complete readiness validator on live Linux evidence. +- [x] Bind immutable artifacts and authorization; isolate publication permissions + and recheck the existing tag before the release side effect. +- [x] Exercise publication refusal and success through the actual validator, + with only the final publication side effect mocked. +- [x] Root: verify manual native campaign dispatch invocation (accepted at + 08c5b7b5; campaign execution outcomes remain pending). +- [ ] Root: complete actual GitHub PR pipeline verification. First selected + archive attempt failed at API PR-head versus merge-SHA origin binding; + causal repair and negative controls passed locally; fresh CI is pending. +- [ ] CI / v3 PR: obtain fresh full local gate and FreeBSD/Darwin/Windows receipts + for the selected commit/archive. Missing evidence is not a release pass. + +Details and commands: [v3 CI](../../docs/v3-ci.md). Historical TASK-185 receipts +are unchanged and cannot replace fresh evidence. + +Local verification: native TLS-off/on suites (239/270 tests, two documented +nonlocal backend skips in each), all eleven sanitizer fuzz targets, genuine +Doxygen/Graphviz, GCC QUIC replay and 100 initial CI/release/fuzz contracts passed. +The subsequent artifact-origin repair adds three focused contracts (103 total). +The fresh Linux installed attempt passed TLS-off but failed TLS-on public TLS at +unprivileged TCP443 binding. Its unsuccessful receipt remains unchanged. A fresh +focused public TLS run using the existing isolated network path passed all four +shared/static and managed/external cells (82 checks each). This proves the +affected prerequisite correction; it does not turn the failed whole matrix into +a passing receipt. The fresh complete 18-gate release run remains pending CI. diff --git a/test/Makefile.am b/test/Makefile.am index 96fcdb497..8ce10bad4 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -1647,3 +1647,6 @@ protocol_limits_SOURCES = unit/protocol_limits_test.cpp check_PROGRAMS += protocol_compression TESTS += protocol_compression protocol_compression_SOURCES = unit/protocol_compression_test.cpp + +TESTS += integ/release_ci_test.py integ/release_platform_bundle_test.py +EXTRA_DIST += integ/release_ci_test.py integ/release_platform_bundle_test.py diff --git a/test/fuzz/quic_parser_fuzz.cpp b/test/fuzz/quic_parser_fuzz.cpp index fb3cc5e17..c37af4d5f 100644 --- a/test/fuzz/quic_parser_fuzz.cpp +++ b/test/fuzz/quic_parser_fuzz.cpp @@ -168,7 +168,7 @@ std::uint32_t quic_codec_fuzz_input(std::span input) { require(hd::decode_quic_varint(out).value == scalar.value); for (std::size_t width = 1; width <= 4; ++width) { hd::encode_quic_packet_number(scalar.value, width, out); - hd::quic_cursor c(std::span(out).first(width)); + hd::quic_cursor c{std::span(out).first(width)}; require(hd::reconstruct_quic_packet_number(c.fixed(width), width, scalar.value).code == hd::quic_codec_code::ok); } } diff --git a/test/integ/protocol_fuzz_runner_test.py b/test/integ/protocol_fuzz_runner_test.py index 8999f1988..9e30eeb17 100755 --- a/test/integ/protocol_fuzz_runner_test.py +++ b/test/integ/protocol_fuzz_runner_test.py @@ -139,6 +139,50 @@ def test_http3_qpack_build_and_invocation_contract(self): self.assertTrue(list((build / target / 'seeds').glob('*.seed'))) self.assertTrue((build / target / 'reproduce.sh').is_file()) + def test_seed_and_corpus_arguments_reject_invalid_inputs_before_compilation(self): + for seed in ('0', '-1', '4294967296', '$(touch marker)', '12x'): + with self.subTest(seed=seed), tempfile.TemporaryDirectory() as build: + result = subprocess.run(['bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'), + '--build-dir', build, '--seed', seed, '--compiler', '/missing/compiler'], + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 2) + self.assertIn('invalid fuzz seed', result.stderr) + + def test_restored_corpus_is_merged_without_following_links_or_overwriting_seeds(self): + import os + with tempfile.TemporaryDirectory() as directory: + location = pathlib.Path(directory) + compiler = location / 'compiler' + compiler.write_text("#!/usr/bin/env python3\nimport pathlib,sys\n" + "if '--version' in sys.argv: print('recorder'); sys.exit()\n" + "p=pathlib.Path(sys.argv[sys.argv.index('-o')+1]); " + "p.write_text('#!/bin/sh\\nexit 0\\n'); p.chmod(0o755)\n") + compiler.chmod(0o755) + corpus = location / 'corpus' + (corpus / 'http1_head').mkdir(parents=True) + restored = corpus / 'http1_head' / 'extra' + restored.write_bytes(b'unique restored input') + build = location / 'build' + result = subprocess.run(['bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'), + '--build-dir', str(build), '--compiler', str(compiler), '--targets', 'http1_head', + '--seed', '987', '--corpus-dir', str(corpus)], capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + inputs = [p.read_bytes() for p in (corpus / 'http1_head').iterdir()] + self.assertIn(b'unique restored input', inputs) + for fixture in (ROOT / 'test/conformance/http1').glob('*.wire'): + self.assertIn(fixture.read_bytes(), inputs) + reproduction = (build / 'http1_head/reproduce.sh').read_text() + self.assertIn('-seed=987', reproduction) + self.assertIn(str(corpus / 'http1_head'), reproduction) + restored.unlink() + restored.symlink_to(ROOT / 'configure.ac') + bad = subprocess.run(['bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'), + '--build-dir', str(location / 'bad'), '--compiler', str(compiler), + '--targets', 'http1_head', '--corpus-dir', str(corpus)], + capture_output=True, text=True, timeout=10) + self.assertNotEqual(bad.returncode, 0) + self.assertIn('symlink', bad.stderr) + def test_requested_unavailable_compiler_fails(self): with tempfile.TemporaryDirectory() as build: result = subprocess.run(['bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'), diff --git a/test/integ/release_ci_test.py b/test/integ/release_ci_test.py new file mode 100755 index 000000000..61134b26a --- /dev/null +++ b/test/integ/release_ci_test.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Selected archive and publication decisions exercise the real readiness gate.""" +import copy +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts/lib')) +import release_gate_runner_test as release_contracts + + +class PublicationContracts(unittest.TestCase): + def setUp(self): + import hashlib + release_contracts.ReleaseContracts.setUp(self) + self.artifact_identity['source']['working_diff_sha256']=hashlib.sha256(b'').hexdigest() + self.identity_path.write_text(json.dumps(self.artifact_identity)) + @classmethod + def setUpClass(cls): + release_contracts.ReleaseContracts.setUpClass() + cls.gate = release_contracts.ReleaseContracts.gate + cls.ci_path = ROOT / 'scripts/lib/release_ci.py' + + def ci(self): + self.assertTrue(self.ci_path.exists(), 'selected-archive CI acceptance helper missing') + spec = importlib.util.spec_from_file_location('release_ci', self.ci_path) + module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + return module + + def publisher(self): + import ast + import textwrap + text=(ROOT/'.github/workflows/v3-release-acceptance.yml').read_text() + self.assertIn(' publish:\n',text,'isolated publisher job missing') + job=text.split(' publish:\n',1)[1] + self.assertNotIn('checkout@',job) + script=textwrap.dedent(job.split("python3 - <<'PYTHON'\n",1)[1]).rsplit('\nPYTHON',1)[0] + tree=ast.parse(script) + tree.body=[node for node in tree.body if isinstance(node,(ast.Import,ast.ImportFrom,ast.FunctionDef))] + namespace={}; exec(compile(tree,'publisher','exec'),namespace) + return namespace['publish_checked'] + + def publish(self, authorization, binding, archive, publication): + self.publisher()(authorization,binding,archive,self.complete(),'d'*40,publication) + + def complete(self): + manifest = {'schema_version': 1, 'inventory': self.gate.inventory_document(), + 'release_artifact': self.artifact_identity, 'artifact_identity_path': str(self.identity_path), + 'artifacts': self.receipt['artifacts'], 'stages': [], 'platforms': []} + for name in self.gate.LOCAL_GATES + self.gate.PLATFORMS: + identity = self.gate.expected_identity(manifest, name) + row = {'id': name, 'status': 'passed', 'artifacts': self.receipt['artifacts'], + 'identity': identity, 'receipt': dict(self.receipt, identity=identity, gate=name)} + manifest['stages' if name in self.gate.LOCAL_GATES else 'platforms'].append(row) + return manifest + + def binding(self): + return dict(repository='etr/libhttpserver', run_id='12', run_attempt='1', + source_commit='d' * 40, tag='3.0.0', archive_sha256=self.gate.digest(self.binary), + event='push', artifact_id='23', workflow_source='d'*40, producer_ids=['FreeBSD', 'Darwin', 'Windows'], + producer_artifacts={name:dict(id='24',digest='a'*64) for name in ('freebsd','darwin','windows')}) + + def test_valid_publication_requires_actual_readiness_then_calls_side_effect_once(self): + ci = self.ci(); manifest = self.complete(); binding = self.binding() + publish = Mock() + authorization = ci.authorize(self.gate, manifest, binding) + self.publish(authorization, binding, self.binary, publish) + publish.assert_called_once() + + def test_every_missing_or_unsuccessful_gate_and_platform_blocks_publication(self): + ci = self.ci() + for axis in ('stages', 'platforms'): + for index in range(len(self.complete()[axis])): + for damage in ('missing', 'failed', 'duplicate'): + manifest = self.complete(); publish = Mock() + if damage == 'missing': manifest[axis].pop(index) + if damage == 'failed': manifest[axis][index]['status'] = 'failed' + if damage == 'duplicate': manifest[axis].append(copy.deepcopy(manifest[axis][index])) + with self.subTest(axis=axis, index=index, damage=damage), self.assertRaises(ValueError): + self.publish(ci.authorize(self.gate, manifest, self.binding()), self.binding(), self.binary, publish) + publish.assert_not_called() + + def test_wrong_source_archive_run_tag_and_pr_cannot_publish(self): + ci = self.ci(); manifest = self.complete(); binding = self.binding() + authorization = ci.authorize(self.gate, manifest, binding) + for key in ('repository', 'run_id', 'run_attempt', 'source_commit', 'tag', 'artifact_id'): + altered = dict(binding, **{key: 'wrong'}); publish = Mock() + with self.subTest(key=key), self.assertRaises(ValueError): + self.publish(authorization, altered, self.binary, publish) + publish.assert_not_called() + for alteration in ({'event': 'pull_request'}, {'source_commit': 'a' * 40}, + {'archive_sha256': 'a' * 64}, {'tag': '3.0.0; touch marker'}, + {'producer_ids': ['Darwin']}): + with self.subTest(alteration=alteration), self.assertRaises(ValueError): + ci.authorize(self.gate, manifest, dict(binding, **alteration)) + self.binary.write_text('changed after acceptance') + publish = Mock() + with self.assertRaises(ValueError): self.publish(authorization, binding, self.binary, publish) + publish.assert_not_called() + + def test_ci_entrypoint_rejects_unresolved_request_without_side_effects(self): + import subprocess + script = ROOT / 'scripts/run-v3-release-ci.py' + self.assertTrue(script.exists(), 'selected archive CI entrypoint missing') + output=self.directory/'ci' + result=subprocess.run([sys.executable,str(script),'prepare','--output',str(output), + '--commit','$(touch marker)','--version','3.0.0'],cwd=self.directory,capture_output=True,text=True) + self.assertNotEqual(result.returncode,0) + self.assertFalse((self.directory/'marker').exists()) + self.assertFalse(output.exists()) + + def test_authorization_requires_content_bound_platform_producer_artifacts(self): + ci=self.ci() + for damage in ('missing','wrong-id','wrong-digest'): + binding=self.binding() + if damage=='missing': binding['producer_artifacts'].pop('darwin') + if damage=='wrong-id': binding['producer_artifacts']['darwin']['id']='$(touch marker)' + if damage=='wrong-digest': binding['producer_artifacts']['darwin']['digest']='old' + with self.subTest(damage=damage),self.assertRaises(ValueError): ci.authorize(self.gate,self.complete(),binding) + + def test_producer_binding_cannot_change_after_authorization(self): + ci=self.ci(); binding=self.binding(); authorization=ci.authorize(self.gate,self.complete(),binding) + binding['producer_artifacts']['darwin']['id']='25' + with self.assertRaises(ValueError): self.publish(authorization,binding,self.binary,Mock()) + + def test_actual_inline_publisher_calls_once_only_after_real_readiness(self): + publisher=self.publisher() + ci=self.ci(); binding=self.binding(); manifest=self.complete() + authorization=ci.authorize(self.gate,manifest,binding) + release=Mock() + publisher(authorization,binding,self.binary,manifest,'d'*40,release) + release.assert_called_once() + for alteration in ('head','manifest','archive','binding'): + release=Mock(); changed=copy.deepcopy(manifest); expected=copy.deepcopy(binding); tag_commit='d'*40 + if alteration=='head': tag_commit='e'*40 + if alteration=='manifest': changed['platforms'].pop() + if alteration=='archive': self.binary.write_text('different archive') + if alteration=='binding': expected['run_attempt']='2' + with self.subTest(alteration=alteration),self.assertRaises((ValueError,AssertionError)): + publisher(authorization,expected,self.binary,changed,tag_commit,release) + release.assert_not_called() + + def test_immutable_download_checks_source_run_attempt_digest_and_zip_members(self): + import hashlib + import zipfile + ci=self.ci() + archive=self.directory/'artifact.zip' + with zipfile.ZipFile(archive,'w') as stream: stream.writestr('selected.json','selected bytes') + expected=dict(repository='etr/libhttpserver',run_id='12',run_attempt='1',workflow_head='d'*40,artifact_id='23') + metadata=dict(id=23,expired=False,created_at='2026-10-15T03:01:00Z',workflow_run={'id':12,'head_sha':'d'*40},digest='sha256:'+hashlib.sha256(archive.read_bytes()).hexdigest()) + run=dict(id=12,run_attempt=1,run_started_at='2026-10-15T03:00:00Z',repository={'full_name':'etr/libhttpserver'},head_sha='d'*40) + ci.verify_download(archive,metadata,run,expected,metadata['digest'],self.directory/'download') + for field,value in [('id',24),('expired',True),('digest','sha256:'+'a'*64),('created_at','2026-10-14T03:01:00Z')]: + altered=dict(metadata,**{field:value}) + with self.subTest(field=field),self.assertRaises(ValueError): + ci.verify_download(archive,altered,run,expected,metadata['digest'],self.directory/field) + for changed in (dict(run,run_attempt=2),dict(run,head_sha='a'*40),dict(run,repository={'full_name':'other/repo'})): + with self.assertRaises(ValueError): ci.verify_download(archive,metadata,changed,expected,metadata['digest'],self.directory/'wrongrun') + for index,name in enumerate(('../outside','/outside','C:/outside','a\\..\\outside')): + bad=self.directory/('escape-%d.zip'%index) + with zipfile.ZipFile(bad,'w') as stream: stream.writestr(name,'escape') + digest='sha256:'+hashlib.sha256(bad.read_bytes()).hexdigest() + with self.subTest(name=name),self.assertRaises(ValueError): + ci.verify_download(bad,dict(metadata,digest=digest),run,expected,digest,self.directory/('badzip-%d'%index)) + + def test_pr_download_binds_api_branch_head_and_retains_merge_source_descriptor(self): + import hashlib + import zipfile + ci=self.ci() + head='08c5b7b5eb5e992f5a12d88136064010fc435f9d' + merge='b5f8d51fe8b597fb3c720aa5459c04b9147c615b' + archive=self.directory/'pr-artifact.zip' + with zipfile.ZipFile(archive,'w') as stream: + stream.writestr('selected.json',json.dumps({'commit':merge,'source':{'head':merge}})) + digest='sha256:'+hashlib.sha256(archive.read_bytes()).hexdigest() + expected=dict(repository='etr/libhttpserver',run_id='38848731938',run_attempt='1',workflow_head=head,artifact_id='12024142302') + metadata=dict(id=12024142302,expired=False,created_at='2026-10-16T04:05:15Z', + workflow_run=dict(id=38848731938,head_sha=head),digest=digest) + run=dict(id=38848731938,run_attempt=1,head_sha=head,event='pull_request', + repository={'full_name':'etr/libhttpserver'},run_started_at='2026-10-16T04:04:45Z') + destination=self.directory/'pr-download' + ci.verify_download(archive,metadata,run,expected,digest,destination) + self.assertEqual(json.loads((destination/'selected.json').read_text())['commit'],merge) + for index, altered in enumerate((dict(expected,workflow_head=merge),dict(expected,workflow_head='a'*40), + dict(expected,run_id='38848731939'),dict(expected,run_attempt='2'),dict(expected,repository='other/repo'))): + with self.subTest(expected=altered),self.assertRaises(ValueError): + ci.verify_download(archive,metadata,run,altered,digest,self.directory/('wrong-pr-'+str(index))) + for field,value in (('id',12024142303),('workflow_run',dict(id=38848731939,head_sha=head)), + ('workflow_run',dict(id=38848731938,head_sha=merge)),('digest','sha256:'+'a'*64)): + with self.subTest(field=field,value=value),self.assertRaises(ValueError): + ci.verify_download(archive,dict(metadata,**{field:value}),run,expected,digest,self.directory/'wrong-origin') + + def test_download_entrypoint_uses_trusted_api_head_instead_of_pr_merge_sha(self): + import argparse + import hashlib + import io + import os + import zipfile + from unittest.mock import patch + spec=importlib.util.spec_from_file_location('release_ci_entrypoint',ROOT/'scripts/run-v3-release-ci.py') + entrypoint=importlib.util.module_from_spec(spec); spec.loader.exec_module(entrypoint) + head='08c5b7b5eb5e992f5a12d88136064010fc435f9d'; merge='b5f8d51fe8b597fb3c720aa5459c04b9147c615b' + stream=io.BytesIO() + with zipfile.ZipFile(stream,'w') as archive: archive.writestr('selected.json',json.dumps({'commit':merge})) + contents=stream.getvalue(); digest='sha256:'+hashlib.sha256(contents).hexdigest() + metadata=dict(id=12024142302,expired=False,created_at='2026-10-16T04:05:15Z', + workflow_run=dict(id=38848731938,head_sha=head),digest=digest) + run=dict(id=38848731938,run_attempt=1,head_sha=head,repository={'full_name':'etr/libhttpserver'},run_started_at='2026-10-16T04:04:45Z') + def transport(argv,stdout,check): + self.assertEqual(argv,['gh','api','repos/etr/libhttpserver/actions/artifacts/12024142302/zip']) + stdout.write(contents) + env=dict(GITHUB_REPOSITORY='etr/libhttpserver',GITHUB_RUN_ID='38848731938',GITHUB_RUN_ATTEMPT='1', + GITHUB_SHA=merge,WORKFLOW_HEAD_SHA=head) + args=argparse.Namespace(artifact_id='12024142302',digest=digest,output=self.directory/'cli-download') + with patch.dict(os.environ,env),patch('subprocess.check_output',side_effect=[json.dumps(metadata),json.dumps(run)]),patch('subprocess.run',side_effect=transport): + entrypoint.download(args) + self.assertEqual(json.loads((args.output/'selected.json').read_text())['commit'],merge) + + def test_exact_commit_request_rejects_modified_and_untracked_source_inputs(self): + import subprocess + ci=self.ci(); root=self.directory/'checkout'; (root/'scripts').mkdir(parents=True) + for name in ('configure.ac','ChangeLog','scripts/validate-version.sh'): + (root/name).write_bytes((ROOT/name).read_bytes()) + def git(*args): + return subprocess.check_output(['git','-C',str(root),'-c','core.hooksPath=/dev/null',*args],stderr=subprocess.PIPE,text=True) + git('init','-q'); git('add','.') + git('-c','user.name=CI fixture','-c','user.email=ci@example.invalid','commit','-qm','fixture') + commit=git('rev-parse','HEAD').strip() + ci.validate_request(root,commit,'3.0.0') + original=(root/'configure.ac').read_bytes(); (root/'configure.ac').write_bytes(original+b'\n') + with self.assertRaises(ValueError): ci.validate_request(root,commit,'3.0.0') + (root/'configure.ac').write_bytes(original); (root/'injected.cpp').write_text('untracked source') + with self.assertRaises(ValueError): ci.validate_request(root,commit,'3.0.0') + + def test_publication_rejects_self_consistent_dirty_source_snapshot(self): + ci=self.ci(); manifest=self.complete() + manifest['release_artifact']['source']['working_diff_sha256']='a'*64 + self.identity_path.write_text(json.dumps(manifest['release_artifact'])) + self.gate.check_tag_ready(manifest) + with self.assertRaises(ValueError): ci.authorize(self.gate,manifest,self.binding()) + + def test_requested_commit_and_version_are_data_and_must_match_checkout(self): + ci = self.ci() + for commit, version in [('d'*40, '3.0.0; touch marker'), ('$(touch marker)', '3.0.0'), ('e'*40, '3.0.0')]: + with self.subTest(commit=commit,version=version), self.assertRaises(ValueError): + ci.validate_request(ROOT, commit, version) + + def test_request_requires_a_git_checkout_even_when_versions_are_valid(self): + ci=self.ci() + with self.assertRaises(ValueError): ci.validate_request(self.directory,'e'*40,'3.0.0') + + def test_selected_archive_relocation_preserves_source_and_rejects_tampering(self): + import io + import tarfile + ci = self.ci() + checkout = self.directory / 'checkout'; checkout.mkdir() + (checkout / 'configure.ac').write_text('source') + (checkout / 'specs').mkdir(); (checkout / 'specs/product.md').write_text('tooling') + archive = self.directory / 'candidate.tar.gz' + with tarfile.open(archive, 'w:gz') as stream: + for name in ('configure.ac',): + stream.add(checkout / name, arcname='libhttpserver-3.0.0/' + name) + source = dict(head='d' * 40, files_sha256={name: self.gate.digest(checkout / name) + for name in ('configure.ac', 'specs/product.md')}, snapshot_sha256='b' * 64) + identity = ci.selected_archive(checkout, source, archive, self.gate.digest(archive), self.directory / 'relocated') + self.assertEqual(identity['files_sha256'], source['files_sha256']) + self.assertEqual((Path(identity['source_root']) / 'specs/product.md').read_text(), 'tooling') + (checkout / 'configure.ac').write_text('another commit') + with self.assertRaises(ValueError): + ci.selected_archive(checkout, source, archive, self.gate.digest(archive), self.directory / 'wrong-source') + with self.assertRaises(ValueError): + ci.selected_archive(checkout, source, archive, 'a' * 64, self.directory / 'wrong-archive') + for kind in ('symlink', 'hardlink', 'absolute', 'traversal', 'duplicate'): + bad = self.directory / (kind + '.tar.gz') + with tarfile.open(bad, 'w:gz') as stream: + item = tarfile.TarInfo('root/configure.ac'); item.size = 1 + stream.addfile(item, io.BytesIO(b'x')) + item = tarfile.TarInfo({'absolute':'/outside', 'traversal':'../outside'}.get(kind, 'root/configure.ac')) + if kind == 'symlink': item.type=tarfile.SYMTYPE; item.linkname='/outside' + if kind == 'hardlink': item.type=tarfile.LNKTYPE; item.linkname='root/configure.ac' + stream.addfile(item) + with self.subTest(kind=kind), self.assertRaises(ValueError): + ci.selected_archive(checkout, source, bad, self.gate.digest(bad), self.directory / kind) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/integ/release_platform_bundle_test.py b/test/integ/release_platform_bundle_test.py new file mode 100755 index 000000000..908a48aa8 --- /dev/null +++ b/test/integ/release_platform_bundle_test.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +import importlib.util +import io +import json +from pathlib import Path +import sys +import tarfile +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts/lib')) + + +class PlatformBundles(unittest.TestCase): + def module(self): + path = ROOT / 'scripts/lib/release_platform.py' + self.assertTrue(path.exists(), 'portable platform receipt implementation missing') + spec = importlib.util.spec_from_file_location('release_platform', path) + module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + return module + + def test_catalog_relocates_original_bytes_and_rejects_tampering_missing_and_symlinks(self): + module = self.module() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); original = root / 'original.log' + original.write_text('retained immutable native results') + bundle = root / 'bundle' + module.retain(bundle, 'Darwin', {'id': 'Darwin'}, [original]) + moved = root / 'moved'; bundle.rename(moved); original.unlink() + resolver = module.Catalog(moved) + self.assertEqual(resolver.path(str(original)).read_text(), 'retained immutable native results') + with self.assertRaises(ValueError): resolver.path('/missing/record') + retained = resolver.path(str(original)); retained.write_text('tampered') + with self.assertRaises(ValueError): module.Catalog(moved) + retained.unlink(); retained.symlink_to(ROOT / 'configure.ac') + with self.assertRaises(ValueError): module.Catalog(moved) + + def test_original_and_resolved_tool_paths_both_retain_exact_bytes(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); binary=root/'binary'; binary.write_text('native selected compiler') + alias=root/'compiler'; alias.symlink_to(binary) + catalog=module.retain(root/'bundle','Windows',{'id':'Windows'},[alias]) + self.assertEqual(catalog.path(str(alias)).read_bytes(),catalog.path(str(binary)).read_bytes()) + + def test_equivalent_relative_dependency_paths_preserve_both_original_labels(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); (root/'other').mkdir(); file=root/'file'; file.write_text('dependency bytes') + alias=root/'other/../file' + catalog=module.retain(root/'bundle','FreeBSD',{'id':'FreeBSD'},[alias]) + self.assertEqual(catalog.path(str(alias)).read_bytes(),catalog.path(str(file)).read_bytes()) + + def test_windows_logical_paths_resolve_without_rewriting_original_receipts(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); original=root/'log'; original.write_text('native Windows bytes') + retained=root/'bundle'; module.retain(retained,'Windows',{'id':'Windows'},[original]) + catalog=json.loads((retained/'catalog.json').read_text()) + record=catalog['paths'].pop(str(original)); catalog['paths'][r'C:\native\matrix\log']=record + (retained/'catalog.json').write_text(json.dumps(catalog)) + resolver=module.Catalog(retained) + self.assertEqual(resolver.path('C:\\native/matrix/../matrix/log').read_text(),'native Windows bytes') + + def test_provider_client_and_dependency_records_require_retained_original_bytes(self): + module=self.module() + import hashlib + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); client=root/'client.py'; client.write_text('selected client module') + document={'nested':{'path':str(client),'sha256':hashlib.sha256(client.read_bytes()).hexdigest()}, + 'compile':{'dependency_sha256':{str(client):hashlib.sha256(client.read_bytes()).hexdigest()}}} + self.assertEqual(module.referenced_files(document),{client}) + client.write_text('substituted client module') + with self.assertRaises(ValueError): module.referenced_files(document) + client.unlink() + with self.assertRaises(ValueError): module.referenced_files(document) + + def test_duplicate_catalog_keys_are_rejected(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); file=root/'file'; file.write_text('bytes') + module.retain(root/'bundle','FreeBSD',{'id':'FreeBSD'},[file]) + catalog=root/'bundle/catalog.json' + catalog.write_text(catalog.read_text().replace('"schema_version": 1','"schema_version": 1, "schema_version": 1')) + with self.assertRaises(ValueError): module.Catalog(root/'bundle') + + def test_receipt_bytes_are_immutable_even_for_json_equivalent_edits(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); file=root/'file'; file.write_text('retained bytes') + module.retain(root/'bundle','Darwin',{'id':'Darwin'},[file]) + receipt=root/'bundle/receipt.json' + receipt.write_bytes(receipt.read_bytes()+b'\n') + with self.assertRaises(ValueError): module.Catalog(root/'bundle') + + def test_existing_sealed_os_leaves_do_not_require_fabricated_binary_files(self): + module=self.module() + for name in ('/usr/lib/libSystem.B.dylib',r'C:\Windows\System32\api-ms-win-core-synch-l1-2-0.dll'): + graph={'nodes':[dict(id=name,sha256=None,provenance='OS/API-set platform leaf')]} + self.assertEqual(module.referenced_files(graph),set()) + with self.assertRaises(ValueError): + module.referenced_files({'nodes':[dict(id='/private/provider/libssl.dylib',sha256=None,provenance='OS/API-set platform leaf')]}) + + def test_bundle_extraction_rejects_escape_duplicates_links_and_extra_files(self): + module = self.module() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, kind in (('../outside', None), ('/outside', None), ('Darwin/link', tarfile.SYMTYPE), + ('Darwin/link', tarfile.LNKTYPE), ('Darwin/extra', None)): + with self.subTest(name=name, kind=kind): + archive = root / 'bundle.tar.gz' + with tarfile.open(archive, 'w:gz') as stream: + item = tarfile.TarInfo(name); item.size=1 + if kind: item.type=kind; item.linkname='/outside'; item.size=0 + stream.addfile(item, io.BytesIO(b'x') if item.size else None) + with self.assertRaises(ValueError): module.unpack(archive, root / ('extract-' + str(kind) + str(len(name)))) + + def test_valid_archive_import_rejects_duplicate_members_and_namespace_collisions(self): + module=self.module() + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); file=root/'file'; file.write_text('retained bytes') + module.retain(root/'Darwin','Darwin',{'id':'Darwin'},[file]) + archive=root/'valid.tar.gz' + with tarfile.open(archive,'w:gz') as stream: stream.add(root/'Darwin',arcname='Darwin') + module.unpack(archive,root/'imported') + with self.assertRaises(ValueError): module.unpack(archive,root/'imported') + duplicate=root/'duplicate.tar.gz' + with tarfile.open(duplicate,'w:gz') as stream: + stream.add(root/'Darwin',arcname='Darwin') + stream.add(root/'Darwin/catalog.json',arcname='Darwin/catalog.json') + with self.assertRaises(ValueError): module.unpack(duplicate,root/'duplicate') + + def test_platform_origin_rejects_stale_summary_pruned_suite_and_provider_substitution(self): + module = self.module() + import package_matrix as matrix + import release_artifact as identity + import release_checks as checks + import hashlib + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); package = root / 'matrix'; package.mkdir() + inventory = matrix.load() + def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + header = root / 'provider.h'; header.write_bytes(b'selected OpenSSL\r\n') + selected = dict(archive_sha256='a'*64, snapshot_sha256='b'*64, extracted_sha256=hashlib.sha256(json.dumps({},sort_keys=True).encode()).hexdigest(), tls='both', sanitizer='none') + write(package/'summary.json', dict(status='passed', host='Darwin', lanes={'no':{},'yes':{}}, + revision='d'*40, provider_header_sha256=identity.sha256(header), + release_artifact={'archive':{'sha256':'a'*64}, 'source':{'snapshot_sha256':'b'*64,'head':'d'*40}})) + for mode in ('no','yes'): + lane=package/('tls-'+mode); prefix=lane/'prefix' + (prefix/'lib').mkdir(parents=True) + (prefix/'lib/libhttpserver.dylib').write_text('selected native library') + public={} + for linkage in ('shared','static'): + graph={'nodes':[{'id':str(prefix/'lib/libhttpserver.dylib'),'sha256':identity.sha256(prefix/'lib/libhttpserver.dylib')}] if linkage=='shared' else []} + public[linkage]=dict(dependencies=graph, compile={'dependencies':[]}, + cells=[{'configurations':inventory['cases']},{'configurations':inventory['cases']}]) + suites={} + for name, definition in matrix.suites(inventory, 'Darwin', mode).items(): + cases=definition.get('required_cases',[]) or ['fixture_case'] + log=lane/('integration-'+linkage)/(name+'.log') + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(''.join('Running test (%d): %s\n'%(i,case) for i,case in enumerate(cases))+ + '%d tests executed\n%d checks\n-> 0 failures\n-> 0 skipped\n'%(len(cases),len(cases))) + suites[name]=dict(matrix.suite_result(log.read_text(),0),status='passed', log=str(log)) + write(lane/('integration-'+linkage)/'suites.json',suites) + write(lane/'audit.json',dict(violations=[],tls_mode=mode,public=public)) + origin=dict(package=str(package), source_root=str(root/'source'), source_commit='d'*40, provider_header=str(header), native={}) + # This test checks installed matrix origin independently; native origin is mandatory in complete bundles. + counts=module.package_origin(ROOT, origin, lambda p: json.loads(Path(p).read_text()), lambda p: Path(p).read_text(), selected, 'Darwin') + self.assertGreater(counts['tests'],0) + wrong_origin=dict(origin,source_commit='e'*40) + with self.assertRaises(ValueError): + module.package_origin(ROOT,wrong_origin,lambda p:json.loads(Path(p).read_text()),lambda p:Path(p).read_text(),selected,'Darwin') + for mode in ('no','yes'): + build=root/('native-'+mode); (build/'test').mkdir(parents=True) + names=['protocol_fuzz_replay','hpack_corpus','http2_fuzz_replay','quic_fuzz_replay', + 'quic_recovery_fuzz_replay','http3_fuzz_replay','qpack_fuzz_replay','io_kqueue_backend'] + if mode=='yes': names.append('quic_tls_callbacks_fuzz_replay') + for name in names: (build/'test'/(name+'.trs')).write_text(':test-result: PASS\n') + (build/'test/test-suite.log').write_text('# TOTAL: %d\n# PASS: %d\n# SKIP: 0\n# XFAIL: 0\n# FAIL: 0\n# XPASS: 0\n# ERROR: 0\n'%(len(names),len(names))) + origin['native'][mode]=dict(build=str(build), tests=names) + counts=checks.counts(counts['tests']+len(names),counts['checks']+len(names)) + artifacts=[dict(path=str(header),sha256=identity.sha256(header),bytes=header.stat().st_size)] + row=dict(id='Darwin',identity=selected,archive_sha256='a'*64,origin=origin,status='passed',artifacts=artifacts, + receipt=dict(status='passed',gate='Darwin',identity=selected,counts=counts,artifacts=artifacts)) + retained=root/'retained' + catalog=module.retain(retained,'Darwin',row,[p for p in root.rglob('*') if p.is_file()]) + module.validate_origin(ROOT,row,catalog) + manifest=dict(release_artifact={'archive':{'sha256':'a'*64},'source':{'snapshot_sha256':'b'*64,'head':'d'*40},'files_sha256':{}}, + platforms=[dict(id=p,status='pending') for p in module.PLATFORMS]) + module.import_bundle(ROOT, manifest, catalog, expected=selected) + self.assertEqual(manifest['platforms'][1]['status'],'passed') + gate=checks.load_runner('run-v3-release-gates.py') + gate.verify_gate_rows(manifest,[manifest['platforms'][1]]) + raw_catalog=(catalog.directory/'catalog.json').read_bytes() + (catalog.directory/'catalog.json').write_bytes(raw_catalog+b'\n') + with self.assertRaisesRegex(ValueError,'changed artifact'): + gate.verify_gate_rows(manifest,[manifest['platforms'][1]]) + (catalog.directory/'catalog.json').write_bytes(raw_catalog) + with self.assertRaises(ValueError): module.import_bundle(ROOT, manifest, catalog, expected=selected) + wrong=json.loads(json.dumps(row)); wrong['archive_sha256']='e'*64 + (catalog.directory/'receipt.json').write_text(json.dumps(wrong)) + fresh=dict(release_artifact={'source':{'head':'d'*40}},platforms=[dict(id=p,status='pending') for p in module.PLATFORMS]) + with self.assertRaises(ValueError): module.import_bundle(ROOT, fresh, catalog, expected=selected) + (catalog.directory/'receipt.json').write_text(json.dumps(row)) + incomplete=json.loads(json.dumps(row)); incomplete['origin']['native']['yes']['tests'].remove('quic_tls_callbacks_fuzz_replay') + with self.assertRaises(ValueError): module.validate_origin(ROOT,incomplete,catalog) + catalog.path(str(header)).write_text('corrupt retained provider') + with self.assertRaises(ValueError): module.validate_origin(ROOT,row,catalog) + def windows(value): + if isinstance(value,dict): return {k:windows(v) for k,v in value.items()} + if isinstance(value,list): return [windows(v) for v in value] + if isinstance(value,str) and value.startswith(str(root)): + return value.replace(str(root),'C:/native').replace('/','\\') + return value + def physical(value): + return Path(str(value).replace('\\','/').replace('C:/native',str(root))) + module.package_origin(ROOT,windows(origin),lambda p:windows(json.loads(physical(p).read_text())), + lambda p:physical(p).read_text(),selected,'Darwin',lambda p:physical(p).read_bytes()) + suites_path=package/'tls-no/integration-shared/suites.json' + suites=json.loads(suites_path.read_text()); first=next(iter(suites)) + altered=json.loads(suites_path.read_text()); altered[first]['tests']+=1; write(suites_path,altered) + with self.assertRaises(ValueError): module.package_origin(ROOT,origin,lambda p:json.loads(Path(p).read_text()),lambda p:Path(p).read_text(),selected,'Darwin') + write(suites_path,suites); removed=suites.pop(first); write(suites_path,suites) + with self.assertRaises(ValueError): module.package_origin(ROOT,origin,lambda p:json.loads(Path(p).read_text()),lambda p:Path(p).read_text(),selected,'Darwin') + suites[first]=removed; write(suites_path,suites); header.write_text('different provider') + with self.assertRaises(ValueError): module.package_origin(ROOT,origin,lambda p:json.loads(Path(p).read_text()),lambda p:Path(p).read_text(),selected,'Darwin') + + +if __name__ == '__main__': unittest.main()