diff --git a/.agents/skills/fory-release/SKILL.md b/.agents/skills/fory-release/SKILL.md index 62f4453b96..d61d43d3be 100644 --- a/.agents/skills/fory-release/SKILL.md +++ b/.agents/skills/fory-release/SKILL.md @@ -1,56 +1,36 @@ --- name: fory-release -description: Prepare an Apache Fory release candidate from a clean release branch, including the version bump, RC tag, JVM staging, ASF source artifacts, SVN upload, and vote email. Use when creating or rerunning a Fory release candidate. +description: Stage or verify an Apache Fory release candidate. Use the GitHub Actions release workflow by default for ATR source staging and Nexus JVM staging; use the retained local manual workflow only when the user explicitly requests manual publishing. --- # Apache Fory Release -Use the repository release script for the release work. Do not manually reproduce its version-bump, JVM-publication, or source-build logic, and do not add unrelated test runs. +Use `.github/workflows/stage-release-candidate.yml` for source and JVM staging +unless the user explicitly asks for a manual release. The workflow uses +`ci/release.py`; do not reproduce its source-build, JVM-publication, Nexus +closure, or artifact-verification logic in shell commands. + +Do not add unrelated test runs. An invalid candidate gets a higher RC number; +never move or reuse an RC tag. ## Required Inputs -Collect these values before starting: +Collect these values: - `release_version`: final version without `v` or an RC suffix, such as `1.7.0`. - `rc`: RC suffix, such as `rc3`. - `previous_version`: previous release tag version, such as `1.6.1`. -- Release discussion URL, if already known. If it is not supplied, find the - exact release thread in the Fory development-list archive as described below. +- Release discussion URL, if already known. -Load release-manager details from `.local/fory-release.env`. If it does not exist, ask for the following values once, create the ignored local file, and continue. Never commit this file. +Derive the release values: ```bash -FORY_RELEASE_MANAGER_NAME="..." -FORY_RELEASE_APACHE_EMAIL="..." -FORY_RELEASE_GPG_FINGERPRINT="..." -FORY_DIST_DEV_WC="..." -``` - -Load the cached values and derive the release values: - -```bash -repo_root="$(git rev-parse --show-toplevel)" -release_config="$repo_root/.local/fory-release.env" -test -f "$release_config" -. "$release_config" - release_branch="releases-${release_version}" rc_tag="v${release_version}-${rc}" -dist_version="${release_version}" -release_manager_name="${FORY_RELEASE_MANAGER_NAME:?missing release manager name}" -apache_email="${FORY_RELEASE_APACHE_EMAIL:?missing Apache email}" -gpg_fingerprint="${FORY_RELEASE_GPG_FINGERPRINT:?missing GPG fingerprint}" -svn_wc="${FORY_DIST_DEV_WC:?missing ASF Subversion working-copy path}" +release_candidate_url="https://release-test.apache.org/vote/fory/${release_version}" ``` -Use the same `dist_version` in Subversion and the vote email. - -### Find the release discussion - -If the release discussion URL was not supplied, search the -[Fory development-list archive](https://lists.apache.org/list.html?dev@fory.apache.org) -for the exact release version and a `[DISCUSS]` subject. Use the deterministic -helper, which accepts exactly one root discussion thread: +If the discussion URL was not supplied, find the exact root discussion thread: ```bash discussion_url="$( @@ -59,98 +39,78 @@ discussion_url="$( )" ``` -Open the resulting URL and verify that its subject and body discuss the exact -`release_version`. If the automated search finds zero or multiple roots, use -the archive UI to search the same exact version and `[DISCUSS]`; ask the release -manager only if the result remains absent or ambiguous. When a URL is supplied, -open and verify it instead of assuming it matches this release. +Open the URL and verify that its subject and body discuss the exact version. If +the helper finds zero or multiple roots, search the Fory development-list +archive for the exact version and a `[DISCUSS]` subject. Ask the release manager +only if the result remains absent or ambiguous. -## Release Workflow +## Default CI Workflow -### 1. Create a clean release branch +### 1. Create the release branch and commit -Run from the repository root. Clean means no staged, modified, or untracked files. +Run from the repository root. Clean means no staged, modified, or untracked +files. ```bash test -z "$(git status --porcelain)" test "$(git remote get-url apache)" = "git@github.com:apache/fory.git" git fetch apache main --tags git switch -c "$release_branch" apache/main -test -z "$(git status --porcelain)" -``` - -Stop if the branch already exists or either cleanliness check fails. Do not remove or hide user files to make the check pass. - -### 2. Bump the version - -```bash python3 ci/release.py bump_version -version "$release_version" -l all git diff --check git status --short -``` - -Review the version diff. Use this command directly; do not substitute another version-bump workflow. - -### 3. Commit the release version - -```bash git add -u git commit -m "prepare release for ${release_version}" test -z "$(git status --porcelain)" release_commit="$(git rev-parse HEAD)" ``` -Stage only the version changes produced by the release script. - -### 4. Create and push the RC tag +Stop if the branch already exists or the tree is not clean. Review the version +diff and stage only the files changed by `bump_version`. -Confirm that neither the local nor remote tag already exists. An invalid RC gets a new RC number; never move or reuse an RC tag. +### 2. Create and push the RC tag ```bash test -z "$(git tag --list "$rc_tag")" test -z "$(git ls-remote --tags apache "refs/tags/${rc_tag}")" test "$(git rev-parse HEAD)" = "$release_commit" -git tag "$rc_tag" && git push apache "$rc_tag" +git tag "$rc_tag" +git push apache "$rc_tag" test "$(git rev-parse "${rc_tag}^{commit}")" = "$release_commit" ``` -The tag starts the ecosystem package-release workflows. Do not wait for them -here: start JVM publication immediately so the remote workflows and JVM staging -run in parallel. Once the tag has been pushed, any JVM or later release failure -invalidates this RC and requires a higher RC number; never move or reuse the -tag. +The tag starts the ecosystem package workflows. Once pushed, any staging or +verification failure invalidates this RC. -### 5. Publish JVM artifacts +### 3. Stage source and JVM artifacts in CI + +Dispatch the workflow on the RC tag with both default jobs enabled: ```bash -python3 ci/release.py publish_jvm +gh workflow run stage-release-candidate.yml \ + --repo apache/fory \ + --ref "$rc_tag" \ + -f source=true \ + -f jvm=true ``` -The command publishes from a temporary worktree at the committed `HEAD` and -removes that worktree afterward. Record the distinct Java/Kotlin and Scala -Nexus staging repository IDs from the output: +Find the newly created `workflow_dispatch` run for this exact tag, record its +run ID and URL, and wait for it with `gh run watch --exit-status`. Do not select +a run only by commit SHA because main-branch runs can share the same commit. -```bash -java_kotlin_staging_id="orgapachefory-..." -scala_staging_id="orgapachefory-..." -test -n "$java_kotlin_staging_id" -test -n "$scala_staging_id" -test "$java_kotlin_staging_id" != "$scala_staging_id" -``` +The source job builds, signs, and checksum-verifies the existing source-release +archive before uploading `dist/` to Apache Trusted Release (ATR) through OIDC. +It does not use SVN credentials. The JVM job publishes from the tag commit, +identifies only the Nexus repositories created by that run, closes both in one +request, and verifies representative artifacts through anonymous downloads. -After recording both IDs, read and follow -[Nexus staging closure](references/nexus-staging.md). It contains the -credential rules, authenticated state checks, `/bulk/close` request, HTTP 201 -gate, close polling, failure inspection, and anonymous artifact checks. Do not -close any repository ID that was not recorded from this publication. Keep both -repositories closed during the vote; do not promote them until the vote passes. +From the successful run log or job summary, record the distinct +`java_kotlin_staging_id` and `scala_staging_id`. Open the ATR candidate URL and +both closed Nexus repository URLs before drafting the vote. -### 6. Check the tag-triggered workflows +### 4. Check tag-triggered workflows -After JVM publication and Nexus closure, inspect the workflows that have been -running since the tag was pushed. Filter by the tag rather than only by commit -SHA so main-branch runs at the same commit are not mixed into the result. By -default, wait for every tag-triggered run and require successful conclusions: +Require all workflows triggered by the RC tag to succeed: ```bash python3 .agents/skills/fory-release/scripts/check_tag_workflows.py \ @@ -160,70 +120,121 @@ python3 .agents/skills/fory-release/scripts/check_tag_workflows.py \ --watch ``` -The helper re-queries by tag after waiting to catch later-created runs. If the -release manager explicitly waives workflow monitoring for a particular RC, run -the same command with `--allow-incomplete` instead of `--watch`, and record the -snapshot IDs, states, and reason. Do not cancel the remote workflows or report -incomplete runs as successful. +If the release manager explicitly waives workflow monitoring for this RC, use +`--allow-incomplete` instead of `--watch` and record the run states and reason. +Do not report incomplete workflows as successful. -### 7. Build the ASF source release +### 5. Draft the vote email -Start from the clean release branch. The build temporarily commits release-archive changes and resets them, so verify that it restores the original commit and clean tree. +Load release-manager identity from `.local/fory-release.env` only when drafting +the email. Never commit this ignored file. ```bash -test -z "$(git status --porcelain)" -before_build="$(git rev-parse HEAD)" -python3 ci/release.py build -v "$release_version" -test "$(git rev-parse HEAD)" = "$before_build" -test -z "$(git status --porcelain)" -test -f "dist/apache-fory-${release_version}-src.tar.gz" -test -f "dist/apache-fory-${release_version}-src.tar.gz.asc" -test -f "dist/apache-fory-${release_version}-src.tar.gz.sha512" +release_config="$(git rev-parse --show-toplevel)/.local/fory-release.env" +test -f "$release_config" +. "$release_config" +release_manager_name="${FORY_RELEASE_MANAGER_NAME:?missing release manager name}" +apache_email="${FORY_RELEASE_APACHE_EMAIL:?missing Apache email}" +gpg_fingerprint="${FORY_RELEASE_GPG_FINGERPRINT:?missing GPG fingerprint}" ``` -The build command verifies the generated PGP signature and SHA-512 checksum. +If the file does not exist, ask once for the three missing values and create +the ignored local file. Do not store the GitHub or Nexus secrets in it. -### 8. Commit the source release to ASF Subversion +Read [the vote email template](assets/vote-email.txt) and produce a complete, +copyable email. Fill every placeholder from verified output, confirm that no +`${...}` placeholder remains, use an explicit UTC deadline at least 72 hours +after sending, and do not send the email unless requested. + +## Explicit Manual Workflow + +Only when the user explicitly requests manual publishing, read and follow +[the manual release workflow](references/manual-release.md). Do not fall back +to it automatically after a CI failure. A failed CI candidate still requires a +higher RC tag. -Use a clean, updated working copy of the ASF development distribution repository. +## Verification-Only Requests + +Verification belongs to this skill; do not create or invoke a separate +release-verification workflow. For an existing candidate, inspect the exact +workflow run and tag, then download and verify the remote source artifacts: ```bash -test -d "$svn_wc/.svn" || svn checkout https://dist.apache.org/repos/dist/dev/fory "$svn_wc" -svn update "$svn_wc" -mkdir -p "$svn_wc/$dist_version" -cp dist/* "$svn_wc/$dist_version/" -svn add --force "$svn_wc/$dist_version" -svn status "$svn_wc/$dist_version" -svn commit "$svn_wc/$dist_version" -m "Prepare Apache Fory ${rc_tag}" -test -z "$(svn status "$svn_wc/$dist_version")" -svn log -l 1 "$svn_wc/$dist_version" -svn ls "https://dist.apache.org/repos/dist/dev/fory/${dist_version}/" +set -euo pipefail +: "${release_version:?missing release version}" +: "${release_candidate_url:?missing release candidate URL}" +: "${gpg_fingerprint:?missing expected GPG fingerprint}" +release_candidate_url="${release_candidate_url%/}" + +archive="apache-fory-${release_version}-src.tar.gz" +verify_root="$(mktemp -d)" +verify_dist="$verify_root/dist" +verify_gnupg="$verify_root/gnupg" +mkdir -m 700 "$verify_gnupg" +mkdir "$verify_dist" +trap 'rm -rf "$verify_root"' EXIT + +curl -fL "$release_candidate_url/$archive" -o "$verify_dist/$archive" +curl -fL "$release_candidate_url/$archive.asc" -o "$verify_dist/$archive.asc" +curl -fL "$release_candidate_url/$archive.sha512" \ + -o "$verify_dist/$archive.sha512" +curl -fL https://downloads.apache.org/fory/KEYS -o "$verify_root/KEYS" +GNUPGHOME="$verify_gnupg" gpg --batch --import "$verify_root/KEYS" + +( + cd "$verify_dist" + if command -v sha512sum >/dev/null; then + sha512sum --check "$archive.sha512" + else + shasum -a 512 -c "$archive.sha512" + fi + GNUPGHOME="$verify_gnupg" gpg --batch --verify "$archive.asc" "$archive" +) + +# VALIDSIG may append the primary-key fingerprint for subkey signatures. +signer_primary_fingerprint="$( + GNUPGHOME="$verify_gnupg" gpg --batch --status-fd 1 \ + --verify "$verify_dist/$archive.asc" "$verify_dist/$archive" 2>/dev/null | + awk '$2 == "VALIDSIG" { print (NF >= 12 ? $12 : $3); exit }' +)" +expected_fingerprint="$( + printf '%s' "$gpg_fingerprint" | tr -d ' ' | tr '[:lower:]' '[:upper:]' +)" +test "$signer_primary_fingerprint" = "$expected_fingerprint" ``` -Inspect `svn status` before committing. The upload is complete only after `svn commit` returns a revision and the remote `svn ls` shows the three release files; local `A` status alone is not an upload. +Use the staging IDs recorded by the exact CI run and the Nexus credentials from +the release manager's secret store to repeat the existing read-only repository +state and anonymous download checks: -### 9. Draft the vote email - -Read [the vote email template](assets/vote-email.txt) and produce a complete, -copyable email. Replace every placeholder from verified output, confirm that no -`${...}` placeholder remains, use an explicit UTC deadline at least 72 hours -after sending, and do not send the email unless requested. +```bash +: "${java_kotlin_staging_id:?missing Java/Kotlin staging ID}" +: "${scala_staging_id:?missing Scala staging ID}" +: "${rc_tag:?missing release-candidate tag}" +: "${NEXUS_USERNAME:?missing Nexus username}" +: "${NEXUS_PASSWORD:?missing Nexus password}" +python3 ci/release.py close_jvm_staging \ + -v "$release_version" \ + --rc-tag "$rc_tag" \ + --java-kotlin-id "$java_kotlin_staging_id" \ + --scala-id "$scala_staging_id" \ + --verify-only +``` -Before sending, verify the tag and commit, all URLs, both closed Maven staging repositories, the remote Subversion files, PGP fingerprint, and UTC deadline against the actual release outputs. +Finally run the tag-workflow checker from the default workflow without +publishing or changing remote state. Report verification only when the remote +checksum, signature, signer fingerprint, Nexus state/download checks, and tag +workflows all pass. ## Stop Conditions -Before pushing the tag, stop if the Git tree is dirty, a command fails, the RC -tag already exists, or the tag target would differ from the release commit. -After pushing the immutable tag, any failed JVM publication, workflow, -artifact verification, or Subversion publication invalidates that candidate; -fix the issue and create a higher RC instead of moving or reusing the tag. -Before sending the vote, require both staging repositories to be closed and -public, the Subversion commit to be remotely visible, and the tag workflows to -be successful unless the release manager explicitly waived monitoring. +Before pushing the tag, stop if the tree is dirty, the tag exists, a command +fails, or the tag target differs from the release commit. After pushing it, +stop on any CI, ATR, Nexus, signature, checksum, artifact, or tag-workflow +failure and create a higher RC after the cause is fixed. Do not promote Nexus +repositories or send the vote until the vote itself has passed. ## References -- [Apache Fory release guide](https://fory.apache.org/docs/community/how_to_release) - [Fory development-list archive](https://lists.apache.org/list.html?dev@fory.apache.org) -- [Sonatype Nexus 2 staging REST example](https://support.sonatype.com/hc/en-us/articles/213465448-Automatically-dropping-old-staging-repositories) +- [Apache Pekko CI release workflow](https://github.com/apache/pekko/blob/main/.github/workflows/stage-release-candidate.yml) diff --git a/.agents/skills/fory-release/assets/vote-email.txt b/.agents/skills/fory-release/assets/vote-email.txt index 87feab7589..95d54e41ba 100644 --- a/.agents/skills/fory-release/assets/vote-email.txt +++ b/.agents/skills/fory-release/assets/vote-email.txt @@ -14,7 +14,7 @@ The change list since Apache Fory ${previous_version}: https://github.com/apache/fory/compare/v${previous_version}...${rc_tag} The release candidate artifacts: -https://dist.apache.org/repos/dist/dev/fory/${dist_version}/ +${release_candidate_url} The Maven staging repositories: Java and Kotlin: diff --git a/.agents/skills/fory-release/references/manual-release.md b/.agents/skills/fory-release/references/manual-release.md new file mode 100644 index 0000000000..0b8ffc2589 --- /dev/null +++ b/.agents/skills/fory-release/references/manual-release.md @@ -0,0 +1,78 @@ +# Manual Apache Fory Release Staging + +Use this path only when the release manager explicitly requests manual +publishing. Complete the release branch, version commit, and immutable RC tag +steps from the main skill first. + +## Local Configuration + +Load these values from the ignored `.local/fory-release.env`. Ask for missing +values once and never commit the file. + +```bash +FORY_RELEASE_MANAGER_NAME="..." +FORY_RELEASE_APACHE_EMAIL="..." +FORY_RELEASE_GPG_FINGERPRINT="..." +FORY_DIST_DEV_WC="..." +``` + +Export `NEXUS_USERNAME` and `NEXUS_PASSWORD` from the release manager's secret +store. Do not print them or save them in the local release environment file. + +## Stage JVM Artifacts + +```bash +python3 ci/release.py publish_jvm +``` + +Record the distinct Java/Kotlin and Scala staging IDs created by this command. +Then read and follow [Nexus staging closure](nexus-staging.md), using only those +two IDs. Keep both repositories closed during the vote and do not promote them +until the vote passes. + +## Build the Source Release + +Start from the clean release branch. The build temporarily commits the release +archive LICENSE change and resets it, so require the original commit and clean +tree afterward. + +```bash +test -z "$(git status --porcelain)" +before_build="$(git rev-parse HEAD)" +python3 ci/release.py build -v "$release_version" +test "$(git rev-parse HEAD)" = "$before_build" +test -z "$(git status --porcelain)" +test -f "dist/apache-fory-${release_version}-src.tar.gz" +test -f "dist/apache-fory-${release_version}-src.tar.gz.asc" +test -f "dist/apache-fory-${release_version}-src.tar.gz.sha512" +``` + +The command verifies the generated PGP signature and SHA-512 checksum. + +## Upload the Source Release to ASF Subversion + +Use a clean, updated working copy of the ASF development distribution +repository. + +```bash +svn_wc="${FORY_DIST_DEV_WC:?missing ASF Subversion working-copy path}" +test -d "$svn_wc/.svn" || \ + svn checkout https://dist.apache.org/repos/dist/dev/fory "$svn_wc" +svn update "$svn_wc" +mkdir -p "$svn_wc/$release_version" +cp dist/* "$svn_wc/$release_version/" +svn add --force "$svn_wc/$release_version" +svn status "$svn_wc/$release_version" +svn commit "$svn_wc/$release_version" -m "Prepare Apache Fory ${rc_tag}" +test -z "$(svn status "$svn_wc/$release_version")" +svn log -l 1 "$svn_wc/$release_version" +svn ls "https://dist.apache.org/repos/dist/dev/fory/${release_version}/" +``` + +Inspect `svn status` before committing. The upload is complete only after the +commit returns a revision and remote `svn ls` shows all three source-release +files. For the vote template, set: + +```bash +release_candidate_url="https://dist.apache.org/repos/dist/dev/fory/${release_version}/" +``` diff --git a/.github/workflows/stage-release-candidate.yml b/.github/workflows/stage-release-candidate.yml new file mode 100644 index 0000000000..29a38537a5 --- /dev/null +++ b/.github/workflows/stage-release-candidate.yml @@ -0,0 +1,210 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# All `uses:` action pins in this workflow must come from the Apache action allowlist: +# https://github.com/apache/infrastructure-actions/blob/main/actions.yml + +name: Stage Release Candidate +run-name: "Stage Fory ${{ github.ref_name }}" + +on: + workflow_dispatch: + inputs: + source: + description: Stage the source release to ATR + default: true + type: boolean + jvm: + description: Stage JVM artifacts to Nexus + default: true + type: boolean + +permissions: + contents: read + +concurrency: + # Nexus repository ownership is discovered by comparing the global repository + # list before and after publication, so release-candidate staging must be serial. + group: stage-release-candidate + cancel-in-progress: false + +jobs: + validate: + if: github.repository == 'apache/fory' + runs-on: ubuntu-24.04 + outputs: + rc-tag: ${{ steps.release.outputs.rc-tag }} + version: ${{ steps.release.outputs.version }} + steps: + - name: Validate release candidate tag + id: release + shell: bash + env: + REF_NAME: ${{ github.ref_name }} + REF_TYPE: ${{ github.ref_type }} + STAGE_JVM: ${{ inputs.jvm }} + STAGE_SOURCE: ${{ inputs.source }} + run: | + set -euo pipefail + if [[ "$STAGE_SOURCE" != "true" && "$STAGE_JVM" != "true" ]]; then + echo "Enable at least one release-candidate staging job." + exit 1 + fi + if [[ "$REF_TYPE" != "tag" ]]; then + echo "Run this workflow on an immutable release-candidate tag." + exit 1 + fi + if [[ "$REF_NAME" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)-rc([0-9]+)$ ]]; then + echo "version=${BASH_REMATCH[1]}" >> "$GITHUB_OUTPUT" + echo "rc-tag=$REF_NAME" >> "$GITHUB_OUTPUT" + else + echo "Expected a tag matching vX.Y.Z-rcN, got: $REF_NAME" + exit 1 + fi + + - name: Checkout release candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5.0.1 + with: + fetch-depth: 0 + fetch-tags: true + persist-credentials: false + + - name: Verify release versions + shell: bash + env: + VERSION: ${{ steps.release.outputs.version }} + run: | + set -euo pipefail + python3 ci/release.py bump_version -version "$VERSION" -l all + if [[ -n "$(git status --porcelain)" ]]; then + echo "The RC tag does not contain the requested release versions." + git diff --stat + exit 1 + fi + + stage-source: + needs: validate + if: ${{ needs.validate.result == 'success' && inputs.source }} + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + steps: + - name: Checkout release candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5.0.1 + with: + fetch-depth: 0 + fetch-tags: true + persist-credentials: false + + - name: Import release signing key + shell: bash + env: + FORY_GPG_SECRET_KEY: ${{ secrets.FORY_GPG_SECRET_KEY }} + run: | + set -euo pipefail + printf '%s' "$FORY_GPG_SECRET_KEY" | \ + gpg --batch --import --import-options import-show + + - name: Build source release + shell: bash + env: + VERSION: ${{ needs.validate.outputs.version }} + run: | + set -euo pipefail + git config user.name "Apache Fory Release Automation" + git config user.email "dev@fory.apache.org" + python3 ci/release.py build -v "$VERSION" + + - name: Upload source release to ATR + uses: apache/tooling-actions/upload-to-atr@ca6ed9e095c40db61c42a90db2599bb2fbc2319f + with: + project: fory + version: ${{ needs.validate.outputs.version }} + + stage-jvm: + needs: validate + if: ${{ needs.validate.result == 'success' && inputs.jvm }} + runs-on: ubuntu-24.04 + steps: + - name: Checkout release candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v5.0.1 + with: + fetch-depth: 0 + fetch-tags: true + persist-credentials: false + + - name: Set up Java 25 and Apache Nexus + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + distribution: temurin + java-version: 25 + server-id: apache.releases.https + server-username: NEXUS_USERNAME + server-password: NEXUS_PASSWORD + + - name: Set up sbt + uses: sbt/setup-sbt@c7d2d6258b4bd0d3ec5129e6b3453199d3c79729 # v1.5.8 + + - name: Import release signing key + shell: bash + env: + FORY_GPG_SECRET_KEY: ${{ secrets.FORY_GPG_SECRET_KEY }} + run: | + set -euo pipefail + printf '%s' "$FORY_GPG_SECRET_KEY" | \ + gpg --batch --import --import-options import-show + + - name: Stage and verify JVM artifacts + id: stage + shell: bash + env: + NEXUS_USERNAME: ${{ secrets.NEXUS_STAGE_DEPLOYER_USER }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_STAGE_DEPLOYER_PW }} + RC_TAG: ${{ needs.validate.outputs.rc-tag }} + VERSION: ${{ needs.validate.outputs.version }} + run: | + set -euo pipefail + python3 ci/release.py stage_jvm \ + -v "$VERSION" \ + --rc-tag "$RC_TAG" \ + --output "$RUNNER_TEMP/fory-jvm-staging.json" + + - name: Summarize Nexus staging repositories + if: ${{ always() }} + shell: bash + run: | + set -euo pipefail + metadata="$RUNNER_TEMP/fory-jvm-staging.json" + if [[ ! -f "$metadata" ]]; then + exit 0 + fi + java_kotlin_id="$(jq -r .java_kotlin_staging_id "$metadata")" + scala_id="$(jq -r '.scala_staging_id // empty' "$metadata")" + { + echo "### Nexus staging repositories" + echo + echo "- Staging result: \`${{ steps.stage.outcome }}\`" + if [[ -n "$java_kotlin_id" && "$java_kotlin_id" != "null" ]]; then + echo "- Java/Kotlin: \`$java_kotlin_id\`" + echo "- Scala: \`$scala_id\`" + else + while IFS= read -r staging_id; do + echo "- Open repository requiring review: \`$staging_id\`" + done < <(jq -r '.new_staging_ids[]?' "$metadata") + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/ci/release.py b/ci/release.py index 6f9f4df24e..e1e2e4f5f8 100644 --- a/ci/release.py +++ b/ci/release.py @@ -280,6 +280,67 @@ def publish_jvm(languages="all", mode="release"): raise NotImplementedError(f"Unsupported JVM release language: {lang}") +def stage_jvm(v, rc_tag, output=None): + """Publish, discover, close, and verify the JVM staging repositories.""" + _validate_release_candidate(v, rc_tag) + _require_jvm_release_version(v) + authorization = _nexus_authorization() + repositories_before = set(_nexus_repositories(authorization)) + + try: + publish_jvm() + repositories = _nexus_repositories(authorization) + java_kotlin_id, scala_id = _discover_jvm_staging_repositories( + v, + repositories_before, + repositories, + authorization, + ) + except Exception: + _record_failed_jvm_staging(repositories_before, authorization, output) + raise + + staging = { + "java_kotlin_staging_id": java_kotlin_id, + "new_staging_ids": sorted( + _new_fory_staging_repositories(repositories_before, repositories) + ), + "scala_staging_id": scala_id, + } + logger.info( + "Created Nexus staging repositories: Java/Kotlin=%s, Scala=%s", + java_kotlin_id, + scala_id, + ) + if output: + _write_staging_metadata(output, staging) + + close_jvm_staging(v, rc_tag, java_kotlin_id, scala_id) + + +def _record_failed_jvm_staging(repositories_before, authorization, output): + try: + repositories = _nexus_repositories(authorization) + staging_ids = sorted( + _new_fory_staging_repositories(repositories_before, repositories) + ) + if staging_ids: + logger.error( + "JVM publication failed after creating open repositories: %s", + staging_ids, + ) + if output: + _write_staging_metadata(output, {"new_staging_ids": staging_ids}) + except Exception as exc: + logger.error("Unable to inspect Nexus repositories after failure: %s", exc) + + +def _write_staging_metadata(output, staging): + with open(output, "w", encoding="utf-8") as f: + json.dump(staging, f, indent=2, sort_keys=True) + f.write("\n") + + def close_jvm_staging( v, rc_tag, @@ -288,11 +349,7 @@ def close_jvm_staging( verify_only=False, ): """Close and verify the two Nexus repositories created by publish_jvm.""" - _check_release_version(v) - if not re.fullmatch(r"\d+\.\d+\.\d+", v): - raise ValueError(f"Invalid final release version: {v}") - if not re.fullmatch(rf"v{re.escape(v)}-rc\d+", rc_tag): - raise ValueError(f"RC tag {rc_tag} does not match release version {v}") + _validate_release_candidate(v, rc_tag) staging_ids = [java_kotlin_staging_id, scala_staging_id] for staging_id in staging_ids: if not re.fullmatch(r"orgapachefory-\d+", staging_id): @@ -325,6 +382,29 @@ def close_jvm_staging( _verify_nexus_downloads(v, java_kotlin_staging_id, scala_staging_id) +def _validate_release_candidate(v, rc_tag): + _check_release_version(v) + if not re.fullmatch(r"\d+\.\d+\.\d+", v): + raise ValueError(f"Invalid final release version: {v}") + if not re.fullmatch(rf"v{re.escape(v)}-rc\d+", rc_tag): + raise ValueError(f"RC tag {rc_tag} does not match release version {v}") + + +def _require_jvm_release_version(v): + versions = { + "Java": _read_java_version(), + "Kotlin": _read_kotlin_version(), + "Scala": _read_scala_version(), + } + mismatches = { + language: version for language, version in versions.items() if version != v + } + if mismatches: + raise RuntimeError( + f"JVM project versions must match release version {v}: {mismatches}" + ) + + def _nexus_authorization(): username = os.environ.get("NEXUS_USERNAME") password = os.environ.get("NEXUS_PASSWORD") @@ -364,7 +444,7 @@ def _nexus_request(path, authorization, method="GET", payload=None): raise RuntimeError(f"Nexus {method} {path} failed: {exc.reason}") from None -def _nexus_states(staging_ids, authorization): +def _nexus_repositories(authorization): status, body = _nexus_request( "/service/local/staging/profile_repositories", authorization ) @@ -379,9 +459,11 @@ def _nexus_states(staging_ids, authorization): data = payload.get("data") if isinstance(payload, dict) else None if not isinstance(data, list): raise RuntimeError("Nexus repository-list response has no data list") - repositories = { - item.get("repositoryId"): item for item in data if isinstance(item, dict) - } + return {item.get("repositoryId"): item for item in data if isinstance(item, dict)} + + +def _nexus_states(staging_ids, authorization): + repositories = _nexus_repositories(authorization) missing = [ staging_id for staging_id in staging_ids if staging_id not in repositories ] @@ -395,6 +477,71 @@ def _nexus_states(staging_ids, authorization): return states +def _discover_jvm_staging_repositories( + v, + repositories_before, + repositories, + authorization, +): + candidates = _new_fory_staging_repositories( + repositories_before, + repositories, + ) + java_kotlin_ids = _matching_staging_repositories( + candidates, + ( + f"org/apache/fory/fory-core/{v}/fory-core-{v}.jar", + f"org/apache/fory/fory-kotlin/{v}/fory-kotlin-{v}.jar", + ), + authorization, + ) + scala_ids = _matching_staging_repositories( + candidates, + ( + f"org/apache/fory/fory-scala_2.13/{v}/fory-scala_2.13-{v}.jar", + f"org/apache/fory/fory-json-scala_3/{v}/fory-json-scala_3-{v}.jar", + ), + authorization, + ) + if len(java_kotlin_ids) != 1 or len(scala_ids) != 1: + raise RuntimeError( + "Expected one new Java/Kotlin and one new Scala staging repository; " + f"found Java/Kotlin={java_kotlin_ids}, Scala={scala_ids}, " + f"new repositories={sorted(candidates)}" + ) + java_kotlin_id = java_kotlin_ids[0] + scala_id = scala_ids[0] + if java_kotlin_id == scala_id: + raise RuntimeError("Java/Kotlin and Scala staging repositories must differ") + states = { + staging_id: candidates[staging_id].get("type") + for staging_id in (java_kotlin_id, scala_id) + } + _require_nexus_state(states, "open") + return java_kotlin_id, scala_id + + +def _new_fory_staging_repositories(repositories_before, repositories): + return { + staging_id: repository + for staging_id, repository in repositories.items() + if staging_id not in repositories_before + and re.fullmatch(r"orgapachefory-\d+", staging_id or "") + } + + +def _matching_staging_repositories(candidates, artifact_paths, authorization): + matches = [] + for staging_id in sorted(candidates): + base_url = f"{NEXUS_BASE_URL}/content/repositories/{staging_id}/" + if all( + _nexus_download_status(base_url + path, authorization) == 200 + for path in artifact_paths + ): + matches.append(staging_id) + return matches + + def _require_nexus_state(states, expected_state): unexpected = { staging_id: state @@ -451,20 +598,7 @@ def _verify_nexus_downloads(v, java_kotlin_staging_id, scala_staging_id): f"{scala_url}org/apache/fory/fory-json-scala_3/{v}/fory-json-scala_3-{v}.jar", ] for url in artifact_urls: - request = urllib.request.Request( - url, headers={"User-Agent": "apache-fory-release-helper/1"} - ) - try: - with urllib.request.urlopen( - request, timeout=NEXUS_TIMEOUT_SECONDS - ) as response: - status = response.status - except urllib.error.HTTPError as exc: - status = exc.code - except urllib.error.URLError as exc: - raise RuntimeError( - f"Anonymous Nexus download failed for {url}: {exc.reason}" - ) + status = _nexus_download_status(url) logger.info("Anonymous Nexus download HTTP %s: %s", status, url) if status != 200: raise RuntimeError( @@ -472,6 +606,21 @@ def _verify_nexus_downloads(v, java_kotlin_staging_id, scala_staging_id): ) +def _nexus_download_status(url, authorization=None): + headers = {"User-Agent": "apache-fory-release-helper/1"} + if authorization: + headers["Authorization"] = authorization + request = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(request, timeout=NEXUS_TIMEOUT_SECONDS) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + except urllib.error.URLError as exc: + access = "Authenticated" if authorization else "Anonymous" + raise RuntimeError(f"{access} Nexus download failed for {url}: {exc.reason}") + + def _jvm_release_langs(languages): if languages in (None, "", "all"): return list(JVM_RELEASE_LANGS) @@ -1029,6 +1178,15 @@ def _read_kotlin_version(): return version +def _read_scala_version(): + build = os.path.join(PROJECT_ROOT_DIR, "scala", "build.sbt") + with open(build, "r", encoding="utf-8") as f: + matches = re.findall(r'^val foryVersion = "([^"]+)"$', f.read(), re.MULTILINE) + if len(matches) != 1: + raise ValueError("Cannot find the unique Scala Fory version") + return matches[0] + + def bump_version(**kwargs): new_version = kwargs["version"] langs = kwargs["l"] @@ -2004,6 +2162,27 @@ def _parse_args(): ) publish_jvm_parser.set_defaults(func=publish_jvm) + stage_jvm_parser = subparsers.add_parser( + "stage_jvm", + description="Publish, discover, close, and verify JVM staging repositories", + ) + stage_jvm_parser.add_argument( + "-v", + dest="v", + required=True, + help="final release version without a v prefix or RC suffix", + ) + stage_jvm_parser.add_argument( + "--rc-tag", + required=True, + help="immutable release-candidate tag", + ) + stage_jvm_parser.add_argument( + "--output", + help="optional JSON path for the discovered staging repository IDs", + ) + stage_jvm_parser.set_defaults(func=stage_jvm) + close_jvm_parser = subparsers.add_parser( "close_jvm_staging", description="Close and verify the two Nexus repositories from publish_jvm",