(PE-47992) Preserve released changelog sections in module release prep

.github/workflows/lint.yml

@@ -8,6 +8,9 @@ on:
     branches:
       - "main"
 
+permissions:
+  contents: read
+
 jobs:
   lint:
     runs-on: "ubuntu-latest"
@@ -21,3 +24,8 @@ jobs:
         with:
           file_or_dir: ".github/"
           config_file: ".yamllint"
+
+      - name: "Test module_release_prep changelog preservation"
+        run: |
+          python3 -c 'import yaml' 2> /dev/null || { sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends python3-yaml; }
+          bash tests/module_release_prep/test_changelog_guard.sh

.github/workflows/module_release_prep.yml

@@ -37,15 +37,109 @@ jobs:
         run: |
           export GH_HOST=github.com
           gh extension install chelnak/gh-changelog
-          gh changelog new --next-version v${{ github.event.inputs.version }}
+          gh changelog new --next-version "v${VERSION#v}"
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VERSION: ${{ inputs.version }}
+
+      # gh-changelog regenerates the whole file from all closed PRs and tags, which
+      # rewrites and backfills sections of versions that were already released.
+      # Keep the generated preamble and newest section, and append the sections
+      # of previously released versions from the committed CHANGELOG.md. The step
+      # fails rather than lose history when the version is not a plain version,
+      # the generated file is unusable, its newest section is not the requested
+      # version, or the committed file has headings that cannot be restored.
+      - name: "Preserve changelog history"
+        env:
+          VERSION: ${{ inputs.version }}
+        run: |
+          # Reject anything but a plain version, which also keeps it safe to use in
+          # the sed expression of the "Update metadata.json" step
+          if [[ "$VERSION" == *$'\n'* ]] || ! [[ "$VERSION" =~ ^v?[0-9]+(\.[0-9]+)*([-+][0-9A-Za-z.-]+)?$ ]]; then
+            echo "::error::Invalid version"
+            exit 1
+          fi
+          # git ls-tree prints nothing when the file is untracked and fails on a real git error
+          tracked=$(git ls-tree --name-only HEAD -- CHANGELOG.md)
+          if [ -z "$tracked" ]; then
+            echo "No committed CHANGELOG.md, nothing to preserve"
+            exit 0
+          fi
+          old=$(mktemp)
+          history=$(mktemp)
+          merged=$(mktemp)
+          trap 'rm -f "$old" "$history" "$merged"' EXIT
+          git show "HEAD:./CHANGELOG.md" > "$old"
+
+          if ! grep -q '^## ' "$old"; then
+            # Nothing to restore when the file is at most a bare title. Any other
+            # heading is history in a style that cannot be restored, so stop.
+            headings=$(grep -Ec '^#{1,6}([[:space:]]|$)' "$old" || true)
+            titles=$(grep -Eic '^#[[:space:]]+(change ?log|changes|release notes|history)[[:space:]]*$' "$old" || true)
+            if [ "$headings" -gt "$titles" ] || [ "$headings" -gt 1 ]; then
+              echo "::error::Committed CHANGELOG.md has headings but no '## ' sections, refusing to drop its history"
+              exit 1
+            fi
+            echo "::warning::Committed CHANGELOG.md has no '## ' sections, leaving generated CHANGELOG.md as is"
+            exit 0
+          fi
+          if [ ! -f CHANGELOG.md ] || ! grep -q '^## \[' CHANGELOG.md; then
+            echo "::error::Generated CHANGELOG.md is missing or has no version sections, refusing to drop the committed history"
+            exit 1
+          fi
+
+          version=${VERSION#v}
+          # Version of a "## [vX.Y.Z](url) - date" header, without the "v"
+          top_version=$(grep -m1 '^## \[' CHANGELOG.md | sed 's/^## \[v\{0,1\}\([^]]*\)\].*/\1/')
+          if [ "$top_version" != "$version" ]; then
+            echo "::error::Newest generated section is $top_version, not $version, refusing to rewrite the released history"
+            exit 1
+          fi
+
+          # Generated preamble and newest section
+          awk '/^## \[/ { n++ } n < 2' CHANGELOG.md > "$merged"
+
+          # Committed sections, minus the one for the version being prepared (a
+          # re-run replaces it) and any "Unreleased" section. A section starts at
+          # any "## " header, e.g. "## [v1.2.3](url)" or "## v1.2.3". Text before
+          # the first section is dropped.
+          PREP_VERSION="$version" awk '
+            /^## / {
+              v = $0
+              sub(/^## \[?[vV]?/, "", v)
+              sub(/\].*/, "", v)
+              sub(/[ \t\r].*/, "", v)
+              sub(/[:,;]+$/, "", v)
+              keep = (v != ENVIRON["PREP_VERSION"] && tolower(v) != "unreleased")
+            }
+            keep
+          ' "$old" > "$history"
+
+          if grep -Eiq '^## \[?v?unreleased([][:space:]:,;]|$)' "$old"; then
+            echo "::warning::Dropping the committed Unreleased section"
+          fi
+
+          if [ -s "$history" ]; then
+            # Ensure a blank line separates the new section from the restored
+            # history, using the line ending of the file
+            if [ -n "$(tail -n 1 "$merged" | tr -d '\r')" ]; then
+              if head -n 1 "$merged" | grep -q "$(printf '\r')"; then
+                printf '\r\n' >> "$merged"
+              else
+                echo >> "$merged"
+              fi
+            fi
+            cat "$history" >> "$merged"
+          fi
+          cp "$merged" CHANGELOG.md
 
       - name: "Update metadata.json"
+        env:
+          VERSION: ${{ inputs.version }}
         run: |
           current_version=$(jq --raw-output .version metadata.json)
           # Update version in metadata.json, only matching first occurrence
-          sed -i "0,/$current_version/s//${{ github.event.inputs.version }}/" $(find . -name 'metadata.json')
+          sed -i "0,/$current_version/s//${VERSION#v}/" $(find . -name 'metadata.json')
 
       - name: "Configure forge authentication"
         env:

CLAUDE.md

@@ -54,7 +54,8 @@ The workflows split along two product lines and a lifecycle:
 Cross-cutting workflows:
 
 - [lint.yml](.github/workflows/lint.yml) — the **only** workflow that runs on
-  this repo's own pushes/PRs (not `workflow_call`); yamllints `.github/`.
+  this repo's own pushes/PRs (not `workflow_call`); yamllints `.github/` and runs
+  the tests under [tests/](tests/).
 - [claude-pr.yml](.github/workflows/claude-pr.yml) — auto PR review (see below).
 - [fork_ci_label_guard.yml](.github/workflows/fork_ci_label_guard.yml) — part of
   the fork-PR security model (see below).
@@ -106,12 +107,17 @@ exact commits**.
 
 ## Linting
 
-CI lints YAML only. To reproduce locally:
+CI yamllints `.github/` and runs the tests under [tests/](tests/). To reproduce
+locally:
 
 ```bash
 yamllint -c .yamllint .github/
 ```
 
+```bash
+tests/module_release_prep/test_changelog_guard.sh  # needs python3 with PyYAML
+```
+
 Config: [.yamllint](.yamllint) extends `default` and disables `document-start`,
 `truthy`, and `line-length`.
 

README.md

@@ -15,7 +15,7 @@ The following are the workflows we currently maintain in this repository:
 * module_ci: runs automated unit testing CI for modules on PRs
 * module_pdk_update: runs `pdk update` on a module and opens/refreshes a `pdk_update` PR with the template changes
 * fork_ci_label_guard: strips the `allowed-for-ci` label from fork PRs on each new commit so privileged acceptance tests re-require CODEOWNER review
-* module_release_prep: prepares the module for release by running necessary pre-release checks and tasks
+* module_release_prep: prepares the module for release by running necessary pre-release checks and tasks, keeping the sections of previously released versions in `CHANGELOG.md` intact when it is regenerated. It fails on an invalid version, and when the regenerated changelog does not start with the requested version or the existing history cannot be restored. A committed `Unreleased` section is dropped, with a warning
 * module_release: handles the release process of the module, including versioning and publishing
 * tooling_mend_ruby: automates the usage of mend for vulnerability scanning on tools
 * workflow-restarter-test: tests the workflow restarter functionality