feat: address migration gaps (WIP)

.github/workflows/cd-monorepo.yml

@@ -198,25 +198,42 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
+      # The branch already holds the version to release. The bump happens after
+      # the publish, so the version on the branch names an unreleased version
+      - name: Read the version to release
+        run: |
+          git pull
+          VERSION=$(python3 syft/version.py)
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+          echo "Releasing syft $VERSION"
+
+      # Artifacts live inside the package, so they must be committed before the
+      # build. Re-running the export writes nothing when they already exist.
+      - name: Freeze release artifacts
+        run: |
+          just export-release-artifacts
+          git add syft/migrations/history
+          git diff --cached --quiet || \
+            git commit -m "Freeze syft v${{ env.VERSION }} release artifacts"
+
       - name: Upload to PyPI
         id: publish
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT }}
         run: |
-          git pull
-          just bump-and-publish ${{ inputs.bump_type }}
-          VERSION=$(python3 syft/version.py)
-          echo "VERSION=$VERSION" >> $GITHUB_ENV
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
-
-      # bump-and-publish already committed the version bump. Root tags continue
-      # the repo's historical vX.Y.Z lineage; sub-packages use <name>/vX.Y.Z.
-      # The tag is annotated and pushed explicitly: `--follow-tags` only pushes
-      # annotated tags, so a lightweight tag would never reach origin.
-      - name: Tag and push
+          just publish
+          echo "version=${{ env.VERSION }}" >> $GITHUB_OUTPUT
+
+      # The tag must name the published version, so it is created before the bump.
+      # Root tags continue the repo's historical vX.Y.Z lineage; sub-packages
+      # use <name>/vX.Y.Z. The tag is annotated and pushed explicitly:
+      # `--follow-tags` only pushes annotated tags, so a lightweight tag would
+      # never reach origin.
+      - name: Tag the release, then bump for the next one
         run: |
           git tag -a "v${{ env.VERSION }}" -m "syft v${{ env.VERSION }}"
+          just bump ${{ inputs.bump_type }}
           git push origin HEAD
           git push origin "refs/tags/v${{ env.VERSION }}"
 

.github/workflows/cd-syft-bg.yml

@@ -43,16 +43,15 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
-      - name: Bump version
+      # The branch already holds the version to release. The bump happens after
+      # the publish, so the version on the branch always names an unreleased
+      # version while people develop.
+      - name: Read the version to release
         run: |
           git pull
-          pip install packaging
-          OUTPUT=$(python scripts/bump_version.py syft-bg ${{ inputs.bump_type }})
-          VERSION=$(echo "$OUTPUT" | sed -n '1p')
-          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-bg/pyproject.toml','rb'))['project']['version'])")
           echo "VERSION=$VERSION" >> $GITHUB_ENV
-          echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV
-          echo "Bumped syft-bg to $VERSION (modified: $MODIFIED)"
+          echo "Releasing syft-bg $VERSION"
 
       - name: Build package
         working-directory: packages/syft-bg
@@ -65,12 +64,17 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_BG }}
         run: uvx twine upload --verbose dist/*
 
-      # Annotated tag + explicit push: `--follow-tags` only pushes annotated
-      # tags, so a lightweight tag would never reach origin.
-      - name: Commit and tag
+      # The tag must name the published version, so it is created before the bump.
+      # The tag is annotated and pushed explicitly: `--follow-tags` only pushes
+      # annotated tags, so a lightweight tag would never reach origin.
+      - name: Tag the release, then bump for the next one
         run: |
-          git add ${{ env.MODIFIED }}
-          git commit -m "Release syft-bg v${{ env.VERSION }}"
           git tag -a "syft-bg/v${{ env.VERSION }}" -m "syft-bg v${{ env.VERSION }}"
+          pip install packaging
+          OUTPUT=$(python scripts/bump_version.py syft-bg ${{ inputs.bump_type }} --dependents published)
+          NEXT=$(echo "$OUTPUT" | sed -n '1p')
+          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          git add $MODIFIED
+          git commit -m "Bump syft-bg to $NEXT for the next release"
           git push origin HEAD
           git push origin "refs/tags/syft-bg/v${{ env.VERSION }}"

.github/workflows/cd-syft-dataset.yml

@@ -43,16 +43,24 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
-      - name: Bump version
+      # The branch already holds the version to release. The bump happens after
+      # the publish, so the version on the branch always names an unreleased
+      # version while people develop.
+      - name: Read the version to release
         run: |
           git pull
-          pip install packaging
-          OUTPUT=$(python scripts/bump_version.py syft-dataset ${{ inputs.bump_type }})
-          VERSION=$(echo "$OUTPUT" | sed -n '1p')
-          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-datasets/pyproject.toml','rb'))['project']['version'])")
           echo "VERSION=$VERSION" >> $GITHUB_ENV
-          echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV
-          echo "Bumped syft-dataset to $VERSION (modified: $MODIFIED)"
+          echo "Releasing syft-dataset $VERSION"
+
+      # Artifacts live inside the package, so they must be committed before the
+      # build. Re-running the export writes nothing when they already exist.
+      - name: Freeze release artifacts
+        run: |
+          uv run python packages/syft-datasets/scripts/export_release_artifact.py
+          git add packages/syft-datasets/src/syft_datasets/migrations/history
+          git diff --cached --quiet || \
+            git commit -m "Freeze syft-dataset v${{ env.VERSION }} release artifacts"
 
       - name: Build package
         working-directory: packages/syft-datasets
@@ -65,12 +73,17 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_DATASET }}
         run: uvx twine upload --verbose dist/*
 
-      # Annotated tag + explicit push: `--follow-tags` only pushes annotated
-      # tags, so a lightweight tag would never reach origin.
-      - name: Commit and tag
+      # The tag must name the published version, so it is created before the bump.
+      # The tag is annotated and pushed explicitly: `--follow-tags` only pushes
+      # annotated tags, so a lightweight tag would never reach origin.
+      - name: Tag the release, then bump for the next one
         run: |
-          git add ${{ env.MODIFIED }}
-          git commit -m "Release syft-dataset v${{ env.VERSION }}"
           git tag -a "syft-dataset/v${{ env.VERSION }}" -m "syft-dataset v${{ env.VERSION }}"
+          pip install packaging
+          OUTPUT=$(python scripts/bump_version.py syft-dataset ${{ inputs.bump_type }} --dependents published)
+          NEXT=$(echo "$OUTPUT" | sed -n '1p')
+          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          git add $MODIFIED
+          git commit -m "Bump syft-dataset to $NEXT for the next release"
           git push origin HEAD
           git push origin "refs/tags/syft-dataset/v${{ env.VERSION }}"

.github/workflows/cd-syft-enclave.yml

@@ -43,16 +43,15 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
-      - name: Bump version
+      # The branch already holds the version to release. The bump happens after
+      # the publish, so the version on the branch always names an unreleased
+      # version while people develop.
+      - name: Read the version to release
         run: |
           git pull
-          pip install packaging
-          OUTPUT=$(python scripts/bump_version.py syft-enclave ${{ inputs.bump_type }})
-          VERSION=$(echo "$OUTPUT" | sed -n '1p')
-          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-enclave/pyproject.toml','rb'))['project']['version'])")
           echo "VERSION=$VERSION" >> $GITHUB_ENV
-          echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV
-          echo "Bumped syft-enclave to $VERSION (modified: $MODIFIED)"
+          echo "Releasing syft-enclave $VERSION"
 
       - name: Build package
         working-directory: packages/syft-enclave
@@ -65,12 +64,17 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_ENCLAVE }}
         run: uvx twine upload --verbose dist/*
 
-      # Annotated tag + explicit push: `--follow-tags` only pushes annotated
-      # tags, so a lightweight tag would never reach origin.
-      - name: Commit and tag
+      # The tag must name the published version, so it is created before the bump.
+      # The tag is annotated and pushed explicitly: `--follow-tags` only pushes
+      # annotated tags, so a lightweight tag would never reach origin.
+      - name: Tag the release, then bump for the next one
         run: |
-          git add ${{ env.MODIFIED }}
-          git commit -m "Release syft-enclave v${{ env.VERSION }}"
           git tag -a "syft-enclave/v${{ env.VERSION }}" -m "syft-enclave v${{ env.VERSION }}"
+          pip install packaging
+          OUTPUT=$(python scripts/bump_version.py syft-enclave ${{ inputs.bump_type }} --dependents published)
+          NEXT=$(echo "$OUTPUT" | sed -n '1p')
+          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          git add $MODIFIED
+          git commit -m "Bump syft-enclave to $NEXT for the next release"
           git push origin HEAD
           git push origin "refs/tags/syft-enclave/v${{ env.VERSION }}"

.github/workflows/cd-syft-job.yml

@@ -43,16 +43,24 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
-      - name: Bump version
+      # The branch already holds the version to release. The bump happens after
+      # the publish, so the version on the branch always names an unreleased
+      # version while people develop.
+      - name: Read the version to release
         run: |
           git pull
-          pip install packaging
-          OUTPUT=$(python scripts/bump_version.py syft-job ${{ inputs.bump_type }})
-          VERSION=$(echo "$OUTPUT" | sed -n '1p')
-          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-job/pyproject.toml','rb'))['project']['version'])")
           echo "VERSION=$VERSION" >> $GITHUB_ENV
-          echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV
-          echo "Bumped syft-job to $VERSION (modified: $MODIFIED)"
+          echo "Releasing syft-job $VERSION"
+
+      # Artifacts live inside the package, so they must be committed before the
+      # build. Re-running the export writes nothing when they already exist.
+      - name: Freeze release artifacts
+        run: |
+          uv run python packages/syft-job/scripts/export_release_artifact.py
+          git add packages/syft-job/src/syft_job/migrations/history
+          git diff --cached --quiet || \
+            git commit -m "Freeze syft-job v${{ env.VERSION }} release artifacts"
 
       - name: Build package
         working-directory: packages/syft-job
@@ -65,12 +73,17 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_JOB }}
         run: uvx twine upload --verbose dist/*
 
-      # Annotated tag + explicit push: `--follow-tags` only pushes annotated
-      # tags, so a lightweight tag would never reach origin.
-      - name: Commit and tag
+      # The tag must name the published version, so it is created before the bump.
+      # The tag is annotated and pushed explicitly: `--follow-tags` only pushes
+      # annotated tags, so a lightweight tag would never reach origin.
+      - name: Tag the release, then bump for the next one
         run: |
-          git add ${{ env.MODIFIED }}
-          git commit -m "Release syft-job v${{ env.VERSION }}"
           git tag -a "syft-job/v${{ env.VERSION }}" -m "syft-job v${{ env.VERSION }}"
+          pip install packaging
+          OUTPUT=$(python scripts/bump_version.py syft-job ${{ inputs.bump_type }} --dependents published)
+          NEXT=$(echo "$OUTPUT" | sed -n '1p')
+          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          git add $MODIFIED
+          git commit -m "Bump syft-job to $NEXT for the next release"
           git push origin HEAD
           git push origin "refs/tags/syft-job/v${{ env.VERSION }}"

.github/workflows/cd-syft-migration.yml

@@ -43,16 +43,15 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
-      - name: Bump version
+      # The branch already holds the version to release. The bump happens after
+      # the publish, so the version on the branch always names an unreleased
+      # version while people develop.
+      - name: Read the version to release
         run: |
           git pull
-          pip install packaging
-          OUTPUT=$(python scripts/bump_version.py syft-migration ${{ inputs.bump_type }})
-          VERSION=$(echo "$OUTPUT" | sed -n '1p')
-          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-migration/pyproject.toml','rb'))['project']['version'])")
           echo "VERSION=$VERSION" >> $GITHUB_ENV
-          echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV
-          echo "Bumped syft-migration to $VERSION (modified: $MODIFIED)"
+          echo "Releasing syft-migration $VERSION"
 
       - name: Build package
         working-directory: packages/syft-migration
@@ -65,12 +64,17 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_MIGRATION }}
         run: uvx twine upload --verbose dist/*
 
-      # Annotated tag + explicit push: `--follow-tags` only pushes annotated
-      # tags, so a lightweight tag would never reach origin.
-      - name: Commit and tag
+      # The tag must name the published version, so it is created before the bump.
+      # The tag is annotated and pushed explicitly: `--follow-tags` only pushes
+      # annotated tags, so a lightweight tag would never reach origin.
+      - name: Tag the release, then bump for the next one
         run: |
-          git add ${{ env.MODIFIED }}
-          git commit -m "Release syft-migration v${{ env.VERSION }}"
           git tag -a "syft-migration/v${{ env.VERSION }}" -m "syft-migration v${{ env.VERSION }}"
+          pip install packaging
+          OUTPUT=$(python scripts/bump_version.py syft-migration ${{ inputs.bump_type }} --dependents published)
+          NEXT=$(echo "$OUTPUT" | sed -n '1p')
+          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          git add $MODIFIED
+          git commit -m "Bump syft-migration to $NEXT for the next release"
           git push origin HEAD
           git push origin "refs/tags/syft-migration/v${{ env.VERSION }}"

.github/workflows/cd-syft-permissions.yml

@@ -43,16 +43,15 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
-      - name: Bump version
+      # The branch already holds the version to release. The bump happens after
+      # the publish, so the version on the branch always names an unreleased
+      # version while people develop.
+      - name: Read the version to release
         run: |
           git pull
-          pip install packaging
-          OUTPUT=$(python scripts/bump_version.py syft-permissions ${{ inputs.bump_type }})
-          VERSION=$(echo "$OUTPUT" | sed -n '1p')
-          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-permissions/pyproject.toml','rb'))['project']['version'])")
           echo "VERSION=$VERSION" >> $GITHUB_ENV
-          echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV
-          echo "Bumped syft-permissions to $VERSION (modified: $MODIFIED)"
+          echo "Releasing syft-permissions $VERSION"
 
       - name: Build package
         working-directory: packages/syft-permissions
@@ -65,12 +64,17 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_PERMISSIONS }}
         run: uvx twine upload --verbose dist/*
 
-      # Annotated tag + explicit push: `--follow-tags` only pushes annotated
-      # tags, so a lightweight tag would never reach origin.
-      - name: Commit and tag
+      # The tag must name the published version, so it is created before the bump.
+      # The tag is annotated and pushed explicitly: `--follow-tags` only pushes
+      # annotated tags, so a lightweight tag would never reach origin.
+      - name: Tag the release, then bump for the next one
         run: |
-          git add ${{ env.MODIFIED }}
-          git commit -m "Release syft-permissions v${{ env.VERSION }}"
           git tag -a "syft-permissions/v${{ env.VERSION }}" -m "syft-permissions v${{ env.VERSION }}"
+          pip install packaging
+          OUTPUT=$(python scripts/bump_version.py syft-permissions ${{ inputs.bump_type }} --dependents published)
+          NEXT=$(echo "$OUTPUT" | sed -n '1p')
+          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          git add $MODIFIED
+          git commit -m "Bump syft-permissions to $NEXT for the next release"
           git push origin HEAD
           git push origin "refs/tags/syft-permissions/v${{ env.VERSION }}"

.github/workflows/cd-syft-perms.yml

@@ -43,16 +43,15 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
-      - name: Bump version
+      # The branch already holds the version to release. The bump happens after
+      # the publish, so the version on the branch always names an unreleased
+      # version while people develop.
+      - name: Read the version to release
         run: |
           git pull
-          pip install packaging
-          OUTPUT=$(python scripts/bump_version.py syft-perms ${{ inputs.bump_type }})
-          VERSION=$(echo "$OUTPUT" | sed -n '1p')
-          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-perms/pyproject.toml','rb'))['project']['version'])")
           echo "VERSION=$VERSION" >> $GITHUB_ENV
-          echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV
-          echo "Bumped syft-perms to $VERSION (modified: $MODIFIED)"
+          echo "Releasing syft-perms $VERSION"
 
       - name: Build package
         working-directory: packages/syft-perms
@@ -65,12 +64,17 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_PERMS }}
         run: uvx twine upload --verbose dist/*
 
-      # Annotated tag + explicit push: `--follow-tags` only pushes annotated
-      # tags, so a lightweight tag would never reach origin.
-      - name: Commit and tag
+      # The tag must name the published version, so it is created before the bump.
+      # The tag is annotated and pushed explicitly: `--follow-tags` only pushes
+      # annotated tags, so a lightweight tag would never reach origin.
+      - name: Tag the release, then bump for the next one
         run: |
-          git add ${{ env.MODIFIED }}
-          git commit -m "Release syft-perms v${{ env.VERSION }}"
           git tag -a "syft-perms/v${{ env.VERSION }}" -m "syft-perms v${{ env.VERSION }}"
+          pip install packaging
+          OUTPUT=$(python scripts/bump_version.py syft-perms ${{ inputs.bump_type }} --dependents published)
+          NEXT=$(echo "$OUTPUT" | sed -n '1p')
+          MODIFIED=$(echo "$OUTPUT" | sed -n '2p')
+          git add $MODIFIED
+          git commit -m "Bump syft-perms to $NEXT for the next release"
           git push origin HEAD
           git push origin "refs/tags/syft-perms/v${{ env.VERSION }}"