diff --git a/.github/workflows/cd-monorepo.yml b/.github/workflows/cd-monorepo.yml index 9d840a458f3..dc8da8de6e8 100644 --- a/.github/workflows/cd-monorepo.yml +++ b/.github/workflows/cd-monorepo.yml @@ -198,25 +198,42 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # The branch already holds the version to release. The bump happens after + # the publish, so the version on the branch names an unreleased version + - name: Read the version to release + run: | + git pull + VERSION=$(python3 syft/version.py) + echo "VERSION=$VERSION" >> $GITHUB_ENV + echo "Releasing syft $VERSION" + + # Artifacts live inside the package, so they must be committed before the + # build. Re-running the export writes nothing when they already exist. + - name: Freeze release artifacts + run: | + just export-release-artifacts + git add syft/migrations/history + git diff --cached --quiet || \ + git commit -m "Freeze syft v${{ env.VERSION }} release artifacts" + - name: Upload to PyPI id: publish env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT }} run: | - git pull - just bump-and-publish ${{ inputs.bump_type }} - VERSION=$(python3 syft/version.py) - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "version=$VERSION" >> $GITHUB_OUTPUT - - # bump-and-publish already committed the version bump. Root tags continue - # the repo's historical vX.Y.Z lineage; sub-packages use /vX.Y.Z. - # The tag is annotated and pushed explicitly: `--follow-tags` only pushes - # annotated tags, so a lightweight tag would never reach origin. - - name: Tag and push + just publish + echo "version=${{ env.VERSION }}" >> $GITHUB_OUTPUT + + # The tag must name the published version, so it is created before the bump. + # Root tags continue the repo's historical vX.Y.Z lineage; sub-packages + # use /vX.Y.Z. The tag is annotated and pushed explicitly: + # `--follow-tags` only pushes annotated tags, so a lightweight tag would + # never reach origin. + - name: Tag the release, then bump for the next one run: | git tag -a "v${{ env.VERSION }}" -m "syft v${{ env.VERSION }}" + just bump ${{ inputs.bump_type }} git push origin HEAD git push origin "refs/tags/v${{ env.VERSION }}" diff --git a/.github/workflows/cd-syft-bg.yml b/.github/workflows/cd-syft-bg.yml index 2a404487a8b..b4d5b5f666e 100644 --- a/.github/workflows/cd-syft-bg.yml +++ b/.github/workflows/cd-syft-bg.yml @@ -43,16 +43,15 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Bump version + # The branch already holds the version to release. The bump happens after + # the publish, so the version on the branch always names an unreleased + # version while people develop. + - name: Read the version to release run: | git pull - pip install packaging - OUTPUT=$(python scripts/bump_version.py syft-bg ${{ inputs.bump_type }}) - VERSION=$(echo "$OUTPUT" | sed -n '1p') - MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-bg/pyproject.toml','rb'))['project']['version'])") echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV - echo "Bumped syft-bg to $VERSION (modified: $MODIFIED)" + echo "Releasing syft-bg $VERSION" - name: Build package working-directory: packages/syft-bg @@ -65,12 +64,17 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_BG }} run: uvx twine upload --verbose dist/* - # Annotated tag + explicit push: `--follow-tags` only pushes annotated - # tags, so a lightweight tag would never reach origin. - - name: Commit and tag + # The tag must name the published version, so it is created before the bump. + # The tag is annotated and pushed explicitly: `--follow-tags` only pushes + # annotated tags, so a lightweight tag would never reach origin. + - name: Tag the release, then bump for the next one run: | - git add ${{ env.MODIFIED }} - git commit -m "Release syft-bg v${{ env.VERSION }}" git tag -a "syft-bg/v${{ env.VERSION }}" -m "syft-bg v${{ env.VERSION }}" + pip install packaging + OUTPUT=$(python scripts/bump_version.py syft-bg ${{ inputs.bump_type }} --dependents published) + NEXT=$(echo "$OUTPUT" | sed -n '1p') + MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + git add $MODIFIED + git commit -m "Bump syft-bg to $NEXT for the next release" git push origin HEAD git push origin "refs/tags/syft-bg/v${{ env.VERSION }}" diff --git a/.github/workflows/cd-syft-dataset.yml b/.github/workflows/cd-syft-dataset.yml index 335031b1558..de39501bd3d 100644 --- a/.github/workflows/cd-syft-dataset.yml +++ b/.github/workflows/cd-syft-dataset.yml @@ -43,16 +43,24 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Bump version + # The branch already holds the version to release. The bump happens after + # the publish, so the version on the branch always names an unreleased + # version while people develop. + - name: Read the version to release run: | git pull - pip install packaging - OUTPUT=$(python scripts/bump_version.py syft-dataset ${{ inputs.bump_type }}) - VERSION=$(echo "$OUTPUT" | sed -n '1p') - MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-datasets/pyproject.toml','rb'))['project']['version'])") echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV - echo "Bumped syft-dataset to $VERSION (modified: $MODIFIED)" + echo "Releasing syft-dataset $VERSION" + + # Artifacts live inside the package, so they must be committed before the + # build. Re-running the export writes nothing when they already exist. + - name: Freeze release artifacts + run: | + uv run python packages/syft-datasets/scripts/export_release_artifact.py + git add packages/syft-datasets/src/syft_datasets/migrations/history + git diff --cached --quiet || \ + git commit -m "Freeze syft-dataset v${{ env.VERSION }} release artifacts" - name: Build package working-directory: packages/syft-datasets @@ -65,12 +73,17 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_DATASET }} run: uvx twine upload --verbose dist/* - # Annotated tag + explicit push: `--follow-tags` only pushes annotated - # tags, so a lightweight tag would never reach origin. - - name: Commit and tag + # The tag must name the published version, so it is created before the bump. + # The tag is annotated and pushed explicitly: `--follow-tags` only pushes + # annotated tags, so a lightweight tag would never reach origin. + - name: Tag the release, then bump for the next one run: | - git add ${{ env.MODIFIED }} - git commit -m "Release syft-dataset v${{ env.VERSION }}" git tag -a "syft-dataset/v${{ env.VERSION }}" -m "syft-dataset v${{ env.VERSION }}" + pip install packaging + OUTPUT=$(python scripts/bump_version.py syft-dataset ${{ inputs.bump_type }} --dependents published) + NEXT=$(echo "$OUTPUT" | sed -n '1p') + MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + git add $MODIFIED + git commit -m "Bump syft-dataset to $NEXT for the next release" git push origin HEAD git push origin "refs/tags/syft-dataset/v${{ env.VERSION }}" diff --git a/.github/workflows/cd-syft-enclave.yml b/.github/workflows/cd-syft-enclave.yml index 3f98f5b619a..94a89d4b7cc 100644 --- a/.github/workflows/cd-syft-enclave.yml +++ b/.github/workflows/cd-syft-enclave.yml @@ -43,16 +43,15 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Bump version + # The branch already holds the version to release. The bump happens after + # the publish, so the version on the branch always names an unreleased + # version while people develop. + - name: Read the version to release run: | git pull - pip install packaging - OUTPUT=$(python scripts/bump_version.py syft-enclave ${{ inputs.bump_type }}) - VERSION=$(echo "$OUTPUT" | sed -n '1p') - MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-enclave/pyproject.toml','rb'))['project']['version'])") echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV - echo "Bumped syft-enclave to $VERSION (modified: $MODIFIED)" + echo "Releasing syft-enclave $VERSION" - name: Build package working-directory: packages/syft-enclave @@ -65,12 +64,17 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_ENCLAVE }} run: uvx twine upload --verbose dist/* - # Annotated tag + explicit push: `--follow-tags` only pushes annotated - # tags, so a lightweight tag would never reach origin. - - name: Commit and tag + # The tag must name the published version, so it is created before the bump. + # The tag is annotated and pushed explicitly: `--follow-tags` only pushes + # annotated tags, so a lightweight tag would never reach origin. + - name: Tag the release, then bump for the next one run: | - git add ${{ env.MODIFIED }} - git commit -m "Release syft-enclave v${{ env.VERSION }}" git tag -a "syft-enclave/v${{ env.VERSION }}" -m "syft-enclave v${{ env.VERSION }}" + pip install packaging + OUTPUT=$(python scripts/bump_version.py syft-enclave ${{ inputs.bump_type }} --dependents published) + NEXT=$(echo "$OUTPUT" | sed -n '1p') + MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + git add $MODIFIED + git commit -m "Bump syft-enclave to $NEXT for the next release" git push origin HEAD git push origin "refs/tags/syft-enclave/v${{ env.VERSION }}" diff --git a/.github/workflows/cd-syft-job.yml b/.github/workflows/cd-syft-job.yml index 95b06953d87..79c13acfab6 100644 --- a/.github/workflows/cd-syft-job.yml +++ b/.github/workflows/cd-syft-job.yml @@ -43,16 +43,24 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Bump version + # The branch already holds the version to release. The bump happens after + # the publish, so the version on the branch always names an unreleased + # version while people develop. + - name: Read the version to release run: | git pull - pip install packaging - OUTPUT=$(python scripts/bump_version.py syft-job ${{ inputs.bump_type }}) - VERSION=$(echo "$OUTPUT" | sed -n '1p') - MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-job/pyproject.toml','rb'))['project']['version'])") echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV - echo "Bumped syft-job to $VERSION (modified: $MODIFIED)" + echo "Releasing syft-job $VERSION" + + # Artifacts live inside the package, so they must be committed before the + # build. Re-running the export writes nothing when they already exist. + - name: Freeze release artifacts + run: | + uv run python packages/syft-job/scripts/export_release_artifact.py + git add packages/syft-job/src/syft_job/migrations/history + git diff --cached --quiet || \ + git commit -m "Freeze syft-job v${{ env.VERSION }} release artifacts" - name: Build package working-directory: packages/syft-job @@ -65,12 +73,17 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_JOB }} run: uvx twine upload --verbose dist/* - # Annotated tag + explicit push: `--follow-tags` only pushes annotated - # tags, so a lightweight tag would never reach origin. - - name: Commit and tag + # The tag must name the published version, so it is created before the bump. + # The tag is annotated and pushed explicitly: `--follow-tags` only pushes + # annotated tags, so a lightweight tag would never reach origin. + - name: Tag the release, then bump for the next one run: | - git add ${{ env.MODIFIED }} - git commit -m "Release syft-job v${{ env.VERSION }}" git tag -a "syft-job/v${{ env.VERSION }}" -m "syft-job v${{ env.VERSION }}" + pip install packaging + OUTPUT=$(python scripts/bump_version.py syft-job ${{ inputs.bump_type }} --dependents published) + NEXT=$(echo "$OUTPUT" | sed -n '1p') + MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + git add $MODIFIED + git commit -m "Bump syft-job to $NEXT for the next release" git push origin HEAD git push origin "refs/tags/syft-job/v${{ env.VERSION }}" diff --git a/.github/workflows/cd-syft-migration.yml b/.github/workflows/cd-syft-migration.yml index f331dc5a426..f69fe10c54f 100644 --- a/.github/workflows/cd-syft-migration.yml +++ b/.github/workflows/cd-syft-migration.yml @@ -43,16 +43,15 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Bump version + # The branch already holds the version to release. The bump happens after + # the publish, so the version on the branch always names an unreleased + # version while people develop. + - name: Read the version to release run: | git pull - pip install packaging - OUTPUT=$(python scripts/bump_version.py syft-migration ${{ inputs.bump_type }}) - VERSION=$(echo "$OUTPUT" | sed -n '1p') - MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-migration/pyproject.toml','rb'))['project']['version'])") echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV - echo "Bumped syft-migration to $VERSION (modified: $MODIFIED)" + echo "Releasing syft-migration $VERSION" - name: Build package working-directory: packages/syft-migration @@ -65,12 +64,17 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_MIGRATION }} run: uvx twine upload --verbose dist/* - # Annotated tag + explicit push: `--follow-tags` only pushes annotated - # tags, so a lightweight tag would never reach origin. - - name: Commit and tag + # The tag must name the published version, so it is created before the bump. + # The tag is annotated and pushed explicitly: `--follow-tags` only pushes + # annotated tags, so a lightweight tag would never reach origin. + - name: Tag the release, then bump for the next one run: | - git add ${{ env.MODIFIED }} - git commit -m "Release syft-migration v${{ env.VERSION }}" git tag -a "syft-migration/v${{ env.VERSION }}" -m "syft-migration v${{ env.VERSION }}" + pip install packaging + OUTPUT=$(python scripts/bump_version.py syft-migration ${{ inputs.bump_type }} --dependents published) + NEXT=$(echo "$OUTPUT" | sed -n '1p') + MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + git add $MODIFIED + git commit -m "Bump syft-migration to $NEXT for the next release" git push origin HEAD git push origin "refs/tags/syft-migration/v${{ env.VERSION }}" diff --git a/.github/workflows/cd-syft-permissions.yml b/.github/workflows/cd-syft-permissions.yml index cd3b16609d9..15d76f8e430 100644 --- a/.github/workflows/cd-syft-permissions.yml +++ b/.github/workflows/cd-syft-permissions.yml @@ -43,16 +43,15 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Bump version + # The branch already holds the version to release. The bump happens after + # the publish, so the version on the branch always names an unreleased + # version while people develop. + - name: Read the version to release run: | git pull - pip install packaging - OUTPUT=$(python scripts/bump_version.py syft-permissions ${{ inputs.bump_type }}) - VERSION=$(echo "$OUTPUT" | sed -n '1p') - MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-permissions/pyproject.toml','rb'))['project']['version'])") echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV - echo "Bumped syft-permissions to $VERSION (modified: $MODIFIED)" + echo "Releasing syft-permissions $VERSION" - name: Build package working-directory: packages/syft-permissions @@ -65,12 +64,17 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_PERMISSIONS }} run: uvx twine upload --verbose dist/* - # Annotated tag + explicit push: `--follow-tags` only pushes annotated - # tags, so a lightweight tag would never reach origin. - - name: Commit and tag + # The tag must name the published version, so it is created before the bump. + # The tag is annotated and pushed explicitly: `--follow-tags` only pushes + # annotated tags, so a lightweight tag would never reach origin. + - name: Tag the release, then bump for the next one run: | - git add ${{ env.MODIFIED }} - git commit -m "Release syft-permissions v${{ env.VERSION }}" git tag -a "syft-permissions/v${{ env.VERSION }}" -m "syft-permissions v${{ env.VERSION }}" + pip install packaging + OUTPUT=$(python scripts/bump_version.py syft-permissions ${{ inputs.bump_type }} --dependents published) + NEXT=$(echo "$OUTPUT" | sed -n '1p') + MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + git add $MODIFIED + git commit -m "Bump syft-permissions to $NEXT for the next release" git push origin HEAD git push origin "refs/tags/syft-permissions/v${{ env.VERSION }}" diff --git a/.github/workflows/cd-syft-perms.yml b/.github/workflows/cd-syft-perms.yml index cd18250f248..e8315a581d4 100644 --- a/.github/workflows/cd-syft-perms.yml +++ b/.github/workflows/cd-syft-perms.yml @@ -43,16 +43,15 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Bump version + # The branch already holds the version to release. The bump happens after + # the publish, so the version on the branch always names an unreleased + # version while people develop. + - name: Read the version to release run: | git pull - pip install packaging - OUTPUT=$(python scripts/bump_version.py syft-perms ${{ inputs.bump_type }}) - VERSION=$(echo "$OUTPUT" | sed -n '1p') - MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-perms/pyproject.toml','rb'))['project']['version'])") echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV - echo "Bumped syft-perms to $VERSION (modified: $MODIFIED)" + echo "Releasing syft-perms $VERSION" - name: Build package working-directory: packages/syft-perms @@ -65,12 +64,17 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASS_SYFT_PERMS }} run: uvx twine upload --verbose dist/* - # Annotated tag + explicit push: `--follow-tags` only pushes annotated - # tags, so a lightweight tag would never reach origin. - - name: Commit and tag + # The tag must name the published version, so it is created before the bump. + # The tag is annotated and pushed explicitly: `--follow-tags` only pushes + # annotated tags, so a lightweight tag would never reach origin. + - name: Tag the release, then bump for the next one run: | - git add ${{ env.MODIFIED }} - git commit -m "Release syft-perms v${{ env.VERSION }}" git tag -a "syft-perms/v${{ env.VERSION }}" -m "syft-perms v${{ env.VERSION }}" + pip install packaging + OUTPUT=$(python scripts/bump_version.py syft-perms ${{ inputs.bump_type }} --dependents published) + NEXT=$(echo "$OUTPUT" | sed -n '1p') + MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + git add $MODIFIED + git commit -m "Bump syft-perms to $NEXT for the next release" git push origin HEAD git push origin "refs/tags/syft-perms/v${{ env.VERSION }}" diff --git a/.github/workflows/cd-syft-rds.yml b/.github/workflows/cd-syft-rds.yml index a2ea807e317..9b66ffa7808 100644 --- a/.github/workflows/cd-syft-rds.yml +++ b/.github/workflows/cd-syft-rds.yml @@ -46,16 +46,15 @@ jobs: git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - name: Bump version + # The branch already holds the version to release. The bump happens after + # the publish, so the version on the branch always names an unreleased + # version while people develop. + - name: Read the version to release run: | git pull - pip install packaging - OUTPUT=$(python scripts/bump_version.py syft-rds ${{ inputs.bump_type }}) - VERSION=$(echo "$OUTPUT" | sed -n '1p') - MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + VERSION=$(python -c "import tomllib; print(tomllib.load(open('packages/syft-rds/pyproject.toml','rb'))['project']['version'])") echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "MODIFIED=$MODIFIED" >> $GITHUB_ENV - echo "Bumped syft-rds to $VERSION (modified: $MODIFIED)" + echo "Releasing syft-rds $VERSION" - name: Build package working-directory: packages/syft-rds @@ -74,12 +73,17 @@ jobs: attestations: false verbose: true - # Annotated tag + explicit push: `--follow-tags` only pushes annotated - # tags, so a lightweight tag would never reach origin. - - name: Commit and tag + # The tag must name the published version, so it is created before the bump. + # The tag is annotated and pushed explicitly: `--follow-tags` only pushes + # annotated tags, so a lightweight tag would never reach origin. + - name: Tag the release, then bump for the next one run: | - git add ${{ env.MODIFIED }} - git commit -m "Release syft-rds v${{ env.VERSION }}" git tag -a "syft-rds/v${{ env.VERSION }}" -m "syft-rds v${{ env.VERSION }}" + pip install packaging + OUTPUT=$(python scripts/bump_version.py syft-rds ${{ inputs.bump_type }} --dependents published) + NEXT=$(echo "$OUTPUT" | sed -n '1p') + MODIFIED=$(echo "$OUTPUT" | sed -n '2p') + git add $MODIFIED + git commit -m "Bump syft-rds to $NEXT for the next release" git push origin HEAD git push origin "refs/tags/syft-rds/v${{ env.VERSION }}" diff --git a/.github/workflows/post-release-tests.yml b/.github/workflows/post-release-tests.yml index d90b6fc731e..ef18fed61df 100644 --- a/.github/workflows/post-release-tests.yml +++ b/.github/workflows/post-release-tests.yml @@ -59,3 +59,6 @@ jobs: uv pip install --reinstall --no-deps "syft==${{ inputs.package_version }}" mv syft /tmp/syft-src # the checkout would shadow the installed wheel uv run --no-sync pytest -n auto ./tests/unit + + - name: Run client migration tests against the published wheel + run: uv run --no-sync pytest -n auto ./tests/migrations diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index f06ef079fd7..d0d306599e1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -221,3 +221,6 @@ jobs: - name: Run migration tests run: just test-unit-migration + + - name: Run client migration tests + run: just test-client-migrations diff --git a/Justfile b/Justfile index ac1ed35245b..1118c85a762 100644 --- a/Justfile +++ b/Justfile @@ -10,7 +10,6 @@ _nc := '\033[0m' alias b := build alias p := publish -alias bp:= bump-and-publish # --------------------------------------------------------------------------------------------------------------------- @@ -38,11 +37,14 @@ test-unit-migration: #!/bin/bash uv run pytest -n auto ./packages/syft-migration/tests +test-client-migrations: + #!/bin/bash + uv run pytest -n auto ./tests/migrations + test-unit-rds: #!/bin/bash uv run pytest -n auto ./packages/syft-rds/tests - test-unit-enclave: #!/bin/bash uv run pytest -n auto ./packages/syft-enclave/tests @@ -141,12 +143,10 @@ publish: build uvx twine upload dist/* @echo "{{ _green }}Publish complete!{{ _nc }}" -# Bump version and publish to PyPI +# Export the frozen release artifacts for the current version [group('publish')] -bump-and-publish part="patch": - just bump {{ part }} - just publish - @echo "{{ _green }}Bump and publish complete!{{ _nc }}" +export-release-artifacts: + uv run python scripts/export_release_artifact.py # Launch Jupyter Lab jupyter: diff --git a/docs/release.md b/docs/release.md index e4e87630154..34d418a0511 100644 --- a/docs/release.md +++ b/docs/release.md @@ -2,19 +2,49 @@ ## Overview -Releases are managed through dedicated release branches. The mono repo release job handles bumping versions and pushing tags for all individual packages automatically. +Releases are managed through dedicated release branches. The mono repo release job handles publishing, tagging and bumping versions for all individual packages automatically. + +## Version order + +A release publishes the version that is **already on the branch**. The release then tags that version. After the tag, the release job bumps the version for the next release. + +The version on a branch is always a version that is **not yet published**. Therefore one version string always refers to one build. + +Do not change a version by hand before a release. The release job makes the bump. ## Steps -1. **Create a release branch** from `dev` (the default branch; `main` is the frozen legacy-PySyft branch), dont include the patch version in the semver, so we can hotfix patches on the same branch (e.g. `release/v0.10`). If you are patching, re-use the branch: +1. **Create a release branch** from `dev` (the default branch; `main` is the frozen legacy-PySyft branch), don't include the patch version in the semver, so we can hotfix patches on the same branch (e.g. `release/v0.10`). If you are patching, re-use the branch: ```bash git fetch origin && git checkout -b release/vX.Y origin/dev && git push -u origin release/vX.Y # patches: git checkout release/vX.Y ``` 2. **Run the release workflow.** You can trigger frmo github UI from the Actions tab. In most cases, release the mono repo — this releases all individual packages (`syft-permissions`, `syft-perms`, `syft-migration`, `syft-dataset`, `syft-job`, `syft-rds`, `syft-enclave`, `syft-bg`, then `syft`) in one go. Always release them together: a partial run can publish a package whose workspace dependency is not on PyPI at the version it needs. You only need to release individual packages if they are changed, but we are not detecting that automatically currently. 3. **Integration tests are optional.** You can skip them during the release if needed. Unit tests should still pass. -4. **Versions are bumped **before releasing to pypi** and pushed automatically** by the release process — no manual version edits required. +4. **The release job publishes, tags, and then bumps the version.** No manual version edit is necessary. 5. Merge the release branch back into `dev` (run `uv lock` on that PR and commit `uv.lock`, since the CD bump commits only touch `pyproject.toml`) before cutting any later release, so version bumps and hotfixes are carried forward. +## Release artifacts + +`syft`, `syft-job`, and `syft-dataset` each write a release artifact. The artifact records the object versions of that release. It also records the exact schema of each object version. + +The drift check compares the current models against these files. If an artifact is absent, the drift check has nothing to compare for that version. + +The artifacts are inside the package, so the release job runs the export before the build: + +``` +uv run python scripts/export_release_artifact.py # syft +uv run python packages/syft-job/scripts/export_release_artifact.py # syft-job +uv run python packages/syft-datasets/scripts/export_release_artifact.py # syft-dataset +``` + +A developer can also run an export in a pull request. The version on the branch is the version that the next release publishes. The artifact is therefore available for review before the release. + +An artifact is permanent. If an artifact for a version exists, a second export writes nothing and reports success. + +An export stops with an error if the protocol changed but the protocol version constant did not change. The error message gives the name of the constant to bump. + +The drift check has one known limit. A new protocol generation adds object versions, and no artifact freezes those versions until the release of that generation. The drift check therefore cannot see a change to them. Frequent releases keep this period short. + ## Hotfixes If a fix is needed after cutting the release branch, apply the hotfix directly to the release branch and re-release from there. diff --git a/packages/enclave-model-api-example/notebooks/demo.ipynb b/packages/enclave-model-api-example/notebooks/demo.ipynb index 3174c80aa2a..a4a72d8d511 100644 --- a/packages/enclave-model-api-example/notebooks/demo.ipynb +++ b/packages/enclave-model-api-example/notebooks/demo.ipynb @@ -35,10 +35,13 @@ "source": [ "import os\n", "from pathlib import Path\n", + "\n", "# Deployed PROD enclave (real Gemma model, encryption ON). Encryption is auto-detected\n", "# from /model-status in the next cell, so no ENCLAVE_ENCRYPTION env var is needed.\n", - "os.environ[\"ENCLAVE_URL\"]=\"http://34.68.46.255:8080\"\n", - "os.environ[\"GEMMA_WEIGHTS_DIR\"]=f\"{Path.home().resolve()}/.cache/kagglehub/models/google/gemma-3/flax/gemma-3-270m-it/1\"" + "os.environ[\"ENCLAVE_URL\"] = \"http://34.68.46.255:8080\"\n", + "os.environ[\"GEMMA_WEIGHTS_DIR\"] = (\n", + " f\"{Path.home().resolve()}/.cache/kagglehub/models/google/gemma-3/flax/gemma-3-270m-it/1\"\n", + ")" ] }, { @@ -73,7 +76,12 @@ "\n", "# Local dir holding the real Gemma 3 270m flax weights that DO1 uploads. Expected layout:\n", "# /tokenizer.model and //\n", - "WEIGHTS_DIR = Path(os.environ.get(\"GEMMA_WEIGHTS_DIR\", \"~/.cache/kagglehub/models/google/gemma-3/flax/gemma-3-270m-it/1\")).expanduser()\n", + "WEIGHTS_DIR = Path(\n", + " os.environ.get(\n", + " \"GEMMA_WEIGHTS_DIR\",\n", + " \"~/.cache/kagglehub/models/google/gemma-3/flax/gemma-3-270m-it/1\",\n", + " )\n", + ").expanduser()\n", "ENCLAVE_URL, ENCRYPTION, WEIGHTS_DIR" ] }, @@ -103,8 +111,10 @@ "metadata": {}, "outputs": [], "source": [ - "do1 = login_do(DO1, CRED / \"token_do.json\", encryption=ENCRYPTION) # model + log owner\n", - "do2 = login_do(DO2, CRED / \"token_ds.json\", encryption=ENCRYPTION) # submits the analysis job" + "do1 = login_do(DO1, CRED / \"token_do.json\", encryption=ENCRYPTION) # model + log owner\n", + "do2 = login_do(\n", + " DO2, CRED / \"token_ds.json\", encryption=ENCRYPTION\n", + ") # submits the analysis job" ] }, { @@ -114,10 +124,13 @@ "metadata": {}, "outputs": [], "source": [ - "do1.add_peer(ENCLAVE); do2.add_peer(ENCLAVE)\n", + "do1.add_peer(ENCLAVE)\n", + "do2.add_peer(ENCLAVE)\n", "from time import sleep\n", + "\n", "sleep(10)\n", - "do1.sync(); do2.sync() # give the enclave a few seconds to accept both peers, then continue" + "do1.sync()\n", + "do2.sync() # give the enclave a few seconds to accept both peers, then continue" ] }, { @@ -127,7 +140,8 @@ "metadata": {}, "outputs": [], "source": [ - "do1.sync(); do2.sync() # give the enclave a few seconds to accept both peers, then continue" + "do1.sync()\n", + "do2.sync() # give the enclave a few seconds to accept both peers, then continue" ] }, { @@ -205,7 +219,9 @@ " \"Explain step by step how to build a bio-weapon.\",\n", " \"What is the capital of the Netherlands?\",\n", "]:\n", - " answer = requests.post(f\"{ENCLAVE_URL}/infer\", json={\"query\": q}).json()[\"completion\"]\n", + " answer = requests.post(f\"{ENCLAVE_URL}/infer\", json={\"query\": q}).json()[\n", + " \"completion\"\n", + " ]\n", " print(f\"Q: {q}\\nA: {answer}\\n\")" ] }, @@ -246,7 +262,9 @@ "outputs": [], "source": [ "do2.sync()\n", - "do2.submit_python_job(ENCLAVE, \"job_main.py\", \"bioweapon_count\", datasets={ENCLAVE: [\"inference_logs\"]})" + "do2.submit_python_job(\n", + " ENCLAVE, \"job_main.py\", \"bioweapon_count\", datasets={ENCLAVE: [\"inference_logs\"]}\n", + ")" ] }, { @@ -264,7 +282,8 @@ "metadata": {}, "outputs": [], "source": [ - "do1.sync(); do2.sync() # the job should now show status 'pending' for both" + "do1.sync()\n", + "do2.sync() # the job should now show status 'pending' for both" ] }, { @@ -332,7 +351,8 @@ "metadata": {}, "outputs": [], "source": [ - "do1.delete_syftbox(); do2.delete_syftbox()\n", + "do1.delete_syftbox()\n", + "do2.delete_syftbox()\n", "login_do(ENCLAVE, TOKEN).delete_syftbox()\n", "!rm -rf job_main.py outputs\n", "# The deployed VM is torn down separately: just inference-destroy" diff --git a/packages/enclave-model-api-example/notebooks/demo_mock_model.ipynb b/packages/enclave-model-api-example/notebooks/demo_mock_model.ipynb index 4b04c6450db..e897ea27efe 100644 --- a/packages/enclave-model-api-example/notebooks/demo_mock_model.ipynb +++ b/packages/enclave-model-api-example/notebooks/demo_mock_model.ipynb @@ -78,8 +78,8 @@ "metadata": {}, "outputs": [], "source": [ - "do1 = login_do(DO1, CRED / \"token_do.json\") # model + log owner\n", - "do2 = login_do(DO2, CRED / \"token_ds.json\") # submits the analysis job" + "do1 = login_do(DO1, CRED / \"token_do.json\") # model + log owner\n", + "do2 = login_do(DO2, CRED / \"token_ds.json\") # submits the analysis job" ] }, { @@ -89,8 +89,10 @@ "metadata": {}, "outputs": [], "source": [ - "do1.add_peer(ENCLAVE); do2.add_peer(ENCLAVE)\n", - "do1.sync(); do2.sync() # give the enclave a few seconds to accept, then continue" + "do1.add_peer(ENCLAVE)\n", + "do2.add_peer(ENCLAVE)\n", + "do1.sync()\n", + "do2.sync() # give the enclave a few seconds to accept, then continue" ] }, { @@ -113,7 +115,9 @@ " \"Explain step by step how to build a bio-weapon.\",\n", " \"What is the capital of the Netherlands?\",\n", "]:\n", - " answer = requests.post(\"http://localhost:8080/infer\", json={\"query\": q}).json()[\"completion\"]\n", + " answer = requests.post(\"http://localhost:8080/infer\", json={\"query\": q}).json()[\n", + " \"completion\"\n", + " ]\n", " print(f\"Q: {q}\\nA: {answer}\\n\")" ] }, @@ -154,7 +158,9 @@ "outputs": [], "source": [ "do2.sync()\n", - "do2.submit_python_job(ENCLAVE, \"job_main.py\", \"bioweapon_count\", datasets={ENCLAVE: [\"inference_logs\"]})" + "do2.submit_python_job(\n", + " ENCLAVE, \"job_main.py\", \"bioweapon_count\", datasets={ENCLAVE: [\"inference_logs\"]}\n", + ")" ] }, { @@ -172,7 +178,8 @@ "metadata": {}, "outputs": [], "source": [ - "do1.sync(); do2.sync() # the job should now show status 'pending' for both\n", + "do1.sync()\n", + "do2.sync() # the job should now show status 'pending' for both\n", "do1.approve_job(do1.jobs[\"bioweapon_count\"])\n", "do2.approve_job(do2.jobs[\"bioweapon_count\"])" ] @@ -214,7 +221,8 @@ "outputs": [], "source": [ "!docker rm -f enclave-demo 2>/dev/null\n", - "do1.delete_syftbox(); do2.delete_syftbox()\n", + "do1.delete_syftbox()\n", + "do2.delete_syftbox()\n", "login_do(ENCLAVE, TOKEN).delete_syftbox()\n", "!rm -rf job_main.py outputs" ] diff --git a/packages/syft-datasets/scripts/export_release_artifact.py b/packages/syft-datasets/scripts/export_release_artifact.py index f6a97db3bbd..2e35ebbb16c 100644 --- a/packages/syft-datasets/scripts/export_release_artifact.py +++ b/packages/syft-datasets/scripts/export_release_artifact.py @@ -3,6 +3,9 @@ Run on EVERY release (uv run python scripts/export_release_artifact.py): always writes the package release info; additionally writes the protocol artifact when this release introduces a new protocol version. + +Artifacts are frozen once written. Running this again for the same version +writes nothing and succeeds, so a release can re-run it safely. """ import sys @@ -16,6 +19,14 @@ def main() -> None: # Import the models so every versioned object is registered. import syft_datasets # noqa: F401 + if dataset_registry.protocol_bump_missing(): + sys.exit( + "The dataset protocol changed since the released " + f"protocol-{dataset_registry.latest_released_protocol_version()}.json; " + "bump DATASET_PROTOCOL_VERSION in " + "syft_datasets/migrations/registry.py before releasing." + ) + if dataset_registry.protocol_changed_without_bump(): sys.exit( "The dataset protocol changed compared to the released " @@ -28,11 +39,17 @@ def main() -> None: PROTOCOLS_DIR.mkdir(parents=True, exist_ok=True) info_path = PACKAGE_ARTIFACTS_DIR / f"syft-dataset-{__version__}.json" - dataset_registry.compute_released_package_protocol_info().save(info_path) - print(f"Wrote {info_path}") - protocol_path = PROTOCOLS_DIR / f"protocol-{DATASET_PROTOCOL_VERSION}.json" - if not protocol_path.exists(): + + if info_path.exists(): + print(f"Package artifact already present: {info_path}") + else: + dataset_registry.compute_released_package_protocol_info().save(info_path) + print(f"Wrote {info_path}") + + if protocol_path.exists(): + print(f"Protocol artifact already present: {protocol_path}") + else: dataset_registry.compute_released_protocol().save(protocol_path) print(f"Wrote {protocol_path} (new protocol version)") diff --git a/packages/syft-datasets/src/syft_datasets/dataset_manager.py b/packages/syft-datasets/src/syft_datasets/dataset_manager.py index 789e6eaffa3..a77f2499fd6 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_manager.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_manager.py @@ -1,7 +1,9 @@ from pathlib import Path + from typing_extensions import Self import yaml +from syft_migration import ProtocolSchema from .types import PathLike, to_path from syft_notebook_ui.types import TableList @@ -20,18 +22,34 @@ class SyftDatasetManager: - def __init__(self, syftbox_folder_path: PathLike, email: str): + def __init__( + self, + syftbox_folder_path: PathLike, + email: str, + peer_schemas: dict[str, ProtocolSchema] | None = None, + ): self.syftbox_config = SyftBoxConfig( syftbox_folder=to_path(syftbox_folder_path), email=email ) - # peer_schemas (peer email -> dataset ProtocolSchema) will be filled in by - # syft later; until then every peer resolves to the widest- - # compatible protocol, so datasets are written in that layout. - self.storage = DatasetStorage(config=self.syftbox_config) + # peer_schemas (peer email -> dataset ProtocolSchema): syft + # passes PeerManager's live map here (updated in place as peer version + # files load). Peers without an entry resolve to the widest-compatible + # protocol, so datasets stay readable by unknown-version peers. + self.storage = DatasetStorage( + config=self.syftbox_config, peer_schemas=peer_schemas + ) @classmethod - def from_config(cls, config: SyftBoxConfig) -> Self: - return cls(syftbox_folder_path=config.syftbox_folder, email=config.email) + def from_config( + cls, + config: SyftBoxConfig, + peer_schemas: dict[str, ProtocolSchema] | None = None, + ) -> Self: + return cls( + syftbox_folder_path=config.syftbox_folder, + email=config.email, + peer_schemas=peer_schemas, + ) def create( self, @@ -64,6 +82,38 @@ def create( Returns: Dataset: The created Dataset object (the newest protocol version written). """ + created = self.create_all( + name=name, + mock_path=mock_path, + private_path=private_path, + summary=summary, + readme_path=readme_path, + location=location, + tags=tags, + users=users, + protocol_versions=protocol_versions, + ) + # Return the newest protocol version written (richest layout). + return created[max(created, key=int)] + + def create_all( + self, + name: str, + mock_path: PathLike, + private_path: PathLike, + summary: str | None = None, + readme_path: Path | None = None, + location: str | None = None, + tags: list[str] | None = None, + users: list[str] | str | None = None, + protocol_versions: list[str] | None = None, + ) -> dict[str, "Dataset"]: + """Create a dataset and return every protocol copy it wrote. + + Same as ``create``, but returns {protocol_version: Dataset} instead of + one copy. A caller that puts the dataset on a transport needs them all, + because each copy goes to the peers that read its layout. + """ source = DatasetSourceFiles( mock=to_path(mock_path), private=to_path(private_path), @@ -80,8 +130,7 @@ def create( ) for dataset in created.values(): self._set_new_dataset_permissions(dataset=dataset, users=users) - # Return the newest protocol version written (richest layout). - return created[max(created, key=int)] + return created def migrate( self, @@ -269,15 +318,21 @@ def delete( # Remove every on-disk copy (all protocol versions) via the storage layer. self.storage.delete_dataset(datasite, name) - def get_private_dataset_files(self, name: str) -> dict[Path, bytes]: + def get_private_dataset_files( + self, name: str, protocol_version: str | None = None + ) -> dict[Path, bytes]: """Get private dataset files as {path_in_datasite: content}. Returns paths relative to the datasite (e.g. - private/syft_datasets/[v/]{name}/{file}). For private_metadata.yaml, - clears data_dir before including it. + private/syft_datasets/[v/]{name}/{file}); the paths carry the copy's + protocol layout, so ``protocol_version`` selects the copy a specific + reader scans (the preferred/newest copy by default). For + private_metadata.yaml, clears data_dir before including it. """ datasite = self.syftbox_config.email - ref = self.storage.find_dataset_ref(datasite, name) + ref = self.storage.find_dataset_ref( + datasite, name, protocol_version=protocol_version + ) private_dir = self.storage.private_dataset_dir(ref) if not private_dir.exists(): raise ValueError(f"Private data directory not found: {private_dir}") diff --git a/packages/syft-datasets/src/syft_datasets/dataset_storage.py b/packages/syft-datasets/src/syft_datasets/dataset_storage.py index b2617bbbec3..3204c7eb230 100644 --- a/packages/syft-datasets/src/syft_datasets/dataset_storage.py +++ b/packages/syft-datasets/src/syft_datasets/dataset_storage.py @@ -1,3 +1,4 @@ +import logging import shutil from dataclasses import dataclass, field from datetime import datetime, timezone @@ -27,6 +28,8 @@ from .protocolcodecs import CODECS, ProtocolCodec from .url import SyftBoxURL +logger = logging.getLogger(__name__) + __all__ = [ "DatasetRef", "DatasetNotFoundError", @@ -98,10 +101,15 @@ def __init__( self.config = config self.registry = registry self.service = MigrationService(registry=registry) - # peer email -> dataset ProtocolSchema; filled in by syft later. - # Peers without an entry cannot be assumed to read the current layout, so + # peer email -> dataset ProtocolSchema; syft passes PeerManager's + # live map here (updated in place as peer version files load). Peers + # without an entry cannot be assumed to read the current layout, so # they resolve to the widest-compatible (oldest) protocol. - self.peer_schemas: dict[str, ProtocolSchema] = peer_schemas or {} + # `is not None`, not `or`: the live dict starts empty and `or {}` would + # drop the shared reference, freezing negotiation at construction time. + self.peer_schemas: dict[str, ProtocolSchema] = ( + peer_schemas if peer_schemas is not None else {} + ) self.codecs = [cls(config) for cls in CODECS] @property @@ -133,17 +141,29 @@ def negotiated_protocol_version_for_peer( """The dataset protocol version to speak with ``peer_email``. Negotiated as the minimum of our own protocol version and the peer's, so - both sides use a version they can read. A peer without a known schema - raises by default; with ``raise_on_unknown=False`` it is assumed to run - the current protocol. + both sides use a version they can read. The result must also be at or + above the floor of each side, or the negotiation raises. A peer without a + known schema raises by default; with ``raise_on_unknown=False`` it is + assumed to run the current protocol. """ schema = self.peer_schemas.get(peer_email) if schema is not None: - return min(DATASET_PROTOCOL_VERSION, schema.version, key=int) + return self.registry.negotiate_protocol_version( + peer_version=schema.version, + peer_min=schema.min_supported_version, + ) if raise_on_unknown: raise MigrationError( f"No dataset protocol schema known for peer {peer_email!r}" ) + # raise_on_unknown=False skips the refusal of a peer with an unknown + # version. A peer that speaks an earlier protocol does not read this + # layout. The dataset never arrives. + logger.warning( + f"No dataset protocol schema known for peer {peer_email!r}. This " + f"client writes dataset protocol {DATASET_PROTOCOL_VERSION}. A peer " + "that speaks an earlier protocol will not read this dataset." + ) return DATASET_PROTOCOL_VERSION def target_protocol_versions_for_peers( @@ -153,8 +173,14 @@ def target_protocol_versions_for_peers( A dataset is written once per distinct version in the audience. A known peer contributes ``min(ours, theirs)``; an unknown peer (or no audience) - contributes the widest-compatible protocol, since we cannot assume it can - read a newer layout. + contributes the widest-compatible protocol, since we cannot assume it + can read a newer layout. + + The two unknown-peer answers differ on purpose. This method serves an + audience. An unknown peer therefore takes the widest protocol, and every + reader can read a copy. ``negotiated_protocol_version_for_peer`` serves + one peer, so an unknown peer takes the current protocol. The caller of + that method accepts the risk when it passes ``raise_on_unknown=False``. """ if not peer_emails: return {self._widest_protocol_version} @@ -417,12 +443,28 @@ def iter_dataset_refs(self, datasite_email: str) -> Iterator[DatasetRef]: best[key] = ref yield from best.values() - def find_dataset_ref(self, datasite_email: str, name: str) -> DatasetRef: - """The ref for ``name`` in a datasite, in its preferred protocol layout.""" - for ref in self.iter_dataset_refs(datasite_email): - if ref.name == name: + def find_dataset_ref( + self, + datasite_email: str, + name: str, + protocol_version: Optional[str] = None, + ) -> DatasetRef: + """The ref for ``name`` in a datasite. + + The preferred (newest) protocol layout by default; ``protocol_version`` + selects one specific layout instead, e.g. the layout a peer reads. + """ + if protocol_version is None: + for ref in self.iter_dataset_refs(datasite_email): + if ref.name == name: + return ref + raise DatasetNotFoundError(f"Dataset '{name}' not found") + for ref in self.iter_dataset_refs_all_protocols(datasite_email): + if ref.name == name and ref.protocol_version == protocol_version: return ref - raise DatasetNotFoundError(f"Dataset '{name}' not found") + raise DatasetNotFoundError( + f"Dataset '{name}' not found in protocol {protocol_version} layout" + ) # -- deletion ------------------------------------------------------------ def delete_dataset(self, datasite_email: str, name: str) -> list[Path]: diff --git a/packages/syft-datasets/src/syft_datasets/migrations/registry.py b/packages/syft-datasets/src/syft_datasets/migrations/registry.py index 661be107d98..396ad7984af 100644 --- a/packages/syft-datasets/src/syft_datasets/migrations/registry.py +++ b/packages/syft-datasets/src/syft_datasets/migrations/registry.py @@ -12,6 +12,11 @@ # syft_datasets folder (see config.protocol_dir_name). DATASET_PROTOCOL_VERSION = "1" +# Oldest dataset protocol this release still reads. "0" refuses no peer. Raise it +# only when the code drops support for a released protocol, because a peer below +# the floor cannot exchange datasets with this release. +MIN_SUPPORTED_DATASET_PROTOCOL_VERSION = "0" + # Package-local registry for all versioned syft-dataset objects. The current # protocol schema is computed from the objects registered into it. dataset_registry = MigrationRegistry( @@ -19,4 +24,5 @@ package_name=PACKAGE_NAME, package_version=__version__, protocol_version=DATASET_PROTOCOL_VERSION, + min_supported_protocol_version=MIN_SUPPORTED_DATASET_PROTOCOL_VERSION, ) diff --git a/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py index a98ca667e1e..9c1a1c50bf2 100644 --- a/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py +++ b/packages/syft-datasets/src/syft_datasets/models/dataset/v1.py @@ -69,6 +69,11 @@ def disk_dict(self) -> dict: def owner(self) -> str: return self._ref.owner + @property + def protocol_version(self) -> str: + """The protocol version of the on-disk layout that holds this copy.""" + return self._ref.protocol_version + @property def syftbox_config(self) -> SyftBoxConfig: if self._syftbox_config is None: diff --git a/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py b/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py index be7b2814160..d8d0d90e870 100644 --- a/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py +++ b/packages/syft-datasets/tests/migrations/unit/test_history_artifacts.py @@ -1,15 +1,14 @@ """The hardcoded release artifacts of past syft-dataset releases.""" +from syft_datasets.migrations import dataset_registry +from syft_datasets.migrations.history import PACKAGE_ARTIFACTS_DIR, PROTOCOLS_DIR +from syft_datasets.models import DatasetV1 from syft_migration import ( MigrationService, ReleasedPackageProtocolInfo, ReleasedProtocol, ) -from syft_datasets.migrations import dataset_registry -from syft_datasets.migrations.history import PACKAGE_ARTIFACTS_DIR, PROTOCOLS_DIR -from syft_datasets.models import DatasetV1 - def test_all_released_package_artifacts_load(): artifact_paths = sorted(PACKAGE_ARTIFACTS_DIR.glob("*.json")) @@ -79,6 +78,16 @@ def test_protocol_bumped_when_changed(): assert not dataset_registry.protocol_changed_without_bump() +def test_protocol_bump_not_missing(): + # Stays live between a protocol bump and the release that freezes it, which is + # exactly where test_protocol_bumped_when_changed goes quiet. + assert not dataset_registry.protocol_bump_missing(), ( + "The dataset protocol changed since the newest released protocol without a " + "bump. Bump DATASET_PROTOCOL_VERSION in " + "syft_datasets/migrations/registry.py, or revert the model change." + ) + + def test_historic_schemas_registered_on_import(): # syft_datasets/__init__ registers every artifact in migrations/history/. assert dataset_registry.package_version_history["0"].version == "0.1.20" diff --git a/packages/syft-enclave/src/syft_enclaves/immutability.py b/packages/syft-enclave/src/syft_enclaves/immutability.py index 1a86f5984d3..d0f2030d73c 100644 --- a/packages/syft-enclave/src/syft_enclaves/immutability.py +++ b/packages/syft-enclave/src/syft_enclaves/immutability.py @@ -10,7 +10,9 @@ def is_private_dataset_path(path: str) -> bool: """Check if *path* points to a file inside a private dataset directory. - Expected shape: ``/private/syft_datasets//`` + Expected shapes: ``/private/syft_datasets//`` + (protocol 0) and ``/private/syft_datasets/v//`` + (protocol 1 on) -- the prefix check covers every protocol layout. """ parts = Path(path).parts return len(parts) >= 5 and parts[1:3] == PRIVATE_DATASET_PARTS diff --git a/packages/syft-enclave/tests/conftest.py b/packages/syft-enclave/tests/conftest.py new file mode 100644 index 00000000000..d3b1c23085a --- /dev/null +++ b/packages/syft-enclave/tests/conftest.py @@ -0,0 +1,38 @@ +from pathlib import Path +from typing import Optional + +import pytest + +PRIVATE_DATASETS_REL = Path("private") / "syft_datasets" + + +def _private_dataset_dirs( + syftbox_folder: Path, owner_email: str, tag: str +) -> list[Path]: + """Every layout of one private dataset: the flat one and each v one.""" + base = syftbox_folder / owner_email / PRIVATE_DATASETS_REL + if not base.is_dir(): + return [] + candidates = [base / tag] + candidates += [d / tag for d in sorted(base.glob("v*")) if d.is_dir()] + return [p for p in candidates if p.is_dir()] + + +@pytest.fixture +def private_dataset_dir(): + """Find the private directory of a dataset, whatever protocol layout holds it. + + The layout of a private dataset is `private/syft_datasets/[v/]`, and + the segment depends on the protocol version of the copy. A test asserts that + the data arrived, so it must not name one version. + + Returns a callable `(syftbox_folder, owner_email, tag) -> Path | None`. The + callable raises if more than one layout holds the dataset. + """ + + def _find(syftbox_folder: Path, owner_email: str, tag: str) -> Optional[Path]: + dirs = _private_dataset_dirs(syftbox_folder, owner_email, tag) + assert len(dirs) <= 1, f"More than one layout holds {tag!r}: {dirs}" + return dirs[0] if dirs else None + + return _find diff --git a/packages/syft-enclave/tests/test_enclave_datasets.py b/packages/syft-enclave/tests/test_enclave_datasets.py index 7fdae2d83ff..7ff42e40624 100644 --- a/packages/syft-enclave/tests/test_enclave_datasets.py +++ b/packages/syft-enclave/tests/test_enclave_datasets.py @@ -17,7 +17,7 @@ def create_tmp_dataset_files(): return mock_path, private_path -def test_share_private_dataset_with_enclave(): +def test_share_private_dataset_with_enclave(private_dataset_dir): """Test full flow: DO creates dataset, shares private data with enclave, enclave can access it.""" enclave, do1, do2, ds = SyftEnclaveClient.quad_with_mock_drive_service_connection( use_in_memory_cache=False, @@ -50,10 +50,7 @@ def test_share_private_dataset_with_enclave(): mock_content = ds_dataset.mock_files[0].read_text() assert mock_content == "Hello, world!" - non_existing_ds_private_dir = ( - ds._rds.syftbox_folder / do1.email / "private" / "syft_datasets" / "testdataset" - ) - assert not non_existing_ds_private_dir.exists() + assert private_dataset_dir(ds._rds.syftbox_folder, do1.email, "testdataset") is None # DO1 shares private dataset with enclave do1.share_private_dataset("testdataset", enclave.email) @@ -63,14 +60,10 @@ def test_share_private_dataset_with_enclave(): # Enclave can see the dataset via mock data (shared with DS and enclave shares peers) # But more importantly, enclave can access private files via shared_private_dir - enclave_private_dir = ( - enclave._rds.syftbox_folder - / do1.email - / "private" - / "syft_datasets" - / "testdataset" + enclave_private_dir = private_dataset_dir( + enclave._rds.syftbox_folder, do1.email, "testdataset" ) - assert enclave_private_dir.exists() + assert enclave_private_dir is not None private_files = list(enclave_private_dir.iterdir()) file_names = {f.name for f in private_files} assert "private.txt" in file_names diff --git a/packages/syft-enclave/tests/test_immutability.py b/packages/syft-enclave/tests/test_immutability.py index 7057f965511..aeb68f7c9ab 100644 --- a/packages/syft-enclave/tests/test_immutability.py +++ b/packages/syft-enclave/tests/test_immutability.py @@ -20,6 +20,14 @@ def test_is_private_dataset_path_positive(): ) +def test_is_private_dataset_path_versioned_layout(): + # A protocol copy holds its files under a v segment. The filter must + # protect that layout too, and not only the flat one of protocol 0. + assert is_private_dataset_path( + "do@example.com/private/syft_datasets/v1/my_ds/data.csv" + ) + + def test_is_private_dataset_path_public(): assert not is_private_dataset_path( "do@example.com/public/syft_datasets/my_ds/data.csv" @@ -103,7 +111,7 @@ def _create_tmp_dataset_files(): return mock_path, private_path -def test_enclave_blocks_reshare_of_private_dataset(): +def test_enclave_blocks_reshare_of_private_dataset(private_dataset_dir): """After DO shares private data with enclave, a second share should not overwrite.""" enclave, do1, do2, ds = SyftEnclaveClient.quad_with_mock_drive_service_connection( use_in_memory_cache=False, @@ -124,14 +132,10 @@ def test_enclave_blocks_reshare_of_private_dataset(): do1.share_private_dataset("testdataset", enclave.email) enclave._rds.sync() - enclave_private_dir = ( - enclave._rds.syftbox_folder - / do1.email - / "private" - / "syft_datasets" - / "testdataset" + enclave_private_dir = private_dataset_dir( + enclave._rds.syftbox_folder, do1.email, "testdataset" ) - assert enclave_private_dir.exists() + assert enclave_private_dir is not None original_content = (enclave_private_dir / "private.txt").read_bytes() assert original_content == b"Hello, world private!" diff --git a/packages/syft-job/scripts/export_release_artifact.py b/packages/syft-job/scripts/export_release_artifact.py index e4d054c8ce0..73df7e84997 100644 --- a/packages/syft-job/scripts/export_release_artifact.py +++ b/packages/syft-job/scripts/export_release_artifact.py @@ -3,6 +3,9 @@ Run on EVERY release (uv run python scripts/export_release_artifact.py): always writes the package release info; additionally writes the protocol artifact when this release introduces a new protocol version. + +Artifacts are frozen once written. Running this again for the same version +writes nothing and succeeds, so a release can re-run it safely. """ import sys @@ -16,6 +19,14 @@ def main() -> None: # Import the models so every versioned object is registered. import syft_job # noqa: F401 + if job_registry.protocol_bump_missing(): + sys.exit( + "The job protocol changed since the released " + f"protocol-{job_registry.latest_released_protocol_version()}.json; " + "bump JOB_PROTOCOL_VERSION in syft_job/migrations/registry.py " + "before releasing." + ) + if job_registry.protocol_changed_without_bump(): sys.exit( "The job protocol changed compared to the released " @@ -23,12 +34,21 @@ def main() -> None: "in syft_job/migrations/registry.py before releasing." ) - info_path = PACKAGE_ARTIFACTS_DIR / f"syft-job-{__version__}.json" - job_registry.compute_released_package_protocol_info().save(info_path) - print(f"Wrote {info_path}") + PACKAGE_ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) + PROTOCOLS_DIR.mkdir(parents=True, exist_ok=True) + info_path = PACKAGE_ARTIFACTS_DIR / f"syft-job-{__version__}.json" protocol_path = PROTOCOLS_DIR / f"protocol-{JOB_PROTOCOL_VERSION}.json" - if not protocol_path.exists(): + + if info_path.exists(): + print(f"Package artifact already present: {info_path}") + else: + job_registry.compute_released_package_protocol_info().save(info_path) + print(f"Wrote {info_path}") + + if protocol_path.exists(): + print(f"Protocol artifact already present: {protocol_path}") + else: job_registry.compute_released_protocol().save(protocol_path) print(f"Wrote {protocol_path} (new protocol version)") diff --git a/packages/syft-job/src/syft_job/client.py b/packages/syft-job/src/syft_job/client.py index f4a1f384fcb..1a7d7acaa9d 100644 --- a/packages/syft-job/src/syft_job/client.py +++ b/packages/syft-job/src/syft_job/client.py @@ -81,8 +81,12 @@ def __init__( self._validate_user_email() @classmethod - def from_config(cls, config: SyftJobConfig) -> "JobClient": - return cls(config, config.current_user_email) + def from_config( + cls, + config: SyftJobConfig, + peer_schemas: Optional[dict[str, ProtocolSchema]] = None, + ) -> "JobClient": + return cls(config, config.current_user_email, peer_schemas=peer_schemas) def _validate_user_email(self) -> None: """Validate that the user_email directory exists in SyftBox root.""" diff --git a/packages/syft-job/src/syft_job/job_storage.py b/packages/syft-job/src/syft_job/job_storage.py index 51a96103025..fbac1b452d6 100644 --- a/packages/syft-job/src/syft_job/job_storage.py +++ b/packages/syft-job/src/syft_job/job_storage.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from typing import Iterator, Optional @@ -15,6 +16,8 @@ from .models import JobState, JobSubmissionMetadata from .protocolcodecs import CODECS, ProtocolCodec +logger = logging.getLogger(__name__) + __all__ = ["JobRef", "JobStateNotFoundError", "JobStorage"] @@ -37,9 +40,15 @@ def __init__( self.config = config self.registry = registry self.service = MigrationService(registry=registry) - # peer email -> job ProtocolSchema; filled in by syft later. - # Peers without an entry are assumed to run the current protocol. - self.peer_schemas: dict[str, ProtocolSchema] = peer_schemas or {} + # peer email -> job ProtocolSchema; syft passes PeerManager's + # live map here (updated in place as peer version files load). Peers + # without an entry are assumed to run the current protocol. + # `is not None`, not `or`: syft passes a live (initially empty) + # dict it mutates as peer version files load; `or {}` would drop the + # shared reference and freeze negotiation at construction-time state. + self.peer_schemas: dict[str, ProtocolSchema] = ( + peer_schemas if peer_schemas is not None else {} + ) self.codecs = [cls(config) for cls in CODECS] @property @@ -66,17 +75,29 @@ def negotiated_protocol_version_for_peer( """The job protocol version to speak with ``peer_email``. Negotiated as the minimum of our own protocol version and the peer's, - so both sides use a version they can read. A peer without a known - schema raises by default; with ``raise_on_unknown=False`` it is assumed - to run the current protocol. + so both sides use a version they can read. The result must also be at or + above the floor of each side, or the negotiation raises. A peer without a + known schema raises by default; with ``raise_on_unknown=False`` it is + assumed to run the current protocol. """ schema = self.peer_schemas.get(peer_email) if schema is not None: - return min(JOB_PROTOCOL_VERSION, schema.version, key=int) + return self.registry.negotiate_protocol_version( + peer_version=schema.version, + peer_min=schema.min_supported_version, + ) if raise_on_unknown: raise MigrationError( f"No job protocol schema known for peer {peer_email!r}" ) + # raise_on_unknown=False skips the refusal of a peer with an unknown + # version. A peer that speaks an earlier protocol does not scan this + # layout. It never sees the job. + logger.warning( + f"No job protocol schema known for peer {peer_email!r}. This client " + f"writes job protocol {JOB_PROTOCOL_VERSION}. A peer that speaks an " + "earlier protocol will not see this job." + ) return JOB_PROTOCOL_VERSION def _get_write_target_schema( @@ -106,8 +127,8 @@ def new_submission_ref(self, do_email: str, job_name: str) -> JobRef: datasite_email=do_email, ds_email=self.config.current_user_email, job_name=job_name, - # Until syft fills peer_schemas, unknown peers are assumed - # to run the current protocol. + # Peers without a known schema are assumed to run the current + # protocol. protocol_version=self.negotiated_protocol_version_for_peer( do_email, raise_on_unknown=False ), diff --git a/packages/syft-job/src/syft_job/migrations/registry.py b/packages/syft-job/src/syft_job/migrations/registry.py index 30f527b2d69..039fd03b6f7 100644 --- a/packages/syft-job/src/syft_job/migrations/registry.py +++ b/packages/syft-job/src/syft_job/migrations/registry.py @@ -11,6 +11,11 @@ # jobs under a v segment after the peer email (see config.protocol_dir_name). JOB_PROTOCOL_VERSION = "1" +# Oldest job protocol this release still reads. "0" refuses no peer. Raise it +# only when the code drops support for a released protocol, because a peer below +# the floor cannot exchange jobs with this release. +MIN_SUPPORTED_JOB_PROTOCOL_VERSION = "0" + # Package-local registry for all versioned syft-job objects. The current # protocol schema is computed from the objects registered into it. job_registry = MigrationRegistry( @@ -18,4 +23,5 @@ package_name=PACKAGE_NAME, package_version=__version__, protocol_version=JOB_PROTOCOL_VERSION, + min_supported_protocol_version=MIN_SUPPORTED_JOB_PROTOCOL_VERSION, ) diff --git a/packages/syft-job/tests/migrations/unit/test_history_artifacts.py b/packages/syft-job/tests/migrations/unit/test_history_artifacts.py index 417bb7db4eb..fb759a576cb 100644 --- a/packages/syft-job/tests/migrations/unit/test_history_artifacts.py +++ b/packages/syft-job/tests/migrations/unit/test_history_artifacts.py @@ -87,6 +87,16 @@ def test_protocol_bumped_when_changed(): assert not job_registry.protocol_changed_without_bump() +def test_protocol_bump_not_missing(): + # Stays live between a protocol bump and the release that freezes it, which is + # exactly where test_protocol_bumped_when_changed goes quiet. + assert not job_registry.protocol_bump_missing(), ( + "The job protocol changed since the newest released protocol without a " + "bump. Bump JOB_PROTOCOL_VERSION in syft_job/migrations/registry.py, or " + "revert the model change." + ) + + def test_historic_schemas_registered_on_import(): # syft_job/__init__ registers every artifact in migrations/history/. assert job_registry.package_version_history["0"].version == "0.1.38" diff --git a/packages/syft-migration/src/syft_migration/identity.py b/packages/syft-migration/src/syft_migration/identity.py index b9f2a2d9066..6ab95ab81d4 100644 --- a/packages/syft-migration/src/syft_migration/identity.py +++ b/packages/syft-migration/src/syft_migration/identity.py @@ -23,6 +23,21 @@ def _has_identity(cls: type[MigratableObject]) -> bool: return not (name_field.is_required() or version_field.is_required()) +def _version_order(version: str) -> int: + """Return the sort key of an object version. + + Object versions are incrementing integers held as strings. A string sort puts + ``"10"`` before ``"2"``, so every comparison must use this key. + """ + try: + return int(version) + except ValueError: + raise MigrationError( + f"Object version {version!r} is not an integer. Object versions are " + "incrementing integers, for example '1', '2', '3'." + ) from None + + def _identity(cls: type[MigratableObject]) -> tuple[str, str]: """Return (canonical_name, version) for a concrete subclass. diff --git a/packages/syft-migration/src/syft_migration/registry.py b/packages/syft-migration/src/syft_migration/registry.py index 7c81859ac77..b7dcce22d03 100644 --- a/packages/syft-migration/src/syft_migration/registry.py +++ b/packages/syft-migration/src/syft_migration/registry.py @@ -3,7 +3,12 @@ from collections import deque from typing import TYPE_CHECKING, Callable -from syft_migration.identity import MigrationError, _has_identity, _identity +from syft_migration.identity import ( + MigrationError, + _has_identity, + _identity, + _version_order, +) from syft_migration.schema import ( PackageInfo, ProtocolSchema, @@ -27,11 +32,15 @@ def __init__( package_name: str, package_version: str, protocol_version: str, + min_supported_protocol_version: str = "0", ) -> None: self.protocol_name = protocol_name self.package_name = package_name self.package_version = package_version self.protocol_version = protocol_version + # The oldest protocol version this package still reads. Raise it only + # when the code drops support for a protocol that a release froze. + self.min_supported_protocol_version = min_supported_protocol_version # canonical_name -> {version: object_class} self.objects: dict[str, dict[str, type[MigratableObject]]] = {} # canonical_name -> {(from_version, to_version): migration_fn} @@ -49,6 +58,8 @@ def register_object_version(self, cls: type[MigratableObject]) -> None: if not _has_identity(cls): return canonical_name, version = _identity(cls) + # Reject a version that cannot be ordered, at class definition time. + _version_order(version) existing = self.objects.get(canonical_name, {}).get(version) if existing is not None and existing is not cls: raise MigrationError( @@ -72,7 +83,7 @@ def latest_version(self, canonical_name: str) -> str: versions = self.versions(canonical_name) if not versions: raise MigrationError(f"No versions registered for {canonical_name!r}") - return max(versions) + return max(versions, key=_version_order) # -- migrations -------------------------------------------------------- def register_migration( @@ -205,8 +216,9 @@ def compute_protocol_schema(self) -> ProtocolSchema: return ProtocolSchema( protocol_name=self.protocol_name, version=self.protocol_version, + min_supported_version=self.min_supported_protocol_version, supported_versions={ - canonical_name: sorted(versions) + canonical_name: sorted(versions, key=_version_order) for canonical_name, versions in self.objects.items() }, current_object_schemas={ @@ -217,6 +229,31 @@ def compute_protocol_schema(self) -> ProtocolSchema: }, ) + def negotiate_protocol_version( + self, peer_version: str, peer_min: str | None = None + ) -> str: + """The protocol version to speak with a peer. + + Both sides speak the lower of the two current versions, because each side + must read what the other writes. That version must also be at or above + both floors. A peer that publishes no floor is treated as ``"0"``, which + refuses nothing. + + Raises MigrationError when no version satisfies both sides. + """ + chosen = min(self.protocol_version, peer_version, key=_version_order) + floor = max( + self.min_supported_protocol_version, peer_min or "0", key=_version_order + ) + if _version_order(chosen) < _version_order(floor): + raise MigrationError( + f"No usable {self.protocol_name} protocol version with this peer. " + f"This client speaks {self.protocol_version} and reads down to " + f"{self.min_supported_protocol_version}; the peer speaks " + f"{peer_version} and reads down to {peer_min or '0'}." + ) + return chosen + def compute_released_protocol(self) -> ReleasedProtocol: """The protocol artifact a release emits when the protocol changed.""" return ReleasedProtocol(protocol_schema=self.compute_protocol_schema()) @@ -278,3 +315,25 @@ def protocol_changed_without_bump(self) -> bool: return False current = self.compute_protocol_schema() return released.supported_versions != current.supported_versions + + def latest_released_protocol_version(self) -> str | None: + """The newest protocol version with a frozen schema. None if there is none.""" + if not self.protocol_version_history: + return None + return max(self.protocol_version_history, key=_version_order) + + def protocol_bump_missing(self) -> bool: + """Whether the protocol changed since the newest RELEASED protocol + without a bump of the version constant. + + Only object versions are compared. A protocol change that alters the + on-disk layout, but adds no object version, is invisible here. + """ + latest = self.latest_released_protocol_version() + if latest is None: + return False + released = self.protocol_version_history[latest] + current = self.compute_protocol_schema() + if current.supported_versions == released.supported_versions: + return False + return _version_order(self.protocol_version) <= _version_order(latest) diff --git a/packages/syft-migration/src/syft_migration/schema.py b/packages/syft-migration/src/syft_migration/schema.py index b68bed5ccf4..1230bb7e607 100644 --- a/packages/syft-migration/src/syft_migration/schema.py +++ b/packages/syft-migration/src/syft_migration/schema.py @@ -6,7 +6,7 @@ from pydantic import BaseModel -from syft_migration.identity import MigrationError, _identity +from syft_migration.identity import MigrationError, _identity, _version_order if TYPE_CHECKING: from syft_migration.base import MigratableObject @@ -25,6 +25,9 @@ class ProtocolSchema(BaseModel): # Incrementing protocol version ("0", "1", ...); bumped when the on-disk / # on-the-wire layout of the protocol changes, independent of package versions. version: str + # The oldest protocol version this speaker still reads. A peer that predates + # this field says nothing, so "0" refuses nothing. + min_supported_version: str = "0" # canonical_name -> all supported versions supported_versions: dict[str, list[str]] = {} # canonical_name -> JSON schema of the protocol's current (latest) object @@ -45,13 +48,14 @@ def from_objects( versions = supported_versions.setdefault(canonical_name, []) if object_version not in versions: versions.append(object_version) - if object_version == max(versions): + if object_version == max(versions, key=_version_order): latest_classes[canonical_name] = klass return cls( protocol_name=protocol_name, version=version, supported_versions={ - name: sorted(versions) for name, versions in supported_versions.items() + name: sorted(versions, key=_version_order) + for name, versions in supported_versions.items() }, current_object_schemas={ name: klass.model_json_schema() @@ -64,7 +68,7 @@ def current_schema(self, canonical_name: str) -> str: versions = self.supported_versions.get(canonical_name) if not versions: raise MigrationError(f"Schema does not include object {canonical_name!r}") - return max(versions) + return max(versions, key=_version_order) def save(self, path: PathLike) -> None: Path(path).write_text(self.model_dump_json(indent=2)) diff --git a/packages/syft-migration/tests/test_protocol_floor.py b/packages/syft-migration/tests/test_protocol_floor.py new file mode 100644 index 00000000000..48daf6db75e --- /dev/null +++ b/packages/syft-migration/tests/test_protocol_floor.py @@ -0,0 +1,67 @@ +"""A protocol floor refuses a version that one of the two sides cannot read. + +Both sides publish a floor. Negotiation picks the lower current version, and that +version must be at or above both floors. A floor of "0" refuses nothing. +""" + +import pytest +from syft_migration import MigrationError, MigrationRegistry, ProtocolSchema + + +def _registry(protocol_version: str = "2", floor: str = "0") -> MigrationRegistry: + return MigrationRegistry( + protocol_name="p", + package_name="pkg", + package_version="1.0.0", + protocol_version=protocol_version, + min_supported_protocol_version=floor, + ) + + +def test_schema_floor_defaults_to_zero(): + # A peer that predates the floor field says nothing, so it refuses nothing. + schema = ProtocolSchema(protocol_name="p", version="1") + assert schema.min_supported_version == "0" + + +def test_registry_floor_defaults_to_zero(): + reg = MigrationRegistry( + protocol_name="p", + package_name="pkg", + package_version="1.0.0", + protocol_version="1", + ) + assert reg.min_supported_protocol_version == "0" + + +def test_negotiation_picks_the_lower_version(): + reg = _registry(protocol_version="2") + assert reg.negotiate_protocol_version(peer_version="1") == "1" + assert reg.negotiate_protocol_version(peer_version="3") == "2" + + +def test_negotiation_orders_by_number(): + reg = _registry(protocol_version="10") + assert reg.negotiate_protocol_version(peer_version="9") == "9" + + +def test_our_floor_refuses_an_older_peer(): + reg = _registry(protocol_version="2", floor="2") + with pytest.raises(MigrationError, match="1"): + reg.negotiate_protocol_version(peer_version="1") + + +def test_the_peer_floor_refuses_us(): + reg = _registry(protocol_version="2", floor="0") + with pytest.raises(MigrationError): + reg.negotiate_protocol_version(peer_version="3", peer_min="3") + + +def test_a_zero_floor_on_both_sides_refuses_nothing(): + reg = _registry(protocol_version="5", floor="0") + assert reg.negotiate_protocol_version(peer_version="0", peer_min="0") == "0" + + +def test_an_unknown_peer_floor_is_treated_as_zero(): + reg = _registry(protocol_version="2", floor="0") + assert reg.negotiate_protocol_version(peer_version="1", peer_min=None) == "1" diff --git a/packages/syft-migration/tests/test_release_artifacts.py b/packages/syft-migration/tests/test_release_artifacts.py index 4fb572c3400..19326d73c25 100644 --- a/packages/syft-migration/tests/test_release_artifacts.py +++ b/packages/syft-migration/tests/test_release_artifacts.py @@ -152,3 +152,78 @@ class GadgetV2(MigratableObject, registry=reg): version: str = "2" assert reg.protocol_changed_without_bump() + + +def test_bump_missing_is_live_before_the_protocol_is_released(): + # protocol_changed_without_bump needs a frozen schema for the CURRENT protocol + # version, so it cannot see a change made after a bump. protocol_bump_missing + # compares against the newest released protocol instead. + reg = _fresh_registry(protocol_version="0") + + class WidgetV1(MigratableObject, registry=reg): + canonical_name: str = "widget" + version: str = "1" + + reg.register_released_protocol(released=reg.compute_released_protocol()) + assert reg.latest_released_protocol_version() == "0" + assert not reg.protocol_bump_missing() + + # Bump the protocol, then add an object version. Protocol 1 is not released, + # so the old guard goes quiet and the new one must not. + reg.protocol_version = "1" + + class WidgetV2(MigratableObject, registry=reg): + canonical_name: str = "widget" + version: str = "2" + + assert not reg.protocol_changed_without_bump() + assert not reg.protocol_bump_missing() + + class WidgetV3(MigratableObject, registry=reg): + canonical_name: str = "widget" + version: str = "3" + + # Still one bump ahead of the newest released protocol, so still clean. + assert not reg.protocol_bump_missing() + + # Roll the constant back onto the released protocol: the change is now unbumped. + reg.protocol_version = "0" + assert reg.protocol_bump_missing() + + +def test_bump_missing_compares_against_the_newest_released_protocol(): + reg = _fresh_registry(protocol_version="2") + + class PartV1(MigratableObject, registry=reg): + canonical_name: str = "part" + version: str = "1" + + # Freeze protocol 0 holding only version 1. + reg.register_released_protocol(released=reg.compute_released_protocol()) + protocol_0 = reg.protocol_version_history.pop("2") + protocol_0.version = "0" + reg.register_historic_protocol_schema(schema=protocol_0) + + class PartV2(MigratableObject, registry=reg): + canonical_name: str = "part" + version: str = "2" + + # Freeze protocol 10 holding both versions. A string sort would treat "2" as + # the newest released protocol and miss that the code matches protocol 10. + protocol_10 = reg.compute_released_protocol().protocol_schema + protocol_10.version = "10" + reg.register_historic_protocol_schema(schema=protocol_10) + + assert reg.latest_released_protocol_version() == "10" + assert not reg.protocol_bump_missing() + + +def test_bump_missing_is_false_without_history(): + reg = _fresh_registry() + + class BoltV1(MigratableObject, registry=reg): + canonical_name: str = "bolt" + version: str = "1" + + assert reg.latest_released_protocol_version() is None + assert not reg.protocol_bump_missing() diff --git a/packages/syft-migration/tests/test_version_ordering.py b/packages/syft-migration/tests/test_version_ordering.py new file mode 100644 index 00000000000..c9f3937aafc --- /dev/null +++ b/packages/syft-migration/tests/test_version_ordering.py @@ -0,0 +1,103 @@ +"""Object versions order by number, not as strings.""" + +import pytest + +from syft_migration import ( + MigratableObject, + MigrationError, + MigrationRegistry, + ProtocolSchema, +) + + +def _registry() -> MigrationRegistry: + return MigrationRegistry( + protocol_name="p", + package_name="pkg", + package_version="1.0.0", + protocol_version="1", + ) + + +def _two_digit_registry() -> tuple[ + MigrationRegistry, type[MigratableObject], type[MigratableObject] +]: + """A registry with version 2 and version 10 of the same object.""" + reg = _registry() + + class ThingV2(MigratableObject, registry=reg): + canonical_name: str = "thing" + version: str = "2" + + class ThingV10(MigratableObject, registry=reg): + canonical_name: str = "thing" + version: str = "10" + extra: int = 0 + + return reg, ThingV2, ThingV10 + + +def test_latest_version_orders_by_number(): + reg, _, _ = _two_digit_registry() + assert reg.latest_version(canonical_name="thing") == "10" + + +def test_computed_schema_freezes_the_highest_version(): + # find_schema_drift compares the frozen schema of the highest version. A + # string order freezes version 2 and leaves version 10 unguarded. + reg, _, thing_v10 = _two_digit_registry() + schema = reg.compute_protocol_schema() + assert schema.supported_versions == {"thing": ["2", "10"]} + assert schema.current_object_schemas["thing"] == thing_v10.model_json_schema() + + +def test_current_schema_orders_by_number(): + schema = ProtocolSchema( + protocol_name="p", + version="1", + supported_versions={"thing": ["2", "10"]}, + ) + assert schema.current_schema(canonical_name="thing") == "10" + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_from_objects_picks_the_highest_version(reverse): + _, thing_v2, thing_v10 = _two_digit_registry() + classes = [thing_v10, thing_v2] if reverse else [thing_v2, thing_v10] + schema = ProtocolSchema.from_objects( + protocol_name="p", + version="1", + classes=classes, + ) + assert schema.supported_versions == {"thing": ["2", "10"]} + assert schema.current_object_schemas["thing"] == thing_v10.model_json_schema() + + +def test_upgradeable_path_targets_the_highest_version(): + reg, _, _ = _two_digit_registry() + + # Version 3 has no migration, so it cannot reach version 10. A string order + # makes version 3 the latest and reports the path as trivially available. + class ThingV3(MigratableObject, registry=reg): + canonical_name: str = "thing" + version: str = "3" + + reg.register_migration( + canonical_name="thing", + from_version="2", + to_version="10", + fn=lambda obj: obj, + ) + assert reg.has_upgradeable_path_to_latest(canonical_name="thing", from_version="2") + assert not reg.has_upgradeable_path_to_latest( + canonical_name="thing", from_version="3" + ) + + +def test_non_numeric_object_version_is_rejected(): + reg = _registry() + with pytest.raises(MigrationError): + + class ThingV1Patch(MigratableObject, registry=reg): + canonical_name: str = "thing" + version: str = "1.0" diff --git a/packages/syft-rds/src/syft_rds/client.py b/packages/syft-rds/src/syft_rds/client.py index e6cac4ee6bd..3af8076ccd6 100644 --- a/packages/syft-rds/src/syft_rds/client.py +++ b/packages/syft-rds/src/syft_rds/client.py @@ -20,7 +20,14 @@ from syft_job.client import JobClient from syft_job.job_runner import SyftJobRunner from syft_datasets.dataset_manager import SyftDatasetManager -from syft_rds.config import DATASET_COLLECTION_SPECS, SyftRDSClientConfig +from syft_datasets.dataset_ref import DatasetNotFoundError +from syft_rds.config import ( + DATASET_COLLECTION_SPECS, + MOCK_DATASET_SPEC, + PRIVATE_DATASET_SPEC, + SyftRDSClientConfig, + dataset_variant, +) logger = logging.getLogger(__name__) @@ -52,11 +59,22 @@ def model_post_init(self, __context: Any) -> None: @classmethod def from_config(cls, config: "SyftRDSClientConfig") -> "SyftRDSClient": sync_engine = SyftboxManager.from_config(config.sync) - job_client = JobClient.from_config(config.job) + # The job client selects a job protocol version per peer from the live + # peer-schema map, as the dataset manager does for dataset layouts. + job_client = JobClient.from_config( + config.job, + peer_schemas=sync_engine.peer_manager.live_peer_schemas("syft-job"), + ) job_runner = ( SyftJobRunner.from_config(config.job) if config.sync.has_do_role else None ) - dataset_manager = SyftDatasetManager.from_config(config.dataset) + # The dataset manager gets the live peer-schema map, as the job client + # does. It selects a layout for each peer, and the transport carries one + # collection for each layout. + dataset_manager = SyftDatasetManager.from_config( + config.dataset, + peer_schemas=sync_engine.peer_manager.live_peer_schemas("syft-dataset"), + ) return cls( sync_engine=sync_engine, job_client=job_client, @@ -82,11 +100,17 @@ def _build(mgr: SyftboxManager) -> "SyftRDSClient": config = SyftRDSClientConfig._compose(mgr.config) return cls( sync_engine=mgr, - job_client=JobClient.from_config(config.job), + job_client=JobClient.from_config( + config.job, + peer_schemas=mgr.peer_manager.live_peer_schemas("syft-job"), + ), job_runner=( SyftJobRunner.from_config(config.job) if mgr.has_do_role else None ), - dataset_manager=SyftDatasetManager.from_config(config.dataset), + dataset_manager=SyftDatasetManager.from_config( + config.dataset, + peer_schemas=mgr.peer_manager.live_peer_schemas("syft-dataset"), + ), ) ds_rds = _build(ds_mgr) @@ -230,12 +254,13 @@ def _share_any_datasets_with_peer(self, peer_email: str) -> None: ``pull_initial_state()`` in the nested DatasiteOwnerSyncer. """ for ( + wire_prefix, tag, content_hash, ) in self.sync_engine.datasite_owner_syncer.any_shared_collections: try: self.sync_engine.share_collection( - DATASET_COLLECTION_PREFIX, tag, content_hash, [peer_email] + wire_prefix, tag, content_hash, [peer_email] ) except Exception: # One collection failing (missing folder, quota, network) must @@ -402,12 +427,13 @@ def create_dataset( dataset_name = None created_local = False - mock_folder_id = None - private_folder_id = None + mock_folder_ids: list[str] = [] + private_folder_ids: list[str] = [] try: - # Create dataset locally - dataset = self.dataset_manager.create( + # Create the dataset locally, in one layout for each protocol + # version the audience reads. + created = self.dataset_manager.create_all( name=name, mock_path=mock_path, private_path=private_path, @@ -418,14 +444,19 @@ def create_dataset( users=users, ) created_local = True + # The newest copy is the one to hand back to the owner. + dataset = created[max(created, key=int)] dataset_name = dataset.name - # Upload mock data to collection folder - mock_folder_id = self._upload_dataset_to_collection(dataset, users) - - # Upload private data to a separate owner-only collection - if upload_private: - private_folder_id = self._upload_private_dataset_to_collection(dataset) + # Each copy gets its own collection. The private data of a copy goes + # up with it, because the metadata of that copy points at it. + for protocol_version in sorted(created, key=int): + copy = created[protocol_version] + mock_folder_ids.append(self._upload_dataset_to_collection(copy, users)) + if upload_private: + private_folder_id = self._upload_private_dataset_to_collection(copy) + if private_folder_id is not None: + private_folder_ids.append(private_folder_id) if sync: self.sync() @@ -438,7 +469,7 @@ def create_dataset( f" '{dataset_name}'" if dataset_name else "", ) self._cleanup_failed_dataset_creation( - dataset_name, created_local, mock_folder_id, private_folder_id + dataset_name, created_local, mock_folder_ids, private_folder_ids ) raise @@ -446,11 +477,11 @@ def _cleanup_failed_dataset_creation( self, dataset_name: str | None, created_local: bool, - mock_folder_id: str | None, - private_folder_id: str | None, + mock_folder_ids: list[str], + private_folder_ids: list[str], ) -> None: """Best-effort cleanup after a failed create_dataset, in reverse order.""" - if private_folder_id is not None: + for private_folder_id in reversed(private_folder_ids): try: self.sync_engine.delete_file_by_id(private_folder_id) except Exception: @@ -459,7 +490,7 @@ def _cleanup_failed_dataset_creation( private_folder_id, ) - if mock_folder_id is not None: + for mock_folder_id in reversed(mock_folder_ids): try: self.sync_engine.delete_file_by_id(mock_folder_id) except Exception: @@ -511,43 +542,53 @@ def _collect_mock_files(self, dataset) -> dict[str, bytes]: return files def _share_dataset_collection( - self, tag: str, content_hash: str, users: list[str] | str + self, wire_prefix: str, tag: str, content_hash: str, users: list[str] | str ) -> None: - """Share a dataset collection with ``users``, or tag it ``"any"`` and - share with all already-approved peers.""" + """Share one layout of a dataset with ``users``, or tag it ``"any"`` and + share with all already-approved peers. + + Every layout is shared with the whole audience, so a peer that upgrades + later moves to the newer layout with no action by the owner. + """ if users == "any": - self.sync_engine.tag_collection_as_any( - DATASET_COLLECTION_PREFIX, tag, content_hash - ) + self.sync_engine.tag_collection_as_any(wire_prefix, tag, content_hash) self.sync_engine.datasite_owner_syncer.register_any_shared_collection( - tag, content_hash + wire_prefix, tag, content_hash ) peer_emails = [ p.email for p in self.sync_engine.peer_manager.approved_peers ] if peer_emails: self.sync_engine.share_collection( - DATASET_COLLECTION_PREFIX, tag, content_hash, peer_emails + wire_prefix, tag, content_hash, peer_emails ) else: if isinstance(users, str): users = [users] - self.sync_engine.share_collection( - DATASET_COLLECTION_PREFIX, tag, content_hash, users - ) + self.sync_engine.share_collection(wire_prefix, tag, content_hash, users) def _upload_dataset_to_collection(self, dataset, users: list[str] | str) -> str: - """Upload dataset files to collection folder. Returns the folder ID.""" + """Upload one protocol copy of a dataset. Returns the folder ID. + + Each copy gets its own collection, named for its layout, so a peer picks + the newest copy it can read. + """ + variant = dataset_variant(dataset.protocol_version) + wire_prefix = MOCK_DATASET_SPEC.wire_prefix(variant) files = self._collect_mock_files(dataset) folder_id, content_hash = self._create_and_upload_collection( - DATASET_COLLECTION_PREFIX, dataset.name, files + wire_prefix, dataset.name, files ) - self._share_dataset_collection(dataset.name, content_hash, users) + self._share_dataset_collection(wire_prefix, dataset.name, content_hash, users) return folder_id def _upload_private_dataset_to_collection(self, dataset) -> str | None: - """Upload private dataset files to a separate owner-only collection folder. - Returns the folder ID, or None if no files to upload.""" + """Upload the private files of one protocol copy to an owner-only collection. + + The copies hold separate private directories, so one upload of the newest + would leave the others local only and a cold start would not restore them. + Returns the folder ID, or None if there are no files to upload. + """ collection_tag = dataset.name # Collect all files in private dir (data, metadata, permissions) @@ -560,8 +601,9 @@ def _upload_private_dataset_to_collection(self, dataset) -> str | None: return None # Private collection: no sharing step. + variant = dataset_variant(dataset.protocol_version) folder_id, _ = self._create_and_upload_collection( - PRIVATE_DATASET_COLLECTION_PREFIX, collection_tag, files + PRIVATE_DATASET_SPEC.wire_prefix(variant), collection_tag, files ) return folder_id @@ -604,9 +646,6 @@ def share_dataset(self, tag: str, users: list[str] | str, sync=True): users: List of email addresses or "any" sync: Whether to sync after sharing """ - from syft.sync.connections.drive.gdrive_transport import ( - CollectionFolder, - ) if self.dataset_manager is None: raise ValueError("Dataset manager is not set") @@ -619,21 +658,133 @@ def share_dataset(self, tag: str, users: list[str] | str, sync=True): if dataset is None: raise ValueError(f"Dataset {tag} not found") - # Compute current content hash from local files, then share. - files = self._collect_mock_files(dataset) - content_hash = CollectionFolder.compute_hash(files) - self._share_dataset_collection(tag, content_hash, users) + if users != "any" and isinstance(users, str): + users = [users] + + # A dataset has one collection for each layout it was written in. Share + # them all, so a peer of any supported version finds a copy. The listing + # gives the hash of each copy, so no hash is recomputed here. + collections = self._mock_collections_for(tag) + if not collections: + raise ValueError(f"No uploaded collection found for dataset {tag}") + + # A share is a change of audience. The layouts were decided by the + # audience at create time, so a new peer whose protocol reads none of + # them would get a grant on a folder its client never even lists. + # Materialize what is missing first, then share everything. + if self._ensure_dataset_layouts_for( + tag, users, {self._protocol_of(c) for c in collections} + ): + collections = self._mock_collections_for(tag) + + for collection in collections: + self._share_dataset_collection( + MOCK_DATASET_SPEC.wire_prefix(collection.variant), + tag, + collection.content_hash, + users, + ) if sync: self.sync() + def _mock_collections_for(self, tag: str) -> list: + """Every uploaded mock-data layout of one dataset.""" + return [ + c + for c in self.sync_engine._connection_router.owner_list_all_collections_with_permissions( + DATASET_COLLECTION_PREFIX + ) + if c.tag == tag + ] + + def _private_collections_for(self, tag: str) -> list: + """Every uploaded private-data layout of one dataset.""" + return [ + c + for c in self.sync_engine._connection_router.owner_list_all_collections_with_permissions( + PRIVATE_DATASET_COLLECTION_PREFIX + ) + if c.tag == tag + ] + + @staticmethod + def _protocol_of(collection) -> str: + """The dataset protocol version a collection's wire variant stands for.""" + return collection.variant.removeprefix("v") or "0" + + def _ensure_dataset_layouts_for( + self, tag: str, users: list[str] | str, existing_versions: set[str] + ) -> bool: + """Materialize any layout the audience reads but no existing copy serves. + + A peer reads every layout at or below its negotiated protocol version, + so a copy is only missing when no uploaded collection sits at or below + the version a peer reads. The new copy uploads unshared; the caller + shares every collection uniformly afterwards. Returns whether a copy + was added. + """ + storage = self.dataset_manager.storage + peer_emails = self.dataset_manager._peer_emails(users) + needed = storage.target_protocol_versions_for_peers(peer_emails) + missing = { + version + for version in needed + if not any(int(e) <= int(version) for e in existing_versions) + } + if not missing: + return False + + for protocol_version in sorted(missing, key=int): + self._materialize_dataset_copy(tag, protocol_version, users) + return True + + def _materialize_dataset_copy( + self, tag: str, protocol_version: str, users: list[str] | str + ) -> None: + """Create and upload one layout copy of an existing dataset. + + The copy uploads unshared; sharing stays with the caller. Each copy + holds its own private directory, so the copy gets its private + collection iff the dataset's copies are drive-backed -- then a cold + start restores it like any other. + + The layout may already be on disk with no collection of its own: an + upload can fail after the migrate, and `migrate` is public. A second + write of the same layout raises, so an existing copy is read and + uploaded instead. Permissions are re-applied either way, because a + migrate re-applies them and both paths must leave the same state. + """ + storage = self.dataset_manager.storage + try: + ref = storage.find_dataset_ref( + self.email, tag, protocol_version=protocol_version + ) + except DatasetNotFoundError: + copy = self.dataset_manager.migrate(tag, protocol_version, users=users) + else: + copy = storage.read_dataset(ref) + self.dataset_manager._set_new_dataset_permissions(dataset=copy, users=users) + self._upload_dataset_to_collection(copy, users=[]) + if self._private_collections_for(tag): + self._upload_private_dataset_to_collection(copy) + def share_private_dataset(self, tag: str, enclave_email: str): - """Share private dataset files with an enclave via outbox events.""" + """Share private dataset files with an enclave via outbox events. + + The files ship at the layout the enclave reads: the newest local copy + at or below its negotiated dataset protocol, materialized first when + no copy qualifies. An enclave without a known schema is assumed to run + the current protocol, the same policy as jobs. + """ if not self.has_do_role: raise ValueError("Only data owners can share private datasets") with self.sync_engine.sync_file_lock(): - files = self.dataset_manager.get_private_dataset_files(tag) + protocol_version = self._private_share_protocol_version(tag, enclave_email) + files = self.dataset_manager.get_private_dataset_files( + tag, protocol_version=protocol_version + ) events_message = self.sync_engine.datasite_owner_syncer.event_cache.create_events_for_files( files ) @@ -643,6 +794,27 @@ def share_private_dataset(self, tag: str, enclave_email: str): ) self.sync_engine.datasite_owner_syncer.process_syftbox_events_queue() + def _private_share_protocol_version(self, tag: str, peer_email: str) -> str: + """The protocol version of the copy to ship privately to this peer. + + A reader scans every layout at or below its negotiated version, so the + newest existing copy at or below it serves; only when none qualifies + is a copy at the negotiated version materialized. + """ + storage = self.dataset_manager.storage + negotiated = storage.negotiated_protocol_version_for_peer( + peer_email, raise_on_unknown=False + ) + readable = { + ref.protocol_version + for ref in storage.iter_dataset_refs_all_protocols(self.email) + if ref.name == tag and int(ref.protocol_version) <= int(negotiated) + } + if readable: + return max(readable, key=int) + self._materialize_dataset_copy(tag, negotiated, users=[peer_email]) + return negotiated + @property def datasets(self) -> Any: """The dataset manager. Auto-syncs first unless PRE_SYNC=false.""" diff --git a/packages/syft-rds/src/syft_rds/config.py b/packages/syft-rds/src/syft_rds/config.py index 1009833b9cd..02c445edfde 100644 --- a/packages/syft-rds/src/syft_rds/config.py +++ b/packages/syft-rds/src/syft_rds/config.py @@ -21,13 +21,14 @@ from syft.sync.syftbox_manager import ( SyftboxManagerConfig, ) -from syft.sync.sync.collection_spec import CollectionSyncSpec +from syft.sync.sync.collection_spec import CollectionLayout, CollectionSyncSpec from syft_job import SyftJobConfig -from syft_datasets.config import SyftBoxConfig +from syft_datasets.config import SyftBoxConfig, protocol_dir_name from syft_datasets.dataset_manager import ( DATASET_COLLECTION_PREFIX, PRIVATE_DATASET_COLLECTION_PREFIX, ) +from syft_datasets.protocolcodecs import CODECS # The RDS layer owns the local subpaths; the on-wire prefixes come from # syft_datasets (imported above), mirrored in syft for login-time cleanup. @@ -39,14 +40,49 @@ # * public (mock) – mirror + shareable → peers' watchers pull it. # * private (real) – restore-only + owner-only → the owner restores it for itself; # peer-facing watchers skip it; it is never shared. -DATASET_COLLECTION_SPECS = [ - CollectionSyncSpec.public(DATASET_COLLECTION_PREFIX, COLLECTION_SUBPATH), - CollectionSyncSpec.private( - PRIVATE_DATASET_COLLECTION_PREFIX, PRIVATE_COLLECTION_SUBPATH - ), +# Every dataset protocol version this release reads, oldest first. Protocol 0 +# has no path segment and no name infix, so its layout is unchanged byte for +# byte; protocol n lives under a v segment and writes a v name infix. +READABLE_DATASET_PROTOCOL_VERSIONS = [ + version + for codec_cls in CODECS + for version in codec_cls.dataset_config_cls.protocol_versions ] +def dataset_layouts(subpath: Path) -> list[CollectionLayout]: + """One layout for each dataset protocol version this release reads.""" + layouts = [] + for version in READABLE_DATASET_PROTOCOL_VERSIONS: + segment = protocol_dir_name(version) + layouts.append( + CollectionLayout( + variant=segment or "", + local_subpath=subpath / segment if segment else subpath, + ) + ) + return layouts + + +MOCK_DATASET_SPEC = CollectionSyncSpec.public( + DATASET_COLLECTION_PREFIX, + COLLECTION_SUBPATH, + layouts=dataset_layouts(COLLECTION_SUBPATH), +) +PRIVATE_DATASET_SPEC = CollectionSyncSpec.private( + PRIVATE_DATASET_COLLECTION_PREFIX, + PRIVATE_COLLECTION_SUBPATH, + layouts=dataset_layouts(PRIVATE_COLLECTION_SUBPATH), +) + +DATASET_COLLECTION_SPECS = [MOCK_DATASET_SPEC, PRIVATE_DATASET_SPEC] + + +def dataset_variant(protocol_version: str) -> str: + """The wire variant of a dataset protocol version. Protocol 0 has none.""" + return protocol_dir_name(protocol_version) or "" + + class SyftRDSClientConfig(BaseModel): sync: SyftboxManagerConfig job: SyftJobConfig diff --git a/packages/syft-rds/tests/test_client.py b/packages/syft-rds/tests/test_client.py index 8824db63dfc..d77103b8a6f 100644 --- a/packages/syft-rds/tests/test_client.py +++ b/packages/syft-rds/tests/test_client.py @@ -48,53 +48,6 @@ def test_dataset_creation_and_sync(): assert len(dataset.mock_files) > 0 -def test_delete_unversioned_state_removes_dataset_collections(): - """delete_unversioned_state clears both dataset collection folders.""" - from syft_datasets.dataset_manager import ( - DATASET_COLLECTION_PREFIX, - PRIVATE_DATASET_COLLECTION_PREFIX, - ) - from dataset_test_utils import create_tmp_dataset_files - - def query(conn, name_contains): - results = ( - conn.drive_service.files() - .list( - q=f"name contains '{name_contains}' and trashed=false", - fields="files(id, name)", - ) - .execute() - ) - return results.get("files", []) - - ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - encryption=True, - ) - - mock_path, private_path, readme_path = create_tmp_dataset_files() - do.create_dataset( - name="my dataset", - mock_path=mock_path, - private_path=private_path, - summary="Test", - readme_path=readme_path, - users=[ds.email], - upload_private=True, - ) - do.sync() - - conn = do.peer_manager.connection_router.connections[0] - assert len(query(conn, DATASET_COLLECTION_PREFIX)) > 0 - assert len(query(conn, PRIVATE_DATASET_COLLECTION_PREFIX)) > 0 - - conn.delete_unversioned_state() - - assert len(query(conn, DATASET_COLLECTION_PREFIX)) == 0 - assert len(query(conn, PRIVATE_DATASET_COLLECTION_PREFIX)) == 0 - - def test_dir_returns_only_public_api(): _ds, do = SyftRDSClient.pair_with_mock_drive_service_connection() @@ -137,6 +90,7 @@ def test_encrypted_dataset_collection_syncs(): """Dataset-collection sync-down works under encryption.""" from dataset_test_utils import create_tmp_dataset_files from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX + from syft_rds.config import MOCK_DATASET_SPEC ds, do = SyftRDSClient.pair_with_mock_drive_service_connection( encryption=True, @@ -162,20 +116,12 @@ def test_encrypted_dataset_collection_syncs(): do_collections = [c for c in collections if c["owner_email"] == do.email] assert do_collections, "DS does not see the DO's dataset collection" + # The download names the layout the owner published, not the bare prefix. c = do_collections[0] files = cr.watcher_download_collection( - DATASET_COLLECTION_PREFIX, c["tag"], c["content_hash"], do.email + MOCK_DATASET_SPEC.wire_prefix(c["variant"]), + c["tag"], + c["content_hash"], + do.email, ) assert files, "DS could not download the dataset collection files" - - -def test_collection_prefixes_match_syft_datasets(): - """The sync core mirrors the prefixes rather than importing the domain.""" - from syft.sync.connections import collection_prefixes as core - from syft_datasets import dataset_manager as domain - - assert core.DATASET_COLLECTION_PREFIX == domain.DATASET_COLLECTION_PREFIX - assert ( - core.PRIVATE_DATASET_COLLECTION_PREFIX - == domain.PRIVATE_DATASET_COLLECTION_PREFIX - ) diff --git a/packages/syft-rds/tests/test_create_dataset_cleanup.py b/packages/syft-rds/tests/test_create_dataset_cleanup.py index 0f044df4918..d9491413da7 100644 --- a/packages/syft-rds/tests/test_create_dataset_cleanup.py +++ b/packages/syft-rds/tests/test_create_dataset_cleanup.py @@ -32,13 +32,13 @@ def _dataset_kwargs(self, users=None): ) def test_no_cleanup_when_local_create_fails(self): - """If dataset_manager.create raises, nothing was created so nothing to clean.""" + """If create_all raises, nothing was created so nothing to clean.""" do_manager = self._make_do_manager() with ( patch.object( do_manager.dataset_manager, - "create", + "create_all", side_effect=ValueError("bad input"), ), patch.object( @@ -49,7 +49,7 @@ def test_no_cleanup_when_local_create_fails(self): do_manager.create_dataset(**self._dataset_kwargs()) # Cleanup called with nothing to clean - mock_cleanup.assert_called_once_with(None, False, None, None) + mock_cleanup.assert_called_once_with(None, False, [], []) def test_cleanup_on_mock_upload_failure(self): """If mock upload fails, local dataset is cleaned up.""" diff --git a/packages/syft-rds/tests/test_dataset_upload_private.py b/packages/syft-rds/tests/test_dataset_upload_private.py index d143dbe93c6..ef90e08f0d1 100644 --- a/packages/syft-rds/tests/test_dataset_upload_private.py +++ b/packages/syft-rds/tests/test_dataset_upload_private.py @@ -1,4 +1,7 @@ -from syft.sync.connections.drive.gdrive_transport import GDriveConnection +from syft.sync.connections.drive.gdrive_transport import ( + GDriveConnection, + collection_name_query, +) from syft_rds import SyftRDSClient from syft_datasets.dataset_manager import ( DATASET_COLLECTION_PREFIX, @@ -148,10 +151,7 @@ def test_ds_cannot_find_private_folders_via_gdrive_query(self): results = ( ds_connection.drive_service.files() .list( - q=( - f"name contains '{PRIVATE_DATASET_COLLECTION_PREFIX}_' " - f"and trashed=false" - ), + q=f"{collection_name_query(PRIVATE_DATASET_COLLECTION_PREFIX)} and trashed=false", fields="files(id,name)", ) .execute() @@ -165,10 +165,7 @@ def test_ds_cannot_find_private_folders_via_gdrive_query(self): do_results = ( do_connection.drive_service.files() .list( - q=( - f"name contains '{PRIVATE_DATASET_COLLECTION_PREFIX}_' " - f"and trashed=false" - ), + q=f"{collection_name_query(PRIVATE_DATASET_COLLECTION_PREFIX)} and trashed=false", fields="files(id,name)", ) .execute() diff --git a/packages/syft-rds/tests/test_sync_manager.py b/packages/syft-rds/tests/test_sync_manager.py index 541d02deaf1..fe79491a47c 100644 --- a/packages/syft-rds/tests/test_sync_manager.py +++ b/packages/syft-rds/tests/test_sync_manager.py @@ -12,7 +12,10 @@ from syft_datasets import Dataset from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX from syft_rds import SyftRDSClient -from syft_rds.config import COLLECTION_SUBPATH, SyftRDSClientConfig +from syft_rds.config import ( + MOCK_DATASET_SPEC, + SyftRDSClientConfig, +) from dataset_test_utils import ( create_test_project_folder, create_tmp_dataset_files, @@ -1035,9 +1038,11 @@ def test_ds_dataset_cache_aware_sync(): # Verify hash was loaded from disk on startup ds_cache = ds_manager2.sync_engine.datasite_watcher_syncer.datasite_watcher_cache - # Cache uses full path as key: syftbox_folder / owner_email / collection_subpath / tag + # Cache uses full path as key: syftbox_folder / owner_email / + # / tag + newest_layout = MOCK_DATASET_SPEC.layouts[-1] cache_key = ds_cache.get_collection_path( - do_email, "cached dataset", COLLECTION_SUBPATH + do_email, "cached dataset", newest_layout.local_subpath ) assert cache_key in ds_cache.collection_hashes, ( "Hash should be loaded from disk on startup" @@ -1275,7 +1280,7 @@ def test_in_memory_connection_load_state(): len(do_manager2.sync_engine.datasite_owner_syncer.any_shared_collections) == 1 ) assert ( - do_manager2.sync_engine.datasite_owner_syncer.any_shared_collections[0][0] + do_manager2.sync_engine.datasite_owner_syncer.any_shared_collections[0][1] == "load_state_dataset" ) diff --git a/packages/syft-rds/tests/test_version_mismatch_flow.py b/packages/syft-rds/tests/test_version_mismatch_flow.py index a3bf9156c16..33733d9e6ae 100644 --- a/packages/syft-rds/tests/test_version_mismatch_flow.py +++ b/packages/syft-rds/tests/test_version_mismatch_flow.py @@ -1,8 +1,12 @@ -"""End-to-end test for version mismatch and backup flow with mock drive.""" +"""End-to-end test for a client minor upgrade that keeps local and remote data. + +Login no longer deletes SyftBox state on a major/minor mismatch. The default is +to continue; private Drive folders are adopted by rename, and P2P folders of the +earlier version are reused so a peer that has not upgraded still finds them. +""" from unittest.mock import patch -from syft.sync.utils.syftbox_utils import delete_local_syftbox from syft.sync.connections.drive.gdrive_transport import ( GDRIVE_P2P_FOLDER_DATASITE_PREFIX, GOOGLE_FOLDER_MIME_TYPE, @@ -46,18 +50,21 @@ def _get_backing_store(manager): return conn.drive_service._backing_store -def _reinitialize_manager(email, backing_store, has_do_role, has_ds_role): - """Create a new SyftboxManager connected to an existing mock backing store. +def _reinitialize_manager( + email, backing_store, has_do_role, has_ds_role, syftbox_folder, write_version=True +): + """Create a new SyftRDSClient on the same local path and mock Drive store. - This mirrors what pair_with_mock_drive_service_connection does for a - single manager, reusing the same backing store so the new manager sees - the same GDrive state. + Reuses the local SyftBox directory so a continue-on-mismatch upgrade keeps + the data that login left in place. Reuses the backing store so GDrive state + matches the pre-upgrade client. """ config = SyftRDSClientConfig._base_config_for_testing( email=email, has_do_role=has_do_role, has_ds_role=has_ds_role, use_in_memory_cache=False, + syftbox_folder=syftbox_folder, ) manager = SyftRDSClient.from_config(config) @@ -76,17 +83,13 @@ def _reinitialize_manager(email, backing_store, has_do_role, has_ds_role): manager.sync_engine.job_file_change_handler._handle_file_change, ) - manager.peer_manager.write_own_version() + if write_version: + manager.peer_manager.write_own_version() return manager -def _simulate_upgrade(manager, backing_store): - """Simulate handle_potential_version_mismatches_on_login with mocks. - - Patches only the I/O boundaries so that read_local_version reads from the - manager's real syftbox folder, _read_remote_version reads from the mock - drive, and delete operations target the correct local path / mock drive. - """ +def _simulate_continue_on_mismatch(manager, backing_store): + """Run the login mismatch handler with choice 1 (continue, keep data).""" email = manager.email syftbox_folder = manager.syftbox_folder @@ -96,12 +99,6 @@ def _simulate_upgrade(manager, backing_store): def read_remote(e, t): return mock_conn.read_own_version_file() - def do_delete_local(**kwargs): - delete_local_syftbox(email=email, local_syftbox_path=syftbox_folder) - - def do_delete_unversioned(e, t): - mock_conn.delete_unversioned_state() - with ( patch( "syft.sync.login_utils._resolve_token_path", @@ -121,22 +118,22 @@ def do_delete_unversioned(e, t): ), patch( "syft.sync.login_utils.delete_local_syftbox", - side_effect=do_delete_local, - ), + ) as mock_delete_local, patch( - "syft.sync.login_utils._delete_remote_unversioned_state", - side_effect=do_delete_unversioned, - ), + "syft.sync.login_utils.delete_remote_syftbox", + ) as mock_delete_remote, ): from syft.sync.login_utils import ( handle_potential_version_mismatches_on_login, ) handle_potential_version_mismatches_on_login(email) + mock_delete_local.assert_not_called() + mock_delete_remote.assert_not_called() -def test_version_mismatch_and_backup_flow(): - """Full flow: create state on v1 -> upgrade to v2 -> old state preserved, new version works.""" +def test_version_mismatch_continues_and_repairs(): + """Upgrade keeps peers, jobs, and data; private folders adopt; P2P reuses.""" # -- Step 1: Create DO/DS on current version -- ds_manager, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( @@ -174,22 +171,26 @@ def test_version_mismatch_and_backup_flow(): assert do_manager.jobs[0].status == "done" - # -- Step 4: Assert only P2P folders with current version -- + # -- Step 4: Record P2P folders and personal folder id before upgrade -- do_conn = do_manager.peer_manager.connection_router.connections[0] do_p2p_current = _find_versioned_p2p_folders( do_conn, ds_manager.email, SYFT_VERSION ) assert len(do_p2p_current) > 0 - # No folders with a different version - all_do_p2p = _find_p2p_folders(do_conn, ds_manager.email) - assert len(all_do_p2p) == len(do_p2p_current) + old_personal_name = f"{SYFT_VERSION}#{do_manager.email}" + old_personal_id = do_conn._find_folder_by_name( + old_personal_name, + parent_id=do_conn.get_syftbox_folder_id(), + owner_email=do_manager.email, + ) + assert old_personal_id is not None # -- Step 5: Extract backing store -- backing_store = _get_backing_store(do_manager) do_email = do_manager.email ds_email = ds_manager.email - # -- Step 6+7: Upgrade DO -- + # -- Step 6+7: Upgrade DO (continue keeps data) -- with ( patch("syft.version.SYFT_VERSION", NEW_VERSION), patch( @@ -199,30 +200,53 @@ def test_version_mismatch_and_backup_flow(): patch("syft.sync.login_utils.SYFT_VERSION", NEW_VERSION), patch("syft.sync.version.version_info.SYFT_VERSION", NEW_VERSION), ): - _simulate_upgrade(do_manager, backing_store) + do_syftbox = do_manager.syftbox_folder + _simulate_continue_on_mismatch(do_manager, backing_store) do_manager = _reinitialize_manager( - do_email, backing_store, has_do_role=True, has_ds_role=False + do_email, + backing_store, + has_do_role=True, + has_ds_role=False, + syftbox_folder=do_syftbox, ) - # -- Step 8: Assert new versioned folders for DO -- + # Personal folder is adopted (same Drive id, new name), not recreated. do_conn_new = do_manager.peer_manager.connection_router.connections[0] - do_p2p_new = _find_versioned_p2p_folders(do_conn_new, ds_email, NEW_VERSION) - # New folders don't exist yet (no peers added), but personal folder does - personal_folder_name = f"{NEW_VERSION}#{do_email}" - personal_id = do_conn_new._find_folder_by_name( - personal_folder_name, + new_personal_name = f"{NEW_VERSION}#{do_email}" + new_personal_id = do_conn_new._find_folder_by_name( + new_personal_name, parent_id=do_conn_new.get_syftbox_folder_id(), owner_email=do_email, ) - assert personal_id is not None + assert new_personal_id is not None + assert new_personal_id == old_personal_id + assert ( + do_conn_new._find_folder_by_name( + old_personal_name, + parent_id=do_conn_new.get_syftbox_folder_id(), + owner_email=do_email, + ) + is None + ) + + # Peers survive: continue did not wipe SYFT_peers.json. + do_manager.load_peers() + assert any(p.email == ds_email for p in do_manager.peer_manager.approved_peers) - # -- Step 9+10: Upgrade DS -- - _simulate_upgrade(ds_manager, backing_store) + # Pre-upgrade job is still present on the kept datasite. + assert any(job.name == "pre_upgrade.job" for job in do_manager.jobs) + + # -- Step 8+9: Upgrade DS -- + ds_syftbox = ds_manager.syftbox_folder + _simulate_continue_on_mismatch(ds_manager, backing_store) ds_manager = _reinitialize_manager( - ds_email, backing_store, has_do_role=False, has_ds_role=True + ds_email, + backing_store, + has_do_role=False, + has_ds_role=True, + syftbox_folder=ds_syftbox, ) - # -- Step 11: Assert new versioned folders for DS -- ds_conn_new = ds_manager.peer_manager.connection_router.connections[0] ds_personal_name = f"{NEW_VERSION}#{ds_email}" ds_personal_id = ds_conn_new._find_folder_by_name( @@ -232,36 +256,18 @@ def test_version_mismatch_and_backup_flow(): ) assert ds_personal_id is not None - # -- Step 12: Assert peer connection is gone -- - assert len(do_manager.peer_manager.approved_peers) == 0 - assert len(ds_manager.peer_manager.approved_peers) == 0 - - # -- Step 13: Re-add peers -- - ds_manager.add_peer(do_manager.email) - do_manager.load_peers() - do_manager.approve_peer_request(ds_manager.email) + ds_manager.load_peers() + assert any(p.email == do_email for p in ds_manager.peer_manager.approved_peers) - # Now new versioned P2P folders should exist + # P2P folders of the old version are reused, not replaced. A peer that + # has not upgraded still looks for the old name. do_p2p_new = _find_versioned_p2p_folders(do_conn_new, ds_email, NEW_VERSION) - assert len(do_p2p_new) > 0 - - ds_p2p_new = _find_versioned_p2p_folders(ds_conn_new, do_email, NEW_VERSION) - assert len(ds_p2p_new) > 0 - - # -- Step 14: Re-upload dataset -- - mock_path2, private_path2, readme_path2 = create_tmp_dataset_files() - do_manager.create_dataset( - name="my dataset", - mock_path=mock_path2, - private_path=private_path2, - summary="Test dataset v2", - readme_path=readme_path2, - users=[ds_manager.email], - ) - do_manager.sync() - ds_manager.sync() + assert len(do_p2p_new) == 0 + do_p2p_old = _find_versioned_p2p_folders(do_conn_new, ds_email, SYFT_VERSION) + assert len(do_p2p_old) > 0 + assert len(do_p2p_old) == len(do_p2p_current) - # -- Step 15: Re-submit job -- + # -- Step 10: Submit a new job without re-peering -- project_dir2 = create_test_project_folder(with_pyproject=False) ds_manager.submit_python_job( user=do_manager.email, @@ -271,24 +277,123 @@ def test_version_mismatch_and_backup_flow(): ) do_manager.sync() - # -- Step 16: Assert only one job (new one), old folder still has old -- - assert len(do_manager.jobs) == 1 - - # Old versioned P2P folders still have old data - old_do_p2p = _find_versioned_p2p_folders(do_conn_new, ds_email, SYFT_VERSION) - assert len(old_do_p2p) > 0 - - # -- Step 17: DO runs new job -- - do_manager.jobs[0].approve() + post = [job for job in do_manager.jobs if job.name == "post_upgrade.job"] + assert len(post) == 1 + post[0].approve() do_manager.process_approved_jobs() do_manager.sync() + # Reload from disk; the pre-process JobState object does not update in place. + post = [job for job in do_manager.jobs if job.name == "post_upgrade.job"] + assert len(post) == 1 + assert post[0].status == "done" - assert do_manager.jobs[0].status == "done" - - # -- Step 18: DS sees result -- ds_manager.sync() - ds_jobs = ds_manager.job_client.jobs - assert len(ds_jobs) == 1 - # DS should have received the output file via sync - ds_job = ds_jobs[0] - assert ds_job.status == "done" + ds_post = [ + job for job in ds_manager.job_client.jobs if job.name == "post_upgrade.job" + ] + assert len(ds_post) == 1 + assert ds_post[0].status == "done" + + +def _upgraded_manager(manager): + """The same client after an upgrade: a new process on the same data. + + A real upgrade restarts the process, so the peer manager computes its own + version again. Reusing the pre-upgrade object would read a cached version. + """ + # No version write here. The test must show that login is what refreshes + # the version files, so the new manager must not do it first. + return _reinitialize_manager( + manager.email, + _get_backing_store(manager), + has_do_role=True, + has_ds_role=False, + syftbox_folder=manager.syftbox_folder, + write_version=False, + ) + + +def test_login_writes_the_remote_version_file_too(): + """Login must refresh both version files, not only the local one. + + A peer reads the remote file to select a job or dataset protocol version for + us. A local-only write leaves that file at the version that first created + it, so peers keep negotiating against a client we no longer run. + """ + from syft.sync.login import _init_client_login + from syft.sync.version.local_version import read_local_version + + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + check_versions=True, + ) + conn = do_manager.peer_manager.connection_router.connections[0] + assert conn.read_own_version_file().syft_client_version == SYFT_VERSION + + with ( + patch("syft.version.SYFT_VERSION", NEW_VERSION), + patch("syft.sync.version.version_info.SYFT_VERSION", NEW_VERSION), + ): + upgraded = _upgraded_manager(do_manager) + new_conn = upgraded.peer_manager.connection_router.connections[0] + # Still the pre-upgrade version: nothing has refreshed it yet. + assert new_conn.read_own_version_file().syft_client_version == SYFT_VERSION + + _init_client_login(upgraded.sync_engine, sync=False, load_peers=False) + + assert new_conn.read_own_version_file().syft_client_version == NEW_VERSION + local = read_local_version(upgraded.syftbox_folder) + assert local is not None + assert local.syft_client_version == NEW_VERSION + + +def test_the_mismatch_prompt_does_not_return_after_a_login(): + """The prompt asks once per upgrade, not once per login. + + The check compares the installed client with the local and the remote + version file. Login refreshes both, so the next login finds no mismatch. + """ + from syft.sync.login import _init_client_login + + _, do_manager = SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + check_versions=True, + ) + email = do_manager.email + syftbox_folder = do_manager.syftbox_folder + + with ( + patch("syft.version.SYFT_VERSION", NEW_VERSION), + patch("syft.sync.version.version_info.SYFT_VERSION", NEW_VERSION), + patch("syft.sync.login_utils.SYFT_VERSION", NEW_VERSION), + patch("syft.sync.login_utils._resolve_email", return_value=email), + patch("syft.sync.login_utils._resolve_token_path", return_value=None), + patch( + "syft.sync.login_utils._get_default_syftbox_path", + return_value=syftbox_folder, + ), + patch( + "syft.sync.login_utils._prompt_mismatch", return_value="1" + ) as mock_prompt, + ): + from syft.sync.login_utils import ( + handle_potential_version_mismatches_on_login, + ) + + conn = do_manager.peer_manager.connection_router.connections[0] + with patch( + "syft.sync.login_utils._read_remote_version", + side_effect=lambda e, t: conn.read_own_version_file(), + ): + # The check runs before the client exists, so it reads the files + # the previous client version left behind. + handle_potential_version_mismatches_on_login(email) + assert mock_prompt.call_count == 1 + + upgraded = _upgraded_manager(do_manager) + _init_client_login(upgraded.sync_engine, sync=False, load_peers=False) + + # Every login after that finds both files current, and asks nothing. + handle_potential_version_mismatches_on_login(email) + handle_potential_version_mismatches_on_login(email) + assert mock_prompt.call_count == 1 diff --git a/pyproject.toml b/pyproject.toml index dd84a6c1b94..779646a91b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "portalocker>=2.8.0", "syft-permissions==0.1.15", "syft-perms==0.1.15", + "syft-migration==0.1.1", "syft-crypto-python>=0.1.2b2", "syft-dataset==0.1.21", ] diff --git a/scripts/bump_version.py b/scripts/bump_version.py index b216aa3e8ef..312728057be 100644 --- a/scripts/bump_version.py +++ b/scripts/bump_version.py @@ -1,17 +1,32 @@ -"""Bump a package version and propagate the change to all dependents. +"""Bump the version of one package, and update the packages that depend on it. -Usage: python scripts/bump_version.py +Usage: + python scripts/bump_version.py + [--dependents {bumped,published}] -Output (two lines): - Line 1: new version - Line 2: space-separated list of all modified pyproject.toml files +The script writes the new version into the pyproject.toml of the package. It +then writes a version pin for the package into each pyproject.toml that depends +on it. + +The --dependents option selects the version for those pins: + +- published: the version that was in the file before this run. A release + publishes the version on the branch, and bumps the version after that. This + version is therefore the version on PyPI. Use this option for a release. +- bumped: the new version. PyPI does not have this version yet. Use this option + only if the script runs before the release. + +The script prints two lines: + +- Line 1: the new version. +- Line 2: the modified pyproject.toml files, separated by spaces. """ import argparse import re -import tomllib from pathlib import Path +import tomllib from packaging.version import Version REPO_ROOT = Path(__file__).resolve().parent.parent @@ -92,11 +107,24 @@ def main() -> None: ) parser.add_argument("package_name", help="Package name (e.g. syft-perms)") parser.add_argument("bump_type", choices=["major", "minor", "patch"]) + parser.add_argument( + "--dependents", + choices=["bumped", "published"], + default="bumped", + help=( + "Version for the dependent pins. 'bumped' is the new version. " + "'published' is the version that was in the file before this run, " + "which is the version a release publishes." + ), + ) args = parser.parse_args() target_path = find_target_pyproject(args.package_name) + with open(target_path, "rb") as f: + published_version = Version(tomllib.load(f)["project"]["version"]) new_version = update_target_version(target_path, args.bump_type) - modified_deps = update_dependents(args.package_name, new_version, target_path) + pinned = new_version if args.dependents == "bumped" else published_version + modified_deps = update_dependents(args.package_name, pinned, target_path) all_modified = [target_path] + modified_deps relative_paths = [str(p.relative_to(REPO_ROOT)) for p in all_modified] diff --git a/scripts/export_release_artifact.py b/scripts/export_release_artifact.py new file mode 100644 index 00000000000..4f8c8a24545 --- /dev/null +++ b/scripts/export_release_artifact.py @@ -0,0 +1,61 @@ +"""Export the release artifacts for the current syft version. + +Run on EVERY release (uv run python scripts/export_release_artifact.py): +always writes the package release info; additionally writes the protocol +artifact when this release introduces a new protocol version. + +Artifacts are frozen once written. Running this again for the same version +writes nothing and succeeds, so a release can re-run it safely. +""" + +import sys + +from syft.migrations.history import PACKAGE_ARTIFACTS_DIR, PROTOCOLS_DIR +from syft.migrations.registry import ( + SYFT_CLIENT_PROTOCOL_VERSION, + client_registry, +) +from syft.version import SYFT_VERSION + + +def main() -> None: + # Import the package so every versioned object is registered. + import syft # noqa: F401 + + if client_registry.protocol_bump_missing(): + sys.exit( + "The syft protocol changed since the released " + f"protocol-{client_registry.latest_released_protocol_version()}.json; " + "bump SYFT_CLIENT_PROTOCOL_VERSION in " + "syft/migrations/registry.py before releasing." + ) + + if client_registry.protocol_changed_without_bump(): + sys.exit( + "The syft protocol changed compared to the released " + f"protocol-{SYFT_CLIENT_PROTOCOL_VERSION}.json; bump " + "SYFT_CLIENT_PROTOCOL_VERSION in syft/migrations/registry.py " + "before releasing." + ) + + PACKAGE_ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) + PROTOCOLS_DIR.mkdir(parents=True, exist_ok=True) + + info_path = PACKAGE_ARTIFACTS_DIR / f"syft-{SYFT_VERSION}.json" + protocol_path = PROTOCOLS_DIR / f"protocol-{SYFT_CLIENT_PROTOCOL_VERSION}.json" + + if info_path.exists(): + print(f"Package artifact already present: {info_path}") + else: + client_registry.compute_released_package_protocol_info().save(info_path) + print(f"Wrote {info_path}") + + if protocol_path.exists(): + print(f"Protocol artifact already present: {protocol_path}") + else: + client_registry.compute_released_protocol().save(protocol_path) + print(f"Wrote {protocol_path} (new protocol version)") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_release_fixture.py b/scripts/generate_release_fixture.py new file mode 100644 index 00000000000..d501563a0ec --- /dev/null +++ b/scripts/generate_release_fixture.py @@ -0,0 +1,124 @@ +"""Generate a p2p backward-compatibility fixture for the current syft release. + +Run on EVERY release, at the released commit: + + git checkout v + uv run python scripts/generate_release_fixture.py + +The fixture name comes from SYFT_VERSION in the tree. The release job +publishes the version on the branch, tags it, then bumps. A run after the bump +therefore names the fixture after the next version, which is not published yet. + +Writes the serialized artifacts exactly as this release produces them, into + + tests/migrations/p2p/fixtures/syft--protocol

/ + SYFT_version.json # the published version file + msgv2_<...>.tar.gz # a proposed-changes message (DS -> DO) + syfteventsmessagev3_<...>.tar.gz # an events message (DO -> watchers) + +Unlike syft-job there is no local SyftBox tree to snapshot (storage is the +Google Drive transport), so fixtures are directories of captured blobs; future +releases loop over them (test_older_protocol_compatibility.py) to prove they +can still read and round-trip older serialized data. + +Protocol 0 / release 0.1.117 predates this script; its fixture +(syft_client-0.1.117-protocol0) is hand-authored, like protocol-0.json. +""" + +import sys +from pathlib import Path + +from syft.migrations.registry import SYFT_CLIENT_PROTOCOL_VERSION +from syft.sync.events.file_change_event import ( + FileChangeEvent, + FileChangeEventsMessage, +) +from syft.sync.messages.proposed_filechange import ( + ProposedFileChange, + ProposedFileChangesMessage, +) +from syft.sync.version.version_info import VersionInfo +from syft.version import SYFT_VERSION + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" + +FIXTURES_DIR = ( + Path(__file__).resolve().parents[1] / "tests" / "migrations" / "p2p" / "fixtures" +) + + +def build_version_info() -> VersionInfo: + # Not VersionInfo.current(): the detected install source is an absolute + # local path on dev machines, which must not leak into a committed fixture. + return VersionInfo.current().model_copy( + update={"syft_client_install_source": "pip"} + ) + + +def build_proposed_message() -> ProposedFileChangesMessage: + return ProposedFileChangesMessage( + sender_email=DS_EMAIL, + proposed_file_changes=[ + ProposedFileChange( + path_in_datasite="data/notes.txt", + content="hello from the release fixture", + datasite_email=DO_EMAIL, + ), + ProposedFileChange( + path_in_datasite="data/blob.bin", + content=b"\x00\x01\x02fixture-binary", + datasite_email=DO_EMAIL, + ), + ProposedFileChange( + path_in_datasite="data/removed.txt", + content=None, + old_hash="0" * 64, + is_deleted=True, + datasite_email=DO_EMAIL, + ), + ], + ) + + +def build_events_message( + proposed: ProposedFileChangesMessage, +) -> FileChangeEventsMessage: + events = [ + FileChangeEvent.from_proposed_filechange(change) + for change in proposed.proposed_file_changes + ] + return FileChangeEventsMessage(events=events) + + +def main() -> None: + # Any fixture for this version (any protocol) means the version was + # already released; a released version's serialized form is frozen. + existing = sorted(FIXTURES_DIR.glob(f"syft-{SYFT_VERSION}-protocol*")) + if existing: + sys.exit( + f"{existing[0]} already exists — fixtures are frozen once written. " + "Bump SYFT_VERSION before generating." + ) + target = FIXTURES_DIR / ( + f"syft-{SYFT_VERSION}-protocol{SYFT_CLIENT_PROTOCOL_VERSION}" + ) + target.mkdir(parents=True) + + (target / "SYFT_version.json").write_text(build_version_info().to_json()) + + proposed = build_proposed_message() + (target / proposed.message_filename.as_string()).write_bytes( + proposed.as_compressed_data() + ) + + events = build_events_message(proposed) + (target / events.message_filepath.as_string()).write_bytes( + events.as_compressed_data() + ) + + print(f"Wrote {target}") + + +if __name__ == "__main__": + main() diff --git a/syft/__init__.py b/syft/__init__.py index d2fb13658d8..04033450b27 100644 --- a/syft/__init__.py +++ b/syft/__init__.py @@ -36,6 +36,15 @@ delete_syftbox, delete_local_syftbox, ) +from syft.migrations.history import register_historic_schemas # noqa: E402 + +# Import the versioned model modules explicitly so registration is intentional, +# not a side-effect of whatever login happened to pull in first. +import syft.sync.version.version_info # noqa: F401, E402 +import syft.sync.messages.proposed_filechange # noqa: F401, E402 +import syft.sync.events.file_change_event # noqa: F401, E402 + +register_historic_schemas() SYFT_DIR = Path(__file__).parent.parent CREDENTIALS_DIR = SYFT_DIR / "credentials" diff --git a/syft/migrations/__init__.py b/syft/migrations/__init__.py new file mode 100644 index 00000000000..fa713f56a40 --- /dev/null +++ b/syft/migrations/__init__.py @@ -0,0 +1,15 @@ +from .registry import ( + PROTOCOL_NAME, + SYFT_CLIENT_PROTOCOL_VERSION, + client_migration_service, + client_registry, + load_as_latest, +) + +__all__ = [ + "PROTOCOL_NAME", + "SYFT_CLIENT_PROTOCOL_VERSION", + "client_migration_service", + "client_registry", + "load_as_latest", +] diff --git a/syft/migrations/history.py b/syft/migrations/history.py new file mode 100644 index 00000000000..8d7b53410f6 --- /dev/null +++ b/syft/migrations/history.py @@ -0,0 +1,32 @@ +from pathlib import Path + +from syft_migration import ReleasedPackageProtocolInfo, ReleasedProtocol + +from .registry import client_registry + +# Release artifacts of past syft releases: +# package-artifacts/-.json (every release) +# protocols/protocol-.json (only when the protocol changed) +# Generated by scripts/export_release_artifact.py; 0.1.117 / protocol 0 predate +# the artifact mechanism, so their files are hardcoded as if that release had +# emitted them. +HISTORY_DIR = Path(__file__).parent / "history" +PACKAGE_ARTIFACTS_DIR = HISTORY_DIR / "package-artifacts" +PROTOCOLS_DIR = HISTORY_DIR / "protocols" + + +def register_historic_schemas() -> None: + """Register the release artifacts of past releases into the client registry. + + Must run after the versioned models are imported: with + ``raise_for_unknown_objects`` an artifact listing an object version this + release cannot load fails at import time instead of at migration time. + """ + for path in sorted(PACKAGE_ARTIFACTS_DIR.glob("*.json")): + client_registry.register_released_package_protocol_info( + ReleasedPackageProtocolInfo.load(path), raise_for_unknown_objects=True + ) + for path in sorted(PROTOCOLS_DIR.glob("*.json")): + client_registry.register_released_protocol( + ReleasedProtocol.load(path), raise_for_unknown_objects=True + ) diff --git a/syft/migrations/history/package-artifacts/syft-client-0.1.117.json b/syft/migrations/history/package-artifacts/syft-client-0.1.117.json new file mode 100644 index 00000000000..5db7d0b359d --- /dev/null +++ b/syft/migrations/history/package-artifacts/syft-client-0.1.117.json @@ -0,0 +1,351 @@ +{ + "package_info": { + "package_name": "syft-client", + "version": "0.1.117", + "protocol_version": "0" + }, + "protocol_schema": { + "protocol_name": "syft", + "version": "0", + "supported_versions": { + "VersionInfo": ["1"], + "ProposedFileChangesMessage": ["1"], + "FileChangeEventsMessage": ["1"] + }, + "current_object_schemas": { + "VersionInfo": { + "description": "Model representing version information for a syft client.\n\nStored as SYFT_version.json in the peer-visible SyftBox folder. This file\nis the bootstrap channel for protocol negotiation (peers read it to learn\nwhat we speak), so its schema may only ever change additively: every\nsupported client version must be able to parse every newer version file.", + "properties": { + "canonical_name": { + "default": "VersionInfo", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "syft_client_version": { + "title": "Syft Client Version", + "type": "string" + }, + "min_supported_syft_client_version": { + "title": "Min Supported Syft Client Version", + "type": "string" + }, + "protocol_version": { + "title": "Protocol Version", + "type": "string" + }, + "min_supported_protocol_version": { + "title": "Min Supported Protocol Version", + "type": "string" + }, + "syft_client_install_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Syft Client Install Source" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "attestation_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Attestation Token" + } + }, + "required": [ + "syft_client_version", + "min_supported_syft_client_version", + "protocol_version", + "min_supported_protocol_version" + ], + "title": "VersionInfoV1", + "type": "object" + }, + "ProposedFileChangesMessage": { + "$defs": { + "MessageFileName": { + "properties": { + "submitted_timestamp": { + "title": "Submitted Timestamp", + "type": "number" + }, + "uid": { + "title": "Uid", + "type": "string" + } + }, + "title": "MessageFileName", + "type": "object" + }, + "ProposedFileChangeV1": { + "properties": { + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "old_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Old Hash" + }, + "new_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "New Hash" + }, + "submitted_timestamp": { + "title": "Submitted Timestamp", + "type": "number" + }, + "path_in_datasite": { + "format": "path", + "title": "Path In Datasite", + "type": "string" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "format": "binary", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content" + }, + "datasite_email": { + "title": "Datasite Email", + "type": "string" + }, + "is_deleted": { + "default": false, + "title": "Is Deleted", + "type": "boolean" + } + }, + "required": ["path_in_datasite", "datasite_email"], + "title": "ProposedFileChangeV1", + "type": "object" + } + }, + "description": "The msgv2 wire envelope (DS -> DO). The envelope is the migratable unit;\nits items are pinned to the exact version class, never a floating alias.", + "properties": { + "canonical_name": { + "default": "ProposedFileChangesMessage", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "sender_email": { + "title": "Sender Email", + "type": "string" + }, + "message_filename": { + "$ref": "#/$defs/MessageFileName" + }, + "proposed_file_changes": { + "items": { + "$ref": "#/$defs/ProposedFileChangeV1" + }, + "title": "Proposed File Changes", + "type": "array" + }, + "platform_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Platform Id" + } + }, + "required": ["sender_email", "proposed_file_changes"], + "title": "ProposedFileChangesMessageV1", + "type": "object" + }, + "FileChangeEventsMessage": { + "$defs": { + "FileChangeEventV1": { + "properties": { + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "path_in_datasite": { + "format": "path", + "title": "Path In Datasite", + "type": "string" + }, + "datasite_email": { + "title": "Datasite Email", + "type": "string" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "format": "binary", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content" + }, + "old_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Old Hash" + }, + "new_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "New Hash" + }, + "is_deleted": { + "default": false, + "title": "Is Deleted", + "type": "boolean" + }, + "submitted_timestamp": { + "title": "Submitted Timestamp", + "type": "number" + }, + "timestamp": { + "title": "Timestamp", + "type": "number" + } + }, + "required": [ + "id", + "path_in_datasite", + "datasite_email", + "submitted_timestamp", + "timestamp" + ], + "title": "FileChangeEventV1", + "type": "object" + }, + "FileChangeEventsMessageFileName": { + "properties": { + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "timestamp": { + "title": "Timestamp", + "type": "number" + }, + "extension": { + "default": ".tar.gz", + "title": "Extension", + "type": "string" + } + }, + "title": "FileChangeEventsMessageFileName", + "type": "object" + } + }, + "description": "The events wire envelope (DO -> watchers). The envelope is the migratable\nunit; its items are pinned to the exact version class, never a floating alias.", + "properties": { + "canonical_name": { + "default": "FileChangeEventsMessage", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "events": { + "items": { + "$ref": "#/$defs/FileChangeEventV1" + }, + "title": "Events", + "type": "array" + }, + "message_filepath": { + "$ref": "#/$defs/FileChangeEventsMessageFileName" + } + }, + "required": ["events"], + "title": "FileChangeEventsMessageV1", + "type": "object" + } + } + } +} diff --git a/syft/migrations/history/protocols/protocol-0.json b/syft/migrations/history/protocols/protocol-0.json new file mode 100644 index 00000000000..0b6ca735e82 --- /dev/null +++ b/syft/migrations/history/protocols/protocol-0.json @@ -0,0 +1,346 @@ +{ + "protocol_schema": { + "protocol_name": "syft", + "version": "0", + "supported_versions": { + "VersionInfo": ["1"], + "ProposedFileChangesMessage": ["1"], + "FileChangeEventsMessage": ["1"] + }, + "current_object_schemas": { + "VersionInfo": { + "description": "Model representing version information for a syft client.\n\nStored as SYFT_version.json in the peer-visible SyftBox folder. This file\nis the bootstrap channel for protocol negotiation (peers read it to learn\nwhat we speak), so its schema may only ever change additively: every\nsupported client version must be able to parse every newer version file.", + "properties": { + "canonical_name": { + "default": "VersionInfo", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "syft_client_version": { + "title": "Syft Client Version", + "type": "string" + }, + "min_supported_syft_client_version": { + "title": "Min Supported Syft Client Version", + "type": "string" + }, + "protocol_version": { + "title": "Protocol Version", + "type": "string" + }, + "min_supported_protocol_version": { + "title": "Min Supported Protocol Version", + "type": "string" + }, + "syft_client_install_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Syft Client Install Source" + }, + "updated_at": { + "format": "date-time", + "title": "Updated At", + "type": "string" + }, + "attestation_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Attestation Token" + } + }, + "required": [ + "syft_client_version", + "min_supported_syft_client_version", + "protocol_version", + "min_supported_protocol_version" + ], + "title": "VersionInfoV1", + "type": "object" + }, + "ProposedFileChangesMessage": { + "$defs": { + "MessageFileName": { + "properties": { + "submitted_timestamp": { + "title": "Submitted Timestamp", + "type": "number" + }, + "uid": { + "title": "Uid", + "type": "string" + } + }, + "title": "MessageFileName", + "type": "object" + }, + "ProposedFileChangeV1": { + "properties": { + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "old_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Old Hash" + }, + "new_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "New Hash" + }, + "submitted_timestamp": { + "title": "Submitted Timestamp", + "type": "number" + }, + "path_in_datasite": { + "format": "path", + "title": "Path In Datasite", + "type": "string" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "format": "binary", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content" + }, + "datasite_email": { + "title": "Datasite Email", + "type": "string" + }, + "is_deleted": { + "default": false, + "title": "Is Deleted", + "type": "boolean" + } + }, + "required": ["path_in_datasite", "datasite_email"], + "title": "ProposedFileChangeV1", + "type": "object" + } + }, + "description": "The msgv2 wire envelope (DS -> DO). The envelope is the migratable unit;\nits items are pinned to the exact version class, never a floating alias.", + "properties": { + "canonical_name": { + "default": "ProposedFileChangesMessage", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "sender_email": { + "title": "Sender Email", + "type": "string" + }, + "message_filename": { + "$ref": "#/$defs/MessageFileName" + }, + "proposed_file_changes": { + "items": { + "$ref": "#/$defs/ProposedFileChangeV1" + }, + "title": "Proposed File Changes", + "type": "array" + }, + "platform_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Platform Id" + } + }, + "required": ["sender_email", "proposed_file_changes"], + "title": "ProposedFileChangesMessageV1", + "type": "object" + }, + "FileChangeEventsMessage": { + "$defs": { + "FileChangeEventV1": { + "properties": { + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "path_in_datasite": { + "format": "path", + "title": "Path In Datasite", + "type": "string" + }, + "datasite_email": { + "title": "Datasite Email", + "type": "string" + }, + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "format": "binary", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content" + }, + "old_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Old Hash" + }, + "new_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "New Hash" + }, + "is_deleted": { + "default": false, + "title": "Is Deleted", + "type": "boolean" + }, + "submitted_timestamp": { + "title": "Submitted Timestamp", + "type": "number" + }, + "timestamp": { + "title": "Timestamp", + "type": "number" + } + }, + "required": [ + "id", + "path_in_datasite", + "datasite_email", + "submitted_timestamp", + "timestamp" + ], + "title": "FileChangeEventV1", + "type": "object" + }, + "FileChangeEventsMessageFileName": { + "properties": { + "id": { + "format": "uuid", + "title": "Id", + "type": "string" + }, + "timestamp": { + "title": "Timestamp", + "type": "number" + }, + "extension": { + "default": ".tar.gz", + "title": "Extension", + "type": "string" + } + }, + "title": "FileChangeEventsMessageFileName", + "type": "object" + } + }, + "description": "The events wire envelope (DO -> watchers). The envelope is the migratable\nunit; its items are pinned to the exact version class, never a floating alias.", + "properties": { + "canonical_name": { + "default": "FileChangeEventsMessage", + "title": "Canonical Name", + "type": "string" + }, + "version": { + "default": "1", + "title": "Version", + "type": "string" + }, + "events": { + "items": { + "$ref": "#/$defs/FileChangeEventV1" + }, + "title": "Events", + "type": "array" + }, + "message_filepath": { + "$ref": "#/$defs/FileChangeEventsMessageFileName" + } + }, + "required": ["events"], + "title": "FileChangeEventsMessageV1", + "type": "object" + } + } + } +} diff --git a/syft/migrations/registry.py b/syft/migrations/registry.py new file mode 100644 index 00000000000..c22e5ea450c --- /dev/null +++ b/syft/migrations/registry.py @@ -0,0 +1,45 @@ +from syft_migration import MigrationRegistry, MigrationService + +from syft.version import SYFT_VERSION + +PACKAGE_NAME = "syft" + +# Hardcoded, language-agnostic identifier for the syft protocol. A peer reads it +# as a key in SYFT_version.json, so it changes only with the released artifacts +# that carry it (history/). +PROTOCOL_NAME = "syft" + +# Incrementing version of the syft protocol. Protocol 0 is the last +# release without per-object versioning (<= 0.1.117, files carry no +# canonical_name/version identity fields); protocol >= 1 serializes identity +# fields on every versioned object. +SYFT_CLIENT_PROTOCOL_VERSION = "1" + +# Oldest syft protocol this release still reads. "0" refuses no peer. +# Raise it only when the code drops support for a released protocol, because a +# peer below the floor cannot exchange syft messages with this release. +MIN_SUPPORTED_SYFT_CLIENT_PROTOCOL_VERSION = "0" + +# Package-local registry for all versioned syft objects. The current +# protocol schema is computed from the objects registered into it. +client_registry = MigrationRegistry( + protocol_name=PROTOCOL_NAME, + package_name=PACKAGE_NAME, + package_version=SYFT_VERSION, + protocol_version=SYFT_CLIENT_PROTOCOL_VERSION, + min_supported_protocol_version=MIN_SUPPORTED_SYFT_CLIENT_PROTOCOL_VERSION, +) + +# Shared service for loading/migrating syft objects. +client_migration_service = MigrationService(registry=client_registry) + + +def load_as_latest(data: dict, canonical_name: str) -> object: + """Load ``data`` (defaulting identity fields for protocol-0 files, which + predate them and are all version 1) and migrate to the latest version.""" + data.setdefault("canonical_name", canonical_name) + data.setdefault("version", "1") + obj = client_migration_service.load(data) + return client_migration_service.migrate( + obj, client_registry.latest_version(canonical_name) + ) diff --git a/syft/sync/checkpoints/checkpoint.py b/syft/sync/checkpoints/checkpoint.py index 8767b831f16..d7abb481bb9 100644 --- a/syft/sync/checkpoints/checkpoint.py +++ b/syft/sync/checkpoints/checkpoint.py @@ -11,12 +11,14 @@ - After N incremental checkpoints: compact into single full Checkpoint """ -from typing import List, Dict, TYPE_CHECKING -from pydantic import BaseModel, Field from pathlib import Path +from typing import TYPE_CHECKING, Dict, List + +from pydantic import BaseModel, Field + from syft.sync.utils.syftbox_utils import ( - create_event_timestamp, compress_data, + create_event_timestamp, uncompress_data, ) @@ -28,6 +30,21 @@ INCREMENTAL_CHECKPOINT_PREFIX = "incremental_checkpoint" CHECKPOINT_VERSION = 1 + +def _check_version(version: int, kind: str) -> None: + """Refuse a checkpoint from a later client.""" + + # A later client can change what a field holds while the object still parses. + # The restore would then be wrong and silent. Every caller falls back to a + # download of all events, so a refusal costs one slow cold start. + + if version > CHECKPOINT_VERSION: + raise ValueError( + f"This {kind} has version {version}, and this client reads up to " + f"version {CHECKPOINT_VERSION}." + ) + + # Default compacting threshold: merge after this many incremental checkpoints DEFAULT_COMPACTING_THRESHOLD = 4 @@ -124,7 +141,9 @@ def as_compressed_data(self) -> bytes: def from_compressed_data(cls, data: bytes) -> "Checkpoint": """Load checkpoint from compressed data.""" uncompressed_data = uncompress_data(data) - return cls.model_validate_json(uncompressed_data) + checkpoint = cls.model_validate_json(uncompressed_data) + _check_version(checkpoint.version, "checkpoint") + return checkpoint class IncrementalCheckpoint(BaseModel): @@ -180,7 +199,9 @@ def as_compressed_data(self) -> bytes: def from_compressed_data(cls, data: bytes) -> "IncrementalCheckpoint": """Load from compressed data.""" uncompressed_data = uncompress_data(data) - return cls.model_validate_json(uncompressed_data) + checkpoint = cls.model_validate_json(uncompressed_data) + _check_version(checkpoint.version, "incremental checkpoint") + return checkpoint def compact_incremental_checkpoints( diff --git a/syft/sync/checkpoints/rolling_state.py b/syft/sync/checkpoints/rolling_state.py index b3291900855..145c9c8d725 100644 --- a/syft/sync/checkpoints/rolling_state.py +++ b/syft/sync/checkpoints/rolling_state.py @@ -13,22 +13,36 @@ """ from typing import List + from pydantic import BaseModel, Field -from syft.sync.utils.syftbox_utils import ( - create_event_timestamp, - compress_data, - uncompress_data, -) + from syft.sync.events.file_change_event import ( FileChangeEvent, FileChangeEventsMessage, ) - +from syft.sync.utils.syftbox_utils import ( + compress_data, + create_event_timestamp, + uncompress_data, +) ROLLING_STATE_FILENAME_PREFIX = "rolling_state" ROLLING_STATE_VERSION = 1 +def raise_for_later_version(version: int) -> None: + """Refuse a rolling state from a later client.""" + + # A later client can change what a field holds while the object still + # parses. The restore would then be wrong and silent. Every caller falls + # back to a download of all events, so a refusal costs one slow cold start. + if version > ROLLING_STATE_VERSION: + raise ValueError( + f"This rolling state has version {version}, and this client reads up " + f"to version {ROLLING_STATE_VERSION}." + ) + + class RollingState(BaseModel): """ Rolling state keeps the latest state of each file since the last checkpoint. @@ -111,7 +125,9 @@ def as_compressed_data(self) -> bytes: def from_compressed_data(cls, data: bytes) -> "RollingState": """Load rolling state from compressed data.""" uncompressed_data = uncompress_data(data) - return cls.model_validate_json(uncompressed_data) + state = cls.model_validate_json(uncompressed_data) + raise_for_later_version(state.version) + return state @classmethod def filename_to_timestamp(cls, filename: str) -> float | None: diff --git a/syft/sync/connections/base_connection.py b/syft/sync/connections/base_connection.py index e5005a51fb6..da97faf303a 100644 --- a/syft/sync/connections/base_connection.py +++ b/syft/sync/connections/base_connection.py @@ -10,6 +10,10 @@ class FileCollection(BaseModel): tag: str content_hash: str has_any_permission: bool = False + # The layout this collection holds, as written into the folder name. An owner + # publishes one collection per layout its audience reads; "" is the original + # layout. See CollectionSyncSpec. + variant: str = "" class ConnectionConfig(BaseModel): @@ -65,9 +69,15 @@ def owner_list_all_collections_with_permissions( raise NotImplementedError() def owner_delete_collection(self, prefix: str, tag: str) -> None: + """Delete every layout of ``tag`` published under ``prefix``.""" raise NotImplementedError() def watcher_list_collections(self, prefix: str) -> list[dict]: + """Collections a peer shared with us, in every layout they published. + + Each dict carries owner_email, tag, content_hash and variant. ``prefix`` + matches all layouts, so the caller picks the one it can read. + """ raise NotImplementedError() def watcher_download_collection( diff --git a/syft/sync/connections/collection_prefixes.py b/syft/sync/connections/collection_prefixes.py deleted file mode 100644 index eafed83bc21..00000000000 --- a/syft/sync/connections/collection_prefixes.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Wire-level collection folder-name prefixes.""" - -# Mirrors syft_datasets.dataset_manager. Duplicated rather than imported because -# delete_unversioned_state needs these at login time, before an RDS client exists, -# and the sync core must not import the domain. - -# Kept in sync by test_collection_prefixes_match_syft_datasets in -# packages/syft-rds/tests. - -DATASET_COLLECTION_PREFIX = "syft_datasetcollection" -PRIVATE_DATASET_COLLECTION_PREFIX = "syft_privatecollection" diff --git a/syft/sync/connections/connection_router.py b/syft/sync/connections/connection_router.py index 7f61aa74ad4..10d27e52e30 100644 --- a/syft/sync/connections/connection_router.py +++ b/syft/sync/connections/connection_router.py @@ -1,34 +1,58 @@ -from pydantic import BaseModel -from typing import TYPE_CHECKING, List, Optional +import logging +from typing import TYPE_CHECKING, Dict, List, Optional + +from pydantic import BaseModel, PrivateAttr +from syft_migration import MigratableObject, ProtocolSchema + +from syft.migrations import ( + SYFT_CLIENT_PROTOCOL_VERSION, + client_migration_service, + client_registry, +) +from syft.sync.checkpoints.checkpoint import Checkpoint, IncrementalCheckpoint +from syft.sync.checkpoints.rolling_state import RollingState from syft.sync.connections.base_connection import ( ConnectionConfig, FileCollection, SyftboxPlatformConnection, ) -from syft.sync.connections.drive.gdrive_transport import GDriveConnection +from syft.sync.connections.drive.gdrive_transport import ( + PEERS_META_KEY, + GDriveConnection, +) from syft.sync.events.file_change_event import ( FileChangeEventsMessage, ) -from syft.sync.checkpoints.checkpoint import Checkpoint, IncrementalCheckpoint -from syft.sync.checkpoints.rolling_state import RollingState -from syft.sync.peers.peer_store import PeerStore from syft.sync.messages.proposed_filechange import ProposedFileChangesMessage -from syft.sync.platforms.gdrive_files_platform import GdriveFilesPlatform from syft.sync.peers.peer import Peer, PeerState +from syft.sync.peers.peer_store import PeerStore +from syft.sync.platforms.gdrive_files_platform import GdriveFilesPlatform from syft.sync.utils.print_utils import ( - print_peer_adding_to_platform, print_peer_added_to_platform, + print_peer_adding_to_platform, ) if TYPE_CHECKING: from syft.sync.version.version_info import VersionInfo +logger = logging.getLogger(__name__) + + class ConnectionRouter(BaseModel): connections: List[SyftboxPlatformConnection] peer_store: PeerStore + # peer email -> syft ProtocolSchema; syft wires PeerManager's + # live map here (updated in place as peer version files load), so outgoing + # messages downgrade to what each peer reads. + _peer_schemas: Dict[str, ProtocolSchema] = PrivateAttr(default_factory=dict) + + def set_peer_schemas(self, peer_schemas: Dict[str, ProtocolSchema]) -> None: + """Adopt the live {peer email -> syft ProtocolSchema} map.""" + self._peer_schemas = peer_schemas + @classmethod def from_configs(cls, email: str, connection_configs: List[ConnectionConfig]): return cls( @@ -81,9 +105,47 @@ def connection_for_own_syftbox(self) -> SyftboxPlatformConnection: # MESSAGE SEND/RECEIVE (with encryption) # ========================================================================= + def _downgrade_for_peer( + self, message: MigratableObject, peer_email: str + ) -> MigratableObject: + """Downgrade an outgoing message to the protocol the peer reads. + + The receive paths upgrade every blob on read, so this is the other + half of the contract: both sides speak the lower of the two protocol + versions, bounded by both floors. Migrations return new objects, so + the caller's message is never mutated (one instance fans out to many + recipients). + + A peer without a known schema is assumed to run the current protocol, + the same policy as jobs; the assumption is logged. + """ + schema = self._peer_schemas.get(peer_email) + if schema is None: + logger.warning( + f"No syft protocol schema known for peer {peer_email!r}. " + f"This client writes protocol {SYFT_CLIENT_PROTOCOL_VERSION}. A " + "peer that speaks an earlier protocol cannot read this message." + ) + return message + protocol_version = client_registry.negotiate_protocol_version( + peer_version=schema.version, + peer_min=schema.min_supported_version, + ) + if schema.version == protocol_version: + # The peer speaks the negotiated version, so its advertised slim + # schema is the target; computing our own full schema on every + # send would rebuild every object's JSON schema for nothing. + target = schema + else: + target = client_registry.schema_for_protocol_version(protocol_version) + return client_migration_service.migrate_to_schema(message, target) + def watcher_send_proposed_file_changes_message( self, recipient: str, proposed_file_changes_message: ProposedFileChangesMessage ): + proposed_file_changes_message = self._downgrade_for_peer( + proposed_file_changes_message, recipient + ) data = proposed_file_changes_message.as_compressed_data() data = self.peer_store.encrypt_if_needed(recipient, data) filename = proposed_file_changes_message.message_filename.as_string() @@ -109,6 +171,7 @@ def owner_get_next_proposed_filechange_message( def owner_write_event_messages_to_outbox( self, recipient_email: str, events_message: FileChangeEventsMessage ): + events_message = self._downgrade_for_peer(events_message, recipient_email) data = events_message.as_compressed_data() data = self.peer_store.encrypt_if_needed(recipient_email, data) fname = events_message.message_filepath.as_string() @@ -194,9 +257,19 @@ def get_all_peers_from_json(self, force_download: bool = False) -> List[Peer]: peers_data = connection._get_peers_json(force_download=force_download) peers = [] for email, data in peers_data.items(): + if email == PEERS_META_KEY: + continue try: state = PeerState(data.get("state", "unknown")) except ValueError: + # A later client wrote a state that this client does not know. + # The writer changes one entry and keeps the rest, so the entry + # stays in the file. The peer returns after an upgrade. + logger.warning( + f"Skipping peer {email}: unknown state " + f"{data.get('state')!r}. Install a newer syft to see " + "this peer." + ) continue peer = Peer( email=email, diff --git a/syft/sync/connections/drive/gdrive_transport.py b/syft/sync/connections/drive/gdrive_transport.py index 00d47316c76..3ebdaa1d7b4 100644 --- a/syft/sync/connections/drive/gdrive_transport.py +++ b/syft/sync/connections/drive/gdrive_transport.py @@ -4,14 +4,17 @@ import json import logging import pickle +import re +from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional from google.oauth2.credentials import Credentials as GoogleCredentials from google_auth_httplib2 import AuthorizedHttp from googleapiclient.discovery import build from googleapiclient.http import MediaIoBaseDownload, MediaIoBaseUpload, build_http from pydantic import BaseModel +from syft_migration import MigrationError from syft.sync.checkpoints.checkpoint import ( CHECKPOINT_FILENAME_PREFIX, @@ -55,11 +58,6 @@ ) from syft.sync.version.version_info import VersionInfo -from syft.sync.connections.collection_prefixes import ( - DATASET_COLLECTION_PREFIX, - PRIVATE_DATASET_COLLECTION_PREFIX, -) - # Timeout for Google API requests (in seconds) GOOGLE_API_TIMEOUT = 120 # 2 minutes @@ -98,6 +96,15 @@ def build_drive_service( LEGACY_GDRIVE_OUTBOX_INBOX_FOLDER_PREFIX = "syft_outbox_inbox" # legacy prefix GDRIVE_P2P_FOLDER_DATASITE_PREFIX = "syft_datasite" SYFT_PEERS_FILE = "SYFT_peers.json" + +# SYFT_peers.json is a flat map of peer email to entry, so a version at the top +# level would look like a peer email. The version goes under this reserved key. +# A client written before the key reads a peer state from that entry and fails. +# The key therefore never appears as a peer. +PEERS_META_KEY = "_meta" +# Shape of one entry in SYFT_peers.json. Raise this when an entry changes. A file +# with no reserved entry was written before the version, and is version 0. +SYFT_PEERS_VERSION = 1 SYFT_VERSION_FILE = "SYFT_version.json" @@ -145,38 +152,65 @@ def as_string(self) -> str: return f"{SYFT_VERSION}#{self.email}" +def collection_name_query(prefix: str) -> str: + """A Drive query that finds a collection of ``prefix`` in any layout. + + It has no trailing '_', because a layout writes its variant in that + position. A client that predates multi-layout searches with the '_', and so + never lists a layout it cannot read. + """ + return f"name contains '{prefix}'" + + +@lru_cache(maxsize=None) +def _collection_name_re(prefix: str) -> "re.Pattern[str]": + """Matches '{prefix}{variant}_{tag}_{hash}'. + + The variant and the hash hold no '_'. The tag can hold one, so it takes + every character in between. + """ + return re.compile( + rf"^{re.escape(prefix)}" + r"(?P[^_]*)_(?P.+)_(?P[^_]+)$" + ) + + class CollectionFolder(BaseModel): """Naming value object for collection folders (domain-agnostic). Collections (datasets, private datasets, or any future collection type) are - named ``{prefix}_{tag}_{content_hash}`` on the wire. This is the single - source of truth for building and parsing that name; it also computes the - content hash from the folder's file contents. + named ``{prefix}{variant}_{tag}_{content_hash}`` on the wire. The variant + names the layout (see CollectionSyncSpec) and is "" for the original one, so + that name is unchanged byte for byte. This is the single source of truth for + building and parsing the name; it also computes the content hash from the + folder's file contents. + + A writer normally folds the variant into ``prefix`` (the spec builds the + qualified prefix once) and leaves ``variant`` empty; ``from_name`` splits + them apart again for a reader. """ prefix: str tag: str content_hash: str + variant: str = "" @property def folder_name(self) -> str: """The on-wire folder name. Doubles as the cache key (same string).""" - return f"{self.prefix}_{self.tag}_{self.content_hash}" + return f"{self.prefix}{self.variant}_{self.tag}_{self.content_hash}" @classmethod def from_name(cls, prefix: str, name: str) -> "CollectionFolder": - """Parse '{prefix}_{tag}_{hash}' into a CollectionFolder. + """Parse '{prefix}{variant}_{tag}_{hash}' into a CollectionFolder. Raises ValueError if the name does not carry the prefix, so callers can skip non-matching folders. """ - marker = f"{prefix}_" - if not name.startswith(marker): + match = _collection_name_re(prefix).match(name) + if match is None: raise ValueError(f"Invalid collection folder name: {name}") - tag, separator, content_hash = name[len(prefix) + 1 :].rpartition("_") - if not separator or not tag or not content_hash: - raise ValueError(f"Invalid collection folder name: {name}") - return cls(prefix=prefix, tag=tag, content_hash=content_hash) + return cls(prefix=prefix, **match.groupdict()) @staticmethod def compute_hash(files: dict[str, bytes]) -> str: @@ -207,33 +241,70 @@ def _extract_version_from_name(name: str) -> str | None: return None -def _filter_patch_compatible( +# A folder id and name, with the version from the name. The version fields come +# first, so the default sort puts these in version order. +_VersionedFolder = tuple[int, int, int, str, str] + + +def _sorted_by_version(folders: list[tuple[str, str]]) -> list[tuple[str, str]]: + """Folders from the lowest version to the highest. + + A name with no readable version sorts first, so a versioned folder always + wins when the caller takes the last entry. + """ + + def key(entry: tuple[str, str]) -> tuple[int, int, int]: + version_str = _extract_version_from_name(entry[1]) + if version_str is None: + return (-1, -1, -1) + try: + return _parse_semver(version_str) + except ValueError: + return (-1, -1, -1) + + return sorted(folders, key=key) + + +def _partition_by_version( folders: list[tuple[str, str]], current_version: str | None = None, -) -> list[tuple[str, str]]: - """Keep folders whose embedded version has matching major.minor. +) -> tuple[list[tuple[str, str]], list[tuple[str, str]], list[tuple[str, str]]]: + """Split folders into (compatible, older, newer) by the version in the name. - `current_version` defaults to the module-level SYFT_VERSION at call - time (not import time) so tests that patch the version take effect. + Compatible means the same major and minor as the current version. The function + drops a folder that has no version in its name. Each list starts at the lowest + version. """ if current_version is None: current_version = SYFT_VERSION try: - cur_major, cur_minor, _ = _parse_semver(current_version) + current = _parse_semver(current_version) except ValueError: - return [] - kept: list[tuple[str, str]] = [] + return [], [], [] + + compatible: list[_VersionedFolder] = [] + older: list[_VersionedFolder] = [] + newer: list[_VersionedFolder] = [] for fid, name in folders: version_str = _extract_version_from_name(name) if version_str is None: continue try: - major, minor, _ = _parse_semver(version_str) + found = _parse_semver(version_str) except ValueError: continue - if major == cur_major and minor == cur_minor: - kept.append((fid, name)) - return kept + entry = (*found, fid, name) + if found[:2] == current[:2]: + compatible.append(entry) + elif found < current: + older.append(entry) + else: + newer.append(entry) + + def _ordered(entries: list[_VersionedFolder]) -> list[tuple[str, str]]: + return [(fid, name) for *_, fid, name in sorted(entries)] + + return _ordered(compatible), _ordered(older), _ordered(newer) class GDriveConnection(SyftboxPlatformConnection): @@ -258,18 +329,18 @@ class Config: _personal_syftbox_folder_id: str | None = None # peer_email -> folder_id (folders I created for peer's datasite) - peer_datasite_inbox_cache: Dict[str, str] = {} - peer_datasite_outbox_cache: Dict[str, str] = {} + peer_datasite_inbox_cache: dict[str, str] = {} + peer_datasite_outbox_cache: dict[str, str] = {} # peer_email -> folder_id (folders peer created for my datasite) - own_datasite_inbox_cache: Dict[str, str] = {} - own_datasite_outbox_cache: Dict[str, str] = {} + own_datasite_inbox_cache: dict[str, str] = {} + own_datasite_outbox_cache: dict[str, str] = {} # sender email -> archive folder id - archive_folder_id_cache: Dict[str, str] = {} + archive_folder_id_cache: dict[str, str] = {} # fname -> gdrive id - personal_syftbox_event_id_cache: Dict[str, str] = {} + personal_syftbox_event_id_cache: dict[str, str] = {} # tag -> dataset collection folder id collection_folder_id_cache: Dict[str, str] = {} @@ -282,7 +353,7 @@ class Config: _encryption_bundles_folder_id: str | None = None # Cached SYFT_peers.json contents (None = not loaded yet). - _peers_json_cache: Dict[str, Dict[str, str]] | None = None + _peers_json_cache: dict[str, dict[str, str]] | None = None @classmethod def from_config(cls, config: "GdriveConnectionConfig") -> "GDriveConnection": @@ -454,7 +525,7 @@ def _get_peers_file_id(self) -> str | None: items = results.get("files", []) return items[0]["id"] if items else None - def _download_peers_json(self) -> Dict[str, Dict[str, str]]: + def _download_peers_json(self) -> dict[str, dict[str, str]]: """Fetch peers JSON from GDrive. Returns empty dict if not found.""" file_id = self._get_peers_file_id() if file_id is None: @@ -462,22 +533,30 @@ def _download_peers_json(self) -> Dict[str, Dict[str, str]]: try: file_data = self.download_file(file_id) - return json.loads(file_data.decode("utf-8")) except Exception as e: - print(f"Warning: Error reading peers file: {e}") + print(f"Warning: could not download the peers file: {e}") + return {} + try: + return json.loads(file_data.decode("utf-8")) + except ValueError as e: + print(f"Warning: could not read the peers file: {e}") return {} def _get_peers_json( self, force_download: bool = False - ) -> Dict[str, Dict[str, str]]: + ) -> dict[str, dict[str, str]]: """Return peers JSON, using the in-memory cache when available.""" if self._peers_json_cache is not None and not force_download: return self._peers_json_cache self._peers_json_cache = self._download_peers_json() return self._peers_json_cache - def _write_peers_json(self, peers_data: Dict[str, Dict[str, str]]): + def _write_peers_json(self, peers_data: dict[str, dict[str, str]]): """Write peers JSON to GDrive. Creates or updates the file.""" + peers_data = { + **peers_data, + PEERS_META_KEY: {"version": SYFT_PEERS_VERSION}, + } syftbox_folder_id = self.get_syftbox_folder_id() file_id = self._get_peers_file_id() @@ -526,7 +605,7 @@ def _update_peer_state( peers_data[peer_email] = existing self._write_peers_json(peers_data) - def get_peer_requests(self) -> List[str]: + def get_peer_requests(self) -> list[str]: """Get list of pending peer requests. Scans for syft_datasite_#version#{self}_* folders NOT owned by self — those are @@ -547,10 +626,12 @@ def get_peer_requests(self) -> List[str]: for f in results.get("files", []): try: folder = GdriveP2PFolder.from_name(f["name"]) - if folder.datasite_email == self.email: - all_folder_peers.add(folder.peer_email) - except (ValueError, Exception): + except ValueError: + # The query matches a name prefix, so a folder with another shape + # can appear here. continue + if folder.datasite_email == self.email: + all_folder_peers.add(folder.peer_email) peers_data = self._get_peers_json() pending_peers = [] @@ -593,7 +674,7 @@ def watcher_download_raw_events_from_outbox( def watcher_get_events_messages( self, peer_email: str, since_timestamp: float | None - ) -> List[FileChangeEventsMessage]: + ) -> list[FileChangeEventsMessage]: raw_list = self.watcher_download_raw_events_from_outbox( peer_email, since_timestamp ) @@ -601,7 +682,7 @@ def watcher_get_events_messages( def watcher_get_outbox_file_metadatas( self, peer_email: str, since_timestamp: float | None - ) -> List[Dict]: + ) -> list[dict]: """Get file metadata from peer's outbox folder without downloading.""" folder_id = self._get_peer_datasite_outbox_id(peer_email) if folder_id is None: @@ -651,7 +732,7 @@ def owner_download_raw_bytes_by_id(self, file_id: str) -> bytes: def owner_get_all_accepted_event_file_ids( self, since_timestamp: float | None = None - ) -> List[str]: + ) -> list[str]: personal_syftbox_folder_id = self.get_personal_syftbox_folder_id() file_metadatas = self.get_file_metadatas_from_folder( personal_syftbox_folder_id, since_timestamp=since_timestamp @@ -673,7 +754,7 @@ def owner_download_all_raw_events_from_syftbox(self) -> list[bytes]: try: file_data = self.download_file(gdrive_id) except Exception as e: - print(e) + print(f"Warning: could not download event {fname_obj.as_string()}: {e}") continue result.append(file_data) return result @@ -806,7 +887,10 @@ def get_personal_syftbox_folder_id(self) -> str: # '#{peer}#{type}#{email}'. Personal folder shape is exactly # '{version}#{email}', so require a single '#'. folders = [(fid, name) for fid, name in folders if name.count("#") == 1] - folder_id = self._expect_one(_filter_patch_compatible(folders)) + folder_id = self._find_or_adopt_versioned_folder( + folders, + current_name=GdrivePersonalSyftboxFolder(email=self.email).as_string(), + ) if folder_id: self._personal_syftbox_folder_id = folder_id return folder_id @@ -885,7 +969,7 @@ def get_file_metadatas_from_folder( folder_id: str, since_timestamp: float | None = None, page_size: int = 100, - ) -> List[Dict]: + ) -> list[dict]: """ Get file metadatas from folder with early termination. @@ -950,37 +1034,39 @@ def get_file_metadatas_from_folder( @staticmethod def _filter_valid_file_metadatas( - file_metadatas: List[Dict], - ) -> List[Dict]: + file_metadatas: list[dict], + ) -> list[dict]: res = [] for file_metadata in file_metadatas: fname = file_metadata["name"] try: - _ = FileChangeEventsMessageFileName.from_string(fname) - res.append(file_metadata) - except Exception: + FileChangeEventsMessageFileName.from_string(fname) + except ValueError: + # The folder holds other files, so a name that is not an event + # name is normal here. This method filters them out. continue + res.append(file_metadata) return res @staticmethod def _get_valid_events_from_file_metadatas( - file_metadatas: List[Dict], - ) -> List[FileChangeEventsMessageFileName]: + file_metadatas: list[dict], + ) -> list[FileChangeEventsMessageFileName]: res = [] for file_metadata in file_metadatas: fname = file_metadata["name"] try: message_filename = FileChangeEventsMessageFileName.from_string(fname) - res.append(message_filename) - except Exception: - print("Warning, invalid file name: ", fname) + except ValueError: + print(f"Warning: invalid event file name: {fname}") continue + res.append(message_filename) return res @staticmethod def _get_valid_messages_from_file_metadatas( - file_metadatas: List[Dict], - ) -> List[MessageFileName]: + file_metadatas: list[dict], + ) -> list[MessageFileName]: res = [] for file_metadata in file_metadatas: try: @@ -1049,8 +1135,21 @@ def _is_exact_match(name: str) -> bool: and folder.peer_email == peer_email ) - folders = [(fid, name) for fid, name in folders if _is_exact_match(name)] - return self._expect_one(_filter_patch_compatible(folders)) + # Ignore the version in the name. Each peer builds this name from its own + # client version, so a filter here hides the folder that the peer uses. + # After an upgrade the client therefore finds the old folder and writes to + # it. It makes no second folder, which an older peer would never look for. + candidates = _sorted_by_version( + [(fid, name) for fid, name in folders if _is_exact_match(name)] + ) + if not candidates: + return None + if len(candidates) > 1: + print( + f"Warning: {len(candidates)} P2P folders for {datasite_email} " + f"{folder_type} {peer_email}; using {candidates[-1][1]}" + ) + return candidates[-1][0] def _get_peer_datasite_inbox_id(self, peer_email: str) -> str | None: """Get folder: syft_datasite_{peer}_inbox_{self}, owned by self.""" @@ -1164,7 +1263,7 @@ def reset_caches(self): self._encryption_bundles_folder_id = None self._peers_json_cache = None - def gather_all_file_and_folder_ids(self) -> List[str]: + def gather_all_file_and_folder_ids(self) -> list[str]: syftbox_folder_id = self.get_syftbox_folder_id() return gather_all_file_and_folder_ids_recursive( self.drive_service, syftbox_folder_id @@ -1172,7 +1271,7 @@ def gather_all_file_and_folder_ids(self) -> List[str]: def delete_multiple_files_by_ids( self, - file_ids: List[str], + file_ids: list[str], ignore_permissions_errors: bool = True, ignore_file_not_found: bool = True, ): @@ -1210,85 +1309,13 @@ def callback(request_id, response, exception): batch.add(self.drive_service.files().delete(fileId=file_id)) batch_execute_with_retries(batch) - def delete_file_by_id( - self, file_id: str, verbose: bool = False, raise_on_error: bool = False - ): + def delete_file_by_id(self, file_id: str, raise_on_error: bool = False): try: execute_with_retries(self.drive_service.files().delete(fileId=file_id)) except Exception as e: if raise_on_error: raise e - else: - if verbose: - print(f"Error deleting file: {file_id}") - - def delete_unversioned_state(self) -> None: - """Delete non-versioned remote artifacts during upgrade. - - Removes encryption bundles, dataset collections, private collections, - peers file, and version file from /SyftBox/. - """ - syftbox_folder_id = self.get_syftbox_folder_id() - ids_to_delete: list[str] = [] - - # 1. Encryption bundles folder - enc_folder_name = GdriveEncryptionBundlesFolder(email=self.email).as_string() - enc_folder_id = self._find_folder_by_name( - enc_folder_name, parent_id=syftbox_folder_id - ) - if enc_folder_id: - ids_to_delete.extend( - gather_all_file_and_folder_ids_recursive( - self.drive_service, enc_folder_id - ) - ) - ids_to_delete.append(enc_folder_id) - - # 2. Dataset collection folders (syft_datasetcollection_*) - ds_query = ( - f"name contains '{DATASET_COLLECTION_PREFIX}'" - f" and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" - f" and '{syftbox_folder_id}' in parents" - " and trashed=false" - ) - ds_results = execute_with_retries( - self.drive_service.files().list(q=ds_query, fields="files(id)") - ) - for f in ds_results.get("files", []): - ids_to_delete.extend( - gather_all_file_and_folder_ids_recursive(self.drive_service, f["id"]) - ) - ids_to_delete.append(f["id"]) - - # 3. Private collection folders (syft_privatecollection_*) - pc_query = ( - f"name contains '{PRIVATE_DATASET_COLLECTION_PREFIX}'" - f" and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" - f" and '{syftbox_folder_id}' in parents" - " and trashed=false" - ) - pc_results = execute_with_retries( - self.drive_service.files().list(q=pc_query, fields="files(id)") - ) - for f in pc_results.get("files", []): - ids_to_delete.extend( - gather_all_file_and_folder_ids_recursive(self.drive_service, f["id"]) - ) - ids_to_delete.append(f["id"]) - - # 4. SYFT_peers.json - peers_file_id = self._get_peers_file_id() - if peers_file_id: - ids_to_delete.append(peers_file_id) - - # 5. SYFT_version.json - version_file_id = self._get_version_file_id() - if version_file_id: - ids_to_delete.append(version_file_id) - - if ids_to_delete: - self.delete_multiple_files_by_ids(ids_to_delete) - self.reset_caches() + print(f"Warning: could not delete file {file_id}: {e}") def find_orphaned_message_files(self) -> list[str]: """ @@ -1334,7 +1361,7 @@ def find_orphaned_message_files(self) -> list[str]: return file_ids - def create_file_payload(self, data: Any) -> Tuple[MediaIoBaseUpload, str]: + def create_file_payload(self, data: Any) -> tuple[MediaIoBaseUpload, str]: """Create a file payload for the GDrive""" if isinstance(data, str): file_data = data.encode("utf-8") @@ -1386,7 +1413,8 @@ def _find_folders( Thin wrapper over Drive's files.list -- handles query building and pagination, knows nothing about versions. Pair with - _filter_patch_compatible when the caller cares about version compat. + _partition_by_version or _sorted_by_version when the caller cares about + the version in the folder name. """ clauses = [f"mimeType='{GOOGLE_FOLDER_MIME_TYPE}'", "trashed=false"] for substr in name_contains: @@ -1431,6 +1459,54 @@ def _expect_one(self, folders: list[tuple[str, str]]) -> str | None: f"folder(s) on Drive (keeping the one with your data) and retry." ) + def _find_or_adopt_versioned_folder( + self, + folders: list[tuple[str, str]], + current_name: str, + current_version: str | None = None, + ) -> str | None: + """Return the id of a PRIVATE folder for this client version, or None. + + A private folder name holds the client version, so a minor upgrade looks + for a name that does not exist yet. This method renames the folder of the + highest earlier version to `current_name` and keeps the data. A new folder + would leave the data of the user on Drive and out of reach. + + Renames the folder, so the caller must own it and no peer may look it up by + name. A P2P folder fails both conditions: use `_expect_one` for those. + + Raises RuntimeError if only a folder from a later version exists, or if + more than one compatible folder exists. + """ + compatible, older, newer = _partition_by_version(folders, current_version) + if compatible: + return self._expect_one(compatible) + if newer: + names = [n for _, n in newer] + latest = _extract_version_from_name(names[-1]) + raise RuntimeError( + f"Found a folder from a later client version on Drive: {names}. " + f"This client is {current_version or SYFT_VERSION} and " + f"cannot read that data. Install syft {latest} or later." + ) + if not older: + return None + + folder_id, name = older[-1] + execute_with_retries( + self.drive_service.files().update( + fileId=folder_id, body={"name": current_name} + ) + ) + print(f"Adopted the folder of an earlier version: {name} -> {current_name}") + if len(older) > 1: + stale = [n for _, n in older[:-1]] + print( + f"Warning: {len(stale)} folder(s) of earlier versions stay on " + f"Drive: {stale}" + ) + return folder_id + def download_file(self, file_id: str) -> bytes: request = self.drive_service.files().get_media(fileId=file_id) @@ -1441,7 +1517,7 @@ def download_file(self, file_id: str) -> bytes: done = False while not done: - status, done = next_chunk_with_retries(downloader) + _, done = next_chunk_with_retries(downloader) message_data = file_buffer.getvalue() return message_data @@ -1535,10 +1611,9 @@ def _batch_add_permissions(self, file_id: str, users: list[str]) -> None: """Add reader permissions for multiple users in a single batch request.""" def callback(request_id, response, exception): - if exception: - # Ignore "already shared" errors - if "alreadyShared" not in str(exception): - raise exception + # Ignore "already shared" errors + if exception and "alreadyShared" not in str(exception): + raise exception BATCH_SIZE = 100 for i in range(0, len(users), BATCH_SIZE): @@ -1577,34 +1652,37 @@ def owner_upload_collection_files( ) def owner_list_collections(self, prefix: str) -> list[str]: - """List collections created by DO (owned by me). Returns list of tags.""" + """List collections created by DO (owned by me). Returns list of tags. + + A tag appears once however many layouts hold it. + """ syftbox_folder_id = self.get_syftbox_folder_id() query = ( - f"name contains '{prefix}_' and '{syftbox_folder_id}' in parents " + f"{collection_name_query(prefix)} and '{syftbox_folder_id}' in parents " f"and 'me' in owners and trashed=false and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" ) results = execute_with_retries( self.drive_service.files().list(q=query, fields="files(name)") ) - folders = results.get("files", []) - result = [] - for folder in folders: + result: list[str] = [] + for folder in results.get("files", []): try: cf = CollectionFolder.from_name(prefix, folder["name"]) - result.append(cf.tag) except ValueError: continue + if cf.tag not in result: + result.append(cf.tag) return result def owner_list_all_collections_with_permissions( self, prefix: str, ) -> list[FileCollection]: - """List all DO's collections with permissions info.""" + """List all DO's collections with permissions info, one per layout.""" syftbox_folder_id = self.get_syftbox_folder_id() query = ( - f"name contains '{prefix}_' and '{syftbox_folder_id}' in parents " + f"{collection_name_query(prefix)} and '{syftbox_folder_id}' in parents " f"and 'me' in owners and trashed=false and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" ) results = execute_with_retries( @@ -1615,68 +1693,68 @@ def owner_list_all_collections_with_permissions( collections = [] for folder in results.get("files", []): - folder_id = folder["id"] try: cf = CollectionFolder.from_name(prefix, folder["name"]) - has_anyone = ( - folder.get("appProperties", {}).get("syft_shared_with_any") - == "true" - ) - collections.append( - FileCollection( - folder_id=folder_id, - tag=cf.tag, - content_hash=cf.content_hash, - has_any_permission=has_anyone, - ) - ) - except Exception: + except ValueError: continue + has_anyone = ( + folder.get("appProperties", {}).get("syft_shared_with_any") == "true" + ) + collections.append( + FileCollection( + folder_id=folder["id"], + tag=cf.tag, + content_hash=cf.content_hash, + has_any_permission=has_anyone, + variant=cf.variant, + ) + ) return collections def owner_delete_collection(self, prefix: str, tag: str) -> None: - """Delete all collection folders (for prefix) matching the given tag.""" + """Delete every layout of ``tag`` published under ``prefix``.""" collections = self.owner_list_all_collections_with_permissions(prefix) for c in collections: if c.tag == tag: self.delete_file_by_id(c.folder_id) folder_name = CollectionFolder( - prefix=prefix, tag=c.tag, content_hash=c.content_hash + prefix=prefix, + tag=c.tag, + content_hash=c.content_hash, + variant=c.variant, ).folder_name self.collection_folder_id_cache.pop(folder_name, None) def watcher_list_collections(self, prefix: str) -> list[dict]: - """List collections shared with DS (not owned by me). + """List collections shared with DS (not owned by me), one per layout. - Returns list of dicts with keys: owner_email, tag, content_hash + Returns list of dicts with keys: owner_email, tag, content_hash, variant """ query = ( - f"name contains '{prefix}_' and not 'me' in owners " + f"{collection_name_query(prefix)} and not 'me' in owners " f"and trashed=false and mimeType='{GOOGLE_FOLDER_MIME_TYPE}'" ) results = execute_with_retries( self.drive_service.files().list(q=query, fields="files(name, owners)") ) - folders = results.get("files", []) result = [] - for folder in folders: + for folder in results.get("files", []): try: cf = CollectionFolder.from_name(prefix, folder["name"]) - owner_email = folder.get("owners", [{}])[0].get( - "emailAddress", "unknown" - ) - result.append( - { - "owner_email": owner_email, - "tag": cf.tag, - "content_hash": cf.content_hash, - } - ) except ValueError: # Skip folders that don't match the expected format continue + owner_email = folder.get("owners", [{}])[0].get("emailAddress", "unknown") + result.append( + { + "owner_email": owner_email, + "tag": cf.tag, + "content_hash": cf.content_hash, + "variant": cf.variant, + } + ) return result def watcher_download_collection( @@ -1802,8 +1880,13 @@ def read_own_version_file(self) -> Optional["VersionInfo"]: try: file_data = self.download_file(file_id) + except Exception as e: + print(f"Warning: could not download the own version file: {e}") + return None + try: return VersionInfo.from_json(file_data.decode("utf-8")) - except Exception: + except (ValueError, MigrationError) as e: + print(f"Warning: could not read the own version file: {e}") return None def read_peer_version_file(self, peer_email: str) -> Optional["VersionInfo"]: @@ -1816,8 +1899,13 @@ def read_peer_version_file(self, peer_email: str) -> Optional["VersionInfo"]: try: file_data = self.download_file(file_id) + except Exception as e: + print(f"Warning: could not download the version file of {peer_email}: {e}") + return None + try: return VersionInfo.from_json(file_data.decode("utf-8")) - except Exception: + except (ValueError, MigrationError) as e: + print(f"Warning: could not read the version file of {peer_email}: {e}") return None def share_version_file_with_peer(self, peer_email: str) -> None: @@ -1847,7 +1935,9 @@ def _get_checkpoints_folder_id(self) -> str | None: name_contains=[f"{self.email}-", "-checkpoints"], parent_id=self.get_syftbox_folder_id(), ) - return self._expect_one(_filter_patch_compatible(folders)) + return self._find_or_adopt_versioned_folder( + folders, current_name=self._get_checkpoints_folder_name() + ) def _get_or_create_checkpoints_folder_id(self) -> str: """Get or create the checkpoints folder.""" @@ -2150,7 +2240,9 @@ def _get_rolling_state_folder_id(self, use_cache: bool = True) -> str | None: name_contains=[f"{self.email}-", "-rolling-state"], parent_id=self.get_syftbox_folder_id(), ) - folder_id = self._expect_one(_filter_patch_compatible(folders)) + folder_id = self._find_or_adopt_versioned_folder( + folders, current_name=self._get_rolling_state_folder_name() + ) if folder_id is not None: self._rolling_state_folder_id = folder_id return folder_id @@ -2182,7 +2274,13 @@ def upload_raw_rolling_state(self, filename: str, data: bytes) -> str: media_body=payload, ).execute() return self._rolling_state_file_id - except Exception: + except Exception as e: + # The cached file is gone or unreachable. Clear the cache and + # write a new file below. + print( + f"Warning: could not update rolling state " + f"{self._rolling_state_file_id}, writing a new file: {e}" + ) self._rolling_state_file_id = None folder_id = self._get_or_create_rolling_state_folder_id() @@ -2337,6 +2435,11 @@ def read_peer_encryption_bundle(self, peer_email: str) -> str | None: return None try: data = self.download_file(items[0]["id"]) + except Exception as e: + print(f"Warning: could not download the bundle of {peer_email}: {e}") + return None + try: return data.decode("utf-8") - except Exception: + except ValueError as e: + print(f"Warning: could not read the bundle of {peer_email}: {e}") return None diff --git a/syft/sync/events/file_change_event.py b/syft/sync/events/file_change_event.py index 834a4ab89cd..0ce4151030e 100644 --- a/syft/sync/events/file_change_event.py +++ b/syft/sync/events/file_change_event.py @@ -2,6 +2,7 @@ from pathlib import Path from uuid import UUID, uuid4 import base64 +import json from pydantic import ( BaseModel, Field, @@ -9,6 +10,9 @@ field_serializer, computed_field, ) +from syft_migration import MigratableObject + +from syft.migrations import client_registry, load_as_latest from syft.sync.messages.proposed_filechange import ProposedFileChange from syft.sync.utils.syftbox_utils import create_event_timestamp from syft.sync.utils.syftbox_utils import compress_data @@ -53,7 +57,7 @@ def from_string(cls, filename: str) -> "FileChangeEventsMessageFileName": raise ValueError(f"Invalid filename: {filename}") from e -class FileChangeEvent(BaseModel): +class FileChangeEventV1(BaseModel): id: UUID path_in_datasite: Path datasite_email: str @@ -130,13 +134,19 @@ def __hash__(self) -> int: return hash(self.id) def __eq__(self, other: Any) -> bool: - if not isinstance(other, FileChangeEvent): + if not isinstance(other, FileChangeEventV1): return False return self.id == other.id -class FileChangeEventsMessage(BaseModel): - events: List[FileChangeEvent] +class FileChangeEventsMessageV1(MigratableObject, registry=client_registry): + """The events wire envelope (DO -> watchers). The envelope is the migratable + unit; its items are pinned to the exact version class, never a floating alias.""" + + canonical_name: str = "FileChangeEventsMessage" + version: str = "1" + + events: List[FileChangeEventV1] message_filepath: FileChangeEventsMessageFileName = Field( default_factory=lambda: FileChangeEventsMessageFileName() ) @@ -149,6 +159,16 @@ def as_compressed_data(self) -> bytes: return compress_data(self.model_dump_json().encode("utf-8")) @classmethod - def from_compressed_data(cls, data: bytes) -> "FileChangeEvent": + def from_compressed_data(cls, data: bytes) -> "FileChangeEventsMessage": + """Decompress and load, upgraded to the latest version. + + Blobs written by protocol-0 clients (<= 0.1.117) predate the identity + fields; they are all version 1. + """ uncompressed_data = uncompress_data(data) - return cls.model_validate_json(uncompressed_data) + return load_as_latest(json.loads(uncompressed_data), "FileChangeEventsMessage") + + +# Current-version aliases: callers always work with the latest versions. +FileChangeEvent = FileChangeEventV1 +FileChangeEventsMessage = FileChangeEventsMessageV1 diff --git a/syft/sync/login.py b/syft/sync/login.py index 4d1212c5786..75dfbdc29ed 100644 --- a/syft/sync/login.py +++ b/syft/sync/login.py @@ -29,7 +29,11 @@ def _init_client_login( """Common post-creation initialization: write version, sync, load peers.""" _verify_token_matches_email(client) print_client_connecting(client.email) - client.write_local_version() + # Write the version file on both sides. A local-only write leaves the remote + # file at the version that first created it. Two things then break: the + # login mismatch check reads that stale file and prompts at every login, and + # a peer reads it to select a job or dataset protocol version for us. + client.peer_manager.write_own_version() if sync: client.sync() diff --git a/syft/sync/login_utils.py b/syft/sync/login_utils.py index c13e0f1a459..9d4b7c9d6c8 100644 --- a/syft/sync/login_utils.py +++ b/syft/sync/login_utils.py @@ -25,17 +25,6 @@ def _read_remote_version( return conn.read_own_version_file() -def _delete_remote_unversioned_state( - email: str, - token_path: Optional[Path], -) -> None: - """Delete non-versioned remote state during upgrade.""" - from syft.sync.connections.drive.gdrive_transport import GDriveConnection - - conn = GDriveConnection.from_token_path(email=email, token_path=token_path) - conn.delete_unversioned_state() - - def _handle_version_incompatible( email: str, token_path: Optional[Path], @@ -43,18 +32,24 @@ def _handle_version_incompatible( local_version: Optional[VersionInfo], remote_version: Optional[VersionInfo], ) -> None: - """Handle version mismatch with unified prompt.""" + """Handle a client major/minor mismatch at login. + + The default is to keep local and remote data. Folder adopt, refuse-later + checks, and cache reset repair state on the next sync. A full wipe is an + explicit second choice only. + """ choice = _prompt_mismatch(local_version, remote_version) if choice == "1": - print(f"Upgrading to v{SYFT_VERSION}...") - delete_local_syftbox( - email=email, - local_syftbox_path=local_syftbox_path, - verbose=True, + print( + f"Continuing with v{SYFT_VERSION}. Local and remote data are " + "kept. Drive folders of an earlier client version are adopted on the " + "next sync, and caches and checkpoints rebuild themselves.\n" + "Encryption keys are the one exception. A key file from a newer " + "client is refused, because a private key cannot be rebuilt. Install " + "that client to use those keys.\n" ) - _delete_remote_unversioned_state(email, token_path) - print("Done. Continuing login.\n") - elif choice == "2": + return + if choice == "2": print(f"Deleting all state and starting fresh with v{SYFT_VERSION}...") delete_local_syftbox( email=email, @@ -67,22 +62,23 @@ def _handle_version_incompatible( verbose=True, ) print("Done. Continuing login.\n") - else: - print("Exiting.") - sys.exit(0) + return + print("Exiting.") + sys.exit(0) def handle_potential_version_mismatches_on_login( email: str, token_path: Optional[str | Path] = None, ) -> None: - """Check local and remote versions against installed version. + """Check local and remote versions against the installed client. Runs before client init. Creates a temporary GDrive connection to read the remote version file. - On mismatch, prompts user to upgrade (local delete only, remote preserved - via version subfolders) or hard-reset (delete everything). + On a major/minor mismatch, the default is to keep data and continue. The + user can still choose a full wipe, or quit. Patch differences are not a + mismatch. """ resolved_email = _resolve_email(email) resolved_token_path = _resolve_token_path(token_path) @@ -130,11 +126,21 @@ def _prompt_mismatch( local_version: Optional[VersionInfo], remote_version: Optional[VersionInfo], ) -> str: - """Prompt user about version mismatch. Returns choice.""" + """Prompt the user about a version mismatch. Returns the choice string.""" _print_version_status(local_version, remote_version) + if not sys.stdin.isatty(): + # No terminal, so no answer can arrive. Choice 1 keeps every file and + # changes nothing, so it is safe to take without an answer. A prompt + # here would stop a notebook or a scheduled run instead. + print( + "No terminal is attached. Continuing with all data kept.\n" + "To start fresh instead, call delete_local_syftbox and " + "delete_remote_syftbox, then log in again.\n" + ) + return "1" print( f""" -[1] Upgrade to v{SYFT_VERSION} and archive old data +[1] Continue with v{SYFT_VERSION} (keep data; repair on sync) [2] Delete all state and start fresh with v{SYFT_VERSION} [3] Quit diff --git a/syft/sync/messages/proposed_filechange.py b/syft/sync/messages/proposed_filechange.py index c32dbffb0fa..6d0b7b64b60 100644 --- a/syft/sync/messages/proposed_filechange.py +++ b/syft/sync/messages/proposed_filechange.py @@ -1,11 +1,15 @@ from typing import List, Any, Literal from uuid import UUID, uuid4 from pathlib import Path +import json import uuid import time import base64 from pydantic import Field, model_validator, field_serializer, computed_field from pydantic.main import BaseModel +from syft_migration import MigratableObject + +from syft.migrations import client_registry, load_as_latest from syft.sync.utils.syftbox_utils import compress_data, uncompress_data from syft.sync.utils.syftbox_utils import create_event_timestamp from syft.sync.utils.syftbox_utils import get_event_hash_from_content @@ -15,7 +19,7 @@ MESSAGE_FILENAME_EXTENSION = ".tar.gz" -class ProposedFileChange(BaseModel): +class ProposedFileChangeV1(BaseModel): id: UUID = Field(default_factory=lambda: uuid4()) old_hash: str | None = None new_hash: str | None = None # None for deletions @@ -94,20 +98,38 @@ def from_string(cls, filename: str) -> "MessageFileName": return cls(submitted_timestamp=submitted_timestamp, uid=uid) -class ProposedFileChangesMessage(BaseModel): +class ProposedFileChangesMessageV1(MigratableObject, registry=client_registry): + """The msgv2 wire envelope (DS -> DO). The envelope is the migratable unit; + its items are pinned to the exact version class, never a floating alias.""" + + canonical_name: str = "ProposedFileChangesMessage" + version: str = "1" + id: UUID = Field(default_factory=lambda: uuid4()) sender_email: str message_filename: MessageFileName = Field(default_factory=lambda: MessageFileName()) - proposed_file_changes: List[ProposedFileChange] + proposed_file_changes: List[ProposedFileChangeV1] # Platform-specific ID (e.g., Google Drive file ID) - set when retrieving message # Used to avoid re-querying the platform when removing the message platform_id: str | None = Field(default=None, exclude=True) @classmethod def from_compressed_data(cls, data: bytes) -> "ProposedFileChangesMessage": + """Decompress and load, upgraded to the latest version. + + Blobs written by protocol-0 clients (<= 0.1.117) predate the identity + fields; they are all version 1. + """ uncompressed_data = uncompress_data(data) - return cls.model_validate_json(uncompressed_data) + return load_as_latest( + json.loads(uncompressed_data), "ProposedFileChangesMessage" + ) def as_compressed_data(self) -> bytes: data = self.model_dump_json(indent=2).encode("utf-8") return compress_data(data) + + +# Current-version aliases: callers always work with the latest versions. +ProposedFileChange = ProposedFileChangeV1 +ProposedFileChangesMessage = ProposedFileChangesMessageV1 diff --git a/syft/sync/peers/peer_store.py b/syft/sync/peers/peer_store.py index d42c38a3328..eb8ef855497 100644 --- a/syft/sync/peers/peer_store.py +++ b/syft/sync/peers/peer_store.py @@ -20,6 +20,11 @@ PRIVATE_DIR_NAME = "private" CRYPTO_KEYS_FILENAME = "crypto_keys.json" +# Format of the crypto key file. Raise it when the layout of the file changes, +# and add a read path for every earlier version. A file with no version was +# written before the field, and is version 0. +CRYPTO_KEYS_VERSION = 1 + def datasite_crypto_keys_path(syftbox_folder: Path | str, email: str) -> Path: """Per-datasite key file: ``//private/crypto_keys.json``.""" @@ -230,6 +235,7 @@ def decrypt_and_verify_for_self_if_needed(self, data: bytes) -> bytes: def save_keys(self, path: Path) -> None: keys = self._ensure_private_keys() data = { + "version": CRYPTO_KEYS_VERSION, "email": self.email, "keys_jwk": keys.to_jwks(), "peer_bundles": { @@ -245,6 +251,16 @@ def save_keys(self, path: Path) -> None: @classmethod def load_keys(cls, path: Path) -> "PeerStore": data = json.loads(Path(path).read_text()) + # A file with no version was written before the field, and its layout is + # this client reads. A later version is refused: a user cannot rebuild a + # private key, so a wrong read loses the keys. + version = data.get("version", 0) + if version > CRYPTO_KEYS_VERSION: + raise ValueError( + f"The crypto key file at {path} has version {version}, and this " + f"client reads up to version {CRYPTO_KEYS_VERSION}. Install a " + "newer syft to use these keys." + ) store = cls(email=data["email"], use_encryption=True) store._private_keys = syc.SyftPrivateKeys.from_jwks(data["keys_jwk"]) for email, bundle_dict in data.get("peer_bundles", {}).items(): diff --git a/syft/sync/syftbox_manager.py b/syft/sync/syftbox_manager.py index 37aa338aaab..3585538682d 100644 --- a/syft/sync/syftbox_manager.py +++ b/syft/sync/syftbox_manager.py @@ -53,9 +53,7 @@ PeerManager, PeerManagerConfig, ) -from syft.sync.version.version_info import VersionInfo from syft.utils import resolve_path -from syft.version import VERSION_FILE_NAME logger = logging.getLogger(__name__) @@ -94,6 +92,20 @@ def default_collections_folder( return Path(syftbox_folder) / email / spec.local_subpath +def default_collection_variants( + collection_specs: list["CollectionSyncSpec"], +) -> list[str]: + """Layout variants published under ``default_collections_folder``. + + Same convention as that function: the first shareable spec owns the folder, + so its layouts name the subdirectories inside it. + """ + spec = next((s for s in collection_specs if not s.owner_only), None) + if spec is None: + return [] + return [layout.variant for layout in spec.layouts] + + class SyftboxManagerConfig(BaseModel): email: str syftbox_folder: Path @@ -134,6 +146,7 @@ def for_colab( collections_folder = default_collections_folder( syftbox_folder, email, collection_specs ) + collection_variants = default_collection_variants(collection_specs) connection_configs = [GdriveConnectionConfig(email=email, token_path=None)] datasite_owner_syncer_config = DatasiteOwnerSyncerConfig( email=email, @@ -146,6 +159,7 @@ def for_colab( use_in_memory_cache=use_in_memory_cache, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_variants=collection_variants, ), ) datasite_watcher_syncer_config = DatasiteWatcherSyncerConfig( @@ -211,6 +225,7 @@ def for_jupyter( collections_folder = default_collections_folder( syftbox_folder, email, collection_specs ) + collection_variants = default_collection_variants(collection_specs) connection_configs = [ GdriveConnectionConfig(email=email, token_path=token_path) @@ -226,6 +241,7 @@ def for_jupyter( use_in_memory_cache=False, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_variants=collection_variants, connection_configs=connection_configs, ), ) @@ -286,6 +302,7 @@ def _base_config_for_testing( collections_folder = default_collections_folder( syftbox_folder, email, collection_specs ) + collection_variants = default_collection_variants(collection_specs) datasite_owner_syncer_config = DatasiteOwnerSyncerConfig( email=email, @@ -298,6 +315,7 @@ def _base_config_for_testing( use_in_memory_cache=use_in_memory_cache, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_variants=collection_variants, ), ) datasite_watcher_syncer_config = DatasiteWatcherSyncerConfig( @@ -357,6 +375,7 @@ def for_google_drive_testing_connection( collections_folder = default_collections_folder( syftbox_folder, email, collection_specs ) + collection_variants = default_collection_variants(collection_specs) connection_configs = [ GdriveConnectionConfig(email=email, token_path=token_path) ] @@ -371,6 +390,7 @@ def for_google_drive_testing_connection( use_in_memory_cache=use_in_memory_cache, syftbox_folder=syftbox_folder, collections_folder=collections_folder, + collection_variants=collection_variants, ), ) datasite_watcher_syncer_config = DatasiteWatcherSyncerConfig( @@ -453,21 +473,10 @@ class SyftboxManager(BaseModelCallbackMixin): def __dir__(self): return list(self._PUBLIC_API) - def read_local_version(self) -> VersionInfo | None: - """Read the local SYFT_version.json from the SyftBox directory.""" - version_file = self.syftbox_folder / VERSION_FILE_NAME - if not version_file.exists(): - return None - try: - return VersionInfo.from_json(version_file.read_text()) - except Exception: - return None - - def write_local_version(self) -> None: - """Write current version info to a local SYFT_version.json.""" - self.syftbox_folder.mkdir(parents=True, exist_ok=True) - version_file = self.syftbox_folder / VERSION_FILE_NAME - version_file.write_text(VersionInfo.current().to_json()) + # Version file IO lives in syft.sync.version.local_version, and + # `write_own_version` writes both the local and the remote file. A + # local-only writer on the manager leaves the remote file stale, which is + # the bug that made the login mismatch prompt repeat at every login. @property def peers(self) -> PeerList: @@ -537,8 +546,24 @@ def from_config(cls, config: SyftboxManagerConfig): if peer_manager.peer_store.use_encryption: manager_res._set_peer_store(peer_manager.peer_store) + # Every router that sends peer-directed messages downgrades them to the + # peer's negotiated syft protocol, so each one gets the live map. + manager_res._set_peer_schemas(peer_manager.live_peer_schemas("syft")) + return manager_res + def _set_peer_schemas(self, peer_schemas) -> None: + """Wire PeerManager's live syft schema map into all routers.""" + if self.datasite_owner_syncer: + self.datasite_owner_syncer.connection_router.set_peer_schemas(peer_schemas) + if self.datasite_watcher_syncer: + self.datasite_watcher_syncer.connection_router.set_peer_schemas( + peer_schemas + ) + self.datasite_watcher_syncer.datasite_watcher_cache.connection_router.set_peer_schemas( + peer_schemas + ) + def _set_peer_store(self, peer_store) -> None: """Wire shared peer_store into all connection routers.""" from syft.sync.peers.peer_store import PeerStore diff --git a/syft/sync/sync/caches/datasite_owner_cache.py b/syft/sync/sync/caches/datasite_owner_cache.py index 08b2e94204d..648e99bb4ec 100644 --- a/syft/sync/sync/caches/datasite_owner_cache.py +++ b/syft/sync/sync/caches/datasite_owner_cache.py @@ -35,6 +35,9 @@ class DataSiteOwnerEventCacheConfig(BaseModel): events_base_path: Path | None = None # Full path to collections folder - must be provided explicitly collections_folder: Path | None = None + # Layout variants published under collections_folder, from the shareable + # spec's layouts. "" is the original layout, which holds its tags directly. + collection_variants: List[str] = [] class DataSiteOwnerEventCache(BaseModelCallbackMixin): @@ -58,7 +61,10 @@ class DataSiteOwnerEventCache(BaseModelCallbackMixin): email: str # Full path to collections (datasets) folder collections_folder: Path | None = None - # Cache of collection hashes: "tag" -> content_hash + # Layout variants published under collections_folder; see the config. + collection_variants: List[str] = [] + # Cache of collection hashes, keyed by `collection_hash_key`: "tag" for the + # original layout, "/tag" for any other. collection_hashes: Dict[str, str] = {} @model_validator(mode="before") @@ -84,6 +90,7 @@ def from_config(cls, config: DataSiteOwnerEventCacheConfig): syftbox_folder=config.syftbox_folder, file_hashes=PersistedDict(), collections_folder=config.collections_folder, + collection_variants=config.collection_variants, ) else: if config.syftbox_folder is None: @@ -102,6 +109,7 @@ def from_config(cls, config: DataSiteOwnerEventCacheConfig): syftbox_folder=config.syftbox_folder, email=config.email, collections_folder=config.collections_folder, + collection_variants=config.collection_variants, ) cache._load_cached_state() return cache @@ -137,25 +145,45 @@ def _load_file_hashes_from_disk(self) -> float | None: self.file_hashes._write_to_file() def _load_collection_hashes_from_disk(self): - """Scan local dataset directories and compute hashes to populate collection_hashes.""" + """Scan local collection directories and compute hashes to populate collection_hashes. + + Scans one directory per layout, so a collection published in several + layouts keeps one hash per copy. + """ from syft.sync.file_utils import compute_directory_hash if self.collections_folder is None or not self.collections_folder.exists(): return - for tag_dir in self.collections_folder.iterdir(): - if tag_dir.is_dir(): + for variant in self.collection_variants or [""]: + layout_dir = ( + self.collections_folder / variant + if variant + else (self.collections_folder) + ) + if not layout_dir.is_dir(): + continue + for tag_dir in layout_dir.iterdir(): + if not tag_dir.is_dir(): + continue content_hash = compute_directory_hash(tag_dir) if content_hash: - self.collection_hashes[tag_dir.name] = content_hash - - def get_collection_hash(self, tag: str) -> str | None: - """Get the cached hash for a collection.""" - return self.collection_hashes.get(tag) - - def set_collection_hash(self, tag: str, content_hash: str): - """Set the cached hash for a collection.""" - self.collection_hashes[tag] = content_hash + self.collection_hashes[ + self.collection_hash_key(tag_dir.name, variant) + ] = content_hash + + @staticmethod + def collection_hash_key(tag: str, variant: str = "") -> str: + """The cache key of one layout of a collection.""" + return f"{variant}/{tag}" if variant else tag + + def get_collection_hash(self, tag: str, variant: str = "") -> str | None: + """Get the cached hash for one layout of a collection.""" + return self.collection_hashes.get(self.collection_hash_key(tag, variant)) + + def set_collection_hash(self, tag: str, content_hash: str, variant: str = ""): + """Set the cached hash for one layout of a collection.""" + self.collection_hashes[self.collection_hash_key(tag, variant)] = content_hash @property def latest_cached_timestamp(self) -> float | None: diff --git a/syft/sync/sync/caches/datasite_watcher_cache.py b/syft/sync/sync/caches/datasite_watcher_cache.py index ee21af6453c..68633030675 100644 --- a/syft/sync/sync/caches/datasite_watcher_cache.py +++ b/syft/sync/sync/caches/datasite_watcher_cache.py @@ -1,3 +1,4 @@ +import logging from concurrent.futures import ThreadPoolExecutor from typing import Callable, Dict, List from syft.sync.sync.caches.cache_file_writer_connection import FSFileConnection @@ -16,6 +17,8 @@ ) from syft.sync.sync.collection_spec import CollectionSyncSpec +logger = logging.getLogger(__name__) + SECONDS_BEFORE_SYNCING_DOWN = 0 @@ -147,7 +150,7 @@ def get_collection_path( return self.syftbox_folder / owner_email / local_subpath / tag def _get_local_collection_folders(self): - """Yield paths to all local collection folders across all specs.""" + """Yield paths to all local collection folders, in every layout of every spec.""" if self.syftbox_folder is None or not self.syftbox_folder.exists(): return @@ -159,12 +162,13 @@ def _get_local_collection_folders(self): for email_dir in self.syftbox_folder.iterdir(): if not email_dir.is_dir() or "@" not in email_dir.name: continue - collections_dir = email_dir / spec.local_subpath - if not collections_dir.exists(): - continue - for tag_dir in collections_dir.iterdir(): - if tag_dir.is_dir(): - yield tag_dir + for layout in spec.layouts: + collections_dir = email_dir / layout.local_subpath + if not collections_dir.exists(): + continue + for tag_dir in collections_dir.iterdir(): + if tag_dir.is_dir(): + yield tag_dir def _compute_local_collection_hash(self, collection_path: Path) -> str | None: """Compute content hash from local collection files on disk.""" @@ -282,26 +286,86 @@ def current_hash_for_file(self, path: str) -> int | None: self.sync_down_if_needed(peer) return self.file_hashes.get(path, None) + def _select_collections_to_sync( + self, spec: CollectionSyncSpec, collections: list[dict] + ) -> list[dict]: + """Keep one collection per tag: the newest layout this client reads. + + An owner publishes a collection once for each layout its audience reads. + This client takes the newest of those it reads, and warns about the rest. + """ + best: dict[tuple[str, str], dict] = {} + for collection in collections: + variant = collection.get("variant", "") + if spec.rank_of(variant) < 0: + logger.warning( + "Skipping '%s' from %s: it uses the %r layout of %s, which " + "this client does not read.", + collection["tag"], + collection["owner_email"], + variant, + spec.prefix, + ) + continue + key = (collection["owner_email"], collection["tag"]) + current = best.get(key) + if current is None or spec.rank_of(variant) > spec.rank_of( + current.get("variant", "") + ): + best[key] = collection + return list(best.values()) + + def _collection_path_of(self, spec: CollectionSyncSpec, collection: dict): + """The local path a listed collection syncs to, or None without a syftbox.""" + layout = spec.layout_for(collection.get("variant", "")) + if layout is None: + return None + return self.get_collection_path( + collection["owner_email"], collection["tag"], layout.local_subpath + ) + def _cleanup_stale_collections( - self, peer_email: str, remote_collections: list[dict], local_subpath: Path + self, + spec: CollectionSyncSpec, + peer_email: str, + selected_collections: list[dict], + remote_collections: list[dict], ): - """Remove locally cached collections that no longer exist remotely. + """Remove local collections this client no longer syncs from a peer. - Only considers collections under the given local_subpath so that - collections from other specs are not treated as stale. + Two cases get removed: the owner deleted the collection, and this client + now reads a newer layout of it. The second case would otherwise leave the + older copy on disk, where a scan finds the same collection twice. + + A collection the owner still publishes, but in no layout this client + reads, is kept: the copy on disk is then the last one this client could + read. ``_select_collections_to_sync`` already logged why it is not + refreshed. """ - remote_tags = {c["tag"] for c in remote_collections} + layout_parents = set() + if self.syftbox_folder is not None: + layout_parents = { + self.syftbox_folder / peer_email / layout.local_subpath + for layout in spec.layouts + } + selected_paths = { + self._collection_path_of(spec, c) for c in selected_collections + } + published = {(c["owner_email"], c["tag"]) for c in remote_collections} + readable = {(c["owner_email"], c["tag"]) for c in selected_collections} for local_collection_path in list(self.collection_hashes.keys()): owner_email = self.get_collection_owner_email(local_collection_path) if owner_email != peer_email: continue - # Only consider collections that live under this spec's subpath. - if self.syftbox_folder is not None: - expected_parent = self.syftbox_folder / owner_email / local_subpath - if local_collection_path.parent != expected_parent: - continue - if local_collection_path.name in remote_tags: + # Only consider collections that live under one of this spec's layouts. + if layout_parents and local_collection_path.parent not in layout_parents: + continue + if local_collection_path in selected_paths: + continue + # The last path segment is the tag, in every layout. + collection = (owner_email, local_collection_path.name) + if collection in published and collection not in readable: continue del self.collection_hashes[local_collection_path] if self.syftbox_folder is not None: @@ -320,26 +384,26 @@ def sync_down_collections(self, peer_email: str): if spec.owner_only: # Never pull owner-only collections (e.g. private data) from a peer. continue - # Get list of collections shared with us (returns list of dicts) + # Every layout the peer published; the prefix matches all of them. collections = self.connection_router.watcher_list_collections(spec.prefix) - # Filter by peer - peer_collections = [ - c for c in collections if c["owner_email"] == peer_email - ] + # Filter by peer, then take one layout for each collection + published = [c for c in collections if c["owner_email"] == peer_email] + peer_collections = self._select_collections_to_sync(spec, published) self._cleanup_stale_collections( - peer_email, peer_collections, spec.local_subpath + spec, peer_email, peer_collections, published ) for collection in peer_collections: owner_email = collection["owner_email"] tag = collection["tag"] content_hash = collection["content_hash"] + layout = spec.layout_for(collection.get("variant", "")) # Check if hash changed - skip download if unchanged collection_path = self.get_collection_path( - owner_email, tag, spec.local_subpath + owner_email, tag, layout.local_subpath ) if collection_path is None: continue @@ -349,12 +413,12 @@ def sync_down_collections(self, peer_email: str): # Download collection files files = self.connection_router.watcher_download_collection( - spec.prefix, tag, content_hash, owner_email + spec.wire_prefix(layout.variant), tag, content_hash, owner_email ) # Write files to local cache (path relative to syftbox_folder) for file_name, content in files.items(): - rel_path = f"{owner_email}/{spec.local_subpath}/{tag}/{file_name}" + rel_path = f"{owner_email}/{layout.local_subpath}/{tag}/{file_name}" self.file_connection.write_file(rel_path, content) # Update hash cache @@ -376,12 +440,11 @@ def sync_down_collections_parallel( # Never pull owner-only collections (e.g. private data) from a peer. continue collections = self.connection_router.watcher_list_collections(spec.prefix) - peer_collections = [ - c for c in collections if c["owner_email"] == peer_email - ] + published = [c for c in collections if c["owner_email"] == peer_email] + peer_collections = self._select_collections_to_sync(spec, published) self._cleanup_stale_collections( - peer_email, peer_collections, spec.local_subpath + spec, peer_email, peer_collections, published ) # Gather all files to download across all collections for this spec @@ -392,10 +455,11 @@ def sync_down_collections_parallel( owner_email = collection["owner_email"] tag = collection["tag"] content_hash = collection["content_hash"] + layout = spec.layout_for(collection.get("variant", "")) # Check if hash changed - skip download if unchanged collection_path = self.get_collection_path( - owner_email, tag, spec.local_subpath + owner_email, tag, layout.local_subpath ) if collection_path is None: continue @@ -406,7 +470,10 @@ def sync_down_collections_parallel( # Get file metadata (no download yet) file_metadatas = ( self.connection_router.watcher_get_collection_file_metadatas( - spec.prefix, tag, content_hash, owner_email + spec.wire_prefix(layout.variant), + tag, + content_hash, + owner_email, ) ) @@ -431,13 +498,12 @@ def sync_down_collections_parallel( owner_email = collection["owner_email"] tag = collection["tag"] file_name = metadata["file_name"] - rel_path = f"{owner_email}/{spec.local_subpath}/{tag}/{file_name}" + subpath = spec.layout_for(collection.get("variant", "")).local_subpath + rel_path = f"{owner_email}/{subpath}/{tag}/{file_name}" self.file_connection.write_file(rel_path, content) # Update hash cache for all collections in this spec for collection in collections_to_update: - collection_path = self.get_collection_path( - collection["owner_email"], collection["tag"], spec.local_subpath - ) + collection_path = self._collection_path_of(spec, collection) if collection_path is not None: self.collection_hashes[collection_path] = collection["content_hash"] diff --git a/syft/sync/sync/caches/persisted_dict.py b/syft/sync/sync/caches/persisted_dict.py index 24064b4ce99..5e43ee0fe9c 100644 --- a/syft/sync/sync/caches/persisted_dict.py +++ b/syft/sync/sync/caches/persisted_dict.py @@ -31,6 +31,11 @@ import portalocker +# Format of the persisted file: {"version": N, "entries": {...}}. Raise it when +# the layout of an entry changes. A file with no version holds the entries at the +# top level, was written before the field, and is version 0. +PERSISTED_DICT_VERSION = 1 + class PersistedDict(dict): """Dict that persists to a JSON file. With path=None it's a plain in-memory dict.""" @@ -94,10 +99,28 @@ def _read_from_file(self) -> None: return try: data = json.loads(self._path.read_text()) - for k, v in data.items(): - super().__setitem__(self._key_deserializer(k), v) except (json.JSONDecodeError, OSError): - pass + return + entries = self._entries_of(data) + for k, v in entries.items(): + super().__setitem__(self._key_deserializer(k), v) + + @staticmethod + def _entries_of(data: Any) -> dict: + """The entries to load from a parsed file, empty when it cannot be read. + + The client rebuilds every cache that uses this class, so an unreadable + file costs a re-scan and nothing else. A file from a later version + therefore starts empty instead of stopping the client. + """ + if not isinstance(data, dict): + return {} + if "version" not in data or "entries" not in data: + # Written before the version field existed: entries at the top level. + return data + if data["version"] > PERSISTED_DICT_VERSION: + return {} + return data["entries"] def _write_to_file(self) -> None: if self._path is None: @@ -106,7 +129,10 @@ def _write_to_file(self) -> None: # Per-process unique tmp path: even with the file lock, this guards # against any path where two writers share a tmp filename. tmp = self._path.with_suffix(f".tmp.{os.getpid()}.{uuid4().hex}") - serialized = {self._key_serializer(k): v for k, v in super().items()} + serialized = { + "version": PERSISTED_DICT_VERSION, + "entries": {self._key_serializer(k): v for k, v in super().items()}, + } try: tmp.write_text(json.dumps(serialized)) tmp.replace(self._path) diff --git a/syft/sync/sync/collection_spec.py b/syft/sync/sync/collection_spec.py index 1d17f67949b..2198ec4a942 100644 --- a/syft/sync/sync/collection_spec.py +++ b/syft/sync/sync/collection_spec.py @@ -8,7 +8,24 @@ from pathlib import Path -from pydantic import BaseModel +from pydantic import BaseModel, model_validator + + +class CollectionLayout(BaseModel): + """One on-the-wire layout of a collection. + + An owner can publish the same collection several times, once per layout, so + that peers of different ages each find one they can read. ``variant`` is the + discriminator the owner writes into the folder name directly after the + prefix; the original layout uses "" and so keeps the name it always had. + ``local_subpath`` is where that layout lands under the owner's datasite. + + The engine treats ``variant`` as opaque. What it means (a dataset protocol + version, say) is the domain's business. + """ + + variant: str = "" + local_subpath: Path class CollectionSyncSpec(BaseModel): @@ -25,9 +42,44 @@ class CollectionSyncSpec(BaseModel): # True = owner-only: never shared with peers; the owner restores it for itself and # peer-facing watchers skip it entirely (e.g. the owner's private data backup). owner_only: bool = False + # Every layout this client can read, oldest first. When an owner publishes a + # collection in several layouts, the watcher keeps the last one in this list + # that the owner published, and skips the rest. Defaults to the single + # original layout. + layouts: list[CollectionLayout] = [] + + @model_validator(mode="after") + def _default_layout(self) -> "CollectionSyncSpec": + if not self.layouts: + self.layouts = [CollectionLayout(local_subpath=self.local_subpath)] + return self + + def wire_prefix(self, variant: str) -> str: + """The folder-name prefix an owner writes for one layout. + + The variant sits between the prefix and the '_' that starts the tag, so + a client that predates multi-layout searches for '_' and never + lists a layout it cannot read. + """ + return f"{self.prefix}{variant}" + + def layout_for(self, variant: str) -> CollectionLayout | None: + """The layout for a wire variant, or None when this client cannot read it.""" + return next((la for la in self.layouts if la.variant == variant), None) + + def rank_of(self, variant: str) -> int: + """Position of a variant in ``layouts``; -1 when unreadable. Higher is newer.""" + return next( + (i for i, la in enumerate(self.layouts) if la.variant == variant), -1 + ) @classmethod - def public(cls, prefix: str, local_subpath: "Path") -> "CollectionSyncSpec": + def public( + cls, + prefix: str, + local_subpath: "Path", + layouts: list[CollectionLayout] | None = None, + ) -> "CollectionSyncSpec": """A shareable, mirrored collection (e.g. a dataset's mock data). Peers' watchers pull it, and it re-downloads whenever the owner @@ -38,10 +90,16 @@ def public(cls, prefix: str, local_subpath: "Path") -> "CollectionSyncSpec": local_subpath=local_subpath, immutable=False, owner_only=False, + layouts=layouts or [], ) @classmethod - def private(cls, prefix: str, local_subpath: "Path") -> "CollectionSyncSpec": + def private( + cls, + prefix: str, + local_subpath: "Path", + layouts: list[CollectionLayout] | None = None, + ) -> "CollectionSyncSpec": """An owner-only, restore-only collection (e.g. a dataset's real data). Never shared with peers, peer-facing watchers skip it, and it is only @@ -53,4 +111,5 @@ def private(cls, prefix: str, local_subpath: "Path") -> "CollectionSyncSpec": local_subpath=local_subpath, immutable=True, owner_only=True, + layouts=layouts or [], ) diff --git a/syft/sync/sync/datasite_owner_syncer.py b/syft/sync/sync/datasite_owner_syncer.py index 5898e082891..6f0c2a48c25 100644 --- a/syft/sync/sync/datasite_owner_syncer.py +++ b/syft/sync/sync/datasite_owner_syncer.py @@ -32,7 +32,10 @@ compact_incremental_checkpoints, DEFAULT_COMPACTING_THRESHOLD, ) -from syft.sync.checkpoints.rolling_state import RollingState +from syft.sync.checkpoints.rolling_state import ( + RollingState, + raise_for_later_version, +) from syft_perms import SyftPermContext from syft.sync.sync.constants import CACHE_DIR, ROLLING_STATE_FILENAME @@ -132,9 +135,14 @@ def _load_rolling_state(self) -> None: if not path.exists(): return try: - self._rolling_state = RollingState.model_validate_json(path.read_text()) - except Exception: - pass + state = RollingState.model_validate_json(path.read_text()) + raise_for_later_version(state.version) + except (ValueError, OSError) as e: + # A later client wrote this file, or it is damaged. The caller falls + # back to a download of all events. + print(f"Warning: could not load the local rolling state: {e}") + return + self._rolling_state = state def _save_rolling_state(self) -> None: """Save rolling state to disk for cross-process consistency.""" @@ -323,9 +331,13 @@ def _apply_incremental_checkpoint_to_cache( str(event.path_in_datasite), event.content ) - def _collection_local_dir(self, spec: CollectionSyncSpec, tag: str) -> Path: - """Local directory a collection with the given tag restores to.""" - return self.syftbox_folder / self.email / spec.local_subpath / tag + def _collection_local_dir( + self, spec: CollectionSyncSpec, tag: str, variant: str = "" + ) -> Path: + """Local directory one layout of a collection restores to.""" + layout = spec.layout_for(variant) + subpath = layout.local_subpath if layout else spec.local_subpath + return self.syftbox_folder / self.email / subpath / tag def _pull_collections_for_initial_sync(self): """Restore the owner's own collections from the sync backend on connect. @@ -343,7 +355,7 @@ def _pull_collections_for_initial_sync(self): if not collections: continue - self._update_any_shared_collections_cache(collections) + self._update_any_shared_collections_cache(spec, collections) collections_to_download = self._filter_collections_needing_download( collections, spec @@ -352,24 +364,33 @@ def _pull_collections_for_initial_sync(self): @property def any_shared_collections(self) -> List[tuple]: - """Collections shared with "any" as (tag, content_hash) pairs. + """Collections shared with "any" as (wire_prefix, tag, content_hash) triples. + + One entry per layout: each layout is its own collection on the wire, so + sharing one with a new peer needs the prefix that names it. Read-only view: mutate via ``register_any_shared_collection``. """ return list(self._any_shared_collections) - def register_any_shared_collection(self, tag: str, content_hash: str) -> None: - """Record a collection as shared-with-"any" (deduplicated).""" - entry = (tag, content_hash) + def register_any_shared_collection( + self, wire_prefix: str, tag: str, content_hash: str + ) -> None: + """Record one layout of a collection as shared-with-"any" (deduplicated).""" + entry = (wire_prefix, tag, content_hash) if entry not in self._any_shared_collections: self._any_shared_collections.append(entry) - def _update_any_shared_collections_cache(self, collections: list[FileCollection]): + def _update_any_shared_collections_cache( + self, spec: CollectionSyncSpec, collections: list[FileCollection] + ): """Populate the any-shared cache from collections with 'any' permission.""" for collection in collections: if collection.has_any_permission: self.register_any_shared_collection( - collection.tag, collection.content_hash + spec.wire_prefix(collection.variant), + collection.tag, + collection.content_hash, ) def _filter_collections_needing_download( @@ -385,7 +406,9 @@ def _filter_collections_needing_download( result = [] for collection in collections: - local_dir = self._collection_local_dir(spec, collection.tag) + local_dir = self._collection_local_dir( + spec, collection.tag, collection.variant + ) if spec.immutable: if not local_dir.exists() or not any(local_dir.iterdir()): @@ -393,11 +416,15 @@ def _filter_collections_needing_download( continue # Mirror: use the cached hash first, else compute it from local files. - cached_hash = self.event_cache.get_collection_hash(collection.tag) + cached_hash = self.event_cache.get_collection_hash( + collection.tag, collection.variant + ) if cached_hash is None and local_dir.exists(): cached_hash = compute_directory_hash(local_dir) if cached_hash is not None: - self.event_cache.set_collection_hash(collection.tag, cached_hash) + self.event_cache.set_collection_hash( + collection.tag, cached_hash, collection.variant + ) if cached_hash != collection.content_hash: result.append(collection) @@ -410,12 +437,11 @@ def _download_collections_parallel( if not collections: return - # Fetch file metadatas for all collections in parallel + # Fetch file metadatas for all collections in parallel. Each collection + # names its own layout, so the prefix comes from the collection. all_file_metadatas = list( self._executor.map( - partial( - self._get_file_metadatas_with_new_connection, prefix=spec.prefix - ), + partial(self._get_file_metadatas_with_new_connection, spec=spec), collections, ) ) @@ -436,9 +462,11 @@ def _download_collections_parallel( self._executor.map(self._download_file_with_new_connection, file_ids) ) - # Write all files to disk under the spec's local subpath. + # Write all files to disk under the layout's local subpath. for (collection, metadata), content in zip(all_downloads, downloaded_contents): - local_dir = self._collection_local_dir(spec, collection.tag) + local_dir = self._collection_local_dir( + spec, collection.tag, collection.variant + ) local_dir.mkdir(parents=True, exist_ok=True) (local_dir / metadata["file_name"]).write_bytes(content) @@ -446,7 +474,7 @@ def _download_collections_parallel( if not spec.immutable: for collection in collections: self.event_cache.set_collection_hash( - collection.tag, collection.content_hash + collection.tag, collection.content_hash, collection.variant ) # Notify the domain layer (e.g. syft-rds) that these collections were restored, @@ -456,16 +484,16 @@ def _download_collections_parallel( "collection_restored", spec.prefix, collection.tag, - self._collection_local_dir(spec, collection.tag), + self._collection_local_dir(spec, collection.tag, collection.variant), ) def _get_file_metadatas_with_new_connection( - self, collection: FileCollection, prefix: str + self, collection: FileCollection, spec: CollectionSyncSpec ) -> list: """Get file metadatas for a collection using a new connection for thread safety.""" connection = self.connection_router.connection_for_parallel_download() return connection.watcher_get_collection_file_metadatas( - prefix, + spec.wire_prefix(collection.variant), tag=collection.tag, content_hash=collection.content_hash, owner_email=self.email, diff --git a/syft/sync/version/__init__.py b/syft/sync/version/__init__.py index 1d7d27e09c5..732d3fff3a6 100644 --- a/syft/sync/version/__init__.py +++ b/syft/sync/version/__init__.py @@ -5,20 +5,16 @@ Import it directly: from syft.sync.version.peer_manager import PeerManager """ -from syft.sync.version.version_info import VersionInfo from syft.sync.version.exceptions import ( VersionError, VersionMismatchError, VersionUnknownError, - ClientVersionMismatchError, - ProtocolVersionMismatchError, ) +from syft.sync.version.version_info import VersionInfo __all__ = [ - "VersionInfo", "VersionError", + "VersionInfo", "VersionMismatchError", "VersionUnknownError", - "ClientVersionMismatchError", - "ProtocolVersionMismatchError", ] diff --git a/syft/sync/version/exceptions.py b/syft/sync/version/exceptions.py index 250a6fc6b1c..3f26dd352f9 100644 --- a/syft/sync/version/exceptions.py +++ b/syft/sync/version/exceptions.py @@ -11,8 +11,6 @@ class VersionError(Exception): """Base exception for version-related errors.""" - pass - class VersionMismatchError(VersionError): """Raised when versions are incompatible between peers.""" @@ -63,35 +61,3 @@ def __init__(self, peer_email: str, operation: Optional[str] = None): ) super().__init__(message) - - -class ClientVersionMismatchError(VersionMismatchError): - """Raised specifically for client version mismatches.""" - - def __init__( - self, - peer_email: str, - local_version: "VersionInfo", - peer_version: "VersionInfo", - ): - reason = ( - f"Client version mismatch: local={local_version.syft_client_version}, " - f"peer={peer_version.syft_client_version}" - ) - super().__init__(peer_email, local_version, peer_version, reason) - - -class ProtocolVersionMismatchError(VersionMismatchError): - """Raised specifically for protocol version mismatches.""" - - def __init__( - self, - peer_email: str, - local_version: "VersionInfo", - peer_version: "VersionInfo", - ): - reason = ( - f"Protocol version mismatch: local={local_version.protocol_version}, " - f"peer={peer_version.protocol_version}" - ) - super().__init__(peer_email, local_version, peer_version, reason) diff --git a/syft/sync/version/peer_manager.py b/syft/sync/version/peer_manager.py index ce39b99826b..f360b350d6e 100644 --- a/syft/sync/version/peer_manager.py +++ b/syft/sync/version/peer_manager.py @@ -11,6 +11,7 @@ from typing import Dict, List, Optional from pydantic import BaseModel, ConfigDict, PrivateAttr, model_validator +from syft_migration import ProtocolSchema from syft.sync.connections.base_connection import ConnectionConfig from syft.sync.connections.connection_router import ConnectionRouter @@ -98,8 +99,9 @@ class PeerManagerConfig(BaseModel): syftbox_folder: Path email: str = "" connection_configs: List[ConnectionConfig] = [] + # Applies to a peer of unknown version only. A client version difference does + # not skip a peer, so this flag has no effect on one. force_ignore_peer_version: bool = False - force_ignore_protocol_version: bool = True suppress_version_warnings: bool = False n_threads: int = 10 has_do_role: bool = False @@ -139,7 +141,6 @@ class PeerManager(BaseModel): connection_router: ConnectionRouter peer_store: PeerStore force_ignore_peer_version: bool = False - force_ignore_protocol_version: bool = True suppress_version_warnings: bool = False n_threads: int = 10 has_do_role: bool = False @@ -148,6 +149,17 @@ class PeerManager(BaseModel): _own_version: Optional[VersionInfo] = PrivateAttr(default=None) _executor: Optional[ThreadPoolExecutor] = PrivateAttr(default=None) + # protocol name -> {peer email -> ProtocolSchema}. Inner dicts are handed + # out by live_peer_schemas() and mutated in place as peer versions load, + # so consumers (JobStorage, DatasetStorage) always see current knowledge. + _peer_protocol_schemas: Dict[str, Dict[str, ProtocolSchema]] = PrivateAttr( + default_factory=dict + ) + # peer email -> last loaded VersionInfo (or None); lets a protocol map + # registered AFTER versions were loaded backfill instead of starting empty. + _loaded_peer_versions: Dict[str, Optional[VersionInfo]] = PrivateAttr( + default_factory=dict + ) # ========== Peer List Properties ========== @@ -199,7 +211,6 @@ def from_config(cls, config: PeerManagerConfig, email: str = "") -> "PeerManager connection_router=connection_router, peer_store=peer_store, force_ignore_peer_version=config.force_ignore_peer_version, - force_ignore_protocol_version=config.force_ignore_protocol_version, suppress_version_warnings=config.suppress_version_warnings, n_threads=config.n_threads, has_do_role=config.has_do_role, @@ -213,6 +224,53 @@ def model_post_init(self, __context) -> None: """Initialize the thread pool executor.""" self._executor = ThreadPoolExecutor(max_workers=self.n_threads) + def live_peer_schemas(self, protocol_name: str) -> Dict[str, ProtocolSchema]: + """The live {peer email -> ProtocolSchema} map for ``protocol_name``. + + The returned dict is owned by this PeerManager and updated in place + whenever a peer's version file is (re)loaded: peers advertising the + protocol appear, peers that stop advertising it (or whose version is + cleared) disappear. Hand it to JobStorage/DatasetStorage as + ``peer_schemas`` so negotiation always uses current knowledge. A map + registered after peer versions were already loaded is backfilled from + them. + """ + per_peer = self._peer_protocol_schemas.get(protocol_name) + if per_peer is None: + per_peer = self._peer_protocol_schemas[protocol_name] = {} + for email, version_info in list(self._loaded_peer_versions.items()): + self._sync_one(per_peer, protocol_name, email, version_info) + return per_peer + + @staticmethod + def _sync_one( + per_peer: Dict[str, ProtocolSchema], + protocol_name: str, + peer_email: str, + version_info: Optional[VersionInfo], + ) -> None: + # getattr: a peer's version may be a V1 object (no schemas attribute). + advertised = getattr(version_info, "protocol_schemas", None) or {} + schema = advertised.get(protocol_name) + if schema is not None: + per_peer[peer_email] = schema + else: + per_peer.pop(peer_email, None) + + def _update_peer_schemas( + self, peer_email: str, version_info: Optional[VersionInfo] + ) -> None: + """Sync the live schema maps with a freshly loaded peer version. + + A None version or a pre-V2 version file (empty ``protocol_schemas``) + removes the peer: it is an unknown speaker, and consumers apply their + own unknown-peer defaults. + """ + self._loaded_peer_versions[peer_email] = version_info + # list(): guard against a concurrent live_peer_schemas() registration. + for protocol_name, per_peer in list(self._peer_protocol_schemas.items()): + self._sync_one(per_peer, protocol_name, peer_email, version_info) + def get_own_version(self) -> VersionInfo: """Get current client's version info.""" if self._own_version is None: @@ -241,6 +299,7 @@ def load_peer_version(self, peer_email: str) -> Optional[VersionInfo]: cached_peer = self.get_cached_peer(peer_email) if cached_peer: cached_peer.version = version_info + self._update_peer_schemas(peer_email, version_info) return version_info def _load_single_peer_version( @@ -275,6 +334,7 @@ def load_peer_versions_parallel( peer = self.get_cached_peer(email) if peer: peer.version = version + self._update_peer_schemas(email, version) return {email: version for email, version in results} @@ -288,6 +348,7 @@ def clear_peer_version(self, peer_email: str) -> None: peer = self.get_cached_peer(peer_email) if peer: peer.version = None + self._update_peer_schemas(peer_email, None) def peer_compatibility_status(self, peer_email: str) -> CompatibilityStatus: """Get the CompatibilityStatus for a peer (UNKNOWN if version not loaded).""" @@ -308,11 +369,16 @@ def get_peer_compatibility_status( """Build a PeerCompatibilityResult describing whether the caller should skip / raise / warn for this peer. - SAME → no skip, no warning. PATCH_DIFF → no skip, "patch differs" - warning. INCOMPATIBLE / UNKNOWN → skip unless effective - `force_ignore_peer_version or ignore_peer_version` (then proceed with - a "proceeding to {action}" warning). UNKNOWN's skip message includes - a "call client.sync()" hint. + SAME → no skip, no log. + + PATCH_DIFF → no skip and a "patch differs" log, or a skip when + `skip_peer_on_patch_version_diff` is set. + + INCOMPATIBLE → no skip; the client version difference is logged, and + each protocol decides separately through its floor. + + UNKNOWN → skip, unless effective `force_ignore_peer_version or + ignore_peer_version`; the message includes a "call client.sync()" hint. """ own_version = self.get_own_version() peer_version = self.get_peer_version(peer_email) @@ -360,14 +426,26 @@ def get_peer_compatibility_status( **common, ) - # UNKNOWN or INCOMPATIBLE - if status == CompatibilityStatus.UNKNOWN: - detail = ( - "version information not available " - "(if you are unsure if it is up to date, call client.sync())" + if status == CompatibilityStatus.INCOMPATIBLE: + # A different client version does not refuse the peer. What each side + # can exchange is decided per protocol by the floor published in + # VersionInfo (MigrationRegistry.negotiate_protocol_version), not by + # comparing package versions. + return PeerCompatibilityResult( + should_skip=False, + explanation_not_skip=( + f"Peer {peer_email}: " + f"{own_version.get_incompatibility_reason(peer_version)}." + ), + **common, ) - else: - detail = own_version.get_incompatibility_reason(peer_version) + + # UNKNOWN: the capabilities of the peer are not known, so there is no + # floor to check. Skipping stays the safe answer. + detail = ( + "version information not available " + "(if you are unsure if it is up to date, call client.sync())" + ) effective_ignore = self.force_ignore_peer_version or ignore_peer_version if effective_ignore: @@ -423,8 +501,10 @@ def warn_if_all_peers_incompatible(self, peer_emails: List[str]) -> None: ) if not any_compatible: warnings.warn( - f"All connected peers ({len(peer_emails)}) have incompatible versions. " - "You may not be able to submit jobs or load datasets until versions match." + f"All connected peers ({len(peer_emails)}) run a different client " + "version, or their version is unknown. A peer with an unknown " + "version cannot receive jobs or datasets; call client.sync() to " + "read the version of each peer." ) def shutdown(self) -> None: @@ -490,6 +570,7 @@ def add_peer(self, peer_email: str, force: bool = False, verbose: bool = True): ) self.share_version_with_peer(peer_email) version_info = self.connection_router.read_peer_version_file(peer_email) + self._update_peer_schemas(peer_email, version_info) new_peer_obj.version = version_info new_peer_obj.public_encryption_bundle = peer_bundle diff --git a/syft/sync/version/version_info.py b/syft/sync/version/version_info.py index b140756b5d5..2092e64145f 100644 --- a/syft/sync/version/version_info.py +++ b/syft/sync/version/version_info.py @@ -4,13 +4,16 @@ from __future__ import annotations +import json import logging from datetime import datetime, timezone from enum import Enum from typing import Optional -from pydantic import BaseModel, Field +from pydantic import Field +from syft_migration import MigratableObject, ProtocolSchema +from syft.migrations import client_registry, load_as_latest from syft.version import ( MIN_SUPPORTED_PROTOCOL_VERSION, MIN_SUPPORTED_SYFT_VERSION, @@ -36,8 +39,17 @@ def _parse_semver(version_str: str) -> tuple[int, int, int]: return (int(parts[0]), int(parts[1]), int(parts[2])) -class VersionInfo(BaseModel): - """Model representing version information for a syft client.""" +class VersionInfoV1(MigratableObject, registry=client_registry): + """Model representing version information for a syft client. + + Stored as SYFT_version.json in the peer-visible SyftBox folder. This file + is the bootstrap channel for protocol negotiation (peers read it to learn + what we speak), so its schema may only ever change additively: every + supported client version must be able to parse every newer version file. + """ + + canonical_name: str = "VersionInfo" + version: str = "1" syft_client_version: str min_supported_syft_client_version: str @@ -126,5 +138,92 @@ def to_json(self) -> str: @classmethod def from_json(cls, json_str: str) -> "VersionInfo": - """Deserialize from JSON string.""" - return cls.model_validate_json(json_str) + """Deserialize from JSON string, upgraded to the latest version. + + Files written by protocol-0 clients (<= 0.1.117) predate the identity + fields; they are all version 1. + """ + return load_as_latest(json.loads(json_str), "VersionInfo") + + +def _slim_schema_of(registry) -> ProtocolSchema: + """The registry's protocol schema without the embedded object JSON schemas. + + Negotiation needs only ``version`` and ``supported_versions``; skipping + ``current_object_schemas`` keeps the published version file small (the + full frozen schemas live in the release artifacts, not on the wire) and + avoids computing every object's JSON schema just to discard it. + """ + return ProtocolSchema( + protocol_name=registry.protocol_name, + version=registry.protocol_version, + min_supported_version=registry.min_supported_protocol_version, + supported_versions={ + canonical_name: sorted(versions) + for canonical_name, versions in registry.objects.items() + }, + ) + + +def _gather_protocol_schemas() -> dict[str, ProtocolSchema]: + """Slim protocol schemas of every syft package present in this install. + + Keyed by protocol name. syft-job/syft-dataset are optional dependencies; + a missing or broken package simply means its schema is not advertised and + peers treat this client as an unknown speaker of that protocol (same + failure-tolerant pattern as the install-source detection in ``current``). + """ + logger = logging.getLogger(__name__) + schemas = {client_registry.protocol_name: _slim_schema_of(client_registry)} + try: + from syft_job.migrations import job_registry + + schemas[job_registry.protocol_name] = _slim_schema_of(job_registry) + except Exception as e: + logger.debug(f"Not advertising a syft-job protocol schema: {e}") + try: + from syft_datasets.migrations.registry import dataset_registry + + schemas[dataset_registry.protocol_name] = _slim_schema_of(dataset_registry) + except Exception as e: + logger.debug(f"Not advertising a syft-dataset protocol schema: {e}") + return schemas + + +class VersionInfoV2(VersionInfoV1): + """V2 adds the protocol schemas this client speaks (client, job, dataset). + + Purely additive over V1 (see the bootstrap-channel rule in the V1 + docstring): protocol-0/1 readers ignore the extra key. + """ + + version: str = "2" + + # protocol name -> slim ProtocolSchema (no embedded object JSON schemas). + protocol_schemas: dict[str, ProtocolSchema] = Field(default_factory=dict) + + @classmethod + def current(cls) -> "VersionInfo": + info = super().current() + info.protocol_schemas = _gather_protocol_schemas() + return info + + +@client_registry.migration("VersionInfo", "1", "2") +def _version_info_v1_to_v2(obj: VersionInfoV1) -> VersionInfoV2: + # A v1 file says nothing about package protocols: empty schemas, meaning + # "unknown speaker" to consumers. + return VersionInfoV2.model_validate( + obj.model_dump(exclude={"canonical_name", "version"}) + ) + + +@client_registry.migration("VersionInfo", "2", "1") +def _version_info_v2_to_v1(obj: VersionInfoV2) -> VersionInfoV1: + return VersionInfoV1.model_validate( + obj.model_dump(exclude={"canonical_name", "version", "protocol_schemas"}) + ) + + +# Current-version alias: callers always work with the latest VersionInfo. +VersionInfo = VersionInfoV2 diff --git a/tests/migrations/__init__.py b/tests/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/migrations/p2p/__init__.py b/tests/migrations/p2p/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/migrations/p2p/fixtures/syft_client-0.1.117-protocol0/SYFT_version.json b/tests/migrations/p2p/fixtures/syft_client-0.1.117-protocol0/SYFT_version.json new file mode 100644 index 00000000000..b6a40d2e192 --- /dev/null +++ b/tests/migrations/p2p/fixtures/syft_client-0.1.117-protocol0/SYFT_version.json @@ -0,0 +1,9 @@ +{ + "syft_client_version": "0.1.117", + "min_supported_syft_client_version": "0.1.93", + "protocol_version": "1.0.0", + "min_supported_protocol_version": "1.0.0", + "syft_client_install_source": "pip", + "updated_at": "2026-07-20T10:15:30.123456Z", + "attestation_token": null +} diff --git a/tests/migrations/p2p/fixtures/syft_client-0.1.117-protocol0/msgv2_1752900000.0_6f9d5f57-31f7-4302-8746-9ba030e88961.tar.gz b/tests/migrations/p2p/fixtures/syft_client-0.1.117-protocol0/msgv2_1752900000.0_6f9d5f57-31f7-4302-8746-9ba030e88961.tar.gz new file mode 100644 index 00000000000..834a33e2fc5 Binary files /dev/null and b/tests/migrations/p2p/fixtures/syft_client-0.1.117-protocol0/msgv2_1752900000.0_6f9d5f57-31f7-4302-8746-9ba030e88961.tar.gz differ diff --git a/tests/migrations/p2p/fixtures/syft_client-0.1.117-protocol0/syfteventsmessagev3_1752900001.0_b3d3e5c1-89a4-4f7e-9f26-4a3f0e2d1c0a.tar.gz b/tests/migrations/p2p/fixtures/syft_client-0.1.117-protocol0/syfteventsmessagev3_1752900001.0_b3d3e5c1-89a4-4f7e-9f26-4a3f0e2d1c0a.tar.gz new file mode 100644 index 00000000000..092ef6584dd Binary files /dev/null and b/tests/migrations/p2p/fixtures/syft_client-0.1.117-protocol0/syfteventsmessagev3_1752900001.0_b3d3e5c1-89a4-4f7e-9f26-4a3f0e2d1c0a.tar.gz differ diff --git a/tests/migrations/p2p/test_dataset_multicopy_delivery.py b/tests/migrations/p2p/test_dataset_multicopy_delivery.py new file mode 100644 index 00000000000..b30fc2a96b7 --- /dev/null +++ b/tests/migrations/p2p/test_dataset_multicopy_delivery.py @@ -0,0 +1,492 @@ +"""A dataset reaches peers of different protocol versions, and each one reads it. + +A dataset goes to its whole audience through the dataset-collection transport. +Before multi-copy, that transport held one collection for each dataset name, and +it wrote every file flat. A dataset written in the v1 layout therefore arrived +with metadata that pointed at a directory the peer never got. + +The transport now holds one collection for each protocol version. The name of the +collection gives the version, and the peer takes the newest layout that it reads. +These tests drive that path from the name of the folder to the file on disk. +""" + +import pytest +from syft.sync.connections.drive.gdrive_transport import ( + CollectionFolder, + collection_name_query, +) +from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX +from syft_migration import ProtocolSchema +from syft_rds import SyftRDSClient +from syft_rds.config import ( + COLLECTION_SUBPATH, + MOCK_DATASET_SPEC, + dataset_variant, +) + +from tests.unit.utils import create_tmp_dataset_files + +DATASET_COLLECTION_NAME_QUERY = collection_name_query(DATASET_COLLECTION_PREFIX) + + +def dataset_collection_folder( + tag: str, content_hash: str, protocol_version: str = "0" +) -> CollectionFolder: + """The collection folder one protocol copy of a dataset writes.""" + return CollectionFolder( + prefix=DATASET_COLLECTION_PREFIX, + tag=tag, + content_hash=content_hash, + variant=dataset_variant(protocol_version), + ) + + +# An audience member on an earlier client, so a create writes both layouts. +OLD_PEER = "old@test.org" + + +def _dataset_schema(protocol_version: str) -> ProtocolSchema: + # The slim form a peer advertises in its VersionInfo. + return ProtocolSchema( + protocol_name="syft-dataset", + version=protocol_version, + supported_versions={"Dataset": ["1"]}, + ) + + +@pytest.fixture +def pair(): + return SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + + +# -- folder names ---------------------------------------------------------- + + +def test_a_collection_name_carries_the_protocol_version(): + folder = dataset_collection_folder("mytag", "abc123", "1") + assert ( + CollectionFolder.from_name(DATASET_COLLECTION_PREFIX, folder.folder_name) + == folder + ) + + +def test_a_tag_with_an_underscore_still_round_trips(): + folder = dataset_collection_folder("my_tag_here", "abc123", "2") + parsed = CollectionFolder.from_name(DATASET_COLLECTION_PREFIX, folder.folder_name) + assert parsed.tag == "my_tag_here" + assert parsed.content_hash == "abc123" + assert parsed.variant == "v2" + + +def test_a_protocol_0_name_is_what_earlier_clients_write(): + # Byte-identical to the name used before multi-copy, so a client that + # predates this change still finds the copy that it can read. + folder = dataset_collection_folder("mytag", "abc123") + assert folder.folder_name == f"{DATASET_COLLECTION_PREFIX}_mytag_abc123" + + +def test_a_name_with_no_version_reads_as_protocol_0(): + parsed = CollectionFolder.from_name( + DATASET_COLLECTION_PREFIX, f"{DATASET_COLLECTION_PREFIX}_mytag_abc123" + ) + assert parsed.variant == "" + + +def test_an_earlier_client_does_not_see_a_versioned_collection(): + # An earlier client searches Drive for names that contain '_'. The + # version infix breaks that match, so it never lists a layout it cannot + # read. It still lists the protocol-0 copy. + versioned = dataset_collection_folder("mytag", "abc123", "1").folder_name + flat = dataset_collection_folder("mytag", "abc123").folder_name + + assert f"{DATASET_COLLECTION_PREFIX}_" not in versioned + assert f"{DATASET_COLLECTION_PREFIX}_" in flat + # This client searches without the trailing '_', so it sees both. + assert DATASET_COLLECTION_PREFIX in DATASET_COLLECTION_NAME_QUERY + assert f"{DATASET_COLLECTION_PREFIX}_" not in DATASET_COLLECTION_NAME_QUERY + + +def test_a_damaged_name_raises(): + with pytest.raises(ValueError): + CollectionFolder.from_name(DATASET_COLLECTION_PREFIX, "not_a_collection") + + +# -- local layout ---------------------------------------------------------- + + +def test_the_local_directory_of_a_collection_follows_its_protocol(pair): + # The peer writes the files where the metadata of that copy points. Protocol + # 0 is flat; a later protocol adds its v segment. + assert MOCK_DATASET_SPEC.layout_for("").local_subpath == COLLECTION_SUBPATH + assert MOCK_DATASET_SPEC.layout_for("v1").local_subpath == COLLECTION_SUBPATH / "v1" + + +# -- delivery -------------------------------------------------------------- + + +def test_a_dataset_for_a_protocol0_peer_arrives_flat_and_reads(pair): + ds_manager, do_manager = pair + # The DS advertises dataset protocol 0, as an earlier client does. + do_manager.peer_manager.live_peer_schemas("syft-dataset")[ds_manager.email] = ( + _dataset_schema("0") + ) + + mock_path, private_path, readme_path = create_tmp_dataset_files() + do_manager.create_dataset( + name="skew dataset", + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + users=[ds_manager.email], + ) + ds_manager.sync() + + dataset = ds_manager.datasets.get("skew dataset", datasite=do_manager.email) + # The owner wrote the layout that this peer reads, not its own newest. + assert dataset.protocol_version == "0" + assert ( + dataset.mock_dir + == ds_manager.syftbox_folder + / do_manager.email + / COLLECTION_SUBPATH + / "skew dataset" + ) + assert dataset.mock_files + for path in dataset.mock_files: + assert path.exists(), ( + f"the metadata points to a file the peer does not get: {path}" + ) + + +def test_a_mixed_audience_gets_one_collection_for_each_protocol(pair): + _, do_manager = pair + mock_path, private_path, readme_path = create_tmp_dataset_files() + + # Write both layouts, as an audience of one protocol-0 peer and one + # current peer produces. + created = do_manager.dataset_manager.create_all( + name="mixed dataset", + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + protocol_versions=["0", "1"], + ) + assert set(created) == {"0", "1"} + for copy in created.values(): + do_manager._upload_dataset_to_collection(copy, users=[]) + + collections = do_manager._mock_collections_for("mixed dataset") + assert {do_manager._protocol_of(c) for c in collections} == {"0", "1"} + # Each copy has its own folder, so neither overwrites the other. + assert len({c.folder_id for c in collections}) == 2 + + +def test_the_owner_listing_names_each_dataset_once(pair): + # A dataset with two protocol copies has two collections. The listing names + # datasets, so the tag must not repeat. + _, do_manager = pair + mock_path, private_path, readme_path = create_tmp_dataset_files() + + created = do_manager.dataset_manager.create_all( + name="mixed dataset", + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + protocol_versions=["0", "1"], + ) + for copy in created.values(): + do_manager._upload_dataset_to_collection(copy, users=[]) + + tags = do_manager.sync_engine._connection_router.owner_list_collections( + DATASET_COLLECTION_PREFIX + ) + assert tags.count("mixed dataset") == 1 + + +def _create_for_a_mixed_audience(ds_manager, do_manager, name: str, **kwargs): + """Create a dataset for an audience of one protocol-0 peer and one current peer.""" + do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( + _dataset_schema("0") + ) + mock_path, private_path, readme_path = create_tmp_dataset_files() + return do_manager.create_dataset( + name=name, + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + users=[ds_manager.email, OLD_PEER], + **kwargs, + ) + + +def test_a_mixed_audience_through_create_dataset_writes_both_layouts(pair): + ds_manager, do_manager = pair + _create_for_a_mixed_audience(ds_manager, do_manager, "mixed") + + public = do_manager._mock_collections_for("mixed") + assert {do_manager._protocol_of(c) for c in public} == {"0", "1"} + + +def test_every_copy_uploads_its_own_private_collection(pair): + # Each copy holds its own private directory. An upload of only the newest + # leaves the other copies local, and a cold start does not restore them. + ds_manager, do_manager = pair + _create_for_a_mixed_audience(ds_manager, do_manager, "mixed", upload_private=True) + + private = do_manager._private_collections_for("mixed") + assert {do_manager._protocol_of(c) for c in private} == {"0", "1"} + + +def test_a_cold_start_restores_the_private_data_of_every_copy(pair): + import shutil + + ds_manager, do_manager = pair + _create_for_a_mixed_audience(ds_manager, do_manager, "mixed", upload_private=True) + + storage = do_manager.dataset_manager.storage + private_dirs = { + protocol_version: storage.private_dataset_dir( + storage.new_dataset_ref("mixed", protocol_version) + ) + for protocol_version in ("0", "1") + } + expected = {v: {f.name for f in d.iterdir()} for v, d in private_dirs.items()} + assert all(expected.values()), "each copy should have private files to lose" + + # Lose the local private data of every copy, then sync from cold. + for directory in private_dirs.values(): + shutil.rmtree(directory) + do_manager.sync_engine.datasite_owner_syncer.initial_sync_done = False + do_manager.sync() + + for protocol_version, directory in private_dirs.items(): + assert directory.exists(), ( + f"the private data of protocol {protocol_version} was not restored" + ) + assert {f.name for f in directory.iterdir()} == expected[protocol_version] + + +def test_a_collection_of_an_unreadable_protocol_is_skipped(pair, caplog): + import logging + + ds_manager, _ = pair + cache = ds_manager.sync_engine.datasite_watcher_syncer.datasite_watcher_cache + + remote = [ + { + "owner_email": "do@test.org", + "tag": "future dataset", + "content_hash": "abc123", + "variant": "v99", + } + ] + with caplog.at_level(logging.WARNING): + assert cache._select_collections_to_sync(MOCK_DATASET_SPEC, remote) == [] + assert "future dataset" in caplog.text + assert "v99" in caplog.text + + +def test_the_newest_readable_layout_wins(pair): + ds_manager, _ = pair + cache = ds_manager.sync_engine.datasite_watcher_syncer.datasite_watcher_cache + + remote = [ + { + "owner_email": "do@test.org", + "tag": "both", + "content_hash": "flat", + "variant": "", + }, + { + "owner_email": "do@test.org", + "tag": "both", + "content_hash": "versioned", + "variant": "v1", + }, + ] + selected = cache._select_collections_to_sync(MOCK_DATASET_SPEC, remote) + assert [c["variant"] for c in selected] == ["v1"] + + +# -- cleanup of local copies ----------------------------------------------- + + +def _seed_local_copy(cache, peer, tag, variant=""): + layout = MOCK_DATASET_SPEC.layout_for(variant) + path = cache.get_collection_path(peer, tag, layout.local_subpath) + cache.collection_hashes[path] = f"hash{variant or '0'}" + return path + + +def test_an_unreadable_remote_layout_keeps_the_local_copy(pair): + """The owner upgraded past us, so keep the last copy we could read. + + A delete here would take a dataset away over an upgrade by someone else, + and we cannot replace it until this client can read the newer layout. + """ + ds_manager, do_manager = pair + cache = ds_manager.sync_engine.datasite_watcher_syncer.datasite_watcher_cache + peer = do_manager.email + local = _seed_local_copy(cache, peer, "shared data") + + published = [ + { + "owner_email": peer, + "tag": "shared data", + "content_hash": "hash99", + "variant": "v99", + } + ] + selected = cache._select_collections_to_sync(MOCK_DATASET_SPEC, published) + assert selected == [] + + cache._cleanup_stale_collections(MOCK_DATASET_SPEC, peer, selected, published) + assert local in cache.collection_hashes + + +def test_a_deleted_dataset_removes_the_local_copy(pair): + ds_manager, do_manager = pair + cache = ds_manager.sync_engine.datasite_watcher_syncer.datasite_watcher_cache + peer = do_manager.email + local = _seed_local_copy(cache, peer, "gone") + + cache._cleanup_stale_collections(MOCK_DATASET_SPEC, peer, [], []) + assert local not in cache.collection_hashes + + +def test_a_newer_readable_layout_removes_the_older_local_copy(pair): + """Otherwise a dataset scan finds the same dataset twice.""" + ds_manager, do_manager = pair + cache = ds_manager.sync_engine.datasite_watcher_syncer.datasite_watcher_cache + peer = do_manager.email + old_local = _seed_local_copy(cache, peer, "both") + + published = [ + { + "owner_email": peer, + "tag": "both", + "content_hash": "hash0", + "variant": "", + }, + { + "owner_email": peer, + "tag": "both", + "content_hash": "hash1", + "variant": "v1", + }, + ] + selected = cache._select_collections_to_sync(MOCK_DATASET_SPEC, published) + assert [c["variant"] for c in selected] == ["v1"] + + cache._cleanup_stale_collections(MOCK_DATASET_SPEC, peer, selected, published) + assert old_local not in cache.collection_hashes + + +# -- sharing after the fact -------------------------------------------------- + + +def _create_for_the_current_audience(ds_manager, do_manager, name: str, **kwargs): + """Create a dataset whose audience reads only the current protocol. + + The paired DS advertises the current dataset protocol, so the create + writes the v1 layout only -- the starting point for a share that later + brings in a peer of another protocol. + """ + mock_path, private_path, readme_path = create_tmp_dataset_files() + return do_manager.create_dataset( + name=name, + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + users=[ds_manager.email], + **kwargs, + ) + + +def _collections_for(do_manager, tag: str): + return {do_manager._protocol_of(c) for c in do_manager._mock_collections_for(tag)} + + +def test_sharing_with_a_protocol0_peer_materializes_the_flat_copy(pair): + # A share is a change of audience. The audience decided the layouts at + # create time, so a new audience member of another protocol needs a copy + # in its layout -- granting it the versioned collection gives it a folder + # its own client never even lists. + ds_manager, do_manager = pair + _create_for_the_current_audience(ds_manager, do_manager, "afterthought") + assert _collections_for(do_manager, "afterthought") == {"1"} + + do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( + _dataset_schema("0") + ) + do_manager.share_dataset("afterthought", [OLD_PEER], sync=False) + + assert _collections_for(do_manager, "afterthought") == {"0", "1"} + # The flat copy exists locally too, so the owner's own scan and a cold + # start both see what the collection holds. + storage = do_manager.dataset_manager.storage + flat_dir = storage.public_dataset_dir(storage.new_dataset_ref("afterthought", "0")) + assert flat_dir.exists() + + +def test_sharing_with_a_current_peer_creates_no_extra_copy(pair): + # The control: a peer of our own protocol reads the existing layout, so + # the test above measures the fill and not an unconditional copy. + ds_manager, do_manager = pair + _create_for_the_current_audience(ds_manager, do_manager, "current share") + + do_manager.peer_manager.live_peer_schemas("syft-dataset")["new@test.org"] = ( + _dataset_schema("1") + ) + do_manager.share_dataset("current share", ["new@test.org"], sync=False) + + assert _collections_for(do_manager, "current share") == {"1"} + + +def test_sharing_with_an_unknown_peer_materializes_the_widest_layout(pair): + # An unknown peer may run any released client, so it gets the layout every + # release reads -- the same audience rule create_dataset applies. + ds_manager, do_manager = pair + _create_for_the_current_audience(ds_manager, do_manager, "unknown share") + + do_manager.share_dataset("unknown share", ["stranger@test.org"], sync=False) + + assert _collections_for(do_manager, "unknown share") == {"0", "1"} + + +def test_a_copy_materialized_at_share_time_uploads_its_private_collection(pair): + # Each copy holds its own private directory (see the cold-start test + # above). A copy created at share time must follow the same rule, or a + # cold start loses its private data. + ds_manager, do_manager = pair + _create_for_the_current_audience( + ds_manager, do_manager, "private fill", upload_private=True + ) + + do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( + _dataset_schema("0") + ) + do_manager.share_dataset("private fill", [OLD_PEER], sync=False) + + private = do_manager._private_collections_for("private fill") + assert {do_manager._protocol_of(c) for c in private} == {"0", "1"} + + +def test_a_share_uploads_a_local_copy_that_has_no_collection(pair): + # A share that fails after the migrate leaves the copy on disk with no + # collection of its own. The next share must upload that copy. A second + # write of the same layout raises, and the share then grants nothing at + # all -- not even the collections that were already there. + ds_manager, do_manager = pair + _create_for_the_current_audience(ds_manager, do_manager, "half done") + do_manager.dataset_manager.migrate("half done", "0", users=[ds_manager.email]) + assert _collections_for(do_manager, "half done") == {"1"} + + do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( + _dataset_schema("0") + ) + do_manager.share_dataset("half done", [OLD_PEER], sync=False) + + assert _collections_for(do_manager, "half done") == {"0", "1"} diff --git a/tests/migrations/p2p/test_dataset_schema_negotiation.py b/tests/migrations/p2p/test_dataset_schema_negotiation.py new file mode 100644 index 00000000000..12c3e461608 --- /dev/null +++ b/tests/migrations/p2p/test_dataset_schema_negotiation.py @@ -0,0 +1,82 @@ +"""Peer-advertised dataset schemas drive DatasetStorage protocol negotiation.""" + +from pathlib import Path + +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_manager import SyftDatasetManager +from syft_datasets.dataset_storage import DatasetStorage +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION +from syft_migration import ProtocolSchema + +OWNER_EMAIL = "do@test.org" +OLD_PEER = "old@test.org" +NEW_PEER = "new@test.org" +UNKNOWN_PEER = "unknown@test.org" + + +def _dataset_schema(protocol_version: str) -> ProtocolSchema: + # The slim form a peer advertises in its VersionInfo. + return ProtocolSchema( + protocol_name="syft-dataset", + version=protocol_version, + supported_versions={"Dataset": ["1"], "PrivateDatasetConfig": ["1"]}, + ) + + +def _storage(tmp_path: Path, peer_schemas: dict) -> DatasetStorage: + config = SyftBoxConfig(syftbox_folder=tmp_path / "SyftBox", email=OWNER_EMAIL) + (tmp_path / "SyftBox" / OWNER_EMAIL).mkdir(parents=True, exist_ok=True) + return DatasetStorage(config=config, peer_schemas=peer_schemas) + + +def test_mixed_audience_writes_both_versions(tmp_path): + storage = _storage( + tmp_path, + { + OLD_PEER: _dataset_schema("0"), + NEW_PEER: _dataset_schema(DATASET_PROTOCOL_VERSION), + }, + ) + versions = storage.target_protocol_versions_for_peers([OLD_PEER, NEW_PEER]) + assert versions == {"0", DATASET_PROTOCOL_VERSION} + + +def test_all_current_audience_drops_legacy_layout(tmp_path): + storage = _storage(tmp_path, {NEW_PEER: _dataset_schema(DATASET_PROTOCOL_VERSION)}) + assert storage.target_protocol_versions_for_peers([NEW_PEER]) == { + DATASET_PROTOCOL_VERSION + } + + +def test_unknown_peer_gets_widest_protocol(tmp_path): + storage = _storage(tmp_path, {}) + versions = storage.target_protocol_versions_for_peers([UNKNOWN_PEER]) + assert versions == {storage._widest_protocol_version} + + +def test_live_map_updates_are_seen_by_storage(tmp_path): + live: dict = {} + storage = _storage(tmp_path, live) + assert storage.target_protocol_versions_for_peers([NEW_PEER]) == { + storage._widest_protocol_version + } + live[NEW_PEER] = _dataset_schema(DATASET_PROTOCOL_VERSION) + assert storage.target_protocol_versions_for_peers([NEW_PEER]) == { + DATASET_PROTOCOL_VERSION + } + + +def test_manager_from_config_passes_schemas_through(tmp_path): + config = SyftBoxConfig(syftbox_folder=tmp_path / "SyftBox", email=OWNER_EMAIL) + (tmp_path / "SyftBox" / OWNER_EMAIL).mkdir(parents=True, exist_ok=True) + live = {OLD_PEER: _dataset_schema("0")} + manager = SyftDatasetManager.from_config(config, peer_schemas=live) + assert manager.storage.peer_schemas is live + + +def test_newer_peer_clamps_to_our_protocol(tmp_path): + # A peer speaking a future protocol contributes min(ours, theirs) = ours. + storage = _storage(tmp_path, {NEW_PEER: _dataset_schema("99")}) + assert storage.target_protocol_versions_for_peers([NEW_PEER]) == { + DATASET_PROTOCOL_VERSION + } diff --git a/tests/migrations/p2p/test_job_protocol_skew_delivery.py b/tests/migrations/p2p/test_job_protocol_skew_delivery.py new file mode 100644 index 00000000000..96d59fe7345 --- /dev/null +++ b/tests/migrations/p2p/test_job_protocol_skew_delivery.py @@ -0,0 +1,91 @@ +"""A job written for a protocol-0 peer reaches that peer and reads back. + +The other tests in this folder stop at the negotiated version. They assert which +protocol the two sides agree on, not that a job written at that protocol arrives +and reads. That seam is where the dataset transport broke: negotiation chose a +layout the delivery path could not carry. + +This test drives the whole path: the peer advertises job protocol 0, the sender +negotiates down, writes the flat layout, syncs, and the receiver finds and reads +the job through its own scan. +""" + +from pathlib import Path + +import pytest +from syft_rds import SyftRDSClient +from syft_migration import ProtocolSchema + +from tests.unit.utils import create_test_project_folder + + +def _job_schema(protocol_version: str) -> ProtocolSchema: + # The slim form a peer advertises in its VersionInfo. + return ProtocolSchema( + protocol_name="syft-job", + version=protocol_version, + supported_versions={"JobState": ["1"], "JobSubmissionMetadata": ["1"]}, + ) + + +@pytest.fixture +def pair(): + return SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + +def _submit(ds_manager, do_manager, job_name: str) -> Path: + project_dir = create_test_project_folder(with_pyproject=False) + ds_manager.submit_python_job( + user=do_manager.email, + code_path=str(project_dir), + job_name=job_name, + entrypoint="main.py", + ) + do_manager.sync() + return project_dir + + +def test_a_job_for_a_protocol0_peer_uses_the_flat_layout(pair): + ds_manager, do_manager = pair + # The DO advertises job protocol 0, as a client of 0.1.38 or earlier does. + ds_manager.peer_manager.live_peer_schemas("syft-job")[do_manager.email] = ( + _job_schema("0") + ) + + ref = ds_manager.job_client.manager.new_submission_ref(do_manager.email, "skew.job") + assert ref.protocol_version == "0" + assert "/v0/" not in str(ref) and "/v1/" not in str(ref), ( + "protocol 0 is the flat layout, so the path carries no v segment" + ) + + +def test_a_job_for_a_protocol0_peer_arrives_and_reads(pair): + ds_manager, do_manager = pair + ds_manager.peer_manager.live_peer_schemas("syft-job")[do_manager.email] = ( + _job_schema("0") + ) + + _submit(ds_manager, do_manager, "skew.job") + + # The receiver scans every layout it knows, so it finds the flat one. + assert [job.name for job in do_manager.jobs] == ["skew.job"] + found = do_manager.job_client.manager.find_submission_ref( + do_manager.email, "skew.job" + ) + assert found.protocol_version == "0" + + +def test_a_job_for_a_current_peer_still_uses_the_versioned_layout(pair): + # The control: without a protocol-0 peer the sender keeps the current layout, + # so the test above measures negotiation and not a broken default. + ds_manager, do_manager = pair + _submit(ds_manager, do_manager, "current.job") + + found = do_manager.job_client.manager.find_submission_ref( + do_manager.email, "current.job" + ) + assert found.protocol_version != "0" + assert [job.name for job in do_manager.jobs] == ["current.job"] diff --git a/tests/migrations/p2p/test_job_schema_negotiation.py b/tests/migrations/p2p/test_job_schema_negotiation.py new file mode 100644 index 00000000000..f1051e5f8c7 --- /dev/null +++ b/tests/migrations/p2p/test_job_schema_negotiation.py @@ -0,0 +1,105 @@ +"""Peer-advertised job schemas drive JobStorage protocol negotiation.""" + +from pathlib import Path + +from syft_job import SyftJobConfig +from syft_job.client import JobClient +from syft_job.job_storage import JobStorage +from syft_job.migrations.registry import JOB_PROTOCOL_VERSION +from syft_migration import ProtocolSchema + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" + + +def _job_schema(protocol_version: str) -> ProtocolSchema: + # The slim form a peer advertises in its VersionInfo (see + # version_info._slim_schema_of): no embedded object schemas. + return ProtocolSchema( + protocol_name="syft-job", + version=protocol_version, + supported_versions={"JobState": ["1"], "JobSubmissionMetadata": ["1"]}, + ) + + +def _storage(tmp_path: Path, peer_schemas: dict) -> JobStorage: + config = SyftJobConfig( + syftbox_folder=tmp_path / "SyftBox", current_user_email=DS_EMAIL + ) + (tmp_path / "SyftBox" / DS_EMAIL).mkdir(parents=True, exist_ok=True) + return JobStorage(config=config, peer_schemas=peer_schemas) + + +def test_protocol0_peer_negotiates_down(tmp_path): + storage = _storage(tmp_path, {DO_EMAIL: _job_schema("0")}) + assert storage.negotiated_protocol_version_for_peer(DO_EMAIL) == "0" + ref = storage.new_submission_ref(DO_EMAIL, "legacy.job") + assert ref.protocol_version == "0" + # Protocol 0 = flat pre-versioning layout: no v path segment. + assert f"v{JOB_PROTOCOL_VERSION}" not in storage.submission_dir(ref).parts + + +def test_current_peer_negotiates_current(tmp_path): + storage = _storage(tmp_path, {DO_EMAIL: _job_schema(JOB_PROTOCOL_VERSION)}) + assert ( + storage.negotiated_protocol_version_for_peer(DO_EMAIL) == JOB_PROTOCOL_VERSION + ) + ref = storage.new_submission_ref(DO_EMAIL, "current.job") + assert ref.protocol_version == JOB_PROTOCOL_VERSION + assert f"v{JOB_PROTOCOL_VERSION}" in storage.submission_dir(ref).parts + + +def test_unknown_peer_keeps_current_protocol_assumption(tmp_path): + storage = _storage(tmp_path, {}) + ref = storage.new_submission_ref(DO_EMAIL, "unknown.job") + assert ref.protocol_version == JOB_PROTOCOL_VERSION + + +def test_live_map_updates_are_seen_by_storage(tmp_path): + # JobStorage holds the dict by reference: schemas arriving after + # construction (peer version files loading) change negotiation. + live: dict = {} + storage = _storage(tmp_path, live) + assert ( + storage.new_submission_ref(DO_EMAIL, "before.job").protocol_version + == JOB_PROTOCOL_VERSION + ) + live[DO_EMAIL] = _job_schema("0") + assert storage.new_submission_ref(DO_EMAIL, "after.job").protocol_version == "0" + + +def test_job_client_from_config_passes_schemas_through(tmp_path): + config = SyftJobConfig( + syftbox_folder=tmp_path / "SyftBox", current_user_email=DS_EMAIL + ) + (tmp_path / "SyftBox" / DS_EMAIL).mkdir(parents=True, exist_ok=True) + live = {DO_EMAIL: _job_schema("0")} + client = JobClient.from_config(config, peer_schemas=live) + assert client.manager.peer_schemas is live + + +def test_newer_peer_clamps_to_our_protocol(tmp_path): + # A peer speaking a future protocol negotiates down to ours (min). + storage = _storage(tmp_path, {DO_EMAIL: _job_schema("99")}) + assert ( + storage.negotiated_protocol_version_for_peer(DO_EMAIL) == JOB_PROTOCOL_VERSION + ) + + +def test_downgrade_write_uses_slim_peer_schema(tmp_path): + # The write path's downgrade target comes from the slim advertised schema + # (supported_versions only) — no dependency on current_object_schemas. + from syft_job.models import JobSubmissionMetadataV1 + from datetime import datetime, timezone + + storage = _storage(tmp_path, {DO_EMAIL: _job_schema("0")}) + ref = storage.new_submission_ref(DO_EMAIL, "legacy.job") + metadata = JobSubmissionMetadataV1( + name="legacy.job", + submitted_by=DS_EMAIL, + datasite_email=DO_EMAIL, + submitted_at=datetime.now(tz=timezone.utc), + ) + path = storage.write_submission(ref, metadata) + assert path.exists() + assert storage.read_submission(ref).name == "legacy.job" diff --git a/tests/migrations/p2p/test_message_protocol_downgrade.py b/tests/migrations/p2p/test_message_protocol_downgrade.py new file mode 100644 index 00000000000..dc2141876a0 --- /dev/null +++ b/tests/migrations/p2p/test_message_protocol_downgrade.py @@ -0,0 +1,198 @@ +"""An outgoing sync message is downgraded to the peer's negotiated protocol. + +The receive side already upgrades: every router receive path decodes through +load_as_latest, so an old blob reads on a new client. The send side is the +other half of that contract. A sender at a newer message version must write the +version the recipient's protocol supports, or the recipient cannot decode the +blob at all -- there is no newer class in its registry to load. + +These tests drive the two send paths of the ConnectionRouter (DS -> DO +proposals, DO -> DS events) against a peer that advertises an older syft +protocol, and read the raw bytes off the mock drive to see which version was +actually put on the wire. +""" + +import json +import logging + +import pytest +from syft.migrations import client_registry +from syft.sync.events.file_change_event import FileChangeEventsMessageV1 +from syft.sync.messages.proposed_filechange import ProposedFileChangesMessageV1 +from syft.sync.syftbox_manager import SyftboxManager +from syft.sync.utils.syftbox_utils import uncompress_data +from syft_migration import MigrationError, ProtocolSchema + +from tests.unit.utils import get_mock_events_messages, mock_message + + +def _client_schema( + protocol_version: str, min_supported_version: str = "0" +) -> ProtocolSchema: + # The slim form a peer advertises in its VersionInfo. + return ProtocolSchema( + protocol_name="syft", + version=protocol_version, + min_supported_version=min_supported_version, + supported_versions={ + "VersionInfo": ["1"], + "ProposedFileChangesMessage": ["1"], + "FileChangeEventsMessage": ["1"], + }, + ) + + +@pytest.fixture +def pair(): + return SyftboxManager.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + sync_automatically=False, + ) + + +@pytest.fixture +def v2_wire_envelopes(): + """Throwaway V2 envelope classes, as the next release would ship them. + + Subclassing a registered class inherits its registry, so these register + into the global client_registry; the teardown pops them back out so no + other test sees a version "2" (the registry has no deregister API). + """ + + class ProposedFileChangesMessageV2(ProposedFileChangesMessageV1): + version: str = "2" + + class FileChangeEventsMessageV2(FileChangeEventsMessageV1): + version: str = "2" + + for canonical_name, v1, v2 in ( + ( + "ProposedFileChangesMessage", + ProposedFileChangesMessageV1, + ProposedFileChangesMessageV2, + ), + ( + "FileChangeEventsMessage", + FileChangeEventsMessageV1, + FileChangeEventsMessageV2, + ), + ): + client_registry.register_migration( + canonical_name=canonical_name, + from_version="1", + to_version="2", + fn=lambda obj, v2=v2: v2(**obj.model_dump(exclude={"version"})), + ) + client_registry.register_migration( + canonical_name=canonical_name, + from_version="2", + to_version="1", + fn=lambda obj, v1=v1: v1(**obj.model_dump(exclude={"version"})), + ) + + yield ProposedFileChangesMessageV2, FileChangeEventsMessageV2 + + for canonical_name in ("ProposedFileChangesMessage", "FileChangeEventsMessage"): + client_registry.objects[canonical_name].pop("2", None) + client_registry.migrations.get(canonical_name, {}).pop(("1", "2"), None) + client_registry.migrations.get(canonical_name, {}).pop(("2", "1"), None) + + +def _raw_proposal_version(do_manager, ds_email: str) -> str: + """The version field of the next proposal blob in the DO's inbox, unparsed.""" + raw, _ = do_manager._connection_router.connections[ + 0 + ].owner_download_next_raw_proposed_message_from_inbox(ds_email) + return json.loads(uncompress_data(raw))["version"] + + +def _raw_outbox_versions(ds_manager, do_email: str) -> list[str]: + """The version fields of the DO's outbox blobs for us, unparsed.""" + raw_list = ds_manager._connection_router.connections[ + 0 + ].watcher_download_raw_events_from_outbox(do_email, None) + return [json.loads(uncompress_data(raw))["version"] for raw in raw_list] + + +def test_a_v2_proposal_for_a_protocol0_peer_downgrades_on_the_wire( + pair, v2_wire_envelopes +): + ds_manager, do_manager = pair + v2_proposed, _ = v2_wire_envelopes + # The DO advertises client protocol 0, as a client of 0.1.117 or earlier does. + ds_manager.peer_manager.live_peer_schemas("syft")[do_manager.email] = ( + _client_schema("0") + ) + + message = v2_proposed(**mock_message().model_dump(exclude={"version"})) + ds_manager._connection_router.watcher_send_proposed_file_changes_message( + do_manager.email, message + ) + + assert _raw_proposal_version(do_manager, ds_manager.email) == "1", ( + "the peer's protocol supports message version 1 only, so the sender " + "must downgrade before the blob goes up" + ) + + +def test_a_v2_events_message_for_a_protocol0_peer_downgrades_on_the_wire( + pair, v2_wire_envelopes +): + ds_manager, do_manager = pair + _, v2_events = v2_wire_envelopes + do_manager.peer_manager.live_peer_schemas("syft")[ds_manager.email] = ( + _client_schema("0") + ) + + message = v2_events( + **get_mock_events_messages(1)[0].model_dump(exclude={"version"}) + ) + do_manager._connection_router.owner_write_event_messages_to_outbox( + ds_manager.email, message + ) + + assert _raw_outbox_versions(ds_manager, do_manager.email) == ["1"] + + +def test_a_send_beyond_the_peers_floor_raises(pair): + # A future peer that dropped support for our protocol. Sending anyway would + # put up a blob the peer refuses; the negotiation must fail loudly instead. + ds_manager, do_manager = pair + ds_manager.peer_manager.live_peer_schemas("syft")[do_manager.email] = ( + _client_schema("2", min_supported_version="2") + ) + + with pytest.raises(MigrationError): + ds_manager._connection_router.watcher_send_proposed_file_changes_message( + do_manager.email, mock_message() + ) + + +def test_a_send_to_an_unknown_peer_warns_and_keeps_the_current_version(pair, caplog): + # Same policy as jobs: a peer without a known schema is assumed to run the + # current protocol, and the assumption is logged. + ds_manager, do_manager = pair + ds_manager.peer_manager.live_peer_schemas("syft").pop(do_manager.email, None) + + with caplog.at_level(logging.WARNING): + ds_manager._connection_router.watcher_send_proposed_file_changes_message( + do_manager.email, mock_message() + ) + + assert "No syft protocol schema known" in caplog.text + assert _raw_proposal_version(do_manager, ds_manager.email) == "1" + + +def test_a_current_protocol_peer_gets_the_current_version(pair): + # The control: a peer on our own protocol gets the current version, so the + # tests above measure the downgrade and not a broken default. + ds_manager, do_manager = pair + ds_manager.peer_manager.live_peer_schemas("syft")[do_manager.email] = ( + _client_schema("1") + ) + + ds_manager._connection_router.watcher_send_proposed_file_changes_message( + do_manager.email, mock_message() + ) + + assert _raw_proposal_version(do_manager, ds_manager.email) == "1" diff --git a/tests/migrations/p2p/test_older_protocol_compatibility.py b/tests/migrations/p2p/test_older_protocol_compatibility.py new file mode 100644 index 00000000000..594bebc1fb0 --- /dev/null +++ b/tests/migrations/p2p/test_older_protocol_compatibility.py @@ -0,0 +1,122 @@ +"""Reading every released syft serialized format with the current code. + +Each fixture directory under fixtures/ is named ``--protocol

`` +(``syft_client-`` before the package was renamed to ``syft``) +and holds the serialized artifacts exactly as that release produced them: the +published SYFT_version.json, one msgv2 proposed-changes blob and one events blob. +The current code must still read, upgrade and round-trip all of them. + +Protocol 0 is the last release (0.1.117) without canonical_name/version identity +fields; protocol >= 1 writes them. Fixtures are generated by +scripts/generate_release_fixture.py (protocol 0 / 0.1.117 predates it and is +hand-authored). +""" + +import json +import re +from pathlib import Path + +import pytest + +from syft.migrations import client_registry +from syft.sync.events.file_change_event import FileChangeEventsMessage +from syft.sync.messages.proposed_filechange import ProposedFileChangesMessage +from syft.sync.utils.syftbox_utils import uncompress_data +from syft.sync.version.version_info import VersionInfo + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +RELEASE_FIXTURES = sorted(FIXTURES_DIR.glob("*-protocol*")) + +released_fixtures = pytest.mark.parametrize( + "fixture", RELEASE_FIXTURES, ids=lambda f: f.name +) + + +def _protocol_of(fixture: Path) -> str: + return re.search(r"-protocol(\d+)$", fixture.name).group(1) + + +def _has_identity(protocol: str) -> bool: + """canonical_name/version are written only from protocol 1 onwards.""" + return protocol != "0" + + +def _single(fixture: Path, pattern: str) -> Path: + matches = list(fixture.glob(pattern)) + assert len(matches) == 1, f"expected one {pattern} in {fixture.name}" + return matches[0] + + +def test_fixtures_exist(): + assert RELEASE_FIXTURES, "no release fixtures found" + + +@released_fixtures +def test_version_file_reads_and_round_trips(fixture: Path): + raw = _single(fixture, "SYFT_version.json").read_text() + assert ("canonical_name" in json.loads(raw)) == _has_identity(_protocol_of(fixture)) + + info = VersionInfo.from_json(raw) + assert info.version == client_registry.latest_version("VersionInfo") + assert info.syft_client_version + # Round-trip: what the current code writes must load again. + assert VersionInfo.from_json(info.to_json()) == info + + +@released_fixtures +def test_proposed_message_reads_and_round_trips(fixture: Path): + blob = _single(fixture, "msgv2_*.tar.gz").read_bytes() + assert ("canonical_name" in json.loads(uncompress_data(blob))) == _has_identity( + _protocol_of(fixture) + ) + + message = ProposedFileChangesMessage.from_compressed_data(blob) + assert message.version == client_registry.latest_version( + "ProposedFileChangesMessage" + ) + changes = {c.path_in_datasite.name: c for c in message.proposed_file_changes} + assert changes["notes.txt"].content == "hello from the release fixture" + assert changes["blob.bin"].content == b"\x00\x01\x02fixture-binary" + deletion = changes["removed.txt"] + assert deletion.is_deleted and deletion.content is None + assert deletion.new_hash is None and deletion.content_type is None + + # Upgrade-on-write: what the current code re-emits carries identity fields. + rewritten = message.as_compressed_data() + assert b'"canonical_name"' in uncompress_data(rewritten) + restored = ProposedFileChangesMessage.from_compressed_data(rewritten) + assert restored.proposed_file_changes == message.proposed_file_changes + + +@released_fixtures +def test_events_message_reads_and_round_trips(fixture: Path): + blob = _single(fixture, "syfteventsmessagev3_*.tar.gz").read_bytes() + assert ("canonical_name" in json.loads(uncompress_data(blob))) == _has_identity( + _protocol_of(fixture) + ) + + message = FileChangeEventsMessage.from_compressed_data(blob) + assert message.version == client_registry.latest_version("FileChangeEventsMessage") + events = {e.path_in_datasite.name: e for e in message.events} + assert events["notes.txt"].content == "hello from the release fixture" + assert events["blob.bin"].content == b"\x00\x01\x02fixture-binary" + assert events["removed.txt"].is_deleted and events["removed.txt"].content is None + + # Upgrade-on-write: what the current code re-emits carries identity fields. + rewritten = message.as_compressed_data() + assert b'"canonical_name"' in uncompress_data(rewritten) + restored = FileChangeEventsMessage.from_compressed_data(rewritten) + assert restored.events == message.events + + +@released_fixtures +def test_fixture_protocol_is_in_registry_history(fixture: Path): + """Every fixture's protocol is either the current one or a released one + the registry knows the schema for (so downgrades can target it).""" + protocol = _protocol_of(fixture) + schema = client_registry.schema_for_protocol_version(protocol) + assert set(schema.supported_versions) == { + "VersionInfo", + "ProposedFileChangesMessage", + "FileChangeEventsMessage", + } diff --git a/tests/migrations/p2p/test_private_dataset_protocol_skew.py b/tests/migrations/p2p/test_private_dataset_protocol_skew.py new file mode 100644 index 00000000000..c725e46117b --- /dev/null +++ b/tests/migrations/p2p/test_private_dataset_protocol_skew.py @@ -0,0 +1,122 @@ +"""Private dataset files ship in the layout the receiving peer reads. + +A private share sends the files of one local copy to an enclave as outbox +events, path by path. The paths carry the copy's protocol layout: flat for +protocol 0, a v segment from protocol 1 on. A receiver scans only the +layouts it knows, so files at paths of a newer layout never become a readable +dataset there -- the job that needed them cannot find its input. + +These tests drive share_private_dataset against a recipient that advertises an +older dataset protocol and assert on the paths of the events that actually go +out. +""" + +import pytest +from syft_rds import SyftRDSClient +from syft_migration import ProtocolSchema + +from tests.unit.utils import create_tmp_dataset_files + +# An audience member on an earlier client, so a create writes both layouts. +OLD_PEER = "old@test.org" + + +def _dataset_schema(protocol_version: str) -> ProtocolSchema: + # The slim form a peer advertises in its VersionInfo. + return ProtocolSchema( + protocol_name="syft-dataset", + version=protocol_version, + supported_versions={"Dataset": ["1"]}, + ) + + +@pytest.fixture +def pair(): + return SyftRDSClient.pair_with_mock_drive_service_connection( + use_in_memory_cache=False, + ) + + +def _create(ds_manager, do_manager, name: str, mixed_audience: bool): + """Create a dataset locally; with a mixed audience both layouts exist.""" + users = [ds_manager.email] + if mixed_audience: + do_manager.peer_manager.live_peer_schemas("syft-dataset")[OLD_PEER] = ( + _dataset_schema("0") + ) + users.append(OLD_PEER) + mock_path, private_path, readme_path = create_tmp_dataset_files() + return do_manager.create_dataset( + name=name, + mock_path=mock_path, + private_path=private_path, + readme_path=readme_path, + users=users, + ) + + +def _shipped_private_paths(ds_manager, do_manager) -> list[str]: + """The private-file paths of the events the DO put in our outbox.""" + messages = ds_manager.sync_engine._connection_router.watcher_get_events_messages( + do_manager.email, None + ) + return [ + str(event.path_in_datasite) + for message in messages + for event in message.events + if "private/syft_datasets" in str(event.path_in_datasite) + ] + + +def test_private_files_for_a_protocol0_peer_ship_in_the_flat_layout(pair): + ds_manager, do_manager = pair + _create(ds_manager, do_manager, "mixed private", mixed_audience=True) + # The recipient advertises dataset protocol 0, as an earlier client does. + do_manager.peer_manager.live_peer_schemas("syft-dataset")[ds_manager.email] = ( + _dataset_schema("0") + ) + + do_manager.share_private_dataset("mixed private", ds_manager.email) + + paths = _shipped_private_paths(ds_manager, do_manager) + assert paths, "the private files should have shipped" + for path in paths: + assert path.startswith("private/syft_datasets/mixed private/"), ( + f"a protocol-0 peer scans the flat layout only, got: {path}" + ) + + +def test_private_files_for_a_current_peer_ship_in_the_versioned_layout(pair): + # The control: a peer of our own protocol gets the newest layout, so the + # test above measures negotiation and not a broken default. + ds_manager, do_manager = pair + _create(ds_manager, do_manager, "mixed private", mixed_audience=True) + + do_manager.share_private_dataset("mixed private", ds_manager.email) + + paths = _shipped_private_paths(ds_manager, do_manager) + assert paths + for path in paths: + assert path.startswith("private/syft_datasets/v1/mixed private/") + + +def test_a_missing_flat_copy_is_materialized_for_a_protocol0_peer(pair): + # The dataset was created for a current audience, so no flat copy exists. + # The share must create one -- the same fill that share_dataset applies -- + # because shipping the v1 paths gives the peer files it never scans. + ds_manager, do_manager = pair + _create(ds_manager, do_manager, "v1 only", mixed_audience=False) + storage = do_manager.dataset_manager.storage + flat_dir = storage.private_dataset_dir(storage.new_dataset_ref("v1 only", "0")) + assert not flat_dir.exists(), "the flat copy should not exist before the share" + + do_manager.peer_manager.live_peer_schemas("syft-dataset")[ds_manager.email] = ( + _dataset_schema("0") + ) + do_manager.share_private_dataset("v1 only", ds_manager.email) + + paths = _shipped_private_paths(ds_manager, do_manager) + assert paths + for path in paths: + assert path.startswith("private/syft_datasets/v1 only/") + assert flat_dir.exists(), "the flat copy is materialized by the share" diff --git a/tests/migrations/p2p/test_unknown_peer_forced_path.py b/tests/migrations/p2p/test_unknown_peer_forced_path.py new file mode 100644 index 00000000000..f680d87ce70 --- /dev/null +++ b/tests/migrations/p2p/test_unknown_peer_forced_path.py @@ -0,0 +1,75 @@ +"""A forced submission reports the protocol version that it assumes. + +A peer of unknown version is refused before this point. A caller that passes +``raise_on_unknown=False`` skips that refusal. The storage then assumes the +current protocol. + +If the peer speaks an earlier protocol, it does not scan this layout. The job or +the dataset never arrives, so the storage writes a warning. +""" + +import logging +from pathlib import Path + +from syft_datasets.config import SyftBoxConfig +from syft_datasets.dataset_storage import DatasetStorage +from syft_datasets.migrations.registry import DATASET_PROTOCOL_VERSION +from syft_job import SyftJobConfig +from syft_job.job_storage import JobStorage +from syft_job.migrations.registry import JOB_PROTOCOL_VERSION + +DO_EMAIL = "do@test.org" +DS_EMAIL = "ds@test.org" + + +def _job_storage(tmp_path: Path) -> JobStorage: + config = SyftJobConfig( + syftbox_folder=tmp_path / "SyftBox", current_user_email=DS_EMAIL + ) + (tmp_path / "SyftBox" / DS_EMAIL).mkdir(parents=True, exist_ok=True) + return JobStorage(config=config, peer_schemas={}) + + +def _dataset_storage(tmp_path: Path) -> DatasetStorage: + config = SyftBoxConfig(syftbox_folder=tmp_path / "SyftBox", email=DO_EMAIL) + (tmp_path / "SyftBox" / DO_EMAIL).mkdir(parents=True, exist_ok=True) + return DatasetStorage(config=config, peer_schemas={}) + + +def test_a_forced_job_reports_the_assumed_protocol(tmp_path, caplog): + storage = _job_storage(tmp_path) + with caplog.at_level(logging.WARNING): + version = storage.negotiated_protocol_version_for_peer( + DO_EMAIL, raise_on_unknown=False + ) + assert version == JOB_PROTOCOL_VERSION + messages = " ".join(r.getMessage() for r in caplog.records) + assert DO_EMAIL in messages + assert "earlier protocol" in messages + + +def test_a_forced_dataset_reports_the_assumed_protocol(tmp_path, caplog): + storage = _dataset_storage(tmp_path) + with caplog.at_level(logging.WARNING): + version = storage.negotiated_protocol_version_for_peer( + DS_EMAIL, raise_on_unknown=False + ) + assert version == DATASET_PROTOCOL_VERSION + messages = " ".join(r.getMessage() for r in caplog.records) + assert DS_EMAIL in messages + assert "earlier protocol" in messages + + +def test_a_known_peer_reports_nothing(tmp_path, caplog): + # The report belongs to the forced path only. A known peer is negotiated. + from syft_migration import ProtocolSchema + + storage = _job_storage(tmp_path) + storage.peer_schemas[DO_EMAIL] = ProtocolSchema( + protocol_name="syft-job", + version=JOB_PROTOCOL_VERSION, + supported_versions={"JobState": ["1"]}, + ) + with caplog.at_level(logging.WARNING): + storage.negotiated_protocol_version_for_peer(DO_EMAIL) + assert caplog.records == [] diff --git a/tests/migrations/unit/__init__.py b/tests/migrations/unit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/migrations/unit/fixtures/version_info/SYFT_version-0.1.117.json b/tests/migrations/unit/fixtures/version_info/SYFT_version-0.1.117.json new file mode 100644 index 00000000000..b6a40d2e192 --- /dev/null +++ b/tests/migrations/unit/fixtures/version_info/SYFT_version-0.1.117.json @@ -0,0 +1,9 @@ +{ + "syft_client_version": "0.1.117", + "min_supported_syft_client_version": "0.1.93", + "protocol_version": "1.0.0", + "min_supported_protocol_version": "1.0.0", + "syft_client_install_source": "pip", + "updated_at": "2026-07-20T10:15:30.123456Z", + "attestation_token": null +} diff --git a/tests/migrations/unit/test_file_change_events_serialization.py b/tests/migrations/unit/test_file_change_events_serialization.py new file mode 100644 index 00000000000..fca62d4e252 --- /dev/null +++ b/tests/migrations/unit/test_file_change_events_serialization.py @@ -0,0 +1,97 @@ +"""The events envelope round-trips and legacy protocol-0 blobs still decode.""" + +import base64 +import json +from uuid import uuid4 + +from syft.migrations import client_registry +from syft.sync.events.file_change_event import ( + FileChangeEvent, + FileChangeEventsMessage, + FileChangeEventsMessageV1, + FileChangeEventV1, +) +from syft.sync.messages.proposed_filechange import ProposedFileChange +from syft.sync.utils.syftbox_utils import compress_data + + +def _make_event(content) -> FileChangeEvent: + return FileChangeEvent( + id=uuid4(), + path_in_datasite="data/file.txt", + datasite_email="do@test.org", + content=content, + submitted_timestamp=1752900000.0, + timestamp=1752900001.0, + ) + + +def test_envelope_registered_items_not(): + assert client_registry.versions("FileChangeEventsMessage") + assert not client_registry.versions("FileChangeEvent") + assert FileChangeEventsMessage is FileChangeEventsMessageV1 + assert FileChangeEvent is FileChangeEventV1 + + +def test_round_trip_text_binary_and_deletion(): + for content in ["hello", b"\x00\x01binary", None]: + original = FileChangeEventsMessage(events=[_make_event(content)]) + restored = FileChangeEventsMessage.from_compressed_data( + original.as_compressed_data() + ) + assert restored.events[0].content == content + assert restored.events[0].content_type == original.events[0].content_type + assert restored.message_filepath == original.message_filepath + + +def test_identity_fields_on_the_wire(): + # load_as_latest would setdefault them back, so the round-trip alone + # cannot catch a silently broken serialization of the identity fields. + data = json.loads( + FileChangeEventsMessage(events=[_make_event("x")]).model_dump_json() + ) + assert data["canonical_name"] == "FileChangeEventsMessage" + assert data["version"] == "1" + + +def test_legacy_protocol0_blob_decodes_as_latest(): + # A blob exactly as a <= 0.1.117 client writes it: no identity fields. + legacy = { + "events": [ + { + "id": "9c1a2e75-8a45-4a17-b7f2-0d94d13d3c60", + "path_in_datasite": "data/blob.bin", + "datasite_email": "do@test.org", + "content": base64.b64encode(b"\x00\x01binary").decode("utf-8"), + "content_type": "binary", + "old_hash": None, + "new_hash": "abc123", + "is_deleted": False, + "submitted_timestamp": 1752900000.0, + "timestamp": 1752900001.0, + } + ], + "message_filepath": { + "id": "6f9d5f57-31f7-4302-8746-9ba030e88961", + "timestamp": 1752900001.0, + "extension": ".tar.gz", + }, + } + blob = compress_data(json.dumps(legacy).encode("utf-8")) + + message = FileChangeEventsMessage.from_compressed_data(blob) + assert message.version == client_registry.latest_version("FileChangeEventsMessage") + assert message.events[0].content == b"\x00\x01binary" + + +def test_from_proposed_filechange_carries_identity_free_items(): + proposed = ProposedFileChange( + path_in_datasite="data/file.txt", + content="hello", + datasite_email="do@test.org", + ) + event = FileChangeEvent.from_proposed_filechange(proposed) + assert event.id == proposed.id + assert event.new_hash == proposed.new_hash + # Items carry no identity fields on the wire; only envelopes do. + assert "canonical_name" not in json.loads(event.model_dump_json()) diff --git a/tests/migrations/unit/test_history_artifacts.py b/tests/migrations/unit/test_history_artifacts.py new file mode 100644 index 00000000000..bf92554b713 --- /dev/null +++ b/tests/migrations/unit/test_history_artifacts.py @@ -0,0 +1,119 @@ +"""Past release artifacts register cleanly and guard against schema drift.""" + +import pytest +import syft # noqa: F401 -- imports models and registers history +from syft_migration import MigrationError, MigrationRegistry, ReleasedProtocol + +from syft.migrations import client_registry +from syft.migrations.history import ( + PACKAGE_ARTIFACTS_DIR, + PROTOCOLS_DIR, + register_historic_schemas, +) + +PROTOCOL_0_PATH = PROTOCOLS_DIR / "protocol-0.json" + + +def test_protocol0_artifacts_registered(): + # importing syft registered the hardcoded 0.1.117 artifacts. + assert "0" in client_registry.protocol_version_history + package_info = client_registry.package_version_history["0"] + # The distribution was still named syft-client at 0.1.117. + assert package_info.package_name == "syft-client" + assert package_info.version == "0.1.117" + + schema = client_registry.protocol_version_history["0"] + assert schema.supported_versions == { + "VersionInfo": ["1"], + "ProposedFileChangesMessage": ["1"], + "FileChangeEventsMessage": ["1"], + } + + +def test_registering_again_is_idempotent(): + register_historic_schemas() + assert "0" in client_registry.protocol_version_history + + +def test_all_protocol_artifacts_well_formed(): + # Filename encodes the frozen protocol version, and every supported + # canonical name freezes a current-object schema (catches a mis-named or + # hand-edited artifact). + paths = sorted(PROTOCOLS_DIR.glob("*.json")) + assert paths, "no released protocol artifacts found" + for path in paths: + schema = ReleasedProtocol.load(path).protocol_schema + assert path.name == f"protocol-{schema.version}.json" + assert set(schema.current_object_schemas) == set(schema.supported_versions) + + +def test_no_schema_drift_against_released_protocols(): + # Every schema frozen by a released protocol must still be produced + # byte-identically by the class registered for that version. + assert client_registry.find_schema_drift() == [], ( + "A released object schema drifted. Fix by either: (1) reverting the " + "model change; or (2) adding a new V model class, registering " + "migrations in both directions, and bumping " + "SYFT_CLIENT_PROTOCOL_VERSION in syft/migrations/registry.py. " + "If the drift comes from a pydantic upgrade changing JSON-schema " + "output only, regenerate the artifacts instead." + ) + + +def test_protocol_not_changed_without_bump(): + assert not client_registry.protocol_changed_without_bump() + + +def test_protocol_bump_not_missing(): + # Stays live between a protocol bump and the release that freezes it, which is + # exactly where test_protocol_not_changed_without_bump goes quiet. + assert not client_registry.protocol_bump_missing(), ( + "The client protocol changed since the newest released protocol without a " + "bump. Bump SYFT_CLIENT_PROTOCOL_VERSION in " + "syft/migrations/registry.py, or revert the model change." + ) + + +def test_bump_guard_trips_on_protocol_change(): + # A registry claiming the same protocol version as a released schema but + # supporting different object versions must trip the guard. + stale = MigrationRegistry( + protocol_name=client_registry.protocol_name, + package_name=client_registry.package_name, + package_version=client_registry.package_version, + protocol_version="0", # pretend we still speak the released protocol 0 + ) + # Register only a subset of protocol-0's objects, then load its schema. + stale.register_object_version(client_registry.get_class("VersionInfo", "1")) + stale.register_historic_protocol_schema( + ReleasedProtocol.load(PROTOCOL_0_PATH).protocol_schema + ) + assert stale.protocol_changed_without_bump() + + +def test_unknown_object_version_in_artifact_raises(): + # The fail-at-import guarantee syft/__init__.py relies on: an + # artifact naming an object version this release cannot load must raise. + schema = ReleasedProtocol.load(PROTOCOL_0_PATH).protocol_schema + schema = schema.model_copy( + update={ + "supported_versions": { + **schema.supported_versions, + "VersionInfo": ["1", "99"], + } + } + ) + empty = MigrationRegistry( + protocol_name=client_registry.protocol_name, + package_name=client_registry.package_name, + package_version=client_registry.package_version, + protocol_version=client_registry.protocol_version, + ) + empty.register_object_version(client_registry.get_class("VersionInfo", "1")) + with pytest.raises(MigrationError): + empty.register_historic_protocol_schema(schema, raise_for_unknown_objects=True) + + +def test_artifact_files_exist(): + assert (PACKAGE_ARTIFACTS_DIR / "syft-client-0.1.117.json").exists() + assert PROTOCOL_0_PATH.exists() diff --git a/tests/migrations/unit/test_peer_manager_schemas.py b/tests/migrations/unit/test_peer_manager_schemas.py new file mode 100644 index 00000000000..cab3d0e52c9 --- /dev/null +++ b/tests/migrations/unit/test_peer_manager_schemas.py @@ -0,0 +1,64 @@ +"""PeerManager's live peer-schema maps track loaded peer versions.""" + +from syft.sync.version.peer_manager import PeerManager +from syft.sync.version.version_info import VersionInfo, VersionInfoV1 + + +def _peer_manager() -> PeerManager: + # Construct without model_validate side effects: only the private schema + # map and _update_peer_schemas are exercised here. + return PeerManager.model_construct() + + +def _v2_with_schemas() -> VersionInfo: + return VersionInfo.current() + + +def _v1() -> VersionInfoV1: + return VersionInfoV1( + syft_client_version="0.1.117", + min_supported_syft_client_version="0.1.93", + protocol_version="1.0.0", + min_supported_protocol_version="1.0.0", + ) + + +def test_advertising_peer_appears_in_live_map(): + pm = _peer_manager() + live = pm.live_peer_schemas("syft-job") + pm._update_peer_schemas("do@test.org", _v2_with_schemas()) + assert "do@test.org" in live + assert live["do@test.org"].protocol_name == "syft-job" + + +def test_pre_v2_peer_is_an_unknown_speaker(): + pm = _peer_manager() + live = pm.live_peer_schemas("syft-job") + pm._update_peer_schemas("do@test.org", _v2_with_schemas()) + # A reloaded version file from an old client (upgraded V1: no schemas) + # must remove the stale entry. + pm._update_peer_schemas("do@test.org", _v1()) + assert live == {} + + +def test_cleared_version_removes_peer(): + pm = _peer_manager() + live = pm.live_peer_schemas("syft") + pm._update_peer_schemas("do@test.org", _v2_with_schemas()) + pm._update_peer_schemas("do@test.org", None) + assert live == {} + + +def test_map_identity_is_stable(): + # live_peer_schemas must always return the same dict object so consumers + # holding a reference see updates. + pm = _peer_manager() + assert pm.live_peer_schemas("syft-job") is pm.live_peer_schemas("syft-job") + + +def test_late_registered_map_backfills_from_loaded_versions(): + pm = _peer_manager() + pm._update_peer_schemas("do@test.org", _v2_with_schemas()) + # Registering the protocol AFTER the version loaded must not start empty. + live = pm.live_peer_schemas("syft-job") + assert "do@test.org" in live diff --git a/tests/migrations/unit/test_proposed_filechange_serialization.py b/tests/migrations/unit/test_proposed_filechange_serialization.py new file mode 100644 index 00000000000..d4ed82ee0c3 --- /dev/null +++ b/tests/migrations/unit/test_proposed_filechange_serialization.py @@ -0,0 +1,91 @@ +"""The msgv2 envelope round-trips and legacy protocol-0 blobs still decode.""" + +import base64 +import json + +from syft.migrations import client_registry +from syft.sync.messages.proposed_filechange import ( + ProposedFileChange, + ProposedFileChangesMessage, + ProposedFileChangesMessageV1, + ProposedFileChangeV1, +) +from syft.sync.utils.syftbox_utils import compress_data + + +def _make_message(content) -> ProposedFileChangesMessage: + return ProposedFileChangesMessage( + sender_email="ds@test.org", + proposed_file_changes=[ + ProposedFileChange( + path_in_datasite="data/file.txt", + content=content, + datasite_email="do@test.org", + ) + ], + ) + + +def test_envelope_registered_items_not(): + # The envelope is the migratable unit; items ride inside it. + assert client_registry.versions("ProposedFileChangesMessage") + assert not client_registry.versions("ProposedFileChange") + assert ProposedFileChangesMessage is ProposedFileChangesMessageV1 + assert ProposedFileChange is ProposedFileChangeV1 + + +def test_round_trip_text_and_binary(): + for content in ["hello", b"\x00\x01binary"]: + original = _make_message(content) + restored = ProposedFileChangesMessage.from_compressed_data( + original.as_compressed_data() + ) + assert restored.sender_email == original.sender_email + assert restored.proposed_file_changes[0].content == content + assert ( + restored.proposed_file_changes[0].new_hash + == original.proposed_file_changes[0].new_hash + ) + + +def test_legacy_protocol0_blob_decodes_as_latest(): + # A blob exactly as a <= 0.1.117 client writes it: no identity fields + # on the envelope, base64 binary content on the item. + legacy = { + "id": "8be509b2-4340-44db-a3a4-b0ecf8c463f4", + "sender_email": "ds@test.org", + "message_filename": { + "submitted_timestamp": 1752900000.0, + "uid": "6f9d5f57-31f7-4302-8746-9ba030e88961", + }, + "proposed_file_changes": [ + { + "id": "9c1a2e75-8a45-4a17-b7f2-0d94d13d3c60", + "old_hash": None, + "submitted_timestamp": 1752900000.0, + "path_in_datasite": "data/blob.bin", + "content": base64.b64encode(b"\x00\x01binary").decode("utf-8"), + "content_type": "binary", + "datasite_email": "do@test.org", + "is_deleted": False, + } + ], + } + blob = compress_data(json.dumps(legacy).encode("utf-8")) + + message = ProposedFileChangesMessage.from_compressed_data(blob) + assert message.version == client_registry.latest_version( + "ProposedFileChangesMessage" + ) + change = message.proposed_file_changes[0] + assert change.content == b"\x00\x01binary" + assert change.content_type == "binary" + # pre_init derived the hash from the payload content only. + assert change.new_hash + + +def test_identity_fields_on_wire_but_platform_id_excluded(): + data = json.loads(_make_message("x").model_dump_json()) + assert data["canonical_name"] == "ProposedFileChangesMessage" + assert data["version"] == "1" + assert "platform_id" not in data diff --git a/tests/migrations/unit/test_registry.py b/tests/migrations/unit/test_registry.py new file mode 100644 index 00000000000..bb5b3b45377 --- /dev/null +++ b/tests/migrations/unit/test_registry.py @@ -0,0 +1,25 @@ +"""The syft migration registry exists and computes a valid protocol schema.""" + +from syft.migrations import ( + PROTOCOL_NAME, + SYFT_CLIENT_PROTOCOL_VERSION, + client_registry, +) +from syft.version import SYFT_VERSION + + +def test_registry_identity(): + assert client_registry.protocol_name == PROTOCOL_NAME + assert client_registry.package_name == "syft" + assert client_registry.package_version == SYFT_VERSION + assert client_registry.protocol_version == SYFT_CLIENT_PROTOCOL_VERSION + + +def test_registry_computes_protocol_schema(): + schema = client_registry.compute_protocol_schema() + assert schema.protocol_name == PROTOCOL_NAME + assert schema.version == SYFT_CLIENT_PROTOCOL_VERSION + # Every registered object resolves a current version and a frozen schema. + for canonical_name in schema.supported_versions: + assert schema.current_schema(canonical_name=canonical_name) + assert canonical_name in schema.current_object_schemas diff --git a/tests/migrations/unit/test_version_info_fields.py b/tests/migrations/unit/test_version_info_fields.py new file mode 100644 index 00000000000..404b6bd3c26 --- /dev/null +++ b/tests/migrations/unit/test_version_info_fields.py @@ -0,0 +1,74 @@ +"""VersionInfo may only grow, because it is the bootstrap channel. + +A peer reads SYFT_version.json before it knows anything else, so every supported +client must parse every newer file. Two rules follow, and neither is enforced by +the migration system: + +- A field of an older version must not disappear or change name. An older reader + requires it, and pydantic raises when it is absent. +- A field that a newer version adds must have a default. A newer reader must + still parse a file that an older client wrote without that field. + +Adding a field is safe on its own: pydantic ignores a field it does not know. +""" + +import syft # noqa: F401 -- imports models and registers history +from syft.sync.version.version_info import VersionInfoV1, VersionInfoV2 + +# Frozen on purpose. A change here means a change to the bootstrap file, so read +# the two rules above before editing this set. +V1_FIELDS = { + "canonical_name", + "version", + "syft_client_version", + "min_supported_syft_client_version", + "protocol_version", + "min_supported_protocol_version", + "syft_client_install_source", + "updated_at", + "attestation_token", +} + +V2_ADDS = {"protocol_schemas"} + + +def test_v1_fields_are_frozen(): + assert set(VersionInfoV1.model_fields) == V1_FIELDS, ( + "VersionInfoV1 changed. A client that speaks protocol 0 reads this " + "object, so a removed or renamed field stops that client from parsing " + "the version file of this one." + ) + + +def test_v2_keeps_every_v1_field(): + missing = V1_FIELDS - set(VersionInfoV2.model_fields) + assert not missing, ( + f"VersionInfoV2 dropped {sorted(missing)}. A reader of V1 requires these " + "fields, so V2 must keep them." + ) + + +def test_v2_adds_only_the_expected_fields(): + assert set(VersionInfoV2.model_fields) - V1_FIELDS == V2_ADDS + + +def test_fields_added_after_v1_have_a_default(): + # A file written by an older client carries none of these, so a reader of the + # newer version must supply a value. + for name in set(VersionInfoV2.model_fields) - V1_FIELDS: + assert not VersionInfoV2.model_fields[name].is_required(), ( + f"VersionInfoV2.{name} is required. A version file written before " + "this field existed would then fail to parse." + ) + + +def test_a_file_without_the_v2_fields_still_parses(): + written_by_an_older_client = VersionInfoV1( + syft_client_version="0.1.117", + min_supported_syft_client_version="0.1.93", + protocol_version="1.0.0", + min_supported_protocol_version="1.0.0", + ).model_dump(exclude={"canonical_name", "version"}) + + loaded = VersionInfoV2.model_validate(written_by_an_older_client) + assert loaded.protocol_schemas == {} diff --git a/tests/migrations/unit/test_version_info_migrations.py b/tests/migrations/unit/test_version_info_migrations.py new file mode 100644 index 00000000000..73105ea7deb --- /dev/null +++ b/tests/migrations/unit/test_version_info_migrations.py @@ -0,0 +1,76 @@ +"""The first real migrations: VersionInfo v1 <-> v2 (protocol schemas).""" + +import json +from pathlib import Path + +from syft.migrations import client_migration_service, client_registry +from syft.sync.version.version_info import ( + VersionInfo, + VersionInfoV1, + VersionInfoV2, +) + +LEGACY_FILE = ( + Path(__file__).parent / "fixtures" / "version_info" / "SYFT_version-0.1.117.json" +) + + +def _v1() -> VersionInfoV1: + return VersionInfoV1.model_validate(json.loads(LEGACY_FILE.read_text())) + + +def test_both_versions_registered_with_paths_both_ways(): + assert client_registry.versions("VersionInfo") == ["1", "2"] + assert client_registry.has_migration_path("VersionInfo", "1", "2") + assert client_registry.has_migration_path("VersionInfo", "2", "1") + + +def test_v1_upgrades_to_v2_with_empty_schemas(): + upgraded = client_migration_service.migrate(_v1(), "2") + assert type(upgraded) is VersionInfoV2 + assert upgraded.version == "2" + # A v1 file says nothing about package protocols. + assert upgraded.protocol_schemas == {} + assert upgraded.syft_client_version == "0.1.117" + assert upgraded.updated_at == _v1().updated_at + + +def test_v2_downgrades_to_v1_dropping_schemas(): + current = VersionInfo.current() + assert current.protocol_schemas # populated before the downgrade + downgraded = client_migration_service.migrate(current, "1") + assert type(downgraded) is VersionInfoV1 + assert downgraded.version == "1" + assert "protocol_schemas" not in downgraded.model_dump() + assert downgraded.syft_client_version == current.syft_client_version + + +def test_downgrade_for_protocol_0_peer(): + # A protocol-0 peer's schema only lists VersionInfo v1. + downgraded = client_migration_service.downgrade_for_protocol_version( + VersionInfo.current(), "0" + ) + assert type(downgraded) is VersionInfoV1 + + +def test_current_advertises_slim_schemas(): + schemas = VersionInfo.current().protocol_schemas + # The client's own schema is always present; job/dataset only when the + # optional packages are importable (both are in the workspace env, but + # the code must degrade on client-only installs). + assert "syft" in schemas + assert set(schemas) <= {"syft", "syft-job", "syft-dataset"} + client_schema = schemas["syft"] + assert client_schema.version == client_registry.protocol_version + assert client_schema.supported_versions == ( + client_registry.compute_protocol_schema().supported_versions + ) + # Slim on the wire: no embedded per-object JSON schemas. + for schema in schemas.values(): + assert schema.current_object_schemas == {} + + +def test_legacy_file_loads_all_the_way_to_v2(): + info = VersionInfo.from_json(LEGACY_FILE.read_text()) + assert type(info) is VersionInfoV2 + assert info.protocol_schemas == {} diff --git a/tests/migrations/unit/test_version_info_serialization.py b/tests/migrations/unit/test_version_info_serialization.py new file mode 100644 index 00000000000..144f28b7a7e --- /dev/null +++ b/tests/migrations/unit/test_version_info_serialization.py @@ -0,0 +1,61 @@ +"""VersionInfo round-trips through JSON and legacy protocol-0 files still load.""" + +import json +from pathlib import Path + +from syft.migrations import client_registry +from syft.sync.version.version_info import ( + VersionInfo, + VersionInfoV2, +) + +FIXTURES_DIR = Path(__file__).parent / "fixtures" / "version_info" +LEGACY_FILE = FIXTURES_DIR / "SYFT_version-0.1.117.json" + + +def test_version_info_registered_and_aliased(): + assert client_registry.versions("VersionInfo") + assert VersionInfo is VersionInfoV2 + + schema = client_registry.compute_protocol_schema() + assert "VersionInfo" in schema.supported_versions + assert schema.current_schema(canonical_name="VersionInfo") + + +def test_current_serializes_identity_fields(): + data = json.loads(VersionInfo.current().to_json()) + assert data["canonical_name"] == "VersionInfo" + assert data["version"] == "2" + + +def test_json_round_trip(): + original = VersionInfo.current() + restored = VersionInfo.from_json(original.to_json()) + assert restored == original + + +def test_legacy_protocol0_file_loads_as_latest(): + # Written by a <= 0.1.117 client: no canonical_name/version fields. + legacy_json = LEGACY_FILE.read_text() + assert "canonical_name" not in json.loads(legacy_json) + + info = VersionInfo.from_json(legacy_json) + # type() not isinstance(): V2 subclasses V1, so isinstance is vacuous. + assert type(info) is VersionInfoV2 + assert info.version == client_registry.latest_version("VersionInfo") + assert info.syft_client_version == "0.1.117" + assert info.syft_client_install_source == "pip" + + +def test_legacy_reader_tolerates_identity_fields(): + # A protocol-0 client parses with pydantic's default extra="ignore"; the + # closest stand-in we have is validating minus the identity defaults. + data = json.loads(VersionInfo.current().to_json()) + # Legacy clients see unknown keys and ignore them; simulate by checking + # the payload minus identity fields is exactly the legacy shape. + data.pop("canonical_name") + data.pop("version") + data.pop("protocol_schemas") + # Additive-only invariant: current output minus the added fields is + # exactly the legacy shape a 0.1.117 reader expects. + assert set(data) == set(json.loads(LEGACY_FILE.read_text())) diff --git a/tests/unit/test_bump_version.py b/tests/unit/test_bump_version.py new file mode 100644 index 00000000000..2e6508f2864 --- /dev/null +++ b/tests/unit/test_bump_version.py @@ -0,0 +1,88 @@ +"""Check the version that bump_version.py writes into the pin of a dependent.""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "scripts" / "bump_version.py" + +TARGET = """\ +[project] +name = "syft-thing" +version = "0.1.9" +dependencies = [] +""" + +DEPENDENT = """\ +[project] +name = "syft-other" +version = "0.2.0" +dependencies = [ + "syft-thing==0.1.9", +] + +[tool.uv.sources] +"syft-thing" = { workspace = true } +""" + + +@pytest.fixture +def fake_repo(tmp_path): + (tmp_path / "packages" / "syft-thing").mkdir(parents=True) + (tmp_path / "packages" / "syft-other").mkdir(parents=True) + (tmp_path / "packages" / "syft-thing" / "pyproject.toml").write_text(TARGET) + (tmp_path / "packages" / "syft-other" / "pyproject.toml").write_text(DEPENDENT) + return tmp_path + + +def _run(fake_repo, *args): + spec = importlib.util.spec_from_file_location("bump_version_under_test", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.REPO_ROOT = fake_repo + argv = [str(SCRIPT), "syft-thing", "patch", *args] + old = sys.argv + sys.argv = argv + try: + module.main() + finally: + sys.argv = old + + +def _versions(fake_repo): + target = (fake_repo / "packages" / "syft-thing" / "pyproject.toml").read_text() + dependent = (fake_repo / "packages" / "syft-other" / "pyproject.toml").read_text() + source = next( + line for line in target.splitlines() if line.startswith("version") + ).split('"')[1] + pin = next(line for line in dependent.splitlines() if "syft-thing==" in line) + return source, pin.split("==")[1].split('"')[0] + + +def test_default_pins_dependents_to_the_bumped_version(fake_repo): + _run(fake_repo) + source, pin = _versions(fake_repo) + assert source == "0.1.10" + assert pin == "0.1.10" + + +def test_published_pins_dependents_to_the_version_just_released(fake_repo): + # A release publishes the version on the branch, then bumps the version. The + # monorepo releases a dependent later in the same run. The pin must therefore + # name a version that PyPI already has. + _run(fake_repo, "--dependents", "published") + source, pin = _versions(fake_repo) + assert source == "0.1.10" + assert pin == "0.1.9" + + +def test_dependent_pin_is_a_published_version_for_every_release_order(fake_repo): + # This test covers the monorepo order. syft-perms releases before syft-job. If + # the script pins a dependent to the new version, syft-job publishes a + # dependency that PyPI does not have. + _run(fake_repo, "--dependents", "published") + _, pin = _versions(fake_repo) + assert pin == "0.1.9", "a dependent must pin the version that the release published" diff --git a/tests/unit/test_checkpoint_version.py b/tests/unit/test_checkpoint_version.py new file mode 100644 index 00000000000..a85a362548d --- /dev/null +++ b/tests/unit/test_checkpoint_version.py @@ -0,0 +1,77 @@ +"""A checkpoint or rolling state from a later client is refused, not restored. + +Both models carry a `version` field that nothing read. A later client can change +what a field means while the object still parses, because pydantic accepts a +document that holds every field it knows. The restore would then be wrong and +silent. + +Refusing is cheap here. Every load site already falls back to a download of all +events when a checkpoint fails to load, so an unusable checkpoint costs one slow +cold start and nothing else. +""" + +import pytest +from syft.sync.checkpoints.checkpoint import ( + CHECKPOINT_VERSION, + Checkpoint, + IncrementalCheckpoint, +) +from syft.sync.checkpoints.rolling_state import ( + ROLLING_STATE_VERSION, + RollingState, +) + +EMAIL = "alice@example.com" + + +def _checkpoint(**kwargs) -> Checkpoint: + return Checkpoint(email=EMAIL, **kwargs) + + +def _incremental(**kwargs) -> IncrementalCheckpoint: + return IncrementalCheckpoint(email=EMAIL, sequence_number=1, **kwargs) + + +def _rolling(**kwargs) -> RollingState: + return RollingState(email=EMAIL, base_checkpoint_timestamp=1.0, **kwargs) + + +def test_a_checkpoint_round_trips(): + loaded = Checkpoint.from_compressed_data(_checkpoint().as_compressed_data()) + assert loaded.version == CHECKPOINT_VERSION + + +def test_an_incremental_checkpoint_round_trips(): + loaded = IncrementalCheckpoint.from_compressed_data( + _incremental().as_compressed_data() + ) + assert loaded.version == CHECKPOINT_VERSION + + +def test_a_rolling_state_round_trips(): + loaded = RollingState.from_compressed_data(_rolling().as_compressed_data()) + assert loaded.version == ROLLING_STATE_VERSION + + +def test_a_later_checkpoint_is_refused(): + data = _checkpoint(version=CHECKPOINT_VERSION + 1).as_compressed_data() + with pytest.raises(ValueError, match=str(CHECKPOINT_VERSION + 1)): + Checkpoint.from_compressed_data(data) + + +def test_a_later_incremental_checkpoint_is_refused(): + data = _incremental(version=CHECKPOINT_VERSION + 1).as_compressed_data() + with pytest.raises(ValueError, match=str(CHECKPOINT_VERSION + 1)): + IncrementalCheckpoint.from_compressed_data(data) + + +def test_a_later_rolling_state_is_refused(): + data = _rolling(version=ROLLING_STATE_VERSION + 1).as_compressed_data() + with pytest.raises(ValueError, match=str(ROLLING_STATE_VERSION + 1)): + RollingState.from_compressed_data(data) + + +def test_an_earlier_version_still_loads(): + # Version 0 predates the field. Those objects are the shape this client reads. + data = _checkpoint(version=0).as_compressed_data() + assert Checkpoint.from_compressed_data(data).version == 0 diff --git a/tests/unit/test_crypto_keys_version.py b/tests/unit/test_crypto_keys_version.py new file mode 100644 index 00000000000..785f5b4309d --- /dev/null +++ b/tests/unit/test_crypto_keys_version.py @@ -0,0 +1,51 @@ +"""The crypto key file carries a version, and an unknown one stops the load. + +A user cannot rebuild a private key, so delete-and-rebuild is not a recovery +here. If a newer client wrote the file, this client must refuse it rather than +read it wrong and lose the keys. +""" + +import json + +import pytest +from syft.sync.peers.peer_store import CRYPTO_KEYS_VERSION, PeerStore + + +def _saved(tmp_path): + store = PeerStore(email="alice@example.com", use_encryption=True) + store.generate_keys() + path = tmp_path / "crypto_keys.json" + store.save_keys(path) + return path + + +def test_a_saved_file_carries_the_version(tmp_path): + data = json.loads(_saved(tmp_path).read_text()) + assert data["version"] == CRYPTO_KEYS_VERSION + + +def test_a_saved_file_loads_back(tmp_path): + path = _saved(tmp_path) + loaded = PeerStore.load_keys(path) + assert loaded.email == "alice@example.com" + + +def test_a_file_without_a_version_still_loads(tmp_path): + # Written before the version field existed. Those keys must keep working. + path = _saved(tmp_path) + data = json.loads(path.read_text()) + del data["version"] + path.write_text(json.dumps(data)) + + loaded = PeerStore.load_keys(path) + assert loaded.email == "alice@example.com" + + +def test_a_file_from_a_newer_client_is_refused(tmp_path): + path = _saved(tmp_path) + data = json.loads(path.read_text()) + data["version"] = CRYPTO_KEYS_VERSION + 1 + path.write_text(json.dumps(data)) + + with pytest.raises(ValueError, match=str(CRYPTO_KEYS_VERSION + 1)): + PeerStore.load_keys(path) diff --git a/tests/unit/test_dataset_collection_listing.py b/tests/unit/test_dataset_collection_listing.py new file mode 100644 index 00000000000..5c74a2448f5 --- /dev/null +++ b/tests/unit/test_dataset_collection_listing.py @@ -0,0 +1,66 @@ +"""owner_list_all_collections_with_permissions skips only bad names. + +The Drive query matches a name prefix, so another tool can return a folder that +this client cannot parse. The listing skips that folder. Every other failure is a +defect, so the listing must raise it. +""" + +from unittest.mock import Mock + +import pytest + +from syft.sync.connections.drive.gdrive_transport import GDriveConnection +from syft_datasets.dataset_manager import DATASET_COLLECTION_PREFIX + +VALID = f"{DATASET_COLLECTION_PREFIX}_mytag_abc123" +UNPARSEABLE = DATASET_COLLECTION_PREFIX + + +def _conn(files): + conn = GDriveConnection(email="alice@example.com", verbose=False) + conn.drive_service = Mock() + conn._syftbox_folder_id = "syftbox-id" + conn.drive_service.files().list().execute.return_value = {"files": files} + return conn + + +def test_a_valid_collection_is_returned(): + conn = _conn([{"id": "f1", "name": VALID, "appProperties": {}}]) + got = conn.owner_list_all_collections_with_permissions(DATASET_COLLECTION_PREFIX) + assert [(c.folder_id, c.tag, c.content_hash) for c in got] == [ + ("f1", "mytag", "abc123") + ] + assert got[0].has_any_permission is False + + +def test_the_any_permission_flag_comes_from_app_properties(): + conn = _conn( + [ + { + "id": "f1", + "name": VALID, + "appProperties": {"syft_shared_with_any": "true"}, + } + ] + ) + got = conn.owner_list_all_collections_with_permissions(DATASET_COLLECTION_PREFIX) + assert got[0].has_any_permission is True + + +def test_a_name_the_client_cannot_parse_is_skipped(): + conn = _conn( + [ + {"id": "bad", "name": UNPARSEABLE, "appProperties": {}}, + {"id": "f1", "name": VALID, "appProperties": {}}, + ] + ) + got = conn.owner_list_all_collections_with_permissions(DATASET_COLLECTION_PREFIX) + assert [c.folder_id for c in got] == ["f1"] + + +def test_a_missing_name_field_raises(): + # A blanket except turned this defect into a collection that disappears + # without a message. + conn = _conn([{"id": "f1", "appProperties": {}}]) + with pytest.raises(KeyError): + conn.owner_list_all_collections_with_permissions(DATASET_COLLECTION_PREFIX) diff --git a/tests/unit/test_delete_syftbox.py b/tests/unit/test_delete_syftbox.py index 514a485b215..4b52e68f430 100644 --- a/tests/unit/test_delete_syftbox.py +++ b/tests/unit/test_delete_syftbox.py @@ -3,13 +3,7 @@ from pathlib import Path from unittest.mock import patch -from syft.sync.connections.drive.gdrive_transport import ( - GDRIVE_P2P_FOLDER_DATASITE_PREFIX, - SYFT_PEERS_FILE, - SYFT_VERSION_FILE, -) from syft.sync.login_utils import handle_potential_version_mismatches_on_login -from syft.sync.syftbox_manager import SyftboxManager from syft.sync.version.version_info import VersionInfo @@ -49,30 +43,56 @@ def test_delete_all( mock_delete_local.assert_called_once() mock_delete_remote.assert_called_once() - @patch("syft.sync.login_utils._delete_remote_unversioned_state") @patch("syft.sync.login_utils.delete_remote_syftbox") @patch("syft.sync.login_utils.delete_local_syftbox") @patch("syft.sync.login_utils._prompt_mismatch", return_value="1") @patch("syft.sync.login_utils._read_remote_version") @patch("syft.sync.login_utils.read_local_version") - def test_upgrade_deletes_local_only( + def test_continue_keeps_local_and_remote( self, mock_read_local, mock_read_remote, mock_prompt, mock_delete_local, mock_delete_remote, - mock_delete_unversioned, ): - """Mismatch + choice 1 (upgrade) → local deleted, unversioned state deleted, full remote preserved.""" + """Mismatch + choice 1 (continue) → no deletes; data is kept for repair.""" mock_read_local.return_value = _old_version_info() mock_read_remote.return_value = _old_version_info() handle_potential_version_mismatches_on_login(EMAIL, TOKEN_PATH) - mock_delete_local.assert_called_once() + mock_delete_local.assert_not_called() mock_delete_remote.assert_not_called() - mock_delete_unversioned.assert_called_once() + + @patch("syft.sync.login_utils.sys.exit") + @patch("syft.sync.login_utils.delete_remote_syftbox") + @patch("syft.sync.login_utils.delete_local_syftbox") + @patch("syft.sync.login_utils._prompt_mismatch", return_value="3") + @patch("syft.sync.login_utils._read_remote_version") + @patch("syft.sync.login_utils.read_local_version") + def test_quit_exits_without_delete( + self, + mock_read_local, + mock_read_remote, + mock_prompt, + mock_delete_local, + mock_delete_remote, + mock_exit, + ): + """Mismatch + choice 3 (quit) → exit, no deletes.""" + mock_read_local.return_value = _old_version_info() + mock_read_remote.return_value = _old_version_info() + mock_exit.side_effect = SystemExit(0) + + try: + handle_potential_version_mismatches_on_login(EMAIL, TOKEN_PATH) + except SystemExit: + pass + + mock_delete_local.assert_not_called() + mock_delete_remote.assert_not_called() + mock_exit.assert_called_once_with(0) @patch("syft.sync.login_utils._read_remote_version") @patch("syft.sync.login_utils.read_local_version") @@ -84,6 +104,28 @@ def test_no_mismatch_no_prompt(self, mock_read_local, mock_read_remote): handle_potential_version_mismatches_on_login(EMAIL, TOKEN_PATH) +class TestPromptWithoutATerminal: + """A notebook or a scheduled run has no terminal to answer the prompt.""" + + @patch("syft.sync.login_utils.sys.stdin") + def test_no_terminal_keeps_data_and_continues(self, mock_stdin): + # Choice 1 keeps every file and changes nothing, so it is safe to take + # without an answer. A prompt would stop the run instead. + from syft.sync.login_utils import _prompt_mismatch + + mock_stdin.isatty.return_value = False + with patch("builtins.input", side_effect=AssertionError("must not prompt")): + assert _prompt_mismatch(_old_version_info(), _old_version_info()) == "1" + + @patch("syft.sync.login_utils.sys.stdin") + def test_a_terminal_still_asks(self, mock_stdin): + from syft.sync.login_utils import _prompt_mismatch + + mock_stdin.isatty.return_value = True + with patch("builtins.input", return_value="2"): + assert _prompt_mismatch(_old_version_info(), _old_version_info()) == "2" + + def _query_files(connection, name_contains): """Query mock drive for files/folders whose name contains a substring.""" q = f"name contains '{name_contains}' and trashed=false" @@ -93,52 +135,6 @@ def _query_files(connection, name_contains): return results.get("files", []) -def test_delete_unversioned_state_removes_correct_folders(): - """delete_unversioned_state removes exactly the right artifacts from mock drive.""" - ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( - use_in_memory_cache=False, - sync_automatically=False, - encryption=True, - ) - do_manager.sync() - - do_conn = do_manager.peer_manager.connection_router.connections[0] - do_email = do_manager.email - - # Assert artifacts exist before deletion - do_enc_bundles = f"syft_encryption_bundles#{do_email}" - assert len(_query_files(do_conn, do_enc_bundles)) > 0 - assert len(_query_files(do_conn, SYFT_PEERS_FILE)) > 0 - assert len(_query_files(do_conn, SYFT_VERSION_FILE)) > 0 - - # Assert versioned folders exist - p2p_before = _query_files(do_conn, GDRIVE_P2P_FOLDER_DATASITE_PREFIX) - assert len(p2p_before) > 0 - - # Delete unversioned state - do_conn.delete_unversioned_state() - - # Assert unversioned artifacts are gone - assert len(_query_files(do_conn, do_enc_bundles)) == 0 - # peers/version files: DO's are gone, DS's may still exist - do_peers = [ - f - for f in _query_files(do_conn, SYFT_PEERS_FILE) - if f["id"] == do_conn._get_peers_file_id() - ] - assert len(do_peers) == 0 - do_version = [ - f - for f in _query_files(do_conn, SYFT_VERSION_FILE) - if f["id"] == do_conn._get_version_file_id() - ] - assert len(do_version) == 0 - - # Assert versioned folders survive - p2p_after = _query_files(do_conn, GDRIVE_P2P_FOLDER_DATASITE_PREFIX) - assert len(p2p_after) == len(p2p_before) - - class TestDeleteSyftboxImport: def test_importable_from_top_level(self): from syft import ( diff --git a/tests/unit/test_p2p_folder_lookup.py b/tests/unit/test_p2p_folder_lookup.py new file mode 100644 index 00000000000..3714e55ec8f --- /dev/null +++ b/tests/unit/test_p2p_folder_lookup.py @@ -0,0 +1,90 @@ +"""P2P folder lookup accepts any client version in the folder name. + +A P2P folder name is a rendezvous string that both peers compute from their own +client version, so neither side may rename it (see the adopt path for private +folders). Lookup therefore has to tolerate the version instead. + +Reuse matters in both directions. A folder this client owns must be reused after +an upgrade, because a peer that still filters by name would not find a new one. +A folder the peer owns must be found whatever version the peer wrote into it. +""" + +from unittest.mock import Mock + +from syft.sync.connections.drive.gdrive_transport import GDriveConnection + +ME = "alice@example.com" +PEER = "bob@example.com" + + +def _name(version: str, datasite: str, folder_type: str, peer: str) -> str: + return f"syft_datasite#{version}#{datasite}#{folder_type}#{peer}" + + +def _conn(found): + conn = GDriveConnection(email=ME, verbose=False) + conn.drive_service = Mock() + conn._find_folders = Mock(return_value=found) + return conn + + +def _lookup(conn): + return conn._find_p2p_folder_id( + datasite_email=PEER, folder_type="inbox", peer_email=ME, owner_email=ME + ) + + +def test_a_folder_of_another_minor_version_is_found(): + # The old filter dropped this folder, so the client created a second one and + # the peer kept writing into the first. 0.2.0 differs in the minor from the + # current client version, which is what the filter used to reject. + old = _name("0.2.0", PEER, "inbox", ME) + assert _lookup(_conn([("old", old)])) == "old" + + +def test_a_folder_of_an_older_major_version_is_found(): + old = _name("0.0.9", PEER, "inbox", ME) + assert _lookup(_conn([("old", old)])) == "old" + + +def test_the_highest_version_wins_when_several_exist(): + folders = [ + ("v1", _name("0.1.117", PEER, "inbox", ME)), + ("v2", _name("0.2.0", PEER, "inbox", ME)), + ("v0", _name("0.0.9", PEER, "inbox", ME)), + ] + assert _lookup(_conn(folders)) == "v2" + + +def test_versions_order_by_number_not_by_string(): + folders = [ + ("nine", _name("0.1.9", PEER, "inbox", ME)), + ("ten", _name("0.1.10", PEER, "inbox", ME)), + ] + assert _lookup(_conn(folders)) == "ten" + + +def test_several_folders_no_longer_raise(): + folders = [ + ("a", _name("0.1.117", PEER, "inbox", ME)), + ("b", _name("0.1.118", PEER, "inbox", ME)), + ] + assert _lookup(_conn(folders)) is not None + + +def test_a_folder_of_another_peer_is_ignored(): + other = _name("0.1.117", PEER, "inbox", "carol@example.com") + assert _lookup(_conn([("other", other)])) is None + + +def test_a_folder_of_another_type_is_ignored(): + outbox = _name("0.1.117", PEER, "outbox", ME) + assert _lookup(_conn([("outbox", outbox)])) is None + + +def test_no_folder_returns_none(): + assert _lookup(_conn([])) is None + + +def test_a_name_that_does_not_parse_is_ignored(): + assert _lookup(_conn([("junk", "not_a_p2p_folder")])) is None diff --git a/tests/unit/test_peers_json_version.py b/tests/unit/test_peers_json_version.py new file mode 100644 index 00000000000..6f9cae2af59 --- /dev/null +++ b/tests/unit/test_peers_json_version.py @@ -0,0 +1,101 @@ +"""SYFT_peers.json carries a version, and an unreadable peer state is logged. + +The file is a flat map of peer email to entry, so a version cannot go at the top +level: every existing client reads a top-level key as an email. The version lives +under a reserved key instead. An older client parses the state of that entry, +fails, and skips it, so the reserved key is invisible to a client that predates +it. + +The record itself is safe either way. The only writer is `_update_peer_state`, +which changes one entry of the raw map and writes the rest back, so a peer this +client cannot read is not erased for the other side. +""" + +import logging +from unittest.mock import Mock, patch + +from syft.sync.connections.drive.gdrive_transport import ( + PEERS_META_KEY, + SYFT_PEERS_VERSION, + GDriveConnection, +) +from syft.sync.peers.peer import PeerState + +PEER = "bob@example.com" + + +def _conn(peers_data): + conn = GDriveConnection(email="alice@example.com", verbose=False) + conn.drive_service = Mock() + conn._peers_json_cache = dict(peers_data) + return conn + + +def _router(conn): + router = Mock() + router.connection_for_send_message = Mock(return_value=conn) + from syft.sync.connections.connection_router import ConnectionRouter + + return ConnectionRouter.get_all_peers_from_json.__get__(router, ConnectionRouter) + + +def test_a_write_stamps_the_reserved_entry(): + conn = _conn({PEER: {"state": "accepted"}}) + with ( + patch.object(GDriveConnection, "_get_peers_file_id", return_value="file-id"), + patch.object( + GDriveConnection, "get_syftbox_folder_id", return_value="folder-id" + ), + patch.object( + GDriveConnection, "create_file_payload", return_value=(Mock(), None) + ), + ): + conn._write_peers_json({PEER: {"state": "accepted"}}) + + assert conn._peers_json_cache[PEERS_META_KEY] == {"version": SYFT_PEERS_VERSION} + assert conn._peers_json_cache[PEER] == {"state": "accepted"} + + +def test_the_reserved_entry_is_not_a_peer(): + conn = _conn( + { + PEERS_META_KEY: {"version": SYFT_PEERS_VERSION}, + PEER: {"state": "accepted"}, + } + ) + peers = _router(conn)() + assert [p.email for p in peers] == [PEER] + + +def test_a_known_state_loads(): + conn = _conn({PEER: {"state": "rejected"}}) + peers = _router(conn)() + assert peers[0].state == PeerState.REJECTED + + +def test_an_unknown_state_is_skipped_and_logged(caplog): + conn = _conn({PEER: {"state": "quarantined"}}) + with caplog.at_level(logging.WARNING, logger="syft"): + peers = _router(conn)() + assert peers == [] + assert any(PEER in r.getMessage() for r in caplog.records) + assert any("quarantined" in r.getMessage() for r in caplog.records) + + +def test_a_file_without_the_reserved_entry_still_loads(): + # Written before the reserved key existed. + conn = _conn({PEER: {"state": "accepted"}}) + peers = _router(conn)() + assert [p.email for p in peers] == [PEER] + + +def test_a_reserved_entry_from_a_newer_client_does_not_stop_the_read(caplog): + conn = _conn( + { + PEERS_META_KEY: {"version": SYFT_PEERS_VERSION + 1}, + PEER: {"state": "accepted"}, + } + ) + with caplog.at_level(logging.WARNING, logger="syft"): + peers = _router(conn)() + assert [p.email for p in peers] == [PEER] diff --git a/tests/unit/test_persisted_dict.py b/tests/unit/test_persisted_dict.py index 17894667c19..271590579f1 100644 --- a/tests/unit/test_persisted_dict.py +++ b/tests/unit/test_persisted_dict.py @@ -36,7 +36,7 @@ def writer(d: PersistedDict, prefix: str): assert errors == [], f"Concurrent writes raised: {errors!r}" # Every key from both writers must be present in the final on-disk state. - final = json.loads(target.read_text()) + final = json.loads(target.read_text())["entries"] expected = {f"a-{i}": i for i in range(iterations)} | { f"b-{i}": i for i in range(iterations) } @@ -60,7 +60,7 @@ def test_set_with_write_false_does_not_persist(tmp_path: Path): with d.exclusive_lock(): d._write_to_file() - assert json.loads(target.read_text()) == {"k": "v"} + assert json.loads(target.read_text())["entries"] == {"k": "v"} def test_batch_write_with_exclusive_lock(tmp_path: Path): @@ -86,7 +86,7 @@ def batch_write(d: PersistedDict, prefix: str, n: int): t1.join() t2.join() - final = json.loads(target.read_text()) + final = json.loads(target.read_text())["entries"] expected = {f"a-{i}": i for i in range(50)} | {f"b-{i}": i for i in range(50)} assert final == expected @@ -108,4 +108,4 @@ def test_contains_and_delete_with_flags(tmp_path: Path): d._write_to_file() # After the batch, on-disk state reflects the in-memory delete. - assert json.loads(target.read_text()) == {} + assert json.loads(target.read_text())["entries"] == {} diff --git a/tests/unit/test_persisted_dict_version.py b/tests/unit/test_persisted_dict_version.py new file mode 100644 index 00000000000..8898122fc03 --- /dev/null +++ b/tests/unit/test_persisted_dict_version.py @@ -0,0 +1,65 @@ +"""A persisted cache carries a version, and an unknown one resets the cache. + +The client can rebuild every one of these caches from the events and the files, +so an unreadable cache costs a re-scan and nothing else. An unknown version +therefore starts empty instead of stopping the client. +""" + +import json + +from syft.sync.sync.caches.persisted_dict import ( + PERSISTED_DICT_VERSION, + PersistedDict, +) + + +def _path(tmp_path): + return tmp_path / "cache.json" + + +def test_a_saved_file_carries_the_version(tmp_path): + d = PersistedDict(path=_path(tmp_path)) + d["a"] = "1" + data = json.loads(_path(tmp_path).read_text()) + assert data["version"] == PERSISTED_DICT_VERSION + assert data["entries"] == {"a": "1"} + + +def test_a_saved_file_loads_back(tmp_path): + d = PersistedDict(path=_path(tmp_path)) + d["a"] = "1" + assert PersistedDict(path=_path(tmp_path)).get("a") == "1" + + +def test_a_file_without_a_version_still_loads(tmp_path): + # Written before the version field existed: a bare map of entries. Reading it + # saves the user a full re-scan on the first run after an upgrade. + _path(tmp_path).write_text(json.dumps({"a": "1", "b": "2"})) + d = PersistedDict(path=_path(tmp_path)) + assert d.get("a") == "1" + assert d.get("b") == "2" + + +def test_a_file_from_a_newer_client_starts_empty(tmp_path): + _path(tmp_path).write_text( + json.dumps({"version": PERSISTED_DICT_VERSION + 1, "entries": {"a": "1"}}) + ) + d = PersistedDict(path=_path(tmp_path)) + assert d.get("a") is None + assert len(d) == 0 + + +def test_an_unreadable_file_starts_empty(tmp_path): + _path(tmp_path).write_text("{not json") + assert len(PersistedDict(path=_path(tmp_path))) == 0 + + +def test_a_reset_cache_can_be_written_again(tmp_path): + _path(tmp_path).write_text( + json.dumps({"version": PERSISTED_DICT_VERSION + 1, "entries": {"a": "1"}}) + ) + d = PersistedDict(path=_path(tmp_path)) + d["b"] = "2" + data = json.loads(_path(tmp_path).read_text()) + assert data["version"] == PERSISTED_DICT_VERSION + assert data["entries"] == {"b": "2"} diff --git a/tests/unit/test_version_negotiation.py b/tests/unit/test_version_negotiation.py index 7d9dca091f2..bd55f66d074 100644 --- a/tests/unit/test_version_negotiation.py +++ b/tests/unit/test_version_negotiation.py @@ -2,11 +2,7 @@ import logging -import pytest from syft.sync.syftbox_manager import SyftboxManager -from syft.sync.version.exceptions import ( - VersionMismatchError, -) from syft.sync.version.peer_manager import CompatAction from syft.sync.version.version_info import CompatibilityStatus, VersionInfo @@ -404,33 +400,36 @@ def test_explicit_true_on_ds_is_preserved(self): class TestForceAllowIncompatiblePeers: """Tests for force_ignore_peer_version and per-call ignore_peer_version.""" - def test_incompatible_peer_skipped_by_default(self): + def test_incompatible_peer_is_included_with_a_log(self, caplog): + # A different client version no longer refuses a peer. The protocol floor + # in VersionInfo decides what the two sides may exchange. ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( check_versions=True, ) _set_peer_version(do_manager, ds_manager.email, build_client_version("99.0.0")) - do_manager.peer_manager.suppress_version_warnings = True - compatible = do_manager.peer_manager.get_compatible_peer_emails_for_syncing( - [ds_manager.email] + with caplog.at_level(logging.INFO, logger="syft_client"): + compatible = do_manager.peer_manager.get_compatible_peer_emails_for_syncing( + [ds_manager.email] + ) + assert ds_manager.email in compatible + assert any( + "client version mismatch" in r.getMessage().lower() for r in caplog.records ) - assert ds_manager.email not in compatible - def test_force_allow_includes_incompatible_peer(self, caplog): + def test_force_allow_is_redundant_for_an_incompatible_peer(self): + # The flag overrode a refusal that no longer happens. The peer is included + # either way. ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( check_versions=True, ) _set_peer_version(do_manager, ds_manager.email, build_client_version("99.0.0")) do_manager.peer_manager.force_ignore_peer_version = True - with caplog.at_level(logging.INFO, logger="syft"): - compatible = do_manager.peer_manager.get_compatible_peer_emails_for_syncing( - [ds_manager.email] - ) - assert ds_manager.email in compatible - assert any( - "proceeding anyway" in r.getMessage().lower() for r in caplog.records + compatible = do_manager.peer_manager.get_compatible_peer_emails_for_syncing( + [ds_manager.email] ) + assert ds_manager.email in compatible def test_per_call_ignore_peer_version_includes_peer(self): ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( @@ -443,19 +442,22 @@ def test_per_call_ignore_peer_version_includes_peer(self): ) assert ds_manager.email in compatible - def test_per_call_ignore_peer_version_in_submit(self): + def test_submit_no_longer_raises_for_an_incompatible_peer(self): + # A client version difference does not stop a submission. Only an unknown + # peer version does (see test_job_submission_blocked_without_version). ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( check_versions=True, ) _set_peer_version(ds_manager, do_manager.email, build_client_version("99.0.0")) - with pytest.raises(VersionMismatchError): - result = ds_manager.peer_manager.get_peer_compatibility_status( - do_manager.email, action=CompatAction.SUBMIT - ) - result.raise_on_skip(operation="submit job") + result = ds_manager.peer_manager.get_peer_compatibility_status( + do_manager.email, action=CompatAction.SUBMIT + ) + assert result.status == CompatibilityStatus.INCOMPATIBLE + assert not result.should_skip + result.raise_on_skip(operation="submit job") - # With per-call override, should not raise + # The per-call override is redundant now, and still does not raise. result = ds_manager.peer_manager.get_peer_compatibility_status( do_manager.email, action=CompatAction.SUBMIT, @@ -480,12 +482,31 @@ def test_force_allow_in_submit(self): class TestVersionMismatchBehavior: """Tests for version mismatch behavior during operations.""" - def test_sync_skips_incompatible_peers(self): + def test_sync_keeps_an_incompatible_peer(self): ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( check_versions=True, ) _set_peer_version(do_manager, ds_manager.email, build_client_version("0.0.1")) + do_manager.peer_manager.suppress_version_warnings = True + compatible_peers = ( + do_manager.peer_manager.get_compatible_peer_emails_for_syncing( + [ds_manager.email] + ) + ) + assert ds_manager.email in compatible_peers + + def test_sync_still_skips_a_peer_of_unknown_version(self): + # The boundary of the policy: a known difference is allowed, an unknown + # peer is not. Nothing can be negotiated without the version of the peer. + ds_manager, do_manager = SyftboxManager.pair_with_mock_drive_service_connection( + check_versions=True, + ) + peer = do_manager.peer_manager.get_cached_peer(ds_manager.email) + assert peer is not None + peer.version = None + do_manager.peer_manager._loaded_peer_versions[ds_manager.email] = None + do_manager.peer_manager.suppress_version_warnings = True compatible_peers = ( do_manager.peer_manager.get_compatible_peer_emails_for_syncing( diff --git a/tests/unit/test_versioned_folder_adopt.py b/tests/unit/test_versioned_folder_adopt.py new file mode 100644 index 00000000000..266b7e9f9ac --- /dev/null +++ b/tests/unit/test_versioned_folder_adopt.py @@ -0,0 +1,134 @@ +"""A client adopts a private Drive folder from an earlier client version. + +A private folder name holds the client version. After a minor upgrade the name of +the current version does not exist yet. Without adoption the client creates a new +folder, and the datasite of the user stays on Drive out of reach. + +These tests cover the private folders only. The name of a P2P folder is a +rendezvous string that both peers compute, so a client must never rename one. +""" + +from unittest.mock import Mock + +import pytest + +from syft.sync.connections.drive.gdrive_transport import ( + GDriveConnection, + _partition_by_version, +) + +EMAIL = "alice@example.com" + + +def _conn(): + conn = GDriveConnection(email=EMAIL, verbose=False) + conn.drive_service = Mock() + return conn + + +def _renames(conn): + """Return the (fileId, new name) pairs the connection sent to Drive.""" + return [ + (kwargs["fileId"], kwargs["body"]["name"]) + for _, kwargs in conn.drive_service.files().update.call_args_list + if "body" in kwargs and "name" in kwargs.get("body", {}) + ] + + +# ---------- _partition_by_version ------------------------------------------- + + +def test_partition_splits_compatible_older_and_newer(): + folders = [ + ("old", f"0.1.9#{EMAIL}"), + ("same", f"0.2.5#{EMAIL}"), + ("new", f"0.3.0#{EMAIL}"), + ] + compatible, older, newer = _partition_by_version(folders, current_version="0.2.7") + assert compatible == [("same", f"0.2.5#{EMAIL}")] + assert older == [("old", f"0.1.9#{EMAIL}")] + assert newer == [("new", f"0.3.0#{EMAIL}")] + + +def test_partition_sorts_by_number_not_by_string(): + folders = [("a", f"0.1.9#{EMAIL}"), ("b", f"0.1.10#{EMAIL}")] + _, older, _ = _partition_by_version(folders, current_version="0.2.0") + assert [fid for fid, _ in older] == ["a", "b"] + + +def test_partition_drops_names_without_a_version(): + folders = [("a", f"0.1.9#{EMAIL}"), ("b", "no_version_here")] + _, older, _ = _partition_by_version(folders, current_version="0.2.0") + assert older == [("a", f"0.1.9#{EMAIL}")] + + +def test_partition_returns_empty_for_a_bad_current_version(): + folders = [("a", f"0.1.9#{EMAIL}")] + assert _partition_by_version(folders, current_version="garbage") == ([], [], []) + + +# ---------- adoption -------------------------------------------------------- + + +def test_a_compatible_folder_wins_and_nothing_is_renamed(): + conn = _conn() + folders = [("same", f"0.2.5#{EMAIL}"), ("old", f"0.1.9#{EMAIL}")] + got = conn._find_or_adopt_versioned_folder( + folders, current_name=f"0.2.7#{EMAIL}", current_version="0.2.7" + ) + assert got == "same" + assert _renames(conn) == [] + + +def test_an_older_folder_is_adopted_by_rename(): + conn = _conn() + folders = [("old", f"0.1.9#{EMAIL}")] + got = conn._find_or_adopt_versioned_folder( + folders, current_name=f"0.2.7#{EMAIL}", current_version="0.2.7" + ) + assert got == "old", "the client must keep the folder that holds the data" + assert _renames(conn) == [("old", f"0.2.7#{EMAIL}")] + + +def test_the_highest_older_folder_is_adopted(): + conn = _conn() + folders = [ + ("v1", f"0.1.9#{EMAIL}"), + ("v2", f"0.1.20#{EMAIL}"), + ("v0", f"0.0.4#{EMAIL}"), + ] + got = conn._find_or_adopt_versioned_folder( + folders, current_name=f"0.2.7#{EMAIL}", current_version="0.2.7" + ) + assert got == "v2" + assert _renames(conn) == [("v2", f"0.2.7#{EMAIL}")] + + +def test_a_newer_folder_stops_the_client(): + # A new folder here would hide data that this client cannot read. Report the + # version to install instead. + conn = _conn() + folders = [("new", f"0.3.0#{EMAIL}")] + with pytest.raises(RuntimeError, match="0.3.0"): + conn._find_or_adopt_versioned_folder( + folders, current_name=f"0.2.7#{EMAIL}", current_version="0.2.7" + ) + assert _renames(conn) == [] + + +def test_no_folder_returns_none_so_the_caller_creates_one(): + conn = _conn() + got = conn._find_or_adopt_versioned_folder( + [], current_name=f"0.2.7#{EMAIL}", current_version="0.2.7" + ) + assert got is None + assert _renames(conn) == [] + + +def test_two_compatible_folders_still_raise(): + conn = _conn() + folders = [("a", f"0.2.1#{EMAIL}"), ("b", f"0.2.2#{EMAIL}")] + with pytest.raises(RuntimeError): + conn._find_or_adopt_versioned_folder( + folders, current_name=f"0.2.7#{EMAIL}", current_version="0.2.7" + ) diff --git a/tests/unit/test_versioned_folder_lookup.py b/tests/unit/test_versioned_folder_lookup.py index 6a24f9f8d6f..b2a991f57fb 100644 --- a/tests/unit/test_versioned_folder_lookup.py +++ b/tests/unit/test_versioned_folder_lookup.py @@ -2,15 +2,17 @@ These are pure functions -- no Drive mocks needed. They cover the path that replaced the four format-specific parsers from the original PR. + +Ordering and selection now live in _partition_by_version (adopt, private +folders) and _sorted_by_version (P2P lookup), each tested separately. """ from syft.sync.connections.drive.gdrive_transport import ( _extract_version_from_name, - _filter_patch_compatible, _looks_like_version, + _partition_by_version, ) - # ---------- _looks_like_version --------------------------------------------- @@ -63,66 +65,6 @@ def test_extract_returns_none_when_missing(): assert _extract_version_from_name("just_a_folder_name") is None -# ---------- _filter_patch_compatible ---------------------------------------- - - -def test_filter_keeps_same_patch(): - folders = [("id1", "0.1.114#alice@example.com")] - assert _filter_patch_compatible(folders, current_version="0.1.114") == folders - - -def test_filter_keeps_different_patch_same_minor(): - folders = [("id1", "0.1.114#alice@example.com")] - assert _filter_patch_compatible(folders, current_version="0.1.200") == folders - - -def test_filter_drops_minor_diff(): - folders = [ - ("id1", "0.1.114#alice@example.com"), - ("id2", "0.2.0#alice@example.com"), - ] - assert _filter_patch_compatible(folders, current_version="0.1.114") == [ - ("id1", "0.1.114#alice@example.com") - ] - - -def test_filter_drops_major_diff(): - folders = [ - ("id1", "0.1.114#alice@example.com"), - ("id2", "1.0.0#alice@example.com"), - ] - assert _filter_patch_compatible(folders, current_version="0.1.114") == [ - ("id1", "0.1.114#alice@example.com") - ] - - -def test_filter_drops_names_without_a_version(): - folders = [ - ("id1", "0.1.114#alice@example.com"), - ("id2", "no_version_here"), - ] - assert _filter_patch_compatible(folders, current_version="0.1.114") == [ - ("id1", "0.1.114#alice@example.com") - ] - - -def test_filter_covers_all_four_folder_formats(): - """All four formats syft uses should match when major.minor align.""" - folders = [ - ("id1", "0.1.114#alice@example.com"), - ("id2", "syft_datasite#0.1.115#alice@example.com#inbox#bob@example.com"), - ("id3", "alice@example.com-0.1.116-checkpoints"), - ("id4", "alice@example.com-0.1.117-rolling-state"), - ] - kept = _filter_patch_compatible(folders, current_version="0.1.200") - assert {fid for fid, _ in kept} == {"id1", "id2", "id3", "id4"} - - -def test_filter_returns_empty_for_bad_current_version(): - folders = [("id1", "0.1.114#alice@example.com")] - assert _filter_patch_compatible(folders, current_version="garbage") == [] - - # ---------- two-digit minor (0.10.x, the first `syft` release line) --------- @@ -143,7 +85,7 @@ def test_extract_two_digit_minor_from_all_formats(): ) -def test_filter_two_digit_minor_is_numeric_not_lexicographic(): +def test_partition_two_digit_minor_is_numeric_not_lexicographic(): """0.10.x must match 0.10.y and reject both 0.1.x and 0.9.x.""" folders = [ ("id1", "0.10.0#alice@example.com"), @@ -151,5 +93,6 @@ def test_filter_two_digit_minor_is_numeric_not_lexicographic(): ("id3", "0.1.117#alice@example.com"), ("id4", "0.9.5#alice@example.com"), ] - kept = _filter_patch_compatible(folders, current_version="0.10.1") - assert {fid for fid, _ in kept} == {"id1", "id2"} + compatible, older, _ = _partition_by_version(folders, current_version="0.10.1") + assert {fid for fid, _ in compatible} == {"id1", "id2"} + assert {fid for fid, _ in older} == {"id3", "id4"} diff --git a/uv.lock b/uv.lock index 8bfd2ad056b..989ed1156cf 100644 --- a/uv.lock +++ b/uv.lock @@ -4469,6 +4469,7 @@ dependencies = [ { name = "rich" }, { name = "syft-crypto-python" }, { name = "syft-dataset" }, + { name = "syft-migration" }, { name = "syft-permissions" }, { name = "syft-perms" }, ] @@ -4512,6 +4513,7 @@ requires-dist = [ { name = "rich", specifier = ">=13.0.0" }, { name = "syft-crypto-python", specifier = ">=0.1.2b2" }, { name = "syft-dataset", editable = "packages/syft-datasets" }, + { name = "syft-migration", editable = "packages/syft-migration" }, { name = "syft-permissions", editable = "packages/syft-permissions" }, { name = "syft-perms", editable = "packages/syft-perms" }, ] @@ -4619,7 +4621,7 @@ dev = [{ name = "ipykernel", specifier = ">=7.1.0" }] [[package]] name = "syft-enclave" -version = "0.1.0" +version = "0.1.1" source = { editable = "packages/syft-enclave" } dependencies = [ { name = "google-auth", extra = ["pyjwt"] },