ci(release): the CodeArtifact release uploads the files the PyPI release published

.circleci/config.yml

@@ -51,20 +51,43 @@ jobs:
           name: Bump, tag, push
           command: bash scripts/circleci_release_bump.sh
 
-  publish:
+  # The files the GitHub release put on PyPI, which it uploads after publishing the images and recording their
+  # digests in the wheel. No AWS access here, so the wait can outlast CircleCI's OIDC token.
+  wait-for-pypi:
     docker:
       - image: cimg/python:3.11
+    resource_class: small
     steps:
       - checkout
+      - run:
+          name: Download the release's files from PyPI
+          no_output_timeout: 15m
+          command: |
+            set -euo pipefail
+            version() { python -c "import sys, tomllib; print(tomllib.load(open(sys.argv[1], 'rb'))['project']['version'])" "$1"; }
+            python .github/scripts/pypi_release.py download dist --wait-minutes 240 \
+              "agentenv-framework-protocol==$(version packages/agentenv-protocol/pyproject.toml)" \
+              "agentenv-framework==$(version pyproject.toml)"
+      - persist_to_workspace:
+          root: .
+          paths: [dist]
+
+  # Uploads PyPI's files, so the CodeArtifact release is the PyPI release.
+  publish:
+    docker:
+      - image: cimg/python:3.11
+    steps:
+      - attach_workspace:
+          at: .
 
       - aws-cli/setup:
           role_arn: ${AWS_PUBLISH_ROLE_ARN}
           region: us-west-2
 
       - run:
-          name: Install build tools
+          name: Install twine
           command: |
-            pip install --upgrade pip build twine
+            pip install --upgrade pip twine
 
       - run:
           name: Login to CodeArtifact
@@ -79,29 +102,11 @@ jobs:
               --region us-west-2
 
       - run:
-          name: Build and publish the protocol package
+          name: Publish to CodeArtifact, the protocol first
           command: |
             set -euo pipefail
-            python -m build packages/agentenv-protocol --outdir packages/agentenv-protocol/dist
-            twine upload --repository codeartifact packages/agentenv-protocol/dist/*
-
-      - run:
-          name: Build package (pin the exact protocol version)
-          command: |
-            set -euo pipefail
-            PV=$(sed -n 's/^version = "\(.*\)"/\1/p' packages/agentenv-protocol/pyproject.toml)
-            # Only the quoted requirement: the directory name also appears in the sdist include list
-            # and the uv workspace members, and the bare name keys [tool.uv.sources]; none may change.
-            sed -i "s/\"agentenv-framework-protocol[^\"]*\"/\"agentenv-framework-protocol==${PV}\"/" pyproject.toml
-            python -c "import tomllib; tomllib.load(open('pyproject.toml','rb'))"
-            grep -q "\"agentenv-framework-protocol==${PV}\"" pyproject.toml
-            # The shipped bundles' requirements too, as the PyPI release does, or this wheel's agents install a floor.
-            python .github/scripts/shipped_bundles.py pin
-            python -m build
-
-      - run:
-          name: Publish to CodeArtifact
-          command: twine upload --repository codeartifact dist/*
+            twine upload --skip-existing --repository codeartifact dist/agentenv-framework-protocol/*
+            twine upload --skip-existing --repository codeartifact dist/agentenv-framework/*
 
 workflows:
   # Auto-bump pyproject version when a PR labeled `bump-version` (or `bump`)
@@ -118,12 +123,19 @@ workflows:
             tags:
               ignore: /.*/    # Don't run on tag pushes (publish handles those)
 
-  # Publish to AWS CodeArtifact on any vX.Y.Z tag push.
+  # Publish to AWS CodeArtifact on any vX.Y.Z tag push, once the release is on PyPI.
   publish:
     jobs:
-      - publish:
+      - wait-for-pypi:
           filters:
             tags:
               only: /^v.*/    # Only run on tags like v0.1.0, v1.2.3
             branches:
               ignore: /.*/    # Don't run on branch commits
+      - publish:
+          requires: [wait-for-pypi]
+          filters:
+            tags:
+              only: /^v.*/
+            branches:
+              ignore: /.*/

.github/scripts/pypi_release.py

@@ -0,0 +1,79 @@
+"""The files a release put on PyPI, for the CodeArtifact publish to upload as they are.
+
+``download <dir> <name>==<version>...`` waits until PyPI lists each version's wheel and sdist, then fetches every file it
+lists into ``<dir>/<name>/`` and checks each against the sha256 PyPI lists. The GitHub release uploads them once it has
+published the images and recorded their digests in the wheel, so the CodeArtifact release ships the same wheel. It uses
+only the standard library, so the CircleCI publish job can run it before installing anything.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+PYPI = "https://pypi.org/pypi"
+POLL_SECONDS = 30
+PACKAGE_TYPES = {"bdist_wheel", "sdist"}
+
+
+def _get(url: str) -> bytes:
+    with urllib.request.urlopen(url, timeout=60) as response:
+        return response.read()
+
+
+def release_files(name: str, version: str) -> list[dict] | None:
+    """The files PyPI lists for ``name==version``, or None until it lists both a wheel and an sdist."""
+    try:
+        files = json.loads(_get(f"{PYPI}/{name}/{version}/json"))["urls"]
+    except urllib.error.HTTPError as error:
+        if error.code != 404:
+            print(f"PyPI answered {error.code} for {name}=={version}", flush=True)
+        return None
+    except (urllib.error.URLError, TimeoutError) as error:
+        print(f"PyPI is unreachable for {name}=={version}: {error}", flush=True)
+        return None
+    return files if PACKAGE_TYPES <= {file["packagetype"] for file in files} else None
+
+
+def download(directory: Path, requirements: list[str], wait_minutes: float) -> None:
+    deadline = time.monotonic() + wait_minutes * 60
+    for requirement in requirements:
+        name, _, version = requirement.partition("==")
+        while (files := release_files(name, version)) is None:
+            if time.monotonic() >= deadline:
+                sys.exit(f"{requirement} is not on PyPI after {wait_minutes:g} minutes")
+            print(f"Waiting for {requirement} on PyPI", flush=True)
+            time.sleep(POLL_SECONDS)
+        target = directory / name
+        target.mkdir(parents=True, exist_ok=True)
+        for file in files:
+            data = _get(file["url"])
+            digest = hashlib.sha256(data).hexdigest()
+            if digest != file["digests"]["sha256"]:
+                sys.exit(f"{file['filename']} has sha256 {digest}, but PyPI lists {file['digests']['sha256']}")
+            (target / file["filename"]).write_bytes(data)
+            print(f"{file['filename']} sha256:{digest}", flush=True)
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    commands = parser.add_subparsers(dest="command", required=True)
+    command = commands.add_parser("download")
+    command.add_argument("directory", type=Path)
+    command.add_argument("requirements", nargs="+", metavar="NAME==VERSION")
+    command.add_argument("--wait-minutes", type=float, default=240)
+    args = parser.parse_args(argv)
+    for requirement in args.requirements:
+        if requirement.count("==") != 1 or requirement.startswith("=") or requirement.endswith("="):
+            parser.error(f"{requirement!r} is not NAME==VERSION")
+    download(args.directory, args.requirements, args.wait_minutes)
+
+
+if __name__ == "__main__":
+    main()

tst/unit/test_pypi_release_script.py

@@ -0,0 +1,110 @@
+"""``.github/scripts/pypi_release.py``, which hands the CodeArtifact release the files the GitHub release put on PyPI."""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import urllib.error
+from pathlib import Path
+
+import pytest
+import yaml
+
+from tst.util.scripts import load_script
+
+REPO = Path(__file__).resolve().parents[2]
+WHEEL = b"wheel bytes"
+SDIST = b"sdist bytes"
+
+
+def _file(name: str, kind: str, data: bytes) -> dict:
+    return {"filename": name, "packagetype": kind, "url": f"https://files.example/{name}",
+            "digests": {"sha256": hashlib.sha256(data).hexdigest()}}
+
+
+class _PyPI:
+    """PyPI answering 404 until ``listed`` polls in, then listing ``files``; a package in ``never`` is never listed."""
+
+    def __init__(self, files: list[dict], listed: int = 0, blobs: dict[str, bytes] | None = None,
+                 never: frozenset[str] = frozenset()):
+        self.files, self.listed, self.never, self.polls = files, listed, never, 0
+        self.blobs = blobs or {"https://files.example/x-1.whl": WHEEL, "https://files.example/x-1.tar.gz": SDIST}
+
+    def get(self, url: str) -> bytes:
+        if url.endswith("/json"):
+            self.polls += 1
+            if self.polls <= self.listed or url.split("/")[-3] in self.never:
+                raise urllib.error.HTTPError(url, 404, "Not Found", {}, io.BytesIO())
+            return json.dumps({"urls": self.files}).encode()
+        return self.blobs[url]
+
+
+@pytest.fixture
+def script(monkeypatch):
+    script = load_script("pypi_release.py")
+    clock = [0.0]
+    monkeypatch.setattr(script.time, "monotonic", lambda: clock[0])
+    monkeypatch.setattr(script.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds))
+    return script
+
+
+def _both() -> list[dict]:
+    return [_file("x-1.whl", "bdist_wheel", WHEEL), _file("x-1.tar.gz", "sdist", SDIST)]
+
+
+def test_it_waits_for_the_release_then_downloads_every_file_into_the_package_folder(script, monkeypatch, tmp_path):
+    pypi = _PyPI(_both(), listed=3)
+    monkeypatch.setattr(script, "_get", pypi.get)
+
+    script.main(["download", str(tmp_path), "x==1"])
+
+    assert pypi.polls == 4
+    assert (tmp_path / "x" / "x-1.whl").read_bytes() == WHEEL
+    assert (tmp_path / "x" / "x-1.tar.gz").read_bytes() == SDIST
+
+
+def test_a_wheel_without_its_sdist_is_not_yet_the_release(script, monkeypatch, tmp_path):
+    pypi = _PyPI([_file("x-1.whl", "bdist_wheel", WHEEL)])
+    monkeypatch.setattr(script, "_get", pypi.get)
+
+    with pytest.raises(SystemExit, match="x==1 is not on PyPI after 1 minutes"):
+        script.main(["download", str(tmp_path), "x==1", "--wait-minutes", "1"])
+
+    assert not (tmp_path / "x").exists()
+
+
+def test_a_file_whose_sha256_is_not_the_one_pypi_lists_is_refused(script, monkeypatch, tmp_path):
+    pypi = _PyPI(_both(), blobs={"https://files.example/x-1.whl": b"tampered", "https://files.example/x-1.tar.gz": SDIST})
+    monkeypatch.setattr(script, "_get", pypi.get)
+
+    with pytest.raises(SystemExit, match="x-1.whl has sha256"):
+        script.main(["download", str(tmp_path), "x==1"])
+
+
+def test_the_wait_covers_every_requirement_together(script, monkeypatch, tmp_path):
+    pypi = _PyPI(_both(), listed=3, never=frozenset({"y"}))
+    monkeypatch.setattr(script, "_get", pypi.get)
+
+    with pytest.raises(SystemExit, match="y==2 is not on PyPI after 2 minutes"):
+        script.main(["download", str(tmp_path), "x==1", "y==2", "--wait-minutes", "2"])
+
+    assert (tmp_path / "x" / "x-1.whl").exists()
+
+
+@pytest.mark.parametrize("requirement", ["x", "x>=1", "==1", "x==", "x==1==2"])
+def test_a_requirement_must_name_one_exact_version(script, requirement, tmp_path):
+    with pytest.raises(SystemExit):
+        script.main(["download", str(tmp_path), requirement])
+
+
+def test_the_codeartifact_release_uploads_the_files_the_pypi_release_put_there():
+    config = yaml.safe_load((REPO / ".circleci" / "config.yml").read_text())
+    wait, publish = (json.dumps(config["jobs"][job]) for job in ("wait-for-pypi", "publish"))
+    jobs = {name: spec for entry in config["workflows"]["publish"]["jobs"] for name, spec in entry.items()}
+
+    assert "pypi_release.py download dist" in wait and "agentenv-framework-protocol==" in wait
+    assert "aws-cli/setup" not in wait
+    assert "python -m build" not in publish and "uv build" not in publish
+    assert -1 < publish.find("dist/agentenv-framework-protocol/*") < publish.find("dist/agentenv-framework/*")
+    assert jobs["publish"]["requires"] == ["wait-for-pypi"]

tst/unit/test_release_protocol_pins.py

@@ -72,13 +72,6 @@ def test_the_release_pins_the_shipped_bundles_with_the_script():
     assert "requirements_files=$(" not in WORKFLOW
 
 
-def test_the_codeartifact_release_pins_the_shipped_bundles_before_it_builds_the_wheel():
-    publish = (REPO / ".circleci" / "config.yml").read_text().split("\n  publish:\n", 1)[1]
-    build = publish.index("            python -m build\n")
-
-    assert -1 < publish.find(".github/scripts/shipped_bundles.py pin") < build
-
-
 def test_each_shipped_requirements_file_the_release_pins_holds_one_protocol_line():
     pinned = {path.relative_to(REPO).as_posix(): [line for line in path.read_text().splitlines()
                                                    if line.startswith("agentenv-framework-protocol")]