diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e15ef8f..97716565 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,20 +51,43 @@ jobs: name: Bump, tag, push command: bash scripts/circleci_release_bump.sh - publish: + # The files the GitHub release put on PyPI, which it uploads after publishing the images and recording their + # digests in the wheel. No AWS access here, so the wait can outlast CircleCI's OIDC token. + wait-for-pypi: docker: - image: cimg/python:3.11 + resource_class: small steps: - checkout + - run: + name: Download the release's files from PyPI + no_output_timeout: 15m + command: | + set -euo pipefail + version() { python -c "import sys, tomllib; print(tomllib.load(open(sys.argv[1], 'rb'))['project']['version'])" "$1"; } + python .github/scripts/pypi_release.py download dist --wait-minutes 240 \ + "agentenv-framework-protocol==$(version packages/agentenv-protocol/pyproject.toml)" \ + "agentenv-framework==$(version pyproject.toml)" + - persist_to_workspace: + root: . + paths: [dist] + + # Uploads PyPI's files, so the CodeArtifact release is the PyPI release. + publish: + docker: + - image: cimg/python:3.11 + steps: + - attach_workspace: + at: . - aws-cli/setup: role_arn: ${AWS_PUBLISH_ROLE_ARN} region: us-west-2 - run: - name: Install build tools + name: Install twine command: | - pip install --upgrade pip build twine + pip install --upgrade pip twine - run: name: Login to CodeArtifact @@ -79,29 +102,11 @@ jobs: --region us-west-2 - run: - name: Build and publish the protocol package + name: Publish to CodeArtifact, the protocol first command: | set -euo pipefail - python -m build packages/agentenv-protocol --outdir packages/agentenv-protocol/dist - twine upload --repository codeartifact packages/agentenv-protocol/dist/* - - - run: - name: Build package (pin the exact protocol version) - command: | - set -euo pipefail - PV=$(sed -n 's/^version = "\(.*\)"/\1/p' packages/agentenv-protocol/pyproject.toml) - # Only the quoted requirement: the directory name also appears in the sdist include list - # and the uv workspace members, and the bare name keys [tool.uv.sources]; none may change. - sed -i "s/\"agentenv-framework-protocol[^\"]*\"/\"agentenv-framework-protocol==${PV}\"/" pyproject.toml - python -c "import tomllib; tomllib.load(open('pyproject.toml','rb'))" - grep -q "\"agentenv-framework-protocol==${PV}\"" pyproject.toml - # The shipped bundles' requirements too, as the PyPI release does, or this wheel's agents install a floor. - python .github/scripts/shipped_bundles.py pin - python -m build - - - run: - name: Publish to CodeArtifact - command: twine upload --repository codeartifact dist/* + twine upload --skip-existing --repository codeartifact dist/agentenv-framework-protocol/* + twine upload --skip-existing --repository codeartifact dist/agentenv-framework/* workflows: # Auto-bump pyproject version when a PR labeled `bump-version` (or `bump`) @@ -118,12 +123,19 @@ workflows: tags: ignore: /.*/ # Don't run on tag pushes (publish handles those) - # Publish to AWS CodeArtifact on any vX.Y.Z tag push. + # Publish to AWS CodeArtifact on any vX.Y.Z tag push, once the release is on PyPI. publish: jobs: - - publish: + - wait-for-pypi: filters: tags: only: /^v.*/ # Only run on tags like v0.1.0, v1.2.3 branches: ignore: /.*/ # Don't run on branch commits + - publish: + requires: [wait-for-pypi] + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ diff --git a/.github/scripts/pypi_release.py b/.github/scripts/pypi_release.py new file mode 100644 index 00000000..0f3a9433 --- /dev/null +++ b/.github/scripts/pypi_release.py @@ -0,0 +1,79 @@ +"""The files a release put on PyPI, for the CodeArtifact publish to upload as they are. + +``download ==...`` waits until PyPI lists each version's wheel and sdist, then fetches every file it +lists into ``//`` and checks each against the sha256 PyPI lists. The GitHub release uploads them once it has +published the images and recorded their digests in the wheel, so the CodeArtifact release ships the same wheel. It uses +only the standard library, so the CircleCI publish job can run it before installing anything. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +PYPI = "https://pypi.org/pypi" +POLL_SECONDS = 30 +PACKAGE_TYPES = {"bdist_wheel", "sdist"} + + +def _get(url: str) -> bytes: + with urllib.request.urlopen(url, timeout=60) as response: + return response.read() + + +def release_files(name: str, version: str) -> list[dict] | None: + """The files PyPI lists for ``name==version``, or None until it lists both a wheel and an sdist.""" + try: + files = json.loads(_get(f"{PYPI}/{name}/{version}/json"))["urls"] + except urllib.error.HTTPError as error: + if error.code != 404: + print(f"PyPI answered {error.code} for {name}=={version}", flush=True) + return None + except (urllib.error.URLError, TimeoutError) as error: + print(f"PyPI is unreachable for {name}=={version}: {error}", flush=True) + return None + return files if PACKAGE_TYPES <= {file["packagetype"] for file in files} else None + + +def download(directory: Path, requirements: list[str], wait_minutes: float) -> None: + deadline = time.monotonic() + wait_minutes * 60 + for requirement in requirements: + name, _, version = requirement.partition("==") + while (files := release_files(name, version)) is None: + if time.monotonic() >= deadline: + sys.exit(f"{requirement} is not on PyPI after {wait_minutes:g} minutes") + print(f"Waiting for {requirement} on PyPI", flush=True) + time.sleep(POLL_SECONDS) + target = directory / name + target.mkdir(parents=True, exist_ok=True) + for file in files: + data = _get(file["url"]) + digest = hashlib.sha256(data).hexdigest() + if digest != file["digests"]["sha256"]: + sys.exit(f"{file['filename']} has sha256 {digest}, but PyPI lists {file['digests']['sha256']}") + (target / file["filename"]).write_bytes(data) + print(f"{file['filename']} sha256:{digest}", flush=True) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("download") + command.add_argument("directory", type=Path) + command.add_argument("requirements", nargs="+", metavar="NAME==VERSION") + command.add_argument("--wait-minutes", type=float, default=240) + args = parser.parse_args(argv) + for requirement in args.requirements: + if requirement.count("==") != 1 or requirement.startswith("=") or requirement.endswith("="): + parser.error(f"{requirement!r} is not NAME==VERSION") + download(args.directory, args.requirements, args.wait_minutes) + + +if __name__ == "__main__": + main() diff --git a/tst/unit/test_pypi_release_script.py b/tst/unit/test_pypi_release_script.py new file mode 100644 index 00000000..98123506 --- /dev/null +++ b/tst/unit/test_pypi_release_script.py @@ -0,0 +1,110 @@ +"""``.github/scripts/pypi_release.py``, which hands the CodeArtifact release the files the GitHub release put on PyPI.""" + +from __future__ import annotations + +import hashlib +import io +import json +import urllib.error +from pathlib import Path + +import pytest +import yaml + +from tst.util.scripts import load_script + +REPO = Path(__file__).resolve().parents[2] +WHEEL = b"wheel bytes" +SDIST = b"sdist bytes" + + +def _file(name: str, kind: str, data: bytes) -> dict: + return {"filename": name, "packagetype": kind, "url": f"https://files.example/{name}", + "digests": {"sha256": hashlib.sha256(data).hexdigest()}} + + +class _PyPI: + """PyPI answering 404 until ``listed`` polls in, then listing ``files``; a package in ``never`` is never listed.""" + + def __init__(self, files: list[dict], listed: int = 0, blobs: dict[str, bytes] | None = None, + never: frozenset[str] = frozenset()): + self.files, self.listed, self.never, self.polls = files, listed, never, 0 + self.blobs = blobs or {"https://files.example/x-1.whl": WHEEL, "https://files.example/x-1.tar.gz": SDIST} + + def get(self, url: str) -> bytes: + if url.endswith("/json"): + self.polls += 1 + if self.polls <= self.listed or url.split("/")[-3] in self.never: + raise urllib.error.HTTPError(url, 404, "Not Found", {}, io.BytesIO()) + return json.dumps({"urls": self.files}).encode() + return self.blobs[url] + + +@pytest.fixture +def script(monkeypatch): + script = load_script("pypi_release.py") + clock = [0.0] + monkeypatch.setattr(script.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(script.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds)) + return script + + +def _both() -> list[dict]: + return [_file("x-1.whl", "bdist_wheel", WHEEL), _file("x-1.tar.gz", "sdist", SDIST)] + + +def test_it_waits_for_the_release_then_downloads_every_file_into_the_package_folder(script, monkeypatch, tmp_path): + pypi = _PyPI(_both(), listed=3) + monkeypatch.setattr(script, "_get", pypi.get) + + script.main(["download", str(tmp_path), "x==1"]) + + assert pypi.polls == 4 + assert (tmp_path / "x" / "x-1.whl").read_bytes() == WHEEL + assert (tmp_path / "x" / "x-1.tar.gz").read_bytes() == SDIST + + +def test_a_wheel_without_its_sdist_is_not_yet_the_release(script, monkeypatch, tmp_path): + pypi = _PyPI([_file("x-1.whl", "bdist_wheel", WHEEL)]) + monkeypatch.setattr(script, "_get", pypi.get) + + with pytest.raises(SystemExit, match="x==1 is not on PyPI after 1 minutes"): + script.main(["download", str(tmp_path), "x==1", "--wait-minutes", "1"]) + + assert not (tmp_path / "x").exists() + + +def test_a_file_whose_sha256_is_not_the_one_pypi_lists_is_refused(script, monkeypatch, tmp_path): + pypi = _PyPI(_both(), blobs={"https://files.example/x-1.whl": b"tampered", "https://files.example/x-1.tar.gz": SDIST}) + monkeypatch.setattr(script, "_get", pypi.get) + + with pytest.raises(SystemExit, match="x-1.whl has sha256"): + script.main(["download", str(tmp_path), "x==1"]) + + +def test_the_wait_covers_every_requirement_together(script, monkeypatch, tmp_path): + pypi = _PyPI(_both(), listed=3, never=frozenset({"y"})) + monkeypatch.setattr(script, "_get", pypi.get) + + with pytest.raises(SystemExit, match="y==2 is not on PyPI after 2 minutes"): + script.main(["download", str(tmp_path), "x==1", "y==2", "--wait-minutes", "2"]) + + assert (tmp_path / "x" / "x-1.whl").exists() + + +@pytest.mark.parametrize("requirement", ["x", "x>=1", "==1", "x==", "x==1==2"]) +def test_a_requirement_must_name_one_exact_version(script, requirement, tmp_path): + with pytest.raises(SystemExit): + script.main(["download", str(tmp_path), requirement]) + + +def test_the_codeartifact_release_uploads_the_files_the_pypi_release_put_there(): + config = yaml.safe_load((REPO / ".circleci" / "config.yml").read_text()) + wait, publish = (json.dumps(config["jobs"][job]) for job in ("wait-for-pypi", "publish")) + jobs = {name: spec for entry in config["workflows"]["publish"]["jobs"] for name, spec in entry.items()} + + assert "pypi_release.py download dist" in wait and "agentenv-framework-protocol==" in wait + assert "aws-cli/setup" not in wait + assert "python -m build" not in publish and "uv build" not in publish + assert -1 < publish.find("dist/agentenv-framework-protocol/*") < publish.find("dist/agentenv-framework/*") + assert jobs["publish"]["requires"] == ["wait-for-pypi"] diff --git a/tst/unit/test_release_protocol_pins.py b/tst/unit/test_release_protocol_pins.py index 7306b228..17739f89 100644 --- a/tst/unit/test_release_protocol_pins.py +++ b/tst/unit/test_release_protocol_pins.py @@ -72,13 +72,6 @@ def test_the_release_pins_the_shipped_bundles_with_the_script(): assert "requirements_files=$(" not in WORKFLOW -def test_the_codeartifact_release_pins_the_shipped_bundles_before_it_builds_the_wheel(): - publish = (REPO / ".circleci" / "config.yml").read_text().split("\n publish:\n", 1)[1] - build = publish.index(" python -m build\n") - - assert -1 < publish.find(".github/scripts/shipped_bundles.py pin") < build - - def test_each_shipped_requirements_file_the_release_pins_holds_one_protocol_line(): pinned = {path.relative_to(REPO).as_posix(): [line for line in path.read_text().splitlines() if line.startswith("agentenv-framework-protocol")]