From 0bc56ff716eda717392f73720c101bee6b1ede6a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Wed, 21 Oct 2026 21:57:30 -0700 Subject: [PATCH] fix(release): build an agent image again while PyPI doesn't list the protocol it installs v0.9.1344's agent-build ran pip 33 s after the protocol job uploaded agentenv-framework-protocol 0.1.359, and PyPI's index, which a CDN caches, still listed up to 0.1.358, so the build failed and the release stopped with the protocol on PyPI and the framework not; re-running the failed jobs finished it. infra_build.py retry now tries a build again, after the same waits as a rate limit, while pip can't find the agentenv-framework-protocol version, and still fails at once on any other missing package. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/scripts/infra_build.py | 37 ++++++++++++++++++++--------- tst/unit/test_infra_build_script.py | 27 ++++++++++++++++++--- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 62bffb4b..4a9fe46b 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -7,7 +7,8 @@ ships, with its Dockerfile, context, repository and a source tag. ``exists REF`` exits 0 when the registry has REF, 1 when it says it doesn't, and 2 when it can't tell. ``smoke NAME IMAGE`` checks that IMAGE, a local build of NAME, runs. ``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the digest the registry serves -under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it. +under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI doesn't serve +yet the agentenv-framework-protocol an agent image's build installs. """ from __future__ import annotations @@ -41,8 +42,11 @@ # What a registry answers when it rate-limits a request: ECR Public's 429, and the toomanyrequests error code GHCR and # Docker Hub send. _RATE_LIMITED = re.compile(r"429 Too Many Requests|toomanyrequests", re.IGNORECASE) +# What pip in an agent image's build answers for the agentenv-framework-protocol its release published moments earlier, +# until PyPI's index, which a CDN caches, lists it. +_NOT_YET_SERVED = re.compile(r"No matching distribution found for agentenv-framework-protocol(\[[^\]]*\])?==") # How long to wait before each try again, so a burst of builds starting together spreads out. -_RATE_LIMIT_WAITS_SECONDS = (30, 60, 120) +_RETRY_WAITS_SECONDS = (30, 60, 120) # Each image's check, beyond its build: the gateway imports its server, Postgres turns healthy, the sidecars and the # browser answer their own commands, and Claude Code's agent runs its CLI and imports its server. @@ -70,7 +74,7 @@ def main() -> int: commands.add_parser("retry").add_argument("argv", nargs=argparse.REMAINDER, metavar="COMMAND") args = parser.parse_args() if args.command == "retry": - return _until_not_rate_limited(lambda: _streamed(args.argv)).returncode + return _tried_again(lambda: _streamed(args.argv)).returncode if args.command == "exists": return _exists(args.ref) if args.command == "smoke": @@ -100,19 +104,30 @@ def _check(entry: str) -> list[str]: return problems -def _until_not_rate_limited(attempt: Callable[[], subprocess.CompletedProcess]) -> subprocess.CompletedProcess: - """``attempt()``, and again after each of ``_RATE_LIMIT_WAITS_SECONDS`` while what it printed says a registry - rate-limited it: public.ecr.aws allows one anonymous pull a second, which builds starting together pass.""" - for wait in _RATE_LIMIT_WAITS_SECONDS: +def _tried_again(attempt: Callable[[], subprocess.CompletedProcess]) -> subprocess.CompletedProcess: + """``attempt()``, and again after each of ``_RETRY_WAITS_SECONDS`` while what it printed says a later try can work + (``_why_again``).""" + for wait in _RETRY_WAITS_SECONDS: result = attempt() - if result.returncode == 0 or not _RATE_LIMITED.search(f"{result.stdout}{result.stderr}"): + if result.returncode == 0 or (why := _why_again(result)) is None: return result - print(f"::warning::a registry rate-limited `{' '.join(result.args)[:_PRINTED_CHARS]}`; trying again in {wait}s", - flush=True) + print(f"::warning::{why}; trying again in {wait}s", flush=True) time.sleep(wait) return attempt() +def _why_again(result: subprocess.CompletedProcess) -> str | None: + """Why a failed run can work later, or None: a registry rate-limited it, as public.ecr.aws does past one anonymous + pull a second, which builds starting together pass; or a build's pip can't find the agentenv-framework-protocol its + release has just published, which PyPI's index lists a little after the upload.""" + printed, command = f"{result.stdout}{result.stderr}", f"`{' '.join(result.args)[:_PRINTED_CHARS]}`" + if _RATE_LIMITED.search(printed): + return f"a registry rate-limited {command}" + if _NOT_YET_SERVED.search(printed): + return f"PyPI doesn't list yet the agentenv-framework-protocol {command} installs" + return None + + def _streamed(argv: list[str]) -> subprocess.CompletedProcess: """Run ``argv``, printing what it prints as it prints it, which is kept too, to look for a rate limit in. Each line is flushed, since the workflow's log is a pipe, which Python would otherwise fill before passing on.""" @@ -128,7 +143,7 @@ def _streamed(argv: list[str]) -> subprocess.CompletedProcess: def _served(ref: str) -> tuple[str | None, str]: """The digest the registry serves under ``ref``, or None, and what docker said when it serves none. The manifest as JSON, since buildx before 0.35 doesn't know the ``.Manifest.Digest`` template and prints the whole listing.""" - result = _until_not_rate_limited(lambda: subprocess.run( + result = _tried_again(lambda: subprocess.run( ["docker", "buildx", "imagetools", "inspect", ref, "--format", "{{json .Manifest}}"], capture_output=True, text=True)) if result.returncode != 0: diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index f56e3451..5e29a6ff 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -262,16 +262,37 @@ def test_retry_tries_again_while_a_registry_rate_limits_it(script, monkeypatch, assert _run(script, monkeypatch, "retry", "docker", "push", "img") == 0 - assert len(runs) == 3 and waits == list(script._RATE_LIMIT_WAITS_SECONDS[:2]) + assert len(runs) == 3 and waits == list(script._RETRY_WAITS_SECONDS[:2]) assert capsys.readouterr().out.count("::warning::a registry rate-limited `docker push img`; trying again in") == 2 +def test_retry_tries_again_while_pypi_doesnt_list_the_protocol_a_build_installs(script, monkeypatch, capsys): + """An agent image's build installs the agentenv-framework-protocol its release published moments earlier.""" + missing = ("#13 1.522 ERROR: No matching distribution found for agentenv-framework-protocol[agent]==0.1.359\n" + "ERROR: failed to solve: process \"/bin/sh -c pip install\" did not complete successfully: exit code: 1\n") + runs, waits = _attempts(script, monkeypatch, [(missing, 1), ("#13 DONE 4.1s\n", 0)]) + + assert _run(script, monkeypatch, "retry", "docker", "build", ".") == 0 + + assert len(runs) == 2 and waits == [script._RETRY_WAITS_SECONDS[0]] + assert ("::warning::PyPI doesn't list yet the agentenv-framework-protocol `docker build .` installs; trying again in" + in capsys.readouterr().out) + + +def test_retry_doesnt_try_again_a_build_missing_another_package(script, monkeypatch, capsys): + runs, waits = _attempts(script, monkeypatch, [("ERROR: No matching distribution found for nonexistent-pkg==1.0\n", 1)]) + + assert _run(script, monkeypatch, "retry", "docker", "build", ".") == 1 + + assert len(runs) == 1 and waits == [] + + def test_retry_gives_up_after_its_waits(script, monkeypatch, capsys): - runs, waits = _attempts(script, monkeypatch, [(_LIMITED, 1)] * (len(script._RATE_LIMIT_WAITS_SECONDS) + 1)) + runs, waits = _attempts(script, monkeypatch, [(_LIMITED, 1)] * (len(script._RETRY_WAITS_SECONDS) + 1)) assert _run(script, monkeypatch, "retry", "docker", "build", ".") == 1 - assert len(runs) == len(script._RATE_LIMIT_WAITS_SECONDS) + 1 and waits == list(script._RATE_LIMIT_WAITS_SECONDS) + assert len(runs) == len(script._RETRY_WAITS_SECONDS) + 1 and waits == list(script._RETRY_WAITS_SECONDS) def test_retry_doesnt_try_again_a_command_that_fails_otherwise(script, monkeypatch, capsys):