From 541a617210e6fec2304339e4ce99bbf68fc75a1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Mon, 9 Nov 2026 08:11:14 -0800 Subject: [PATCH 1/2] test(apps): de-flake catalog provisioning deadline test test_catalog_wait_respects_provisioning_deadline raced a fixed 0.1s asyncio.wait_for against provisioning reaching the catalog fetch, then asserted catalog.started was set. On slower CI the reconcile path often did not reach the fetch within 100ms, so the assertion failed intermittently (seen repeatedly across unrelated PRs). Wait for the fetch to be in flight (catalog.started) before applying the deadline, so the timeout lands on the blocked fetch rather than on the work leading up to it. The deadline then fires deterministically because the fetch is held open, independent of machine speed. Behavior under test is unchanged; only the timing coupling is removed. --- tests/test_apps/test_gpu_catalog.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/test_apps/test_gpu_catalog.py b/tests/test_apps/test_gpu_catalog.py index a4baf4484..fcdd58cf4 100644 --- a/tests/test_apps/test_gpu_catalog.py +++ b/tests/test_apps/test_gpu_catalog.py @@ -138,13 +138,21 @@ async def test_catalog_wait_keeps_event_loop_responsive(catalog, operation): async def test_catalog_wait_respects_provisioning_deadline(catalog, operation): catalog.release.clear() client = AppsApiClient(api_key="explicit-key") - with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(provision(operation, client), timeout=0.1) - assert catalog.started.is_set() - client.save_endpoint.assert_not_called() - cache = next(iter(gpu._catalog_caches.values())) - assert cache.failed_at is None - assert cache.members is None + task = asyncio.create_task(provision(operation, client)) + try: + # wait for the fetch to be in flight so the deadline lands on it, + # rather than racing a fixed timeout against reaching the fetch + await asyncio.wait_for(catalog.started.wait(), timeout=1) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(task), timeout=0.1) + assert catalog.started.is_set() + client.save_endpoint.assert_not_called() + cache = next(iter(gpu._catalog_caches.values())) + assert cache.failed_at is None + assert cache.members is None + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) catalog.release.set() assert MIG in (await client.gpu_pool_members())["AMPERE_24"] From 38c05233f88a7481824edc8accf51da917c99107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Mon, 9 Nov 2026 08:16:34 -0800 Subject: [PATCH 2/2] test(apps): cancel provisioning on the deadline instead of shielding Let the deadline cancel the provisioning task (as a real one would) rather than shielding it, so the cache assertions run after the cancellation and keep verifying that a cancelled deadline does not record a failure or partial members. Determinism is preserved: the test still waits for the fetch to be in flight before applying the deadline. --- tests/test_apps/test_gpu_catalog.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_apps/test_gpu_catalog.py b/tests/test_apps/test_gpu_catalog.py index fcdd58cf4..c6f7b54c2 100644 --- a/tests/test_apps/test_gpu_catalog.py +++ b/tests/test_apps/test_gpu_catalog.py @@ -143,10 +143,12 @@ async def test_catalog_wait_respects_provisioning_deadline(catalog, operation): # wait for the fetch to be in flight so the deadline lands on it, # rather than racing a fixed timeout against reaching the fetch await asyncio.wait_for(catalog.started.wait(), timeout=1) + # the deadline cancels provisioning mid-fetch, as a real one would with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(asyncio.shield(task), timeout=0.1) + await asyncio.wait_for(task, timeout=0.1) assert catalog.started.is_set() client.save_endpoint.assert_not_called() + # a cancelled deadline must not poison the cache cache = next(iter(gpu._catalog_caches.values())) assert cache.failed_at is None assert cache.members is None