Keep staging candidate tests on the deployment runner

.github/workflows/push.yml

@@ -163,7 +163,7 @@ jobs:
     secrets: inherit
 
   deploy-cloud-run-staging:
-    name: Deploy staging Cloud Run candidate
+    name: Deploy and test staging Cloud Run candidate
     runs-on: ubuntu-latest
     needs:
       - ensure-staging-model-version-aligns-with-sim-api
@@ -302,38 +302,18 @@ jobs:
           CLOUD_RUN_TAG: ${{ steps.cloud_run.outputs.revision_tag }}
       - name: Wait for Cloud Run staging health
         run: bash .github/scripts/health_check.sh "${{ steps.candidate.outputs.url }}/readiness-check"
-
-  integration-tests-staging-cloud-run:
-    name: Run Cloud Run staging integration tests
-    runs-on: ubuntu-latest
-    needs: deploy-cloud-run-staging
-    if: |
-      (github.repository == 'PolicyEngine/policyengine-api')
-      && (github.event.head_commit.message == 'Update PolicyEngine API')
-    environment: staging
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-      - name: Install staging test dependencies
-        run: pip install pytest httpx sqlalchemy psycopg[binary]
       - name: Run staging integration tests
         run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py -v
         env:
-          API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}
-          STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }}
+          API_BASE_URL: ${{ steps.candidate.outputs.url }}
+          STAGING_API_TEST_PROBE_ID: cloud-run-${{ steps.cloud_run.outputs.revision_tag }}
           V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }}
 
   parallel-live-simulation-tests-staging-cloud-run:
     name: Run live simulations against the staging candidate
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    needs:
-      - deploy-cloud-run-staging
-      - integration-tests-staging-cloud-run
+    needs: deploy-cloud-run-staging
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')
       && (github.event.head_commit.message == 'Update PolicyEngine API')
@@ -346,6 +326,8 @@ jobs:
           python-version: "3.12"
       - name: Install live test dependencies
         run: pip install pytest pytest-xdist httpx
+      - name: Wait for Cloud Run staging candidate health before live tests
+        run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.url }}/readiness-check"
       - name: Run live staging tests in two pytest workers
         run: bash .github/scripts/run_live_staging_tests.sh
         env:
@@ -357,7 +339,6 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - deploy-cloud-run-staging
-      - integration-tests-staging-cloud-run
       - parallel-live-simulation-tests-staging-cloud-run
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')

changelog.d/3869.fixed.md

@@ -0,0 +1 @@
+Run staging candidate deployment and integration tests on the same GitHub runner, and wait for candidate readiness again before live simulation tests run on a separate runner.

tests/unit/test_alembic_workflows.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import re
 import subprocess
 from pathlib import Path
 
@@ -231,9 +232,7 @@ def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy():
     assert "create_cloud_sql_backup.sh" in orchestration_script
 
     cloud_run_job = workflow[workflow.index("  deploy-cloud-run-staging:") :]
-    cloud_run_job = cloud_run_job[
-        : cloud_run_job.index("\n  integration-tests-staging-cloud-run:")
-    ]
+    cloud_run_job = re.split(r"\n(?=  \S)", cloud_run_job, maxsplit=1)[0]
     assert "migrate-v1-staging-cloud-sql" in cloud_run_job
     assert "make test" in cloud_run_job
     assert cloud_run_job.index("make test") < cloud_run_job.index(

tests/unit/test_app_engine_decommission.py

@@ -64,9 +64,9 @@ def test_cloud_run_is_the_complete_release_sequence() -> None:
     workflow = PUSH_WORKFLOW.read_text(encoding="utf-8")
     staging_seed = _job_block(workflow, "seed-v2-staging-database")
     staging_deploy = _job_block(workflow, "deploy-cloud-run-staging")
-    staging_integration = _job_block(
+    staging_live_tests = _job_block(
         workflow,
-        "integration-tests-staging-cloud-run",
+        "parallel-live-simulation-tests-staging-cloud-run",
     )
     staging_promotion = _job_block(workflow, "promote-cloud-run-staging")
     production_check = _job_block(
@@ -87,8 +87,13 @@ def test_cloud_run_is_the_complete_release_sequence() -> None:
     assert staging_deploy.index("make test") < staging_deploy.index(
         "Build and push Cloud Run image"
     )
-    assert "needs: deploy-cloud-run-staging" in staging_integration
-    assert "- integration-tests-staging-cloud-run" in staging_promotion
+    assert staging_deploy.index("Wait for Cloud Run staging health") < (
+        staging_deploy.index("Run staging integration tests")
+    )
+    assert "tests/integration/test_live_v2_policies.py" in staging_deploy
+    assert "needs: deploy-cloud-run-staging" in staging_live_tests
+    assert "- deploy-cloud-run-staging" in staging_promotion
+    assert "- parallel-live-simulation-tests-staging-cloud-run" in staging_promotion
     assert "exercise-phase10-staging" not in workflow
     assert "exercise-phase11-staging" not in workflow
     assert "needs: promote-cloud-run-staging" in production_check

tests/unit/test_cloud_run_deploy_scripts.py

@@ -1846,13 +1846,9 @@ def test_set_cloud_run_revision_dry_run_targets_service_override():
     )
 
 
-def test_push_workflow_runs_release_and_cloud_run_staging_tests():
+def test_push_workflow_runs_release_and_cloud_run_staging_tests_in_deploy_job():
     workflow = _push_workflow()
     cloud_run_deploy = _workflow_job_block(workflow, "deploy-cloud-run-staging")
-    cloud_run_tests = _workflow_job_block(
-        workflow,
-        "integration-tests-staging-cloud-run",
-    )
     parallel_live_tests = _workflow_job_block(
         workflow,
         "parallel-live-simulation-tests-staging-cloud-run",
@@ -1876,17 +1872,29 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests():
     assert cloud_run_deploy.index("make test") < cloud_run_deploy.index(
         "Build and push Cloud Run image"
     )
-    assert cloud_run_test_command in cloud_run_tests
-    assert (
-        "API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}"
-        in cloud_run_tests
-    )
-    assert "environment: staging" in cloud_run_tests
-    assert "V2_MIGRATION_DATABASE_URL" in cloud_run_tests
+    assert cloud_run_test_command in cloud_run_deploy
+    assert "API_BASE_URL: ${{ steps.candidate.outputs.url }}" in cloud_run_deploy
+    assert "environment: staging" in cloud_run_deploy
+    assert "V2_MIGRATION_DATABASE_URL" in cloud_run_deploy
+    assert cloud_run_deploy.index(
+        "Wait for Cloud Run staging health"
+    ) < cloud_run_deploy.index("Run staging integration tests")
+    assert "integration-tests-staging-cloud-run:" not in workflow
     assert "strategy:" not in parallel_live_tests
     assert "matrix:" not in parallel_live_tests
-    assert "- integration-tests-staging-cloud-run" in parallel_live_tests
+    assert "needs: deploy-cloud-run-staging" in parallel_live_tests
     assert "pip install pytest pytest-xdist httpx" in parallel_live_tests
+    live_readiness_command = (
+        'bash .github/scripts/health_check.sh "'
+        '${{ needs.deploy-cloud-run-staging.outputs.url }}/readiness-check"'
+    )
+    assert live_readiness_command in parallel_live_tests
+    assert parallel_live_tests.index(
+        "Install live test dependencies"
+    ) < parallel_live_tests.index(live_readiness_command)
+    assert parallel_live_tests.index(
+        live_readiness_command
+    ) < parallel_live_tests.index("Run live staging tests in two pytest workers")
     assert "bash .github/scripts/run_live_staging_tests.sh" in parallel_live_tests
     assert "-n 2" in live_test_script
     assert "--dist load" in live_test_script
@@ -1903,7 +1911,7 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests():
     assert "github.run_attempt" in parallel_live_tests
     assert "needs: promote-cloud-run-staging" in production_check
     assert "- integration-tests-staging-cloud-run" not in production_check
-    assert "- integration-tests-staging-cloud-run" in cloud_run_promotion
+    assert "- integration-tests-staging-cloud-run" not in cloud_run_promotion
     assert "- parallel-live-simulation-tests-staging-cloud-run" in cloud_run_promotion
     assert "exercise-phase10-staging" not in workflow
     assert "run_phase10_staging_probe.sh" not in workflow