diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a3a0f2dbe..9213f4fbd 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -163,7 +163,7 @@ jobs: secrets: inherit deploy-cloud-run-staging: - name: Deploy staging Cloud Run candidate + name: Deploy and test staging Cloud Run candidate runs-on: ubuntu-latest needs: - ensure-staging-model-version-aligns-with-sim-api @@ -302,38 +302,18 @@ jobs: CLOUD_RUN_TAG: ${{ steps.cloud_run.outputs.revision_tag }} - name: Wait for Cloud Run staging health run: bash .github/scripts/health_check.sh "${{ steps.candidate.outputs.url }}/readiness-check" - - integration-tests-staging-cloud-run: - name: Run Cloud Run staging integration tests - runs-on: ubuntu-latest - needs: deploy-cloud-run-staging - if: | - (github.repository == 'PolicyEngine/policyengine-api') - && (github.event.head_commit.message == 'Update PolicyEngine API') - environment: staging - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install staging test dependencies - run: pip install pytest httpx sqlalchemy psycopg[binary] - name: Run staging integration tests run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py -v env: - API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} - STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }} + API_BASE_URL: ${{ steps.candidate.outputs.url }} + STAGING_API_TEST_PROBE_ID: cloud-run-${{ steps.cloud_run.outputs.revision_tag }} V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} parallel-live-simulation-tests-staging-cloud-run: name: Run live simulations against the staging candidate runs-on: ubuntu-latest timeout-minutes: 20 - needs: - - deploy-cloud-run-staging - - integration-tests-staging-cloud-run + needs: deploy-cloud-run-staging if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -346,6 +326,8 @@ jobs: python-version: "3.12" - name: Install live test dependencies run: pip install pytest pytest-xdist httpx + - name: Wait for Cloud Run staging candidate health before live tests + run: bash .github/scripts/health_check.sh "${{ needs.deploy-cloud-run-staging.outputs.url }}/readiness-check" - name: Run live staging tests in two pytest workers run: bash .github/scripts/run_live_staging_tests.sh env: @@ -357,7 +339,6 @@ jobs: runs-on: ubuntu-latest needs: - deploy-cloud-run-staging - - integration-tests-staging-cloud-run - parallel-live-simulation-tests-staging-cloud-run if: | (github.repository == 'PolicyEngine/policyengine-api') diff --git a/changelog.d/3869.fixed.md b/changelog.d/3869.fixed.md new file mode 100644 index 000000000..ef8fbf73e --- /dev/null +++ b/changelog.d/3869.fixed.md @@ -0,0 +1 @@ +Run staging candidate deployment and integration tests on the same GitHub runner, and wait for candidate readiness again before live simulation tests run on a separate runner. diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index ffc82dc57..1e2114154 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import re import subprocess from pathlib import Path @@ -231,9 +232,7 @@ def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy(): assert "create_cloud_sql_backup.sh" in orchestration_script cloud_run_job = workflow[workflow.index(" deploy-cloud-run-staging:") :] - cloud_run_job = cloud_run_job[ - : cloud_run_job.index("\n integration-tests-staging-cloud-run:") - ] + cloud_run_job = re.split(r"\n(?= \S)", cloud_run_job, maxsplit=1)[0] assert "migrate-v1-staging-cloud-sql" in cloud_run_job assert "make test" in cloud_run_job assert cloud_run_job.index("make test") < cloud_run_job.index( diff --git a/tests/unit/test_app_engine_decommission.py b/tests/unit/test_app_engine_decommission.py index 0f5f317bc..ff2032467 100644 --- a/tests/unit/test_app_engine_decommission.py +++ b/tests/unit/test_app_engine_decommission.py @@ -64,9 +64,9 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: workflow = PUSH_WORKFLOW.read_text(encoding="utf-8") staging_seed = _job_block(workflow, "seed-v2-staging-database") staging_deploy = _job_block(workflow, "deploy-cloud-run-staging") - staging_integration = _job_block( + staging_live_tests = _job_block( workflow, - "integration-tests-staging-cloud-run", + "parallel-live-simulation-tests-staging-cloud-run", ) staging_promotion = _job_block(workflow, "promote-cloud-run-staging") production_check = _job_block( @@ -87,8 +87,13 @@ def test_cloud_run_is_the_complete_release_sequence() -> None: assert staging_deploy.index("make test") < staging_deploy.index( "Build and push Cloud Run image" ) - assert "needs: deploy-cloud-run-staging" in staging_integration - assert "- integration-tests-staging-cloud-run" in staging_promotion + assert staging_deploy.index("Wait for Cloud Run staging health") < ( + staging_deploy.index("Run staging integration tests") + ) + assert "tests/integration/test_live_v2_policies.py" in staging_deploy + assert "needs: deploy-cloud-run-staging" in staging_live_tests + assert "- deploy-cloud-run-staging" in staging_promotion + assert "- parallel-live-simulation-tests-staging-cloud-run" in staging_promotion assert "exercise-phase10-staging" not in workflow assert "exercise-phase11-staging" not in workflow assert "needs: promote-cloud-run-staging" in production_check diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 26703a0e8..a22289af8 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -1846,13 +1846,9 @@ def test_set_cloud_run_revision_dry_run_targets_service_override(): ) -def test_push_workflow_runs_release_and_cloud_run_staging_tests(): +def test_push_workflow_runs_release_and_cloud_run_staging_tests_in_deploy_job(): workflow = _push_workflow() cloud_run_deploy = _workflow_job_block(workflow, "deploy-cloud-run-staging") - cloud_run_tests = _workflow_job_block( - workflow, - "integration-tests-staging-cloud-run", - ) parallel_live_tests = _workflow_job_block( workflow, "parallel-live-simulation-tests-staging-cloud-run", @@ -1876,17 +1872,29 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): assert cloud_run_deploy.index("make test") < cloud_run_deploy.index( "Build and push Cloud Run image" ) - assert cloud_run_test_command in cloud_run_tests - assert ( - "API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}" - in cloud_run_tests - ) - assert "environment: staging" in cloud_run_tests - assert "V2_MIGRATION_DATABASE_URL" in cloud_run_tests + assert cloud_run_test_command in cloud_run_deploy + assert "API_BASE_URL: ${{ steps.candidate.outputs.url }}" in cloud_run_deploy + assert "environment: staging" in cloud_run_deploy + assert "V2_MIGRATION_DATABASE_URL" in cloud_run_deploy + assert cloud_run_deploy.index( + "Wait for Cloud Run staging health" + ) < cloud_run_deploy.index("Run staging integration tests") + assert "integration-tests-staging-cloud-run:" not in workflow assert "strategy:" not in parallel_live_tests assert "matrix:" not in parallel_live_tests - assert "- integration-tests-staging-cloud-run" in parallel_live_tests + assert "needs: deploy-cloud-run-staging" in parallel_live_tests assert "pip install pytest pytest-xdist httpx" in parallel_live_tests + live_readiness_command = ( + 'bash .github/scripts/health_check.sh "' + '${{ needs.deploy-cloud-run-staging.outputs.url }}/readiness-check"' + ) + assert live_readiness_command in parallel_live_tests + assert parallel_live_tests.index( + "Install live test dependencies" + ) < parallel_live_tests.index(live_readiness_command) + assert parallel_live_tests.index( + live_readiness_command + ) < parallel_live_tests.index("Run live staging tests in two pytest workers") assert "bash .github/scripts/run_live_staging_tests.sh" in parallel_live_tests assert "-n 2" in live_test_script assert "--dist load" in live_test_script @@ -1903,7 +1911,7 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): assert "github.run_attempt" in parallel_live_tests assert "needs: promote-cloud-run-staging" in production_check assert "- integration-tests-staging-cloud-run" not in production_check - assert "- integration-tests-staging-cloud-run" in cloud_run_promotion + assert "- integration-tests-staging-cloud-run" not in cloud_run_promotion assert "- parallel-live-simulation-tests-staging-cloud-run" in cloud_run_promotion assert "exercise-phase10-staging" not in workflow assert "run_phase10_staging_probe.sh" not in workflow