diff --git a/.github/scripts/run_live_staging_tests.sh b/.github/scripts/run_live_staging_tests.sh new file mode 100755 index 000000000..39bc7a3cb --- /dev/null +++ b/.github/scripts/run_live_staging_tests.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +python -m pytest \ + -n 2 \ + --dist load \ + --maxschedchunk=1 \ + tests/integration/test_live_budget_window_cache.py \ + tests/integration/test_live_calculate.py \ + tests/integration/test_live_economy.py \ + -v diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 856b4f570..a3a0f2dbe 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -320,19 +320,45 @@ jobs: python-version: "3.12" - name: Install staging test dependencies run: pip install pytest httpx sqlalchemy psycopg[binary] - - name: Run staging smoke test - run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v + - name: Run staging integration tests + run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py -v env: API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }} V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }} + parallel-live-simulation-tests-staging-cloud-run: + name: Run live simulations against the staging candidate + runs-on: ubuntu-latest + timeout-minutes: 20 + needs: + - deploy-cloud-run-staging + - integration-tests-staging-cloud-run + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install live test dependencies + run: pip install pytest pytest-xdist httpx + - name: Run live staging tests in two pytest workers + run: bash .github/scripts/run_live_staging_tests.sh + env: + API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} + STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }}-${{ github.run_id }}-${{ github.run_attempt }} + promote-cloud-run-staging: name: Promote staging Cloud Run traffic runs-on: ubuntu-latest needs: - deploy-cloud-run-staging - integration-tests-staging-cloud-run + - parallel-live-simulation-tests-staging-cloud-run if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') diff --git a/changelog.d/2400.added.md b/changelog.d/2400.added.md new file mode 100644 index 000000000..e72c44258 --- /dev/null +++ b/changelog.d/2400.added.md @@ -0,0 +1 @@ +Add parallel live staging coverage for US federal and California tax credits and UK Universal Credit calculations. diff --git a/tests/data/california_eitc_reform.json b/tests/data/california_eitc_reform.json new file mode 100644 index 000000000..f64a5e6bb --- /dev/null +++ b/tests/data/california_eitc_reform.json @@ -0,0 +1,7 @@ +{ + "data": { + "gov.states.ca.tax.income.credits.earned_income.phase_in.rate[1].amount": { + "2025-01-01.2100-12-31": 0.35 + } + } +} diff --git a/tests/data/live_uk_universal_credit_household.json b/tests/data/live_uk_universal_credit_household.json new file mode 100644 index 000000000..41560f113 --- /dev/null +++ b/tests/data/live_uk_universal_credit_household.json @@ -0,0 +1,48 @@ +{ + "household": { + "benunits": { + "benefit unit": { + "members": [ + "parent", + "child" + ], + "universal_credit": { + "2025": null + }, + "would_claim_uc": { + "2025": true + } + } + }, + "households": { + "household": { + "members": [ + "parent", + "child" + ], + "region": { + "2025": "SCOTLAND" + } + } + }, + "people": { + "parent": { + "age": { + "2025": 35 + }, + "employment_income": { + "2025": 12000 + } + }, + "child": { + "age": { + "2025": 6 + }, + "employment_income": { + "2025": 0 + } + } + } + }, + "policy": {} +} diff --git a/tests/data/live_us_credits_household.json b/tests/data/live_us_credits_household.json new file mode 100644 index 000000000..ca1e6d02a --- /dev/null +++ b/tests/data/live_us_credits_household.json @@ -0,0 +1,85 @@ +{ + "household": { + "families": { + "family": { + "members": [ + "parent", + "child" + ] + } + }, + "households": { + "household": { + "members": [ + "parent", + "child" + ], + "state_name": { + "2025": "CA" + } + } + }, + "marital_units": { + "parent's marital unit": { + "members": [ + "parent" + ] + }, + "child's marital unit": { + "marital_unit_id": { + "2025": 1 + }, + "members": [ + "child" + ] + } + }, + "people": { + "parent": { + "age": { + "2025": 35 + }, + "employment_income": { + "2025": 20000 + } + }, + "child": { + "age": { + "2025": 6 + }, + "employment_income": { + "2025": 0 + }, + "is_tax_unit_dependent": { + "2025": true + } + } + }, + "spm_units": { + "spm unit": { + "members": [ + "parent", + "child" + ] + } + }, + "tax_units": { + "tax unit": { + "members": [ + "parent", + "child" + ], + "eitc": { + "2025": null + }, + "ctc": { + "2025": null + }, + "ca_eitc": { + "2025": null + } + } + } + }, + "policy": {} +} diff --git a/tests/data/uk_universal_credit_reform.json b/tests/data/uk_universal_credit_reform.json new file mode 100644 index 000000000..28cae5218 --- /dev/null +++ b/tests/data/uk_universal_credit_reform.json @@ -0,0 +1,7 @@ +{ + "data": { + "gov.dwp.universal_credit.standard_allowance.amount.SINGLE_OLD": { + "2025-01-01.2100-12-31": 500 + } + } +} diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d3e3c40bc..78283bd40 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -85,6 +85,16 @@ def _response_summary(response: httpx.Response) -> str: return f"HTTP {response.status_code}: {response.text[:500]}" +def _structured_error_payload(response: httpx.Response) -> dict | None: + try: + payload = response.json() + except ValueError: + return None + if isinstance(payload, dict) and payload.get("status") == "error": + return payload + return None + + def _poll_live_endpoint( api_client: httpx.Client, path: str, @@ -102,6 +112,9 @@ def _poll_live_endpoint( last_response = f"{type(error).__name__}: {error}" else: if response.status_code in TRANSIENT_POLL_STATUS_CODES: + error_payload = _structured_error_payload(response) + if error_payload is not None: + return error_payload last_response = _response_summary(response) else: response.raise_for_status() diff --git a/tests/integration/test_live_calculate.py b/tests/integration/test_live_calculate.py index 092c464ae..af792d5d6 100644 --- a/tests/integration/test_live_calculate.py +++ b/tests/integration/test_live_calculate.py @@ -1,5 +1,8 @@ +import json from pathlib import Path +import pytest + def _load_payload(filename: str) -> str: return (Path(__file__).resolve().parents[1] / "data" / filename).read_text( @@ -7,6 +10,10 @@ def _load_payload(filename: str) -> str: ) +def _load_json_payload(filename: str) -> dict: + return json.loads(_load_payload(filename)) + + def test_live_calculate_us_1(api_client): response = api_client.post( "/us/calculate", @@ -61,3 +68,40 @@ def test_live_calculate_drops_deprecated_medical_input( "medical_out_of_pocket_expenses" in warning and "deprecated" in warning.lower() for warning in payload["warnings"] ) + + +def test_live_calculate_us_federal_and_california_credits( + api_client, + integration_probe_id, +): + request_payload = _load_json_payload("live_us_credits_household.json") + request_payload["integration_probe_id"] = f"{integration_probe_id}-us-credits" + + response = api_client.post("/us/calculate", json=request_payload) + + assert response.status_code == 200, response.text + payload = response.json() + assert payload["status"] == "ok", payload + tax_unit = payload["result"]["tax_units"]["tax unit"] + assert tax_unit["eitc"]["2025"] == pytest.approx(4328.0, abs=0.01) + assert tax_unit["ctc"]["2025"] == pytest.approx(2200.0, abs=0.01) + assert tax_unit["ca_eitc"]["2025"] == pytest.approx(395.86, abs=0.01) + + +def test_live_calculate_uk_universal_credit_in_scotland( + api_client, + integration_probe_id, +): + request_payload = _load_json_payload("live_uk_universal_credit_household.json") + request_payload["integration_probe_id"] = f"{integration_probe_id}-uk-uc" + + response = api_client.post("/uk/calculate", json=request_payload) + + assert response.status_code == 200, response.text + payload = response.json() + assert payload["status"] == "ok", payload + benefit_unit = payload["result"]["benunits"]["benefit unit"] + assert benefit_unit["universal_credit"]["2025"] == pytest.approx( + 6229.80, + abs=0.01, + ) diff --git a/tests/integration/test_live_economy.py b/tests/integration/test_live_economy.py index 7988f3afd..89cc082bb 100644 --- a/tests/integration/test_live_economy.py +++ b/tests/integration/test_live_economy.py @@ -75,3 +75,75 @@ def test_live_utah_macro_reform(api_client, integration_probe_id, poll_live_endp assert isinstance(lose_less_than_5, int | float), result assert math.isfinite(lose_less_than_5), result assert 0 <= lose_less_than_5 <= 1, result + + +def _assert_live_macro_reform( + api_client, + integration_probe_id, + poll_live_endpoint, + *, + country_id: str, + reform_filename: str, + region: str, + probe_suffix: str, +) -> None: + metadata_response = api_client.get(f"/{country_id}/metadata") + metadata_response.raise_for_status() + metadata = metadata_response.json()["result"] + + policy_response = api_client.post( + f"/{country_id}/policy", + json=_load_reform_payload(reform_filename), + ) + assert policy_response.status_code in (200, 201), policy_response.text + policy_id = policy_response.json()["result"]["policy_id"] + + payload = poll_live_endpoint( + api_client, + f"/{country_id}/economy/{policy_id}/over/{metadata['current_law_id']}", + { + "region": region, + "time_period": _pick_time_period(metadata), + "staging_probe": f"{integration_probe_id}-{probe_suffix}", + }, + route_name=f"{country_id}-{probe_suffix}-economy", + ) + + assert payload["status"] == "ok", payload + result = payload["result"] + assert result is not None, payload + budgetary_impact = result["budget"]["budgetary_impact"] + assert isinstance(budgetary_impact, int | float), result + assert math.isfinite(budgetary_impact), result + + +def test_live_california_eitc_macro_reform( + api_client, + integration_probe_id, + poll_live_endpoint, +): + _assert_live_macro_reform( + api_client, + integration_probe_id, + poll_live_endpoint, + country_id="us", + reform_filename="california_eitc_reform.json", + region="state/ca", + probe_suffix="california-eitc", + ) + + +def test_live_uk_universal_credit_macro_reform_in_scotland( + api_client, + integration_probe_id, + poll_live_endpoint, +): + _assert_live_macro_reform( + api_client, + integration_probe_id, + poll_live_endpoint, + country_id="uk", + reform_filename="uk_universal_credit_reform.json", + region="country/scotland", + probe_suffix="uk-scotland-uc", + ) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index dd6842606..26703a0e8 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -1853,6 +1853,10 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): workflow, "integration-tests-staging-cloud-run", ) + parallel_live_tests = _workflow_job_block( + workflow, + "parallel-live-simulation-tests-staging-cloud-run", + ) cloud_run_promotion = _workflow_job_block(workflow, "promote-cloud-run-staging") production_check = _workflow_job_block( workflow, @@ -1861,11 +1865,9 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): cloud_run_test_command = ( "python -m pytest tests/integration/test_cloud_run_candidate.py " "tests/integration/test_live_v2_metadata.py " - "tests/integration/test_live_v2_policies.py " - "tests/integration/test_live_calculate.py " - "tests/integration/test_live_economy.py " - "tests/integration/test_live_budget_window_cache.py -v" + "tests/integration/test_live_v2_policies.py -v" ) + live_test_script = (REPO / ".github/scripts/run_live_staging_tests.sh").read_text() assert "make test" in cloud_run_deploy assert cloud_run_deploy.index("make test") < cloud_run_deploy.index( @@ -1881,9 +1883,28 @@ def test_push_workflow_runs_release_and_cloud_run_staging_tests(): ) assert "environment: staging" in cloud_run_tests assert "V2_MIGRATION_DATABASE_URL" in cloud_run_tests + assert "strategy:" not in parallel_live_tests + assert "matrix:" not in parallel_live_tests + assert "- integration-tests-staging-cloud-run" in parallel_live_tests + assert "pip install pytest pytest-xdist httpx" in parallel_live_tests + assert "bash .github/scripts/run_live_staging_tests.sh" in parallel_live_tests + assert "-n 2" in live_test_script + assert "--dist load" in live_test_script + assert "--maxschedchunk=1" in live_test_script + assert "tests/integration/test_live_budget_window_cache.py" in live_test_script + assert "tests/integration/test_live_calculate.py" in live_test_script + assert "tests/integration/test_live_economy.py" in live_test_script + assert "tests/integration/test_live_v2_policies.py" not in parallel_live_tests + assert ( + "API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}" + in parallel_live_tests + ) + assert "github.run_id" in parallel_live_tests + assert "github.run_attempt" in parallel_live_tests assert "needs: promote-cloud-run-staging" in production_check assert "- integration-tests-staging-cloud-run" not in production_check assert "- integration-tests-staging-cloud-run" in cloud_run_promotion + assert "- parallel-live-simulation-tests-staging-cloud-run" in cloud_run_promotion assert "exercise-phase10-staging" not in workflow assert "run_phase10_staging_probe.sh" not in workflow assert "V2_FAILURE_DATABASE_URL_SECRET_RESOURCE" not in workflow diff --git a/tests/unit/test_live_polling.py b/tests/unit/test_live_polling.py new file mode 100644 index 000000000..7dd33ec55 --- /dev/null +++ b/tests/unit/test_live_polling.py @@ -0,0 +1,58 @@ +from unittest.mock import Mock + +import httpx + +from tests.integration import conftest as live_polling + + +def _response( + status_code: int, + *, + json_payload: dict | None = None, + text: str | None = None, +) -> httpx.Response: + request = httpx.Request("GET", "https://example.test/economy") + if json_payload is not None: + return httpx.Response(status_code, json=json_payload, request=request) + return httpx.Response(status_code, text=text, request=request) + + +def test_structured_bad_gateway_response_is_returned_immediately(monkeypatch): + payload = { + "status": "error", + "message": "Simulation worker exited", + "result": None, + } + client = Mock() + client.get.return_value = _response(502, json_payload=payload) + monkeypatch.setattr(live_polling, "INTEGRATION_TIMEOUT_SECONDS", 0) + + result = live_polling._poll_live_endpoint( + client, + "/us/economy/1/over/2", + {}, + route_name="economy", + ) + + assert result == payload + client.get.assert_called_once() + + +def test_unstructured_bad_gateway_response_is_retried(monkeypatch): + success_payload = {"status": "ok", "result": {"budget": {}}} + client = Mock() + client.get.side_effect = [ + _response(502, text="Temporary proxy failure"), + _response(200, json_payload=success_payload), + ] + monkeypatch.setattr(live_polling.time, "sleep", lambda _: None) + + result = live_polling._poll_live_endpoint( + client, + "/us/economy/1/over/2", + {}, + route_name="economy", + ) + + assert result == success_payload + assert client.get.call_count == 2