From e7a570cf2a0598b416a998c29bcaaf25974b27e5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:43:36 +0400 Subject: [PATCH] Fix live staging test requests and credit expectation --- changelog.d/fix-live-staging-tests.fixed.md | 1 + tests/integration/conftest.py | 9 +- tests/integration/test_live_calculate.py | 4 +- tests/integration/test_live_economy.py | 5 +- tests/unit/test_live_polling.py | 22 +++++ tests/unit/test_live_simulation_requests.py | 94 +++++++++++++++++++++ 6 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fix-live-staging-tests.fixed.md create mode 100644 tests/unit/test_live_simulation_requests.py diff --git a/changelog.d/fix-live-staging-tests.fixed.md b/changelog.d/fix-live-staging-tests.fixed.md new file mode 100644 index 000000000..251e48de3 --- /dev/null +++ b/changelog.d/fix-live-staging-tests.fixed.md @@ -0,0 +1 @@ +Fix the live staging macro tests to use supported query parameters and the California household credit assertion to match the pinned model. Add local regression coverage and include HTTP response details in polling failures. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 78283bd40..dc838e439 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -117,7 +117,14 @@ def _poll_live_endpoint( return error_payload last_response = _response_summary(response) else: - response.raise_for_status() + try: + response.raise_for_status() + except httpx.HTTPStatusError as error: + raise httpx.HTTPStatusError( + f"{error}\n{route_name} response: {_response_summary(response)}", + request=error.request, + response=error.response, + ) from error payload = response.json() if payload["status"] != "computing": diff --git a/tests/integration/test_live_calculate.py b/tests/integration/test_live_calculate.py index af792d5d6..c4ec69c1c 100644 --- a/tests/integration/test_live_calculate.py +++ b/tests/integration/test_live_calculate.py @@ -85,7 +85,9 @@ def test_live_calculate_us_federal_and_california_credits( tax_unit = payload["result"]["tax_units"]["tax unit"] assert tax_unit["eitc"]["2025"] == pytest.approx(4328.0, abs=0.01) assert tax_unit["ctc"]["2025"] == pytest.approx(2200.0, abs=0.01) - assert tax_unit["ca_eitc"]["2025"] == pytest.approx(395.86, abs=0.01) + # policyengine-us 2.2.1 (bundle 6.2.1) computes 388.2118 for this 2025 + # fixture: one qualifying child and $20,000 in both earnings and AGI. + assert tax_unit["ca_eitc"]["2025"] == pytest.approx(388.21, abs=0.01) def test_live_calculate_uk_universal_credit_in_scotland( diff --git a/tests/integration/test_live_economy.py b/tests/integration/test_live_economy.py index 89cc082bb..de363f039 100644 --- a/tests/integration/test_live_economy.py +++ b/tests/integration/test_live_economy.py @@ -91,9 +91,11 @@ def _assert_live_macro_reform( metadata_response.raise_for_status() metadata = metadata_response.json()["result"] + reform_payload = _load_reform_payload(reform_filename) + reform_payload["label"] = f"Live economy {integration_probe_id}-{probe_suffix}" policy_response = api_client.post( f"/{country_id}/policy", - json=_load_reform_payload(reform_filename), + json=reform_payload, ) assert policy_response.status_code in (200, 201), policy_response.text policy_id = policy_response.json()["result"]["policy_id"] @@ -104,7 +106,6 @@ def _assert_live_macro_reform( { "region": region, "time_period": _pick_time_period(metadata), - "staging_probe": f"{integration_probe_id}-{probe_suffix}", }, route_name=f"{country_id}-{probe_suffix}-economy", ) diff --git a/tests/unit/test_live_polling.py b/tests/unit/test_live_polling.py index 7dd33ec55..5a69b9d7a 100644 --- a/tests/unit/test_live_polling.py +++ b/tests/unit/test_live_polling.py @@ -1,6 +1,7 @@ from unittest.mock import Mock import httpx +import pytest from tests.integration import conftest as live_polling @@ -56,3 +57,24 @@ def test_unstructured_bad_gateway_response_is_retried(monkeypatch): assert result == success_payload assert client.get.call_count == 2 + + +def test_invalid_request_error_includes_response_body_without_retrying(): + client = Mock() + client.get.return_value = _response( + 400, + json_payload={ + "status": "error", + "message": "staging_probe: Extra inputs are not permitted", + }, + ) + + with pytest.raises(httpx.HTTPStatusError, match="staging_probe"): + live_polling._poll_live_endpoint( + client, + "/us/economy/1/over/2", + {}, + route_name="economy", + ) + + client.get.assert_called_once() diff --git a/tests/unit/test_live_simulation_requests.py b/tests/unit/test_live_simulation_requests.py new file mode 100644 index 000000000..627660458 --- /dev/null +++ b/tests/unit/test_live_simulation_requests.py @@ -0,0 +1,94 @@ +"""Exercise live test requests locally against the API contract and pinned model.""" + +import json + +import httpx +import pytest + +from policyengine_api.query_parameters import AnnualEconomyQuery, parse_query_items +from tests.integration import conftest as live_polling +from tests.integration import test_live_calculate as live_calculate +from tests.integration import test_live_economy as live_economy + + +@pytest.mark.parametrize( + ("live_test", "country_id", "region"), + [ + (live_economy.test_live_utah_macro_reform, "us", "ut"), + (live_economy.test_live_california_eitc_macro_reform, "us", "state/ca"), + ( + live_economy.test_live_uk_universal_credit_macro_reform_in_scotland, + "uk", + "country/scotland", + ), + ], +) +def test_live_macro_requests_follow_the_query_contract(live_test, country_id, region): + """Run actual live request construction through the server's strict parser.""" + policies = [] + economy_queries = [] + test_year = str(live_economy.CURRENT_YEAR) + + def respond(request: httpx.Request) -> httpx.Response: + if request.method == "GET" and request.url.path == f"/{country_id}/metadata": + return httpx.Response( + 200, + json={ + "result": { + "current_law_id": 2, + "economy_options": {"time_period": [{"name": test_year}]}, + } + }, + ) + if request.method == "POST" and request.url.path == f"/{country_id}/policy": + policies.append(json.loads(request.content)) + return httpx.Response(201, json={"result": {"policy_id": 123}}) + + assert request.method == "GET" + assert request.url.path == f"/{country_id}/economy/123/over/2" + economy_queries.append( + parse_query_items(AnnualEconomyQuery, request.url.params.multi_items()) + ) + return httpx.Response( + 200, + json={ + "status": "ok", + "result": { + "budget": {"budgetary_impact": -1.0}, + "intra_decile": {"all": {"Lose less than 5%": 0.1}}, + }, + }, + ) + + with httpx.Client( + base_url="https://example.test", transport=httpx.MockTransport(respond) + ) as client: + live_test(client, "local-probe-12345678", live_polling._poll_live_endpoint) + + assert len(economy_queries) == 1 + assert economy_queries[0].region == region + assert economy_queries[0].time_period == test_year + assert len(policies) == 1 + assert "local-probe-12345678" in policies[0]["label"] + + +def test_live_credit_assertions_match_the_pinned_us_model(): + """Validate the live assertions using the exact fixture and installed model.""" + from policyengine_us import Simulation + + def respond(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/us/calculate" + household = json.loads(request.content)["household"] + simulation = Simulation(situation=household) + tax_unit = household["tax_units"]["tax unit"] + for variable in ("eitc", "ctc", "ca_eitc"): + tax_unit[variable]["2025"] = float(simulation.calculate(variable, 2025)[0]) + return httpx.Response(200, json={"status": "ok", "result": household}) + + with httpx.Client( + base_url="https://example.test", transport=httpx.MockTransport(respond) + ) as client: + live_calculate.test_live_calculate_us_federal_and_california_credits( + client, "local-probe-12345678" + )