Fix live staging simulation tests

changelog.d/fix-live-staging-tests.fixed.md

@@ -0,0 +1 @@
+Fix the live staging macro tests to use supported query parameters and the California household credit assertion to match the pinned model. Add local regression coverage and include HTTP response details in polling failures.

tests/integration/conftest.py

@@ -117,7 +117,14 @@ def _poll_live_endpoint(
                     return error_payload
                 last_response = _response_summary(response)
             else:
-                response.raise_for_status()
+                try:
+                    response.raise_for_status()
+                except httpx.HTTPStatusError as error:
+                    raise httpx.HTTPStatusError(
+                        f"{error}\n{route_name} response: {_response_summary(response)}",
+                        request=error.request,
+                        response=error.response,
+                    ) from error
                 payload = response.json()
 
                 if payload["status"] != "computing":

tests/integration/test_live_calculate.py

@@ -85,7 +85,9 @@ def test_live_calculate_us_federal_and_california_credits(
     tax_unit = payload["result"]["tax_units"]["tax unit"]
     assert tax_unit["eitc"]["2025"] == pytest.approx(4328.0, abs=0.01)
     assert tax_unit["ctc"]["2025"] == pytest.approx(2200.0, abs=0.01)
-    assert tax_unit["ca_eitc"]["2025"] == pytest.approx(395.86, abs=0.01)
+    # policyengine-us 2.2.1 (bundle 6.2.1) computes 388.2118 for this 2025
+    # fixture: one qualifying child and $20,000 in both earnings and AGI.
+    assert tax_unit["ca_eitc"]["2025"] == pytest.approx(388.21, abs=0.01)
 
 
 def test_live_calculate_uk_universal_credit_in_scotland(

tests/integration/test_live_economy.py

@@ -91,9 +91,11 @@ def _assert_live_macro_reform(
     metadata_response.raise_for_status()
     metadata = metadata_response.json()["result"]
 
+    reform_payload = _load_reform_payload(reform_filename)
+    reform_payload["label"] = f"Live economy {integration_probe_id}-{probe_suffix}"
     policy_response = api_client.post(
         f"/{country_id}/policy",
-        json=_load_reform_payload(reform_filename),
+        json=reform_payload,
     )
     assert policy_response.status_code in (200, 201), policy_response.text
     policy_id = policy_response.json()["result"]["policy_id"]
@@ -104,7 +106,6 @@ def _assert_live_macro_reform(
         {
             "region": region,
             "time_period": _pick_time_period(metadata),
-            "staging_probe": f"{integration_probe_id}-{probe_suffix}",
         },
         route_name=f"{country_id}-{probe_suffix}-economy",
     )

tests/unit/test_live_polling.py

@@ -1,6 +1,7 @@
 from unittest.mock import Mock
 
 import httpx
+import pytest
 
 from tests.integration import conftest as live_polling
 
@@ -56,3 +57,24 @@ def test_unstructured_bad_gateway_response_is_retried(monkeypatch):
 
     assert result == success_payload
     assert client.get.call_count == 2
+
+
+def test_invalid_request_error_includes_response_body_without_retrying():
+    client = Mock()
+    client.get.return_value = _response(
+        400,
+        json_payload={
+            "status": "error",
+            "message": "staging_probe: Extra inputs are not permitted",
+        },
+    )
+
+    with pytest.raises(httpx.HTTPStatusError, match="staging_probe"):
+        live_polling._poll_live_endpoint(
+            client,
+            "/us/economy/1/over/2",
+            {},
+            route_name="economy",
+        )
+
+    client.get.assert_called_once()

tests/unit/test_live_simulation_requests.py

@@ -0,0 +1,94 @@
+"""Exercise live test requests locally against the API contract and pinned model."""
+
+import json
+
+import httpx
+import pytest
+
+from policyengine_api.query_parameters import AnnualEconomyQuery, parse_query_items
+from tests.integration import conftest as live_polling
+from tests.integration import test_live_calculate as live_calculate
+from tests.integration import test_live_economy as live_economy
+
+
+@pytest.mark.parametrize(
+    ("live_test", "country_id", "region"),
+    [
+        (live_economy.test_live_utah_macro_reform, "us", "ut"),
+        (live_economy.test_live_california_eitc_macro_reform, "us", "state/ca"),
+        (
+            live_economy.test_live_uk_universal_credit_macro_reform_in_scotland,
+            "uk",
+            "country/scotland",
+        ),
+    ],
+)
+def test_live_macro_requests_follow_the_query_contract(live_test, country_id, region):
+    """Run actual live request construction through the server's strict parser."""
+    policies = []
+    economy_queries = []
+    test_year = str(live_economy.CURRENT_YEAR)
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        if request.method == "GET" and request.url.path == f"/{country_id}/metadata":
+            return httpx.Response(
+                200,
+                json={
+                    "result": {
+                        "current_law_id": 2,
+                        "economy_options": {"time_period": [{"name": test_year}]},
+                    }
+                },
+            )
+        if request.method == "POST" and request.url.path == f"/{country_id}/policy":
+            policies.append(json.loads(request.content))
+            return httpx.Response(201, json={"result": {"policy_id": 123}})
+
+        assert request.method == "GET"
+        assert request.url.path == f"/{country_id}/economy/123/over/2"
+        economy_queries.append(
+            parse_query_items(AnnualEconomyQuery, request.url.params.multi_items())
+        )
+        return httpx.Response(
+            200,
+            json={
+                "status": "ok",
+                "result": {
+                    "budget": {"budgetary_impact": -1.0},
+                    "intra_decile": {"all": {"Lose less than 5%": 0.1}},
+                },
+            },
+        )
+
+    with httpx.Client(
+        base_url="https://example.test", transport=httpx.MockTransport(respond)
+    ) as client:
+        live_test(client, "local-probe-12345678", live_polling._poll_live_endpoint)
+
+    assert len(economy_queries) == 1
+    assert economy_queries[0].region == region
+    assert economy_queries[0].time_period == test_year
+    assert len(policies) == 1
+    assert "local-probe-12345678" in policies[0]["label"]
+
+
+def test_live_credit_assertions_match_the_pinned_us_model():
+    """Validate the live assertions using the exact fixture and installed model."""
+    from policyengine_us import Simulation
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        assert request.method == "POST"
+        assert request.url.path == "/us/calculate"
+        household = json.loads(request.content)["household"]
+        simulation = Simulation(situation=household)
+        tax_unit = household["tax_units"]["tax unit"]
+        for variable in ("eitc", "ctc", "ca_eitc"):
+            tax_unit[variable]["2025"] = float(simulation.calculate(variable, 2025)[0])
+        return httpx.Response(200, json={"status": "ok", "result": household})
+
+    with httpx.Client(
+        base_url="https://example.test", transport=httpx.MockTransport(respond)
+    ) as client:
+        live_calculate.test_live_calculate_us_federal_and_california_credits(
+            client, "local-probe-12345678"
+        )