Accept null SPM provenance in Stage 12 aggregate artifacts

changelog_entry.yaml

@@ -17,3 +17,4 @@
       - Raised a clear error, never an omission, when a UK constituency or local-authority breakdown is built over a dataset without area codes
       - Bound UK constituency and local-authority weight matrices to the certified data release of the selected dataset instead of the unversioned bucket copy, and stopped runs before they start when the matrix lacks the run year or household dimension
       - Released each Stage 12 database session after its operation instead of holding pooled sessions through calculations, retried opening each Stage 12 coordinator and worker database connection that the pooler refused without ever retrying a statement or commit, and reported Stage 12 failures to telemetry before recording them in the database so a failed write cannot hide the original error
+      - Accepted a null SPM provenance in Stage 12 aggregate artifacts, so UK and other reports without SPM calculations are saved, while every supplied provenance record is still validated

libs/policyengine-simulation-contract/src/policyengine_simulation_contract/stage12_execution.py

@@ -413,8 +413,11 @@ def validate_spm_provenance(self) -> AggregateReportArtifactPayload:
 
         if "spm_config" in self.result:
             raise ValueError("aggregate result must not include legacy spm_config")
-        if "spm_provenance" in self.result:
-            SPMComparisonProvenance.model_validate(self.result["spm_provenance"])
+        # Reports without US SPM calculations serialize this optional field
+        # as null. Validate a supplied record, not the absence of one.
+        provenance = self.result.get("spm_provenance")
+        if provenance is not None:
+            SPMComparisonProvenance.model_validate(provenance)
         return self
 
 

libs/policyengine-simulation-contract/tests/test_stage12_execution.py

@@ -212,6 +212,29 @@ def test_simulation_artifact_schema_uses_the_receipt_without_a_wrapper() -> None
     assert "SPMCalculationProvenance" not in schema_text
 
 
+@pytest.mark.parametrize("result", [{}, {"spm_provenance": None}])
+def test_aggregate_artifact_accepts_absent_spm_provenance(result) -> None:
+    artifact = AggregateReportArtifactPayload(
+        evaluation_id=UUID("00000000-0000-0000-0000-000000000001"),
+        requested_aggregates=(ReportAggregate.BUDGET,),
+        bundle=_bundle(),
+        result=result,
+    )
+
+    assert artifact.model_dump(mode="json")["result"] == result
+
+
+@pytest.mark.parametrize("provenance", [{}, "invalid", [], 0])
+def test_aggregate_artifact_rejects_invalid_present_spm_provenance(provenance) -> None:
+    with pytest.raises(ValueError):
+        AggregateReportArtifactPayload(
+            evaluation_id=UUID("00000000-0000-0000-0000-000000000001"),
+            requested_aggregates=(ReportAggregate.BUDGET,),
+            bundle=_bundle(),
+            result={"spm_provenance": provenance},
+        )
+
+
 def test_aggregate_artifact_accepts_only_canonical_spm_provenance() -> None:
     comparison = _spm_comparison()
     artifact = AggregateReportArtifactPayload(

projects/policyengine-simulation-executor/fixtures/fake_object_store.py

@@ -0,0 +1,24 @@
+"""Shared in-memory stand-in for ``ArtifactStore``'s byte interface.
+
+The Stage 12 artifact tests and the Stage 12 runtime tests both drive the real
+``Stage12ArtifactStore`` over this fake, so its write-once behaviour has one
+definition instead of a per-file copy.
+"""
+
+
+class InMemoryObjectStore:
+    """Keep uploaded objects by path, refusing to replace an existing one."""
+
+    def __init__(self) -> None:
+        self.objects: dict[str, bytes] = {}
+
+    def upload_bytes(self, path, payload, *, content_type):
+        # The real store uploads with ``if_generation_match=0``: an existing
+        # object is left in place and the call reports that it created nothing.
+        if path in self.objects:
+            return False
+        self.objects[path] = payload
+        return True
+
+    def read_bytes(self, path):
+        return self.objects.get(path)

projects/policyengine-simulation-executor/fixtures/test_simulation_api_contracts.py → projects/policyengine-simulation-executor/fixtures/macro_output.py

@@ -1,4 +1,9 @@
-"""Fixtures for simulation API contract tests."""
+"""A complete current single-year macro result and its key sets.
+
+The API contract, output builder, standalone app and Stage 12 runtime tests
+all need a valid ``SingleYearMacroOutput`` payload. This module is its one
+shared home, so no test lane imports another lane's fixture module.
+"""
 
 CURRENT_SINGLE_YEAR_MACRO_KEYS = {
     "spm_provenance",

projects/policyengine-simulation-executor/tests/test_simulation_api_contracts.py

@@ -11,7 +11,7 @@
     SingleYearMacroOutput,
 )
 
-from fixtures.test_simulation_api_contracts import (
+from fixtures.macro_output import (
     CURRENT_REQUIRED_BUDGET_KEYS,
     CURRENT_SINGLE_YEAR_MACRO_KEYS,
     CURRENT_SINGLE_YEAR_MACRO_RESULT,

projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py

@@ -14,7 +14,7 @@
     installed_spm_selection,
     spm_capable_simulation,
 )
-from fixtures.test_simulation_api_contracts import (
+from fixtures.macro_output import (
     CURRENT_SINGLE_YEAR_MACRO_KEYS,
     CURRENT_SINGLE_YEAR_MACRO_RESULT,
 )

projects/policyengine-simulation-executor/tests/test_stage12_artifacts.py

@@ -24,6 +24,7 @@
 )
 from pydantic import ValidationError
 
+from fixtures.fake_object_store import InMemoryObjectStore
 from policyengine_simulation_executor.stage12_artifacts import (
     Stage12ArtifactStore,
     comparison_run_prefix,
@@ -193,22 +194,8 @@ def test_private_layout_is_environment_and_date_scoped() -> None:
     )
 
 
-class FakeObjectStore:
-    def __init__(self):
-        self.values = {}
-
-    def upload_bytes(self, path, payload, *, content_type):
-        if path in self.values:
-            return False
-        self.values[path] = payload
-        return True
-
-    def read_bytes(self, path):
-        return self.values.get(path)
-
-
 def test_immutable_retry_accepts_identical_bytes_and_rejects_other_bytes() -> None:
-    objects = FakeObjectStore()
+    objects = InMemoryObjectStore()
     store = Stage12ArtifactStore("private-stage12", store=objects)
 
     first = store._write_immutable("path", b"same", content_type="application/json")
@@ -220,7 +207,7 @@ def test_immutable_retry_accepts_identical_bytes_and_rejects_other_bytes() -> No
 
 
 def test_aggregate_write_rejects_an_undeclared_payload_shape() -> None:
-    store = Stage12ArtifactStore("private-stage12", store=FakeObjectStore())
+    store = Stage12ArtifactStore("private-stage12", store=InMemoryObjectStore())
 
     with pytest.raises(ValueError, match="evaluation_id"):
         store.write_aggregate(