From afee349d2c14de1b121b0b7e4f03b695d03f418c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 08:01:41 -0400 Subject: [PATCH 01/21] Let UK certification keep a local-area release on an accepted model mismatch certify-data refuses a regional release that neither was built with nor claims the pinned policyengine-uk. The only Microcosm local-area cut is on 2.132.6, a new one takes 2-13 hours plus a two-build sign-off, and area receipts are uncertified anyway (the certificate covers the national dataset only, #685). So each national re-cut on a newer policyengine-uk had to drop area regions or wait for a dense rebuild. --accept-regional-model-mismatch REASON (UK only) keeps the regional release routed and records regional_certification in the bundle: basis accepted_model_mismatch, both model versions, the reason and area_receipts: uncertified. It refuses an empty reason, a regional release that already covers the version, and a run without a regional release; the core must still be covered. The national certificate is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../accept-regional-model-mismatch.added.md | 1 + docs/engineering/skills/data-certification.md | 25 +++ scripts/bundle.py | 13 ++ scripts/certify_data_release.py | 10 ++ src/policyengine/provenance/certification.py | 100 ++++++++++++ src/policyengine/provenance/manifest.py | 18 +++ tests/test_certify_data_release.py | 144 ++++++++++++++++++ 7 files changed, 311 insertions(+) create mode 100644 changelog.d/accept-regional-model-mismatch.added.md diff --git a/changelog.d/accept-regional-model-mismatch.added.md b/changelog.d/accept-regional-model-mismatch.added.md new file mode 100644 index 000000000..a60f85002 --- /dev/null +++ b/changelog.d/accept-regional-model-mismatch.added.md @@ -0,0 +1 @@ +`certify-data --accept-regional-model-mismatch REASON` (UK only) keeps a local-area release routed when it was built on another policyengine-uk version, and records the reviewed mismatch as `regional_certification` in the bundle instead of refusing; area receipts are uncertified. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 23c5d8133..1ce62d882 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -79,6 +79,31 @@ Microcosm national release with one of: claim first. Its `frs_2024_25` is not carried, so keep it loadable with a `dataset_overlays.uk` entry. +- **An older local-area release, on an accepted mismatch.** When the national + cut moves to a new policyengine-uk before a local-area cut on that version + exists, a reviewer can keep the current local-area release routed instead of + dropping area regions: + + ```bash + python scripts/bundle.py certify-data --country uk --data-producer populace \ + --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ + --regional-manifest-uri "" \ + --accept-regional-model-mismatch "" \ + --model-version "" + ``` + + Certification then records `regional_certification` in the bundle: + `compatibility_basis: accepted_model_mismatch`, both model versions, the + reason and `area_receipts: uncertified`. Area receipts are uncertified in any + case, because the certificate covers the national dataset only (#685). + The cost is real and goes in the pull request. The local-area file was built + and calibrated under the older model. policyengine-core ignores, with a + warning, the columns the newer model no longer defines, and inputs the newer + model added take their defaults in area runs. The flag is UK only. It refuses + an empty reason, a regional release that already covers the version (so no + certificate records a mismatch that is not there) and a run without a + regional release. The core must still be covered. + Either way, certification checks that the regional release serves the pinned model by its own build or compatibility claims (a `--qualified-pair-id` names the primary release only), checks its core claims as for the primary release diff --git a/scripts/bundle.py b/scripts/bundle.py index 53b74a26a..5530a0a72 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -48,6 +48,10 @@ def _certify_data(args: argparse.Namespace) -> int: argv.extend(["--regional-release-kind", args.regional_release_kind]) if args.drop_regional_release: argv.append("--drop-regional-release") + if args.accept_regional_model_mismatch: + argv.extend( + ["--accept-regional-model-mismatch", args.accept_regional_model_mismatch] + ) if args.no_generate: argv.append("--no-generate") if args.no_changelog: @@ -226,6 +230,15 @@ def _parser() -> argparse.ArgumentParser: "Cannot be combined with --regional-manifest-uri." ), ) + certify.add_argument( + "--accept-regional-model-mismatch", + metavar="REASON", + help=( + "UK only: route regions to a regional release on another model " + "version, recording this reviewed reason; forwarded to " + "scripts/certify_data_release.py." + ), + ) certify.add_argument( "--regional-artifact-prefix", help="Legacy per-state artifact prefix to import. Defaults to states/.", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index f3a720ee4..5c3bf6874 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -115,6 +115,15 @@ def main(argv=None) -> int: "qualification of this pair (requires --core-version)." ), ) + parser.add_argument( + "--accept-regional-model-mismatch", + metavar="REASON", + help=( + "UK only: route regions to a regional release that does not cover " + "the pinned model, recording this reviewed reason in the bundle " + "instead of refusing. The core must still be covered." + ), + ) parser.add_argument("--no-generate", action="store_true") parser.add_argument("--no-changelog", action="store_true") parser.add_argument( @@ -149,6 +158,7 @@ def main(argv=None) -> int: drop_regional_release=args.drop_regional_release, core_version=args.core_version, qualified_pair_id=args.qualified_pair_id, + regional_model_mismatch_reason=args.accept_regional_model_mismatch, ) print(result.summary()) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index dc773e215..d42c0ca1c 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -410,6 +410,56 @@ def model_compatibility_basis( return basis, [] if warning is None else [warning] +#: A regional release's model coverage accepted by a reviewer, not by the +#: release's own build or claims (``--accept-regional-model-mismatch``). +REGIONAL_BASIS_ACCEPTED_MISMATCH = "accepted_model_mismatch" + + +def accepted_regional_mismatch( + regional_manifest: DataReleaseManifest, + model_package: str, + model_version: str, + reason: str, +) -> tuple[dict, str]: + """Record a reviewed decision to route regions to a release on another model. + + Area runs on such a release are uncertified twice over: the certificate + covers the national dataset only, and the regional file was built and + calibrated under another model version. The record names both versions + and the reason so the bundle, its TRO and every consumer can say so. + Returns ``(record, warning)``. + """ + + reason = reason.strip() + if not reason: + raise CertificationError( + "--accept-regional-model-mismatch needs a reason a reviewer can read." + ) + built = ( + regional_manifest.build.built_with_model_package + if (regional_manifest.build is not None) + else None + ) + built_with = ( + built.version if built is not None and built.name == model_package else None + ) + record = { + "compatibility_basis": REGIONAL_BASIS_ACCEPTED_MISMATCH, + "certified_for_model_version": model_version, + "built_with_model_version": built_with, + "accepted_reason": reason, + "area_receipts": "uncertified", + } + warning = ( + f"regional release: built with {model_package} " + f"{built_with or 'an unrecorded version'} and routed under " + f"{model_version} on an accepted mismatch ({reason}); area runs are " + "uncertified, inputs the newer model added take their defaults there, " + "and columns it dropped are ignored on load" + ) + return record, warning + + def _check_core_compatibility( manifest: DataReleaseManifest, core_version: str, @@ -1107,6 +1157,7 @@ def build_bundle_data_release_payload( regional_manifest_uri: Optional[str] = None, regional_uri_parts: Optional[dict] = None, regional_release_kind: Optional[RegionalReleaseKind] = None, + regional_certification: Optional[dict] = None, ) -> dict: """Map the country certification payload into bundle ``data_releases``.""" payload = copy.deepcopy(country_payload) @@ -1126,6 +1177,8 @@ def build_bundle_data_release_payload( ) if regional_release_kind is not None: payload["regional_release_kind"] = regional_release_kind + if regional_certification is not None: + payload["regional_certification"] = regional_certification payload["default_dataset_uri"] = certified_artifact.get("uri") return payload @@ -1149,6 +1202,7 @@ def certify( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, ) -> tuple[dict, str, list[str]]: raise NotImplementedError @@ -1172,6 +1226,7 @@ def certify( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, ) -> tuple[dict, str, list[str]]: raise CertificationError( "Legacy data-producer certification updates are not implemented in " @@ -1198,6 +1253,7 @@ def certify( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, ) -> tuple[dict, str, list[str]]: manifest, manifest_sha256, uri_parts = fetch_release_manifest( manifest_uri, token=token @@ -1208,6 +1264,18 @@ def certify( recorded_release_kind = None regional_artifacts: dict[str, DataReleaseArtifact] = {} regional_warnings: list[str] = [] + regional_certification: Optional[dict] = None + mismatch_warning: Optional[str] = None + if regional_model_mismatch_reason is not None and regional_manifest_uri is None: + raise CertificationError( + "--accept-regional-model-mismatch applies to a regional release; " + "pass --regional-manifest-uri." + ) + if regional_model_mismatch_reason is not None and country != "uk": + raise CertificationError( + "--accept-regional-model-mismatch is only supported for the UK, " + "whose area receipts are uncertified." + ) if regional_manifest_uri is not None: regional_manifest, _, regional_uri_parts = fetch_release_manifest( regional_manifest_uri, token=token @@ -1219,6 +1287,27 @@ def certify( _, regional_warnings = model_compatibility_basis( regional_manifest, model_package, model_version ) + except CertificationError as exc: + if regional_model_mismatch_reason is None: + raise CertificationError( + f"Regional release {regional_manifest_uri}: {exc}" + ) from exc + regional_certification, mismatch_warning = accepted_regional_mismatch( + regional_manifest, + model_package, + model_version, + regional_model_mismatch_reason, + ) + regional_warnings = [] + else: + if regional_model_mismatch_reason is not None: + raise CertificationError( + f"Regional release {regional_manifest_uri} serves " + f"{model_package} {model_version} by its own build or " + "claims; drop --accept-regional-model-mismatch so the " + "certificate does not record a mismatch that is not there." + ) + try: if core_version is not None: _check_core_compatibility( regional_manifest, core_version, None, regional_warnings @@ -1230,6 +1319,8 @@ def certify( regional_warnings = [ f"regional release: {warning}" for warning in regional_warnings ] + if mismatch_warning is not None: + regional_warnings.append(mismatch_warning) primary_artifact_names = set(manifest.artifacts) manifest, recorded_release_kind = merge_regional_release_manifest( country, @@ -1323,6 +1414,7 @@ def certify( regional_manifest_uri=regional_manifest_uri, regional_uri_parts=regional_uri_parts, regional_release_kind=recorded_release_kind, + regional_certification=regional_certification, ), manifest_sha256, warnings, @@ -1368,6 +1460,7 @@ def certify_data_release( drop_regional_release: bool = False, core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, + regional_model_mismatch_reason: Optional[str] = None, ) -> CertificationResult: """Certify a data release into the bundle manifest. @@ -1383,6 +1476,12 @@ def certify_data_release( given, by its own build or claims: a qualified pair names the primary release only. While the bundle routes regions, a run without ``regional_manifest_uri`` is refused unless ``drop_regional_release``. + + ``regional_model_mismatch_reason`` (UK only) records a reviewer's + acceptance of a regional release that does not cover the pinned model, + instead of refusing it. The core must still be covered. The bundle then + carries ``regional_certification`` naming both model versions and the + reason; area receipts are uncertified either way. """ certified_country = _certified_country(country) if drop_regional_release and regional_manifest_uri is not None: @@ -1429,6 +1528,7 @@ def certify_data_release( bundle_core_version=( (bundle.get("packages") or {}).get("policyengine-core") or {} ).get("version"), + regional_model_mismatch_reason=regional_model_mismatch_reason, ) previous_release = (bundle.get("data_releases") or {}).get(country) or {} bundle.setdefault("data_releases", {})[country] = data_release diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index f3b15fb5d..328db86e2 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -202,6 +202,21 @@ class CertifiedDataArtifact(BaseModel): RegionalReleaseKind = Literal["local_area", "area_coded_national"] +class RegionalCertification(BaseModel): + """A reviewer's acceptance of a regional release on another model version. + + Written only by ``certify-data --accept-regional-model-mismatch``: the + regional release neither was built with nor claims the certified model, + and area runs on it are uncertified. + """ + + compatibility_basis: Literal["accepted_model_mismatch"] + certified_for_model_version: str + built_with_model_version: Optional[str] = None + accepted_reason: str + area_receipts: Literal["uncertified"] = "uncertified" + + class CountryReleaseManifest(BaseModel): schema_version: int = 1 bundle_id: Optional[str] = None @@ -218,6 +233,9 @@ class CountryReleaseManifest(BaseModel): ``local_area`` for a non-default local-area release, and ``area_coded_national`` for another release's national dataset that carries area codes (and its area weight matrices).""" + regional_certification: Optional[RegionalCertification] = None + """Present only when the regional release was routed on an accepted + model mismatch rather than its own build or claims.""" certified_data_artifact: Optional[CertifiedDataArtifact] = None certification: Optional[DataCertification] = None source_sha256: Optional[str] = Field(default=None, exclude=True) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 092420f61..a8c8b5b21 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1412,6 +1412,150 @@ def test__given_regional_release_without_a_claim__then_refuses(self, tmp_path): regional_release_kind="area_coded_national", ) + def _older_dense(self) -> dict: + dense = _microcosm_uk_dense_payload() + dense["build"]["built_with_model_package"]["version"] = "2.120.0" + dense["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": "==2.120.0"} + ] + return dense + + def test__given_older_dense_cut__then_refuses_without_an_accepted_mismatch( + self, tmp_path + ): + with pytest.raises(CertificationError, match="Regional release .*2.126.12"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + ) + + def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): + bundle_path = _tmp_bundle(tmp_path) + reason = "area receipts are uncertified; the dense cut lags one release" + + result = _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=reason, + ) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert release["regional_release_kind"] == "local_area" + assert release["regional_certification"] == { + "compatibility_basis": "accepted_model_mismatch", + "certified_for_model_version": UK_MODEL_VERSION, + "built_with_model_version": "2.120.0", + "accepted_reason": reason, + "area_receipts": "uncertified", + } + # The national certificate itself is untouched by the regional record. + assert release["certification"]["compatibility_basis"] == ( + "built_with_model_package" + ) + assert _sim_api_routes_both_area_types(release) + manifest = CountryReleaseManifest.model_validate(release) + assert manifest.regional_certification is not None + assert manifest.regional_certification.built_with_model_version == "2.120.0" + mismatch = [w for w in result.warnings if "accepted mismatch" in w] + assert len(mismatch) == 1 and "2.120.0" in mismatch[0] + + def test__given_a_covering_regional_release__then_an_accepted_mismatch_refuses( + self, tmp_path + ): + with pytest.raises(CertificationError, match="drop --accept-regional"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason="not needed", + ) + + @pytest.mark.parametrize("reason", ["", " "]) + def test__given_an_empty_reason__then_refuses(self, tmp_path, reason): + with pytest.raises(CertificationError, match="needs a reason"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=reason, + ) + + def test__given_an_accepted_mismatch__then_the_core_must_still_be_covered( + self, tmp_path + ): + national = _microcosm_uk_national_payload() + national["compatible_core_packages"] = [ + {"name": "policyengine-core", "specifier": "==3.33.0"} + ] + with pytest.raises( + CertificationError, match="Regional release .*policyengine-core 3.33.0" + ): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(national), _response(self._older_dense())], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason="area receipts are uncertified", + core_version="3.33.0", + ) + + def test__given_an_accepted_mismatch_without_a_regional_release__then_refuses( + self, tmp_path + ): + with pytest.raises(CertificationError, match="pass --regional-manifest-uri"): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(_microcosm_uk_national_payload())], + drop_regional_release=True, + regional_model_mismatch_reason="area receipts are uncertified", + ) + + def test__given_bundle_wrapper_accepted_mismatch__then_forwards_it( + self, monkeypatch + ): + bundle_script = _load_bundle_script(monkeypatch) + captured: dict[str, list[str]] = {} + fake = ModuleType("certify_data_release") + + def fake_main(argv): + captured["argv"] = argv + return 0 + + fake.main = fake_main + monkeypatch.setitem(sys.modules, "certify_data_release", fake) + reason = "area receipts are uncertified" + assert ( + bundle_script.main( + [ + "certify-data", + "--country", + "uk", + "--manifest-uri", + UK_NATIONAL_URI, + "--regional-manifest-uri", + UK_DENSE_URI, + "--accept-regional-model-mismatch", + reason, + ] + ) + == 0 + ) + argv = captured["argv"] + assert argv[argv.index("--accept-regional-model-mismatch") + 1] == reason + def test__given_pinned_core__then_the_regional_release_must_cover_it( self, tmp_path ): From 80f54cadd6c35229d14ec1d3b72878553d58941b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:23:28 -0400 Subject: [PATCH 02/21] Test bound regional mismatch input evidence and reversal --- tests/test_regional_input_evidence.py | 237 ++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 tests/test_regional_input_evidence.py diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py new file mode 100644 index 000000000..d0f6a31b3 --- /dev/null +++ b/tests/test_regional_input_evidence.py @@ -0,0 +1,237 @@ +"""A regional model exception requires an audit of the exact dense bytes.""" + +import copy +import hashlib +import json + +import pytest + +from policyengine.provenance.certification import CertificationError +from policyengine.provenance.manifest import CountryReleaseManifest +from tests.test_certify_data_release import ( + UK_DENSE_URI, + UK_MODEL_VERSION, + _certify_uk, + _microcosm_uk_dense_payload, + _microcosm_uk_national_payload, + _response, + _tmp_bundle, +) + +REASON = "Budget accepts the audited dense model gap; area runs remain uncertified" +VAT_LIMIT = ( + "the dense cut's VAT predates the policyengine-uk VAT factor fix " + "(#2187, 2.135.7); the dashboard holds a VAT limit" +) +DEFAULTED_INPUTS = [ + "local_bus_uncapped_single_fare", + "salary_sacrifice_returned_to_income", +] + + +def older_dense() -> dict: + dense = _microcosm_uk_dense_payload() + dense["build"]["built_with_model_package"]["version"] = "2.120.0" + dense["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": "==2.120.0"} + ] + return dense + + +def input_evidence(dense: dict | None = None) -> dict: + dense = older_dense() if dense is None else dense + name = "microcosm_uk_2024_25_dense" + artifact = dense["artifacts"][name] + return { + "schema_version": 1, + "regional_manifest_uri": UK_DENSE_URI, + "regional_manifest_sha256": hashlib.sha256( + json.dumps(dense).encode() + ).hexdigest(), + "model_package": "policyengine-uk", + "model_version": UK_MODEL_VERSION, + "artifacts": { + name: { + key: artifact[key] for key in ("repo_id", "path", "revision", "sha256") + } + | {"columns": ["employment_income"]} + }, + "model_variables": ["employment_income", *DEFAULTED_INPUTS], + "model_inputs": ["employment_income", *DEFAULTED_INPUTS], + "dropped_columns": [], + "defaulted_inputs": DEFAULTED_INPUTS.copy(), + "audited_by": "Budget regional input-surface review", + "limitations": [VAT_LIMIT], + } + + +def certify_with_evidence(bundle_path, evidence: dict, dense: dict | None = None): + dense = older_dense() if dense is None else dense + return _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload()), _response(dense)], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=evidence, + ) + + +def test_accepted_mismatch_without_audited_evidence_refuses(tmp_path): + with pytest.raises(CertificationError, match="[Ee]vidence|[Aa]udit"): + _certify_uk( + _tmp_bundle(tmp_path), + [ + _response(_microcosm_uk_national_payload()), + _response(older_dense()), + ], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + ) + + +@pytest.mark.parametrize( + ("field", "wrong"), + [ + ("regional_manifest_uri", UK_DENSE_URI.replace("dense", "other-dense")), + ("regional_manifest_sha256", "0" * 64), + ("model_package", "policyengine-us"), + ("model_version", "2.147.1"), + ], +) +def test_evidence_for_another_cut_bytes_or_model_refuses(tmp_path, field, wrong): + evidence = input_evidence() + evidence[field] = wrong + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +@pytest.mark.parametrize( + ("field", "wrong"), + [ + ("repo_id", "policyengine/another-dense"), + ("path", "different_dense.h5"), + ("revision", "different-cut"), + ("sha256", "0" * 64), + ], +) +def test_evidence_for_other_artifact_bytes_or_identity_refuses(tmp_path, field, wrong): + evidence = input_evidence() + evidence["artifacts"]["microcosm_uk_2024_25_dense"][field] = wrong + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_a_dropped_column_refuses_even_with_an_acceptance_reason(tmp_path): + evidence = input_evidence() + evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns"].append( + "removed_model_variable" + ) + evidence["dropped_columns"] = ["removed_model_variable"] + with pytest.raises(CertificationError, match="[Dd]ropped|removed_model_variable"): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +@pytest.mark.parametrize("field", ["dropped_columns", "defaulted_inputs"]) +def test_the_declared_comparison_must_match_the_measured_surfaces(tmp_path, field): + evidence = input_evidence() + evidence[field] = ["invented_variable"] + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_a_missing_artifact_measurement_refuses(tmp_path): + evidence = input_evidence() + evidence["artifacts"] = {} + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_every_default_and_limitation_survives_validation_and_roundtrip(tmp_path): + bundle_path = _tmp_bundle(tmp_path) + evidence = input_evidence() + certify_with_evidence(bundle_path, evidence) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + manifest = CountryReleaseManifest.model_validate(release) + assert manifest.regional_certification is not None + record = manifest.regional_certification.model_dump(mode="json") + assert record["defaulted_inputs"] == DEFAULTED_INPUTS + assert record["dropped_columns"] == [] + assert record["limitations"] == [VAT_LIMIT] + assert record["evidence"] == evidence + assert record["area_receipts"] == "uncertified" + assert record["accepted_reason"] == REASON + assert ( + release["regional_source_manifest_sha256"] + == evidence["regional_manifest_sha256"] + ) + rebuilt = CountryReleaseManifest.model_validate_json(manifest.model_dump_json()) + assert rebuilt.regional_certification == manifest.regional_certification + assert release["certification"]["compatibility_basis"] == "built_with_model_package" + + +def test_evidence_digest_binds_original_manifest_bytes(tmp_path): + dense = older_dense() + response = _response(dense) + response.content = json.dumps(dense, indent=2).encode() + evidence = input_evidence(dense) + with pytest.raises(CertificationError): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(_microcosm_uk_national_payload()), response], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=evidence, + ) + + evidence["regional_manifest_sha256"] = hashlib.sha256(response.content).hexdigest() + bundle_path = _tmp_bundle(tmp_path) + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload()), response], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=evidence, + ) + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert ( + release["regional_source_manifest_sha256"] + == evidence["regional_manifest_sha256"] + ) + + +def test_drop_regional_release_removes_acceptance_and_preserves_national_cert(tmp_path): + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, input_evidence()) + before = json.loads(bundle_path.read_text())["data_releases"]["uk"] + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload())], + drop_regional_release=True, + ) + after = json.loads(bundle_path.read_text())["data_releases"]["uk"] + assert after["certification"] == before["certification"] + assert "regional_certification" not in after + assert "regional_source_manifest_sha256" not in after + assert "regional_source_manifest_uri" not in after + assert "regional_release_kind" not in after + assert "microcosm_uk_2024_25_dense" not in after["datasets"] + assert set(after["region_datasets"]) == {"national"} + + +def test_limitations_and_evidence_are_in_hashed_bundle_material(tmp_path): + from policyengine.provenance.canonical import canonical_sha256 + + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, input_evidence()) + before = json.loads(bundle_path.read_text()) + after = copy.deepcopy(before) + after["data_releases"]["uk"]["regional_certification"]["limitations"].append( + "An additional reviewed regional limitation" + ) + + def digest(bundle): + manifest = CountryReleaseManifest.model_validate(bundle["data_releases"]["uk"]) + return canonical_sha256(manifest.model_dump(mode="json")) + + assert digest(before) != digest(after) From 17c68b4d8f5774fd8936bf822bf680c812e1a6ec Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:27:23 -0400 Subject: [PATCH 03/21] Document audited UK regional mismatch acceptance --- changelog.d/699.fixed.md | 1 + docs/engineering/skills/data-certification.md | 70 +++++++++++++++---- 2 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 changelog.d/699.fixed.md diff --git a/changelog.d/699.fixed.md b/changelog.d/699.fixed.md new file mode 100644 index 000000000..55eb27701 --- /dev/null +++ b/changelog.d/699.fixed.md @@ -0,0 +1 @@ +Require a byte-bound input-surface audit when accepting a UK regional model mismatch, refuse dropped columns, and record defaulted inputs and limitations in the bundle, TRACE and execution receipts. Preserve the regional manifest digest and area build model while keeping area receipts uncertified; dropping the regional release removes its exception record. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 1ce62d882..91c2dc2ec 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -81,28 +81,68 @@ Microcosm national release with one of: - **An older local-area release, on an accepted mismatch.** When the national cut moves to a new policyengine-uk before a local-area cut on that version - exists, a reviewer can keep the current local-area release routed instead of - dropping area regions: + exists, a reviewer can accept retaining the older regional route. A reason + and a byte-bound input-surface audit are both required. First audit the + actual local H5 with the exact target model installed: ```bash - python scripts/bundle.py certify-data --country uk --data-producer populace \ + .venv/bin/python scripts/audit_regional_input_surface.py \ + --regional-manifest-uri "" \ + --artifact "microcosm_uk_2024_25_dense=/local/dense.h5" \ + --model-version 2.147.2 \ + --audited-by "" \ + --limitation "the dense cut's VAT predates the policyengine-uk VAT factor fix (#2187, 2.135.7); the dashboard holds a VAT limit" \ + --output regional-input-surface.json + + .venv/bin/python scripts/bundle.py certify-data --country uk --data-producer populace \ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ --regional-manifest-uri "" \ --accept-regional-model-mismatch "" \ - --model-version "" + --regional-input-surface-evidence regional-input-surface.json \ + --model-version 2.147.2 ``` - Certification then records `regional_certification` in the bundle: - `compatibility_basis: accepted_model_mismatch`, both model versions, the - reason and `area_receipts: uncertified`. Area receipts are uncertified in any - case, because the certificate covers the national dataset only (#685). - The cost is real and goes in the pull request. The local-area file was built - and calibrated under the older model. policyengine-core ignores, with a - warning, the columns the newer model no longer defines, and inputs the newer - model added take their defaults in area runs. The flag is UK only. It refuses - an empty reason, a regional release that already covers the version (so no - certificate records a mismatch that is not there) and a run without a - regional release. The core must still be covered. + The comparison is ported from Microcosm #1348's + `tools/certify_uk_cut_in_policyengine_py.py`, and strengthened to inspect the + persisted entity-table columns, including each period and country-loader + aliases. Comparing the old model's input inventory alone cannot prove which + columns the file carries. The audit binds the regional release identity, + original manifest bytes, actual H5 bytes and exact target model version. + Certification refuses missing or mismatched evidence and **any dropped + column**. It records every defaulted input and the free-text limitations in + the validated `regional_certification`; the bundle hash binds that record. + + The national certificate never widens. Area status remains `uncertified`: + the certificate still covers the national dataset only (#685). TRACE and + execution receipts expose the regional identity, build model, acceptance, + reason, input-surface result and limitations separately from the national + certification. The flag remains UK only; an empty reason, a release that + already covers the target model, or a missing regional release refuses. + The regional core checks still apply. Re-certifying with + `--drop-regional-release` removes the regional record, routes and artifact. + + For the Budget route, the measured **model-inventory comparison** from + 2.132.6 to 2.147.2 has no dropped names and these 16 new pure inputs: + `additional_residential_property_purchased`, `defined_benefit_pension_share`, + `directly_held_shares`, `early_years_school_start_deferred`, + `is_full_year_category_a_director`, `local_bus_uncapped_single_fare`, + `non_residential_property_purchased`, `non_residential_property_secured_debt`, + `other_residential_property_secured_debt`, + `pension_annual_allowance_taxable_lump_sum_death_benefits`, + `pension_contributions_via_salary_sacrifice_pre_2015`, `pip_dl_points`, + `pip_m_points`, `salary_sacrifice_returned_to_income`, + `unit_and_investment_trusts`, and `war_widows_pension_reported`. + Both `salary_sacrifice_returned_to_income` and + `local_bus_uncapped_single_fare` default in that comparison. This is not an + audit of cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`: + its private H5 was unavailable to this implementation job without + authentication. Certification must use the complete list measured from the + actual cut by the audit above, including any further missing stored inputs. + Record the pre-fix VAT limitation shown above for this dense cut. + + After the receipt schema changes, refresh the Budget dashboard's vendored + `tests/fixtures/w4/execution_receipt.v1.schema.json` from + `src/policyengine/data/schemas/execution_receipt.v1.schema.json`. Either way, certification checks that the regional release serves the pinned model by its own build or compatibility claims (a `--qualified-pair-id` names From 451f106c798f30462243d453e2dce64be920c589 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:28:08 -0400 Subject: [PATCH 04/21] Test measurement of regional H5 inputs against the exact UK model --- tests/test_measure_regional_inputs.py | 345 ++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 tests/test_measure_regional_inputs.py diff --git a/tests/test_measure_regional_inputs.py b/tests/test_measure_regional_inputs.py new file mode 100644 index 000000000..8cf4006af --- /dev/null +++ b/tests/test_measure_regional_inputs.py @@ -0,0 +1,345 @@ +"""Measure the actual dense H5 surface, never an older model's inventory.""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +import json +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from policyengine.provenance.manifest import RegionalInputArtifactEvidence + +MODEL_VERSION = "2.147.2" +URI = ( + "hf://dataset/policyengine/regional@dense-cut/releases/dense/release_manifest.json" +) +LIMITATION = "Reviewed dense VAT limitation; dashboard holds a VAT limit" +DEFAULTS = ["local_bus_uncapped_single_fare", "salary_sacrifice_returned_to_income"] + + +class FakeSimulation: + def __init__(self): + self.move_values("employment_income", "employment_income_before_lsr") + + def move_values(self, source, target): + pass + + +class FakeVariable: + def __init__(self, is_input): + self.is_input = is_input + + def is_input_variable(self): + return self.is_input + + +class FakeCountryTaxBenefitSystem: + def __init__(self): + inputs = { + "employment_income_before_lsr", + "household_weight", + "benunit_id", + *DEFAULTS, + } + self.variables = { + name: FakeVariable(name in inputs) + for name in sorted(inputs | {"employment_income"}) + } + + +def _module(): + return importlib.import_module("policyengine.provenance.regional_input_surface") + + +def _h5(path, *, periods=None, fmt="fixed", missing=None): + with pd.HDFStore(path, mode="w") as store: + if periods is None: + store.put("time_period", pd.Series([2025])) + periods = {None: True} + for period, has_income in periods.items(): + for entity, columns in { + "person": {"employment_income": [1.0]} if has_income else {}, + "benunit": {"benunit_id": [0]}, + "household": {"household_weight": [1.0]}, + }.items(): + if (period, entity) == missing: + continue + key = entity if period is None else f"{entity}/{period}" + if not columns: + columns = {"benunit_id": [0]} + store.put(key, pd.DataFrame(columns), format=fmt) + return path + + +def _response(artifact_paths): + payload = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "1"}, + "dataset_role": "non_default_local_area", + "is_default": False, + "default_datasets": {}, + "artifacts": { + name: { + "kind": "microdata", + "path": f"{name}.h5", + "repo_id": "policyengine/regional", + "revision": "dense-cut", + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + for name, path in artifact_paths.items() + }, + } + response = MagicMock(status_code=200) + response.content = json.dumps(payload, indent=2).encode() + return response + + +@pytest.fixture +def model(monkeypatch): + module = _module() + monkeypatch.setattr(module, "version", lambda name: MODEL_VERSION) + monkeypatch.setitem( + sys.modules, + "policyengine_uk", + SimpleNamespace( + CountryTaxBenefitSystem=FakeCountryTaxBenefitSystem, + Simulation=FakeSimulation, + ), + ) + return module + + +def _measure(module, paths, response=None, **kwargs): + response = _response(paths) if response is None else response + with patch( + "policyengine.provenance.certification.requests.get", return_value=response + ): + return module.measure_regional_input_surface( + URI, + paths, + MODEL_VERSION, + "Budget operator", + [LIMITATION], + **kwargs, + ) + + +@pytest.mark.parametrize("fmt", ["fixed", "table"]) +def test_actual_h5_columns_and_aliases_produce_bound_audit(tmp_path, model, fmt): + paths = {"dense": _h5(tmp_path / "dense.h5", fmt=fmt)} + response = _response(paths) + result = _measure(model, paths, response) + assert result.model_version == MODEL_VERSION + assert result.model_package == "policyengine-uk" + assert result.regional_manifest_uri == URI + assert ( + result.regional_manifest_sha256 == hashlib.sha256(response.content).hexdigest() + ) + artifact = result.artifacts["dense"] + assert artifact.sha256 == hashlib.sha256(paths["dense"].read_bytes()).hexdigest() + assert artifact.repo_type == "dataset" + assert artifact.columns == ["benunit_id", "employment_income", "household_weight"] + assert artifact.columns_by_period == {"2025": artifact.columns} + assert result.input_aliases == {"employment_income": "employment_income_before_lsr"} + assert result.defaulted_inputs == DEFAULTS + assert result.dropped_columns == [] + assert result.limitations == [LIMITATION] + assert result.audited_by == "Budget operator" + + +def test_a_default_in_one_period_is_not_hidden_by_another_period(tmp_path, model): + paths = {"dense": _h5(tmp_path / "dense.h5", periods={"2025": True, "2026": False})} + result = _measure(model, paths) + assert result.defaulted_inputs == [ + "employment_income_before_lsr", + *DEFAULTS, + ] + assert "employment_income" in result.artifacts["dense"].columns_by_period["2025"] + assert ( + "employment_income" not in result.artifacts["dense"].columns_by_period["2026"] + ) + + +@pytest.mark.parametrize("entity", ["person", "benunit", "household"]) +def test_each_period_requires_every_entity(tmp_path, model, entity): + paths = { + "dense": _h5( + tmp_path / "dense.h5", + periods={"2025": True, "2026": True}, + missing=("2026", entity), + ) + } + with pytest.raises(ValueError, match="2026|entity"): + _measure(model, paths) + + +def test_single_year_layout_requires_a_time_period(tmp_path, model): + path = _h5(tmp_path / "dense.h5") + with pd.HDFStore(path, mode="a") as store: + store.remove("time_period") + with pytest.raises(ValueError, match="time_period|period"): + _measure(model, {"dense": path}) + + +def test_extra_metadata_is_not_mistaken_for_a_model_column(tmp_path, model): + path = _h5(tmp_path / "dense.h5") + with pd.HDFStore(path, mode="a") as store: + store.put("metadata", pd.Series(["producer build notes"])) + assert _measure(model, {"dense": path}).dropped_columns == [] + + +def test_an_actual_dropped_column_blocks_audit(tmp_path, model): + path = _h5(tmp_path / "dense.h5") + with pd.HDFStore(path, mode="a") as store: + store.put("person", pd.DataFrame({"removed_column": [1]})) + with pytest.raises(ValueError, match="[Dd]ropped|removed_column"): + _measure(model, {"dense": path}) + + +def test_installed_model_must_be_the_requested_exact_version( + tmp_path, model, monkeypatch +): + monkeypatch.setattr(model, "version", lambda name: "2.146.4") + paths = {"dense": _h5(tmp_path / "dense.h5")} + with pytest.raises(ValueError, match="2.147.2|2.146.4"): + _measure(model, paths) + + +def test_artifact_bytes_must_match_the_manifest(tmp_path, model): + paths = {"dense": _h5(tmp_path / "dense.h5")} + response = _response(paths) + payload = json.loads(response.content) + payload["artifacts"]["dense"]["sha256"] = "0" * 64 + response.content = json.dumps(payload).encode() + with pytest.raises(ValueError, match="sha256|[Hh]ash|bytes"): + _measure(model, paths, response) + + +def test_file_changing_during_metadata_read_refuses(tmp_path, model, monkeypatch): + paths = {"dense": _h5(tmp_path / "dense.h5")} + response = _response(paths) + original = pd.HDFStore.select + mutated = False + + def select_and_modify(store, *args, **kwargs): + nonlocal mutated + result = original(store, *args, **kwargs) + if not mutated and args[0] != "/time_period": + with paths["dense"].open("ab") as file: + file.write(b"changed while inspecting metadata") + mutated = True + return result + + monkeypatch.setattr(pd.HDFStore, "select", select_and_modify) + with pytest.raises(ValueError, match="chang|sha256|[Hh]ash|bytes"): + _measure(model, paths, response) + + +@pytest.mark.parametrize("names", [[], ["other"], ["dense", "other"]]) +def test_every_routed_h5_requires_exactly_one_supplied_path(tmp_path, model, names): + path = _h5(tmp_path / "dense.h5") + response = _response({"dense": path}) + with pytest.raises(ValueError, match="artifact|[Hh]5"): + _measure(model, {name: path for name in names}, response) + + +def test_comparison_uses_each_artifact_and_every_period(): + module = _module() + artifact = RegionalInputArtifactEvidence( + repo_id="r", + path="dense.h5", + revision="cut", + sha256="a" * 64, + columns=["income", "weight"], + columns_by_period={"2025": ["income", "weight"], "2026": ["weight"]}, + ) + result = module.compare_input_surface( + {"dense": artifact}, + ["income", "income_reported", "weight", "new_input"], + ["income_reported", "weight", "new_input"], + {"income": "income_reported"}, + ) + assert result == { + "dropped_columns": [], + "defaulted_inputs": ["income_reported", "new_input"], + } + + +def test_comparison_refuses_a_period_union_that_hides_columns(): + module = _module() + artifact = RegionalInputArtifactEvidence( + repo_id="r", + path="dense.h5", + revision="cut", + sha256="a" * 64, + columns=["weight"], + columns_by_period={"2025": ["weight", "removed"]}, + ) + with pytest.raises(ValueError, match="columns|union|period"): + module.compare_input_surface({"dense": artifact}, ["weight"], ["weight"]) + + +def test_conditional_alias_cannot_silently_claim_a_supplied_input( + tmp_path, model, monkeypatch +): + class ConditionalSimulation: + def __init__(self): + if self.condition: + self.move_values("employment_income", "employment_income_before_lsr") + + monkeypatch.setattr( + sys.modules["policyengine_uk"], "Simulation", ConditionalSimulation + ) + paths = {"dense": _h5(tmp_path / "dense.h5")} + with pytest.raises(ValueError, match="alias|move_values|conditional"): + _measure(model, paths) + + +def test_audit_cli_writes_pydantic_evidence_json(tmp_path, model, monkeypatch): + script_path = ( + Path(__file__).parents[1] / "scripts" / "audit_regional_input_surface.py" + ) + spec = importlib.util.spec_from_file_location( + "audit_regional_input_surface", script_path + ) + script = importlib.util.module_from_spec(spec) + spec.loader.exec_module(script) + path = _h5(tmp_path / "dense.h5") + result = _measure(model, {"dense": path}) + captured = {} + + def measure(**kwargs): + captured.update(kwargs) + return result + + monkeypatch.setattr(script, "measure_regional_input_surface", measure) + output = tmp_path / "evidence.json" + assert ( + script.main( + [ + "--regional-manifest-uri", + URI, + "--artifact", + f"dense={path}", + "--model-version", + MODEL_VERSION, + "--audited-by", + "Budget operator", + "--limitation", + LIMITATION, + "--output", + str(output), + ] + ) + == 0 + ) + assert captured["artifact_paths"] == {"dense": path} + assert captured["limitations"] == [LIMITATION] + assert json.loads(output.read_text()) == result.model_dump(mode="json") From 97ce735ebfffd146bea5182ed4fa06a56ff34c95 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:28:17 -0400 Subject: [PATCH 05/21] Expose audited regional acceptance in TRACE and execution receipts --- .../schemas/execution_receipt.v1.schema.json | 287 +++++++++++++++++- src/policyengine/provenance/receipt.py | 78 ++++- src/policyengine/provenance/trace.py | 6 + tests/test_execution_receipt.py | 162 ++++++++++ 4 files changed, 522 insertions(+), 11 deletions(-) diff --git a/src/policyengine/data/schemas/execution_receipt.v1.schema.json b/src/policyengine/data/schemas/execution_receipt.v1.schema.json index d10d1955b..e8a449b87 100644 --- a/src/policyengine/data/schemas/execution_receipt.v1.schema.json +++ b/src/policyengine/data/schemas/execution_receipt.v1.schema.json @@ -457,6 +457,279 @@ "title": "ReformIdentity", "type": "object" }, + "RegionalCertification": { + "description": "A reviewer's acceptance of a regional release on another model version.\n\nWritten only by ``certify-data --accept-regional-model-mismatch``: the\nregional release neither was built with nor claims the certified model,\nand area runs on it are uncertified.", + "properties": { + "accepted_reason": { + "title": "Accepted Reason", + "type": "string" + }, + "area_receipts": { + "const": "uncertified", + "default": "uncertified", + "title": "Area Receipts", + "type": "string" + }, + "built_with_model_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Built With Model Version" + }, + "certified_for_model_version": { + "title": "Certified For Model Version", + "type": "string" + }, + "compatibility_basis": { + "const": "accepted_model_mismatch", + "title": "Compatibility Basis", + "type": "string" + }, + "defaulted_inputs": { + "items": { + "type": "string" + }, + "title": "Defaulted Inputs", + "type": "array" + }, + "dropped_columns": { + "items": { + "type": "string" + }, + "title": "Dropped Columns", + "type": "array" + }, + "evidence": { + "$ref": "#/$defs/RegionalInputSurfaceEvidence" + }, + "limitations": { + "items": { + "type": "string" + }, + "title": "Limitations", + "type": "array" + }, + "regional_manifest_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Regional Manifest Sha256", + "type": "string" + }, + "regional_manifest_uri": { + "title": "Regional Manifest Uri", + "type": "string" + } + }, + "required": [ + "compatibility_basis", + "certified_for_model_version", + "accepted_reason", + "regional_manifest_uri", + "regional_manifest_sha256", + "defaulted_inputs", + "dropped_columns", + "limitations", + "evidence" + ], + "title": "RegionalCertification", + "type": "object" + }, + "RegionalInputArtifactEvidence": { + "additionalProperties": false, + "description": "Measured columns of one byte-verified regional H5 artifact.", + "properties": { + "columns": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Columns", + "type": "array" + }, + "columns_by_period": { + "anyOf": [ + { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Columns By Period" + }, + "path": { + "minLength": 1, + "title": "Path", + "type": "string" + }, + "repo_id": { + "minLength": 1, + "title": "Repo Id", + "type": "string" + }, + "repo_type": { + "anyOf": [ + { + "enum": [ + "model", + "dataset" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Repo Type" + }, + "revision": { + "minLength": 1, + "title": "Revision", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "repo_id", + "path", + "revision", + "sha256", + "columns" + ], + "title": "RegionalInputArtifactEvidence", + "type": "object" + }, + "RegionalInputSurfaceEvidence": { + "additionalProperties": false, + "description": "An operator's audit of actual regional columns against an exact model.\n\nSurfaces are retained so certification recomputes the comparison rather\nthan trusting a declared zero-loss result. The artifact hashes identify\nthe bytes inspected by the auditor, not merely a model's possible inputs.", + "properties": { + "artifacts": { + "additionalProperties": { + "$ref": "#/$defs/RegionalInputArtifactEvidence" + }, + "minProperties": 1, + "title": "Artifacts", + "type": "object" + }, + "audited_by": { + "minLength": 1, + "pattern": "\\S", + "title": "Audited By", + "type": "string" + }, + "defaulted_inputs": { + "items": { + "type": "string" + }, + "title": "Defaulted Inputs", + "type": "array" + }, + "dropped_columns": { + "items": { + "type": "string" + }, + "title": "Dropped Columns", + "type": "array" + }, + "input_aliases": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Aliases" + }, + "limitations": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Limitations", + "type": "array" + }, + "model_inputs": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Model Inputs", + "type": "array" + }, + "model_package": { + "minLength": 1, + "title": "Model Package", + "type": "string" + }, + "model_variables": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Model Variables", + "type": "array" + }, + "model_version": { + "minLength": 1, + "title": "Model Version", + "type": "string" + }, + "regional_manifest_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Regional Manifest Sha256", + "type": "string" + }, + "regional_manifest_uri": { + "minLength": 1, + "title": "Regional Manifest Uri", + "type": "string" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "regional_manifest_uri", + "regional_manifest_sha256", + "model_package", + "model_version", + "artifacts", + "model_variables", + "model_inputs", + "dropped_columns", + "defaulted_inputs", + "audited_by", + "limitations" + ], + "title": "RegionalInputSurfaceEvidence", + "type": "object" + }, "RuntimeContext": { "additionalProperties": false, "description": "Where the execution ran. Part of no digest.", @@ -655,6 +928,18 @@ "reform": { "$ref": "#/$defs/ReformIdentity" }, + "regional_certification": { + "anyOf": [ + { + "$ref": "#/$defs/RegionalCertification" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The bundle's accepted regional model mismatch, including its exact regional manifest and artifact identity, measured input surface, acceptance reason and limitations. This records an exception for uncertified area runs and does not widen the national certificate." + }, "runtime": { "anyOf": [ { @@ -711,7 +996,7 @@ "x-policyengine-digests": { "canonicalization": "RFC 8785 (JSON Canonicalization Scheme) over the stated value after removing object members whose value is null (array elements are kept). Numbers are IEEE 754 doubles; integers must lie within +/-(2**53 - 1).", "certificate.certificate_sha256": "the bundle manifest's data_releases. record", - "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", + "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?, regional_certification?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "outputs_sha256": "the canonical output payload the receipt binds", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", "reform.reform_sha256": "RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {\"$nonfinite\": \"Infinity\"} or {\"$nonfinite\": \"-Infinity\"}, and a payload may not use the key $nonfinite itself" diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index 1c7b43e04..2492e706f 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -133,6 +133,7 @@ drop_null_members, ) from policyengine.provenance.loaded import loaded_versions +from policyengine.provenance.manifest import RegionalCertification from policyengine.utils.hashing import sha256_file __all__ = [ @@ -223,7 +224,7 @@ #: How each digest is computed. DIGEST_RULES: dict[str, str] = { "canonicalization": "RFC 8785 (JSON Canonicalization Scheme) over the stated value after removing object members whose value is null (array elements are kept). Numbers are IEEE 754 doubles; integers must lie within +/-(2**53 - 1).", - "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", + "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?, regional_certification?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", "certificate.certificate_sha256": "the bundle manifest's data_releases. record", "reform.reform_sha256": 'RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {"$nonfinite": "Infinity"} or {"$nonfinite": "-Infinity"}, and a payload may not use the key $nonfinite itself', @@ -529,6 +530,15 @@ class ExecutionReceipt(_Contract): data: DataIdentity certification_status: Literal["certified", "uncertified"] certificate: Optional[CertificateIdentity] = None + regional_certification: Optional[RegionalCertification] = Field( + default=None, + description=( + "The bundle's accepted regional model mismatch, including its exact " + "regional manifest and artifact identity, measured input surface, " + "acceptance reason and limitations. This records an exception for " + "uncertified area runs and does not widen the national certificate." + ), + ) reform: ReformIdentity baseline: ReformIdentity fiscal_year: FiscalYearConvention @@ -549,6 +559,13 @@ class ExecutionReceipt(_Contract): ) runtime: Optional[RuntimeContext] = None + @model_serializer(mode="wrap") + def _omit_absent_regional_certification(self, handler): + payload = handler(self) + if self.regional_certification is None: + payload.pop("regional_certification", None) + return payload + @field_validator("schema_version", mode="before") @classmethod def _schema_version_is_an_integer(cls, value: Any) -> Any: @@ -627,6 +644,10 @@ def execution_identity(self) -> dict[str, Any]: for key in CertificateIdentity.IDENTITY_FIELDS if key in payload } + if self.regional_certification is not None: + identity["regional_certification"] = self.regional_certification.model_dump( + mode="json", exclude_none=True + ) return identity @property @@ -3811,12 +3832,25 @@ def _data_identity( datasets = record.get("datasets") or {} entry = datasets.get(dataset_name) primary = datasets.get(record.get("default_dataset")) or {} - # A regional release's artifacts are certified beside the primary release - # but are not part of it, so only entries at the primary release's own - # revision take its build and manifest identity. - release = entry is not None and entry.get("revision") in ( - None, - primary.get("revision"), + # A regional release's artifacts keep their own build provenance, even + # when acceptance allows them to run beside another national model. + regional_certification = manifest.regional_certification + regional = regional_certification is not None and any( + artifact.repo_id == spec.repo_id + and artifact.path == spec.path + and artifact.revision == spec.revision + and artifact.sha256 == spec.sha256 + and artifact.repo_type in (None, spec.repo_type) + for artifact in regional_certification.evidence.artifacts.values() + ) + release = ( + not regional + and entry is not None + and entry.get("revision") + in ( + None, + primary.get("revision"), + ) ) data_package = record["data_package"] certified_artifact = record.get("certified_data_artifact") or {} @@ -3832,9 +3866,13 @@ def _data_identity( build_fingerprint=certification.get("data_build_fingerprint") if release else None, - build_model_version=certification.get("built_with_model_version") - if release - else None, + build_model_version=( + regional_certification.built_with_model_version + if regional and regional_certification is not None + else certification.get("built_with_model_version") + if release + else None + ), release_manifest_revision=data_package.get("release_manifest_revision") if release else None, @@ -3946,6 +3984,7 @@ def identity(name: str) -> dict[str, Any]: data=data, certification_status=status, certificate=certificate, + regional_certification=record.get("regional_certification"), reform=reform, baseline=baseline, fiscal_year=uk_fiscal_year_convention(), @@ -3993,6 +4032,25 @@ def verify_certificate( if not isinstance(record, Mapping): return [f"the bundle has no {receipt.country_id} data release"] problems = [] + regional_record = record.get("regional_certification") + try: + expected_regional = ( + RegionalCertification.model_validate(regional_record).model_dump( + mode="json", exclude_none=True + ) + if regional_record is not None + else None + ) + except ValueError as exc: + problems.append(f"the bundle cannot determine regional_certification: {exc}") + else: + claimed_regional = ( + receipt.regional_certification.model_dump(mode="json", exclude_none=True) + if receipt.regional_certification is not None + else None + ) + if claimed_regional != expected_regional: + problems.append("regional_certification differs from the bundle's record") for package in (receipt.wrapper, receipt.model, receipt.core): if not _package_can_be_certified(package): problems.append( diff --git a/src/policyengine/provenance/trace.py b/src/policyengine/provenance/trace.py index bcffda78a..4a3128ceb 100644 --- a/src/policyengine/provenance/trace.py +++ b/src/policyengine/provenance/trace.py @@ -556,6 +556,12 @@ def _build_bundle_performance( performance["pe:dataBuildId"] = certification.data_build_id if certification.certified_by is not None: performance["pe:certifiedBy"] = certification.certified_by + if country_manifest.regional_certification is not None: + performance["pe:regionalCertification"] = ( + country_manifest.regional_certification.model_dump( + mode="json", exclude_none=True + ) + ) performance.update(dict(emission_context or _emission_context())) return performance diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 66fe4d220..b7f03a3bc 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -433,6 +433,168 @@ def test_a_regional_release_dataset_keeps_its_own_identity(dataset_file): assert receipt.data.release_manifest_revision is None +def _bundle_with_regional_acceptance(dataset_file): + """A synthetic, audited area exception beside a national certificate.""" + + bundle = _bundle_for(dataset_file.read_bytes()) + record = bundle["data_releases"]["uk"] + artifact = { + "path": "area_file.h5", + "repo_id": record["data_package"]["repo_id"], + "revision": "regional-release-cut", + "sha256": hashlib.sha256(dataset_file.read_bytes()).hexdigest(), + } + record["datasets"]["area_file"] = artifact + manifest_uri = ( + "hf://dataset/policyengine/populace-uk-private@regional-release-cut/" + "releases/dense/release_manifest.json" + ) + defaulted = [ + "local_bus_uncapped_single_fare", + "salary_sacrifice_returned_to_income", + ] + limitations = [ + "The dense cut's VAT predates the policyengine-uk VAT factor fix " + "(#2187, 2.135.7); the dashboard holds a VAT limit." + ] + record["regional_certification"] = { + "compatibility_basis": "accepted_model_mismatch", + "certified_for_model_version": INSTALLED_UK, + "built_with_model_version": "2.132.6", + "accepted_reason": "UK hub accepted this measured input surface.", + "area_receipts": "uncertified", + "regional_manifest_uri": manifest_uri, + "regional_manifest_sha256": "b" * 64, + "defaulted_inputs": defaulted, + "dropped_columns": [], + "limitations": limitations, + "evidence": { + "model_package": "policyengine-uk", + "model_version": INSTALLED_UK, + "regional_manifest_uri": manifest_uri, + "regional_manifest_sha256": "b" * 64, + "artifacts": { + "area_file": { + **artifact, + "columns": ["employment_income", "household_weight"], + } + }, + "model_variables": sorted( + ["employment_income", "household_weight", *defaulted] + ), + "model_inputs": sorted( + ["employment_income", "household_weight", *defaulted] + ), + "dropped_columns": [], + "defaulted_inputs": defaulted, + "audited_by": "UK hub review", + "limitations": limitations, + }, + } + return bundle + + +@pytest.mark.parametrize("area", [False, True], ids=["national", "area"]) +def test_receipt_exposes_regional_acceptance_without_widening_certification( + dataset_file, schema_validator, area +): + from policyengine.provenance.manifest import RegionalCertification + + bundle = _bundle_with_regional_acceptance(dataset_file) + record = bundle["data_releases"]["uk"] + + receipt = _build(dataset_file, bundle, dataset="area_file" if area else None) + + payload = json.loads(receipt.model_dump_json(exclude_none=True)) + assert payload["regional_certification"] == RegionalCertification.model_validate( + record["regional_certification"] + ).model_dump(mode="json", exclude_none=True) + assert receipt.certification_status == ("uncertified" if area else "certified") + assert (receipt.certificate is None) is area + assert receipt.data.build_model_version == ( + "2.132.6" if area else record["certification"]["built_with_model_version"] + ) + schema_validator.validate(payload) + restored = ExecutionReceipt.model_validate_json(receipt.model_dump_json()) + assert restored.regional_certification == receipt.regional_certification + assert restored.receipt_sha256 == receipt.receipt_sha256 + + +def test_regional_limitations_are_bound_to_receipt_identity(dataset_file): + bundle = _bundle_with_regional_acceptance(dataset_file) + receipt = _build(dataset_file, bundle, dataset="area_file") + changed_bundle = copy.deepcopy(bundle) + record = changed_bundle["data_releases"]["uk"]["regional_certification"] + record["limitations"].append("Additional accepted area limitation.") + record["evidence"]["limitations"].append("Additional accepted area limitation.") + + changed = _build(dataset_file, changed_bundle, dataset="area_file") + + assert changed.execution_identity_sha256 != receipt.execution_identity_sha256 + assert changed.receipt_sha256 != receipt.receipt_sha256 + + +def test_certificate_verification_checks_the_exposed_regional_record(dataset_file): + bundle = _bundle_with_regional_acceptance(dataset_file) + receipt = _build(dataset_file, bundle) + payload = receipt.model_dump(mode="json") + payload["regional_certification"]["accepted_reason"] = "A different acceptance." + changed = ExecutionReceipt.model_validate(payload) + + assert any( + "regional_certification" in issue + for issue in verify_certificate(changed, bundle) + ) + + +def test_trace_exposes_regional_acceptance_separately_from_national(dataset_file): + from policyengine.provenance.manifest import ( + CountryReleaseManifest, + RegionalCertification, + ) + from policyengine.provenance.trace import build_trace_tro_from_release_bundle + + record = _bundle_with_regional_acceptance(dataset_file)["data_releases"]["uk"] + tro = build_trace_tro_from_release_bundle( + CountryReleaseManifest.model_validate(record), + None, + fetch_pypi=lambda name, version: {}, + ) + performance = tro["@graph"][0]["trov:hasPerformance"] + + assert performance[ + "pe:regionalCertification" + ] == RegionalCertification.model_validate( + record["regional_certification"] + ).model_dump(mode="json", exclude_none=True) + assert ( + performance["pe:compatibilityBasis"] + == record["certification"]["compatibility_basis"] + ) + assert performance["pe:regionalCertification"]["area_receipts"] == "uncertified" + + +def test_removing_regional_acceptance_removes_downstream_record(dataset_file): + from policyengine.provenance.manifest import CountryReleaseManifest + from policyengine.provenance.trace import build_trace_tro_from_release_bundle + + bundle = _bundle_with_regional_acceptance(dataset_file) + record = bundle["data_releases"]["uk"] + del record["regional_certification"] + del record["datasets"]["area_file"] + + receipt = _build(dataset_file, bundle) + tro = build_trace_tro_from_release_bundle( + CountryReleaseManifest.model_validate(record), + None, + fetch_pypi=lambda name, version: {}, + ) + + assert "regional_certification" not in receipt.canonical_payload() + assert "pe:regionalCertification" not in tro["@graph"][0]["trov:hasPerformance"] + assert receipt.certification_status == "certified" + + def test_an_omitted_artifact_revision_is_the_release_manifest_revision(dataset_file): """Round-4 review M5: the receipt names what the materializer downloads.""" From 68cf966da7930f131acd792a6223bb136f589bba Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:31:42 -0400 Subject: [PATCH 06/21] Measure byte-bound regional inputs against the installed UK model --- scripts/audit_regional_input_surface.py | 54 +++ .../provenance/regional_input_surface.py | 345 ++++++++++++++++++ 2 files changed, 399 insertions(+) create mode 100644 scripts/audit_regional_input_surface.py create mode 100644 src/policyengine/provenance/regional_input_surface.py diff --git a/scripts/audit_regional_input_surface.py b/scripts/audit_regional_input_surface.py new file mode 100644 index 000000000..635a610da --- /dev/null +++ b/scripts/audit_regional_input_surface.py @@ -0,0 +1,54 @@ +"""Write a byte-bound audit of regional H5 columns for an installed UK model. + +Run in an environment with the exact requested policyengine-uk version and +PyTables installed. Supply the H5 files from the immutable regional manifest; +the audit verifies their hashes without installing models or downloading data. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "src")) + +from policyengine.provenance.regional_input_surface import ( # noqa: E402 + measure_regional_input_surface, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--regional-manifest-uri", required=True) + parser.add_argument( + "--artifact", required=True, action="append", metavar="NAME=PATH" + ) + parser.add_argument("--model-version", required=True) + parser.add_argument("--audited-by", required=True) + parser.add_argument("--limitation", required=True, action="append") + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + paths: dict[str, Path] = {} + for entry in args.artifact: + name, separator, path = entry.partition("=") + if not separator or not name.strip() or not path.strip() or name in paths: + parser.error("Each --artifact must be a unique nonblank NAME=PATH.") + paths[name] = Path(path) + evidence = measure_regional_input_surface( + regional_manifest_uri=args.regional_manifest_uri, + artifact_paths=paths, + model_version=args.model_version, + audited_by=args.audited_by, + limitations=args.limitation, + token=os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN"), + ) + args.output.write_text(evidence.model_dump_json(indent=2) + "\n") + print(f"Saved {len(evidence.artifacts)} artifact input audits to {args.output}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/policyengine/provenance/regional_input_surface.py b/src/policyengine/provenance/regional_input_surface.py new file mode 100644 index 000000000..d156d64c0 --- /dev/null +++ b/src/policyengine/provenance/regional_input_surface.py @@ -0,0 +1,345 @@ +"""Audit regional H5 bytes against the exact installed UK country model. + +The set comparison ports Microcosm PR #1348's handoff comparison, replacing +its older-model inventory proxy with the columns actually stored in every +artifact and period. Country-model input moves are read from that installed +model's constructor, so an alias cannot make a supplied input look defaulted. +No household values are included in the resulting audit. +""" + +from __future__ import annotations + +import ast +import hashlib +import inspect +import re +import textwrap +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any + +from policyengine.provenance.manifest import ( + RegionalInputArtifactEvidence, + RegionalInputSurfaceEvidence, +) + +UK_ENTITIES = ("person", "benunit", "household") + + +class RegionalInputSurfaceError(ValueError): + """A complete, byte-bound regional surface cannot be established.""" + + +def _names(values: list[str], label: str) -> set[str]: + if not values or any( + not isinstance(value, str) or not value.strip() or value != value.strip() + for value in values + ): + raise RegionalInputSurfaceError(f"{label} must contain nonblank names.") + names = set(values) + if len(names) != len(values): + raise RegionalInputSurfaceError(f"{label} contains duplicate names.") + return names + + +def compare_input_surface( + artifacts: dict[str, RegionalInputArtifactEvidence], + model_variables: list[str], + model_inputs: list[str], + input_aliases: dict[str, str] | None = None, +) -> dict[str, list[str]]: + """Recompute losses and every default, including defaults in just one year. + + Inputs supplied by the model's unconditional ``move_values`` calls use the + destination name after the move. Raw columns still have to be variables + defined by the target model. Every artifact and period must pass: the + union of supplied columns cannot hide a missing input in another period. + """ + variables = _names(model_variables, "model_variables") + inputs = _names(model_inputs, "model_inputs") + if not inputs <= variables: + raise RegionalInputSurfaceError("model_inputs must be model_variables.") + aliases = input_aliases or {} + for donor, target in aliases.items(): + if ( + donor not in variables + or target not in inputs + or donor == target + or target in aliases + ): + raise RegionalInputSurfaceError( + f"Unsupported input alias {donor!r} -> {target!r}." + ) + if not artifacts: + raise RegionalInputSurfaceError("No regional artifact columns measured.") + dropped: set[str] = set() + defaulted: set[str] = set() + for name, artifact in artifacts.items(): + columns = _names(artifact.columns, f"{name} columns") + if not artifact.columns_by_period: + raise RegionalInputSurfaceError(f"{name} has no measured periods.") + periods = [] + for period, period_columns in artifact.columns_by_period.items(): + if not re.fullmatch(r"[0-9]{4}", period): + raise RegionalInputSurfaceError( + f"{name} has an invalid period {period!r}." + ) + periods.append(_names(period_columns, f"{name}/{period} columns")) + if set().union(*periods) != columns: + raise RegionalInputSurfaceError( + f"{name} columns must equal the union of columns_by_period." + ) + for supplied in periods: + dropped.update(supplied - variables) + effective = set(supplied) + for donor, target in aliases.items(): + if donor in effective: + effective.remove(donor) + effective.add(target) + defaulted.update(inputs - effective) + return { + "dropped_columns": sorted(dropped), + "defaulted_inputs": sorted(defaulted), + } + + +def _input_aliases(simulation_class: Any) -> dict[str, str]: + """Read unconditional literal moves; refuse unsupported dynamic semantics. + + policyengine-uk has no public alias inventory. Its public Simulation + constructor declares the input moves. Reading that source avoids running + a simulation or encoding one version's aliases as a permanent allowlist. + """ + try: + tree = ast.parse(textwrap.dedent(inspect.getsource(simulation_class.__init__))) + except (OSError, TypeError, SyntaxError) as exc: + raise RegionalInputSurfaceError( + "Cannot inspect the installed UK Simulation input aliases." + ) from exc + constructor = tree.body[0] + if not isinstance(constructor, (ast.FunctionDef, ast.AsyncFunctionDef)): + raise RegionalInputSurfaceError("Cannot locate the UK Simulation constructor.") + unconditional = { + id(statement.value) + for statement in constructor.body + if isinstance(statement, ast.Expr) + } + aliases: dict[str, str] = {} + for node in ast.walk(constructor): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "self" + and node.func.attr == "move_values" + ): + continue + if ( + id(node) not in unconditional + or len(node.args) != 2 + or node.keywords + or any( + not isinstance(argument, ast.Constant) + or not isinstance(argument.value, str) + for argument in node.args + ) + ): + raise RegionalInputSurfaceError( + "UK Simulation has a conditional or dynamic move_values input alias; " + "an explicit audit of its semantics is required." + ) + donor = str(node.args[0].value) # type: ignore[attr-defined] + target = str(node.args[1].value) # type: ignore[attr-defined] + if donor in aliases: + raise RegionalInputSurfaceError(f"Duplicate move_values alias {donor!r}.") + aliases[donor] = target + return aliases + + +def _model_surface(model_version: str) -> tuple[list[str], list[str], dict[str, str]]: + try: + installed = version("policyengine-uk") + except PackageNotFoundError as exc: + raise RegionalInputSurfaceError( + "Install the exact policyengine-uk target first." + ) from exc + if installed != model_version: + raise RegionalInputSurfaceError( + f"Installed policyengine-uk {installed} differs from requested {model_version}." + ) + # Keep country-model loading out of certification's evidence-validation path. + import policyengine_uk + + system = policyengine_uk.CountryTaxBenefitSystem() + variables = sorted(system.variables) + inputs = sorted( + name + for name, variable in system.variables.items() + if variable.is_input_variable() + ) + return variables, inputs, _input_aliases(policyengine_uk.Simulation) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _hdf_columns(path: Path) -> dict[str, list[str]]: + import pandas as pd + + try: + with pd.HDFStore(path, mode="r") as store: + keys = set(store.keys()) + flat = {f"/{entity}" for entity in UK_ENTITIES} & keys + nested = { + key + for key in keys + if any(key.startswith(f"/{entity}/") for entity in UK_ENTITIES) + } + if flat and nested: + raise RegionalInputSurfaceError( + "H5 mixes single-year and multi-year entity keys." + ) + entity_keys: dict[str, dict[str, str]] = {} + if flat: + if "/time_period" not in keys: + raise RegionalInputSurfaceError( + "Single-year H5 requires /time_period." + ) + metadata = store.select("/time_period") + values = metadata.to_numpy().ravel().tolist() + if len(values) != 1 or not re.fullmatch(r"[0-9]{4}", str(values[0])): + raise RegionalInputSurfaceError( + "H5 /time_period must name one year." + ) + entity_keys[str(values[0])] = { + entity: f"/{entity}" + for entity in UK_ENTITIES + if f"/{entity}" in flat + } + else: + for key in sorted(nested): + parts = key.strip("/").split("/") + if len(parts) != 2 or not re.fullmatch(r"[0-9]{4}", parts[1]): + raise RegionalInputSurfaceError( + f"Invalid H5 entity period key {key!r}." + ) + entity_keys.setdefault(parts[1], {})[parts[0]] = key + if not entity_keys: + raise RegionalInputSurfaceError("H5 has no UK entity periods.") + columns_by_period = {} + for period, entities in sorted(entity_keys.items()): + missing = set(UK_ENTITIES) - set(entities) + if missing: + raise RegionalInputSurfaceError( + f"H5 period {period} is missing entity tables: {sorted(missing)}." + ) + columns: set[str] = set() + for entity in UK_ENTITIES: + # This obtains the same column metadata for fixed and table + # stores without selecting household or person rows. + frame = store.select(entities[entity], start=0, stop=0) + columns.update( + _names(list(frame.columns), f"{entity}/{period} columns") + ) + columns_by_period[period] = sorted(columns) + return columns_by_period + except RegionalInputSurfaceError: + raise + except (OSError, KeyError, ValueError, ImportError) as exc: + raise RegionalInputSurfaceError( + f"Cannot read H5 entity metadata from {path.name}." + ) from exc + + +def measure_regional_input_surface( + regional_manifest_uri: str, + artifact_paths: dict[str, Path], + model_version: str, + audited_by: str, + limitations: list[str], + token: str | None = None, +) -> RegionalInputSurfaceEvidence: + """Create an audit from actual H5 columns and an exact installed UK model. + + The operator supplies local files named by the immutable regional release. + Every microdata H5 must be measured and match its manifest's SHA-256 both + before and after metadata inspection. This reads no unit values for output. + """ + # Imported lazily: the certification gate reuses compare_input_surface. + from policyengine.provenance.certification import fetch_release_manifest + + if ( + not audited_by.strip() + or not limitations + or any(not item.strip() for item in limitations) + ): + raise RegionalInputSurfaceError( + "An auditor and nonblank limitations are required." + ) + manifest, manifest_sha256, parts = fetch_release_manifest( + regional_manifest_uri, token=token + ) + expected = { + name: artifact + for name, artifact in manifest.artifacts.items() + if artifact.kind == "microdata" and Path(artifact.path).suffix.lower() == ".h5" + } + if not expected or set(artifact_paths) != set(expected): + raise RegionalInputSurfaceError( + "Supplied artifact paths must exactly cover the regional microdata H5 artifacts: " + f"{sorted(expected)}." + ) + variables, inputs, aliases = _model_surface(model_version) + artifacts = {} + for name, artifact in sorted(expected.items()): + path = Path(artifact_paths[name]) + if not artifact.sha256: + raise RegionalInputSurfaceError( + f"Artifact {name!r} has no manifest sha256." + ) + before = _sha256_file(path) + if before != artifact.sha256: + raise RegionalInputSurfaceError( + f"Artifact {name!r} bytes differ from its manifest sha256." + ) + periods = _hdf_columns(path) + after = _sha256_file(path) + if after != before: + raise RegionalInputSurfaceError( + f"Artifact {name!r} bytes changed during metadata inspection." + ) + artifacts[name] = RegionalInputArtifactEvidence( + repo_id=artifact.repo_id, + repo_type=artifact.repo_type or parts["repo_type"], + path=artifact.path, + revision=artifact.revision, + sha256=after, + columns=sorted( + {column for columns in periods.values() for column in columns} + ), + columns_by_period=periods, + ) + comparison = compare_input_surface(artifacts, variables, inputs, aliases) + if comparison["dropped_columns"]: + raise RegionalInputSurfaceError( + f"Dropped regional input columns refuse acceptance: {comparison['dropped_columns']}." + ) + return RegionalInputSurfaceEvidence( + regional_manifest_uri=regional_manifest_uri, + regional_manifest_sha256=manifest_sha256, + model_package="policyengine-uk", + model_version=model_version, + artifacts=artifacts, + model_variables=variables, + model_inputs=inputs, + input_aliases=aliases, + dropped_columns=comparison["dropped_columns"], + defaulted_inputs=comparison["defaulted_inputs"], + audited_by=audited_by, + limitations=limitations, + ) From 355fd34ba1f8f891ffdcc440b67a97e72871a5e4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:33:06 -0400 Subject: [PATCH 07/21] Preserve regional audit evidence through JSON-LD expansion --- .../schemas/execution_receipt.v1.schema.json | 26 +++++------ src/policyengine/provenance/trace.py | 11 +++-- tests/test_execution_receipt.py | 44 +++++++++++++++++-- 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/src/policyengine/data/schemas/execution_receipt.v1.schema.json b/src/policyengine/data/schemas/execution_receipt.v1.schema.json index e8a449b87..f2e467cc3 100644 --- a/src/policyengine/data/schemas/execution_receipt.v1.schema.json +++ b/src/policyengine/data/schemas/execution_receipt.v1.schema.json @@ -552,22 +552,15 @@ "type": "array" }, "columns_by_period": { - "anyOf": [ - { - "additionalProperties": { - "items": { - "type": "string" - }, - "type": "array" - }, - "type": "object" + "additionalProperties": { + "items": { + "type": "string" }, - { - "type": "null" - } - ], - "default": null, - "title": "Columns By Period" + "type": "array" + }, + "minProperties": 1, + "title": "Columns By Period", + "type": "object" }, "path": { "minLength": 1, @@ -611,7 +604,8 @@ "path", "revision", "sha256", - "columns" + "columns", + "columns_by_period" ], "title": "RegionalInputArtifactEvidence", "type": "object" diff --git a/src/policyengine/provenance/trace.py b/src/policyengine/provenance/trace.py index 4a3128ceb..0e79b34e8 100644 --- a/src/policyengine/provenance/trace.py +++ b/src/policyengine/provenance/trace.py @@ -557,11 +557,14 @@ def _build_bundle_performance( if certification.certified_by is not None: performance["pe:certifiedBy"] = certification.certified_by if country_manifest.regional_certification is not None: - performance["pe:regionalCertification"] = ( - country_manifest.regional_certification.model_dump( + # An explicit JSON literal preserves evidence member names through + # JSON-LD expansion; they are JSON contract keys, not RDF predicates. + performance["pe:regionalCertification"] = { + "@type": "@json", + "@value": country_manifest.regional_certification.model_dump( mode="json", exclude_none=True - ) - ) + ), + } performance.update(dict(emission_context or _emission_context())) return performance diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index b7f03a3bc..6589bff47 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -477,6 +477,9 @@ def _bundle_with_regional_acceptance(dataset_file): "area_file": { **artifact, "columns": ["employment_income", "household_weight"], + "columns_by_period": { + "2025": ["employment_income", "household_weight"] + }, } }, "model_variables": sorted( @@ -562,8 +565,9 @@ def test_trace_exposes_regional_acceptance_separately_from_national(dataset_file ) performance = tro["@graph"][0]["trov:hasPerformance"] - assert performance[ - "pe:regionalCertification" + assert performance["pe:regionalCertification"]["@type"] == "@json" + assert performance["pe:regionalCertification"][ + "@value" ] == RegionalCertification.model_validate( record["regional_certification"] ).model_dump(mode="json", exclude_none=True) @@ -571,7 +575,41 @@ def test_trace_exposes_regional_acceptance_separately_from_national(dataset_file performance["pe:compatibilityBasis"] == record["certification"]["compatibility_basis"] ) - assert performance["pe:regionalCertification"]["area_receipts"] == "uncertified" + assert ( + performance["pe:regionalCertification"]["@value"]["area_receipts"] + == "uncertified" + ) + + +def test_trace_regional_acceptance_survives_jsonld_expansion(dataset_file): + rdflib = pytest.importorskip("rdflib") + from policyengine.provenance.manifest import CountryReleaseManifest + from policyengine.provenance.trace import ( + POLICYENGINE_TRACE_NAMESPACE, + build_trace_tro_from_release_bundle, + ) + + record = _bundle_with_regional_acceptance(dataset_file)["data_releases"]["uk"] + manifest = CountryReleaseManifest.model_validate(record) + tro = build_trace_tro_from_release_bundle( + manifest, + None, + fetch_pypi=lambda name, version: {}, + ) + + graph = rdflib.Graph().parse(data=json.dumps(tro), format="json-ld") + values = list( + graph.objects( + None, rdflib.URIRef(POLICYENGINE_TRACE_NAMESPACE + "regionalCertification") + ) + ) + + assert len(values) == 1 + assert isinstance(values[0], rdflib.Literal) + assert values[0].datatype == rdflib.RDF.JSON + assert json.loads(str(values[0])) == manifest.regional_certification.model_dump( + mode="json", exclude_none=True + ) def test_removing_regional_acceptance_removes_downstream_record(dataset_file): From 04e0450a32e16a156cb8c2e2fa8e725061367202 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:41:29 -0400 Subject: [PATCH 08/21] Require bound zero-loss evidence for UK regional model exceptions --- scripts/bundle.py | 8 + scripts/certify_data_release.py | 14 +- src/policyengine/provenance/certification.py | 161 +++++++++++++++++-- src/policyengine/provenance/manifest.py | 47 ++++++ tests/test_certify_data_release.py | 19 +++ tests/test_regional_input_evidence.py | 52 +++++- 6 files changed, 289 insertions(+), 12 deletions(-) diff --git a/scripts/bundle.py b/scripts/bundle.py index 5530a0a72..125635cd6 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -52,6 +52,10 @@ def _certify_data(args: argparse.Namespace) -> int: argv.extend( ["--accept-regional-model-mismatch", args.accept_regional_model_mismatch] ) + if args.regional_input_surface_evidence: + argv.extend( + ["--regional-input-surface-evidence", args.regional_input_surface_evidence] + ) if args.no_generate: argv.append("--no-generate") if args.no_changelog: @@ -239,6 +243,10 @@ def _parser() -> argparse.ArgumentParser: "scripts/certify_data_release.py." ), ) + certify.add_argument( + "--regional-input-surface-evidence", + help="Audited regional input-surface JSON; required for an accepted mismatch.", + ) certify.add_argument( "--regional-artifact-prefix", help="Legacy per-state artifact prefix to import. Defaults to states/.", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index 5c3bf6874..7cdc1a5a1 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -24,6 +24,7 @@ from __future__ import annotations import argparse +import json import os import sys from pathlib import Path @@ -121,10 +122,16 @@ def main(argv=None) -> int: help=( "UK only: route regions to a regional release that does not cover " "the pinned model, recording this reviewed reason in the bundle " - "instead of refusing. The core must still be covered." + "instead of refusing. Requires --regional-input-surface-evidence; " + "dropped columns refuse. The core must still be covered." ), ) parser.add_argument("--no-generate", action="store_true") + parser.add_argument( + "--regional-input-surface-evidence", + type=Path, + help="Audited JSON evidence bound to the regional bytes and exact target model.", + ) parser.add_argument("--no-changelog", action="store_true") parser.add_argument( "--skip-artifact-check", @@ -159,6 +166,11 @@ def main(argv=None) -> int: core_version=args.core_version, qualified_pair_id=args.qualified_pair_id, regional_model_mismatch_reason=args.accept_regional_model_mismatch, + regional_input_surface_evidence=( + json.loads(args.regional_input_surface_evidence.read_text()) + if args.regional_input_surface_evidence is not None + else None + ), ) print(result.summary()) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index edae62391..e773ba900 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -47,6 +47,8 @@ from typing import Literal, Optional, cast, get_args import requests +from packaging.version import InvalidVersion, Version +from pydantic import ValidationError from policyengine.data.uk_geography_assets import ( CONSTITUENCY_ASSET_SPEC, @@ -57,6 +59,7 @@ HF_REQUEST_TIMEOUT_SECONDS, DataReleaseArtifact, DataReleaseManifest, + RegionalInputSurfaceEvidence, RegionalReleaseKind, _specifier_matches, fetch_pypi_wheel_metadata, @@ -416,6 +419,10 @@ def model_compatibility_basis( #: A regional release's model coverage accepted by a reviewer, not by the #: release's own build or claims (``--accept-regional-model-mismatch``). REGIONAL_BASIS_ACCEPTED_MISMATCH = "accepted_model_mismatch" +REGIONAL_VAT_LIMITATION = ( + "the dense cut's VAT predates the policyengine-uk VAT factor fix " + "(#2187, 2.135.7); the dashboard holds a VAT limit" +) def accepted_regional_mismatch( @@ -423,6 +430,10 @@ def accepted_regional_mismatch( model_package: str, model_version: str, reason: str, + *, + regional_manifest_uri: str, + regional_manifest_sha256: str, + evidence: Optional[RegionalInputSurfaceEvidence], ) -> tuple[dict, str]: """Record a reviewed decision to route regions to a release on another model. @@ -438,6 +449,99 @@ def accepted_regional_mismatch( raise CertificationError( "--accept-regional-model-mismatch needs a reason a reviewer can read." ) + if evidence is None: + raise CertificationError( + "An accepted regional model mismatch requires audited input-surface " + "evidence (--regional-input-surface-evidence)." + ) + try: + audit = RegionalInputSurfaceEvidence.model_validate(evidence) + except ValidationError as exc: + raise CertificationError( + f"Invalid regional input-surface evidence: {exc}" + ) from exc + if ( + audit.regional_manifest_uri != regional_manifest_uri + or audit.regional_manifest_sha256 != regional_manifest_sha256 + or audit.model_package != model_package + or audit.model_version != model_version + ): + raise CertificationError( + "Regional input-surface evidence must bind the exact regional cut, " + "raw manifest bytes and target model package/version." + ) + expected_artifacts = { + name: artifact + for name, artifact in regional_manifest.artifacts.items() + if artifact.kind == "microdata" and artifact.path.endswith(".h5") + } + if not expected_artifacts or set(audit.artifacts) != set(expected_artifacts): + raise CertificationError( + "Evidence must measure every regional microdata artifact." + ) + for name, artifact in expected_artifacts.items(): + measured = audit.artifacts[name] + if any( + getattr(measured, key) != getattr(artifact, key) + for key in ("repo_id", "path", "revision", "sha256") + ) or ( + measured.repo_type is not None + and measured.repo_type + != ( + artifact.repo_type + or parse_manifest_uri(regional_manifest_uri)["repo_type"] + ) + ): + raise CertificationError( + f"Regional evidence artifact {name!r} does not match its identity/bytes." + ) + if not all(column.strip() for column in measured.columns): + raise CertificationError("Regional evidence has an empty column name.") + if measured.columns_by_period is not None: + if ( + not measured.columns_by_period + or any(not columns for columns in measured.columns_by_period.values()) + or set(measured.columns) + != { + column + for columns in measured.columns_by_period.values() + for column in columns + } + ): + raise CertificationError( + "Regional evidence has inconsistent period columns." + ) + if not set(audit.model_inputs).issubset(audit.model_variables): + raise CertificationError( + "Evidence model inputs must belong to its variable surface." + ) + if not all(item.strip() for item in audit.limitations): + raise CertificationError("Regional evidence limitations must not be blank.") + from policyengine.provenance.regional_input_surface import compare_input_surface + + try: + comparison = compare_input_surface( + audit.artifacts, + audit.model_variables, + audit.model_inputs, + audit.input_aliases, + ) + except ValueError as exc: + raise CertificationError( + f"Invalid regional input-surface evidence: {exc}" + ) from exc + if ( + audit.dropped_columns != comparison["dropped_columns"] + or audit.defaulted_inputs != comparison["defaulted_inputs"] + ): + raise CertificationError( + "Regional evidence comparison disagrees with its measured surfaces." + ) + if comparison["dropped_columns"]: + raise CertificationError( + "Dropped regional input columns are refused: " + + ", ".join(comparison["dropped_columns"]) + ) built = ( regional_manifest.build.built_with_model_package if (regional_manifest.build is not None) @@ -446,19 +550,37 @@ def accepted_regional_mismatch( built_with = ( built.version if built is not None and built.name == model_package else None ) + limitations = list(audit.limitations) + try: + predates_vat_fix = built_with is not None and Version(built_with) < Version( + "2.135.7" + ) <= Version(model_version) + except InvalidVersion as exc: + raise CertificationError( + "Regional model versions must be valid versions." + ) from exc + if predates_vat_fix and REGIONAL_VAT_LIMITATION not in limitations: + limitations.append(REGIONAL_VAT_LIMITATION) record = { "compatibility_basis": REGIONAL_BASIS_ACCEPTED_MISMATCH, "certified_for_model_version": model_version, "built_with_model_version": built_with, "accepted_reason": reason, "area_receipts": "uncertified", + "regional_manifest_uri": regional_manifest_uri, + "regional_manifest_sha256": regional_manifest_sha256, + "defaulted_inputs": comparison["defaulted_inputs"], + "dropped_columns": comparison["dropped_columns"], + "limitations": limitations, + "evidence": audit.model_dump(mode="json", exclude_none=True), } warning = ( f"regional release: built with {model_package} " f"{built_with or 'an unrecorded version'} and routed under " f"{model_version} on an accepted mismatch ({reason}); area runs are " - "uncertified, inputs the newer model added take their defaults there, " - "and columns it dropped are ignored on load" + f"uncertified; audited dropped columns: none; defaulted inputs: " + f"{', '.join(comparison['defaulted_inputs']) or 'none'}. " + + " ".join(limitations) ) return record, warning @@ -1148,6 +1270,7 @@ def build_bundle_data_release_payload( regional_uri_parts: Optional[dict] = None, regional_release_kind: Optional[RegionalReleaseKind] = None, regional_certification: Optional[dict] = None, + regional_manifest_sha256: Optional[str] = None, ) -> dict: """Map the country certification payload into bundle ``data_releases``.""" payload = copy.deepcopy(country_payload) @@ -1161,6 +1284,8 @@ def build_bundle_data_release_payload( payload["release_manifest_uri"] = https_manifest_url(uri_parts) if regional_manifest_uri is not None: payload["regional_source_manifest_uri"] = regional_manifest_uri + if regional_manifest_sha256 is not None: + payload["regional_source_manifest_sha256"] = regional_manifest_sha256 if regional_uri_parts is not None: payload["regional_release_manifest_uri"] = https_manifest_url( regional_uri_parts @@ -1193,6 +1318,7 @@ def certify( qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> tuple[dict, str, list[str]]: raise NotImplementedError @@ -1217,6 +1343,7 @@ def certify( qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> tuple[dict, str, list[str]]: raise CertificationError( "Legacy data-producer certification updates are not implemented in " @@ -1244,12 +1371,14 @@ def certify( qualified_pair_id: Optional[str] = None, bundle_core_version: Optional[str] = None, regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> tuple[dict, str, list[str]]: manifest, manifest_sha256, uri_parts = fetch_release_manifest( manifest_uri, token=token ) primary_manifest = manifest regional_manifest = None + regional_manifest_sha256 = None regional_uri_parts = None recorded_release_kind = None regional_artifacts: dict[str, DataReleaseArtifact] = {} @@ -1267,8 +1396,8 @@ def certify( "whose area receipts are uncertified." ) if regional_manifest_uri is not None: - regional_manifest, _, regional_uri_parts = fetch_release_manifest( - regional_manifest_uri, token=token + regional_manifest, regional_manifest_sha256, regional_uri_parts = ( + fetch_release_manifest(regional_manifest_uri, token=token) ) # The regional release runs on the same model and core, so it must # cover the pinned versions by its own build or claims. A qualified @@ -1287,6 +1416,9 @@ def certify( model_package, model_version, regional_model_mismatch_reason, + regional_manifest_uri=regional_manifest_uri, + regional_manifest_sha256=regional_manifest_sha256, + evidence=regional_input_surface_evidence, ) regional_warnings = [] else: @@ -1405,6 +1537,7 @@ def certify( regional_uri_parts=regional_uri_parts, regional_release_kind=recorded_release_kind, regional_certification=regional_certification, + regional_manifest_sha256=regional_manifest_sha256, ), manifest_sha256, warnings, @@ -1451,6 +1584,7 @@ def certify_data_release( core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, regional_model_mismatch_reason: Optional[str] = None, + regional_input_surface_evidence: Optional[RegionalInputSurfaceEvidence] = None, ) -> CertificationResult: """Certify a data release into the bundle manifest. @@ -1467,13 +1601,21 @@ def certify_data_release( release only. While the bundle routes regions, a run without ``regional_manifest_uri`` is refused unless ``drop_regional_release``. - ``regional_model_mismatch_reason`` (UK only) records a reviewer's - acceptance of a regional release that does not cover the pinned model, - instead of refusing it. The core must still be covered. The bundle then - carries ``regional_certification`` naming both model versions and the - reason; area receipts are uncertified either way. + ``regional_model_mismatch_reason`` (UK only) requires + ``regional_input_surface_evidence`` audited against the exact target model + and regional manifest/artifact bytes. Dropped columns are always refused; + every defaulted input and limitation is recorded in + ``regional_certification`` alongside the reason and both model versions. + The core must still be covered; area receipts remain uncertified. """ certified_country = _certified_country(country) + if ( + regional_input_surface_evidence is not None + and regional_model_mismatch_reason is None + ): + raise CertificationError( + "Regional input-surface evidence requires an accepted mismatch reason." + ) if drop_regional_release and regional_manifest_uri is not None: raise CertificationError( "--drop-regional-release removes the bundle's regional routing, so it " @@ -1519,6 +1661,7 @@ def certify_data_release( (bundle.get("packages") or {}).get("policyengine-core") or {} ).get("version"), regional_model_mismatch_reason=regional_model_mismatch_reason, + regional_input_surface_evidence=regional_input_surface_evidence, ) previous_release = (bundle.get("data_releases") or {}).get(country) or {} bundle.setdefault("data_releases", {})[country] = data_release diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index f309636cd..1e00f5fce 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -211,6 +211,45 @@ class CertifiedDataArtifact(BaseModel): RegionalReleaseKind = Literal["local_area", "area_coded_national"] +class RegionalInputArtifactEvidence(BaseModel): + """Measured columns of one byte-verified regional H5 artifact.""" + + model_config = {"extra": "forbid"} + + repo_id: str = Field(min_length=1) + path: str = Field(min_length=1) + revision: str = Field(min_length=1) + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + repo_type: Optional[Literal["model", "dataset"]] = None + columns: list[str] = Field(min_length=1) + columns_by_period: dict[str, list[str]] = Field(min_length=1) + + +class RegionalInputSurfaceEvidence(BaseModel): + """An operator's audit of actual regional columns against an exact model. + + Surfaces are retained so certification recomputes the comparison rather + than trusting a declared zero-loss result. The artifact hashes identify + the bytes inspected by the auditor, not merely a model's possible inputs. + """ + + model_config = {"extra": "forbid"} + + schema_version: Literal[1] = 1 + regional_manifest_uri: str = Field(min_length=1) + regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + model_package: str = Field(min_length=1) + model_version: str = Field(min_length=1) + artifacts: dict[str, RegionalInputArtifactEvidence] = Field(min_length=1) + model_variables: list[str] = Field(min_length=1) + model_inputs: list[str] = Field(min_length=1) + input_aliases: Optional[dict[str, str]] = None + dropped_columns: list[str] + defaulted_inputs: list[str] + audited_by: str = Field(min_length=1, pattern=r"\S") + limitations: list[str] = Field(min_length=1) + + class RegionalCertification(BaseModel): """A reviewer's acceptance of a regional release on another model version. @@ -224,6 +263,12 @@ class RegionalCertification(BaseModel): built_with_model_version: Optional[str] = None accepted_reason: str area_receipts: Literal["uncertified"] = "uncertified" + regional_manifest_uri: str + regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + defaulted_inputs: list[str] + dropped_columns: list[str] + limitations: list[str] + evidence: RegionalInputSurfaceEvidence class CountryReleaseManifest(BaseModel): @@ -245,6 +290,8 @@ class CountryReleaseManifest(BaseModel): regional_certification: Optional[RegionalCertification] = None """Present only when the regional release was routed on an accepted model mismatch rather than its own build or claims.""" + regional_source_manifest_sha256: Optional[str] = None + """SHA-256 of the original regional release manifest bytes.""" certified_data_artifact: Optional[CertifiedDataArtifact] = None certification: Optional[DataCertification] = None source_sha256: Optional[str] = Field(default=None, exclude=True) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index a8c8b5b21..b00893fca 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1436,6 +1436,9 @@ def test__given_older_dense_cut__then_refuses_without_an_accepted_mismatch( def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): bundle_path = _tmp_bundle(tmp_path) reason = "area receipts are uncertified; the dense cut lags one release" + from tests.test_regional_input_evidence import input_evidence + + evidence = input_evidence(self._older_dense()) result = _certify_uk( bundle_path, @@ -1445,6 +1448,7 @@ def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): ], regional_manifest_uri=UK_DENSE_URI, regional_model_mismatch_reason=reason, + regional_input_surface_evidence=evidence, ) release = json.loads(bundle_path.read_text())["data_releases"]["uk"] @@ -1455,6 +1459,12 @@ def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): "built_with_model_version": "2.120.0", "accepted_reason": reason, "area_receipts": "uncertified", + "regional_manifest_uri": UK_DENSE_URI, + "regional_manifest_sha256": evidence["regional_manifest_sha256"], + "defaulted_inputs": evidence["defaulted_inputs"], + "dropped_columns": [], + "limitations": evidence["limitations"], + "evidence": evidence, } # The national certificate itself is untouched by the regional record. assert release["certification"]["compatibility_basis"] == ( @@ -1497,6 +1507,8 @@ def test__given_an_empty_reason__then_refuses(self, tmp_path, reason): def test__given_an_accepted_mismatch__then_the_core_must_still_be_covered( self, tmp_path ): + from tests.test_regional_input_evidence import input_evidence + national = _microcosm_uk_national_payload() national["compatible_core_packages"] = [ {"name": "policyengine-core", "specifier": "==3.33.0"} @@ -1509,6 +1521,7 @@ def test__given_an_accepted_mismatch__then_the_core_must_still_be_covered( [_response(national), _response(self._older_dense())], regional_manifest_uri=UK_DENSE_URI, regional_model_mismatch_reason="area receipts are uncertified", + regional_input_surface_evidence=input_evidence(self._older_dense()), core_version="3.33.0", ) @@ -1549,12 +1562,18 @@ def fake_main(argv): UK_DENSE_URI, "--accept-regional-model-mismatch", reason, + "--regional-input-surface-evidence", + "regional-audit.json", ] ) == 0 ) argv = captured["argv"] assert argv[argv.index("--accept-regional-model-mismatch") + 1] == reason + assert ( + argv[argv.index("--regional-input-surface-evidence") + 1] + == "regional-audit.json" + ) def test__given_pinned_core__then_the_regional_release_must_cover_it( self, tmp_path diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py index d0f6a31b3..5efdd99a0 100644 --- a/tests/test_regional_input_evidence.py +++ b/tests/test_regional_input_evidence.py @@ -54,7 +54,10 @@ def input_evidence(dense: dict | None = None) -> dict: name: { key: artifact[key] for key in ("repo_id", "path", "revision", "sha256") } - | {"columns": ["employment_income"]} + | { + "columns": ["employment_income"], + "columns_by_period": {"2025": ["employment_income"]}, + } }, "model_variables": ["employment_income", *DEFAULTED_INPUTS], "model_inputs": ["employment_income", *DEFAULTED_INPUTS], @@ -126,6 +129,9 @@ def test_a_dropped_column_refuses_even_with_an_acceptance_reason(tmp_path): evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns"].append( "removed_model_variable" ) + evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns_by_period"][ + "2025" + ].append("removed_model_variable") evidence["dropped_columns"] = ["removed_model_variable"] with pytest.raises(CertificationError, match="[Dd]ropped|removed_model_variable"): certify_with_evidence(_tmp_bundle(tmp_path), evidence) @@ -154,7 +160,7 @@ def test_every_default_and_limitation_survives_validation_and_roundtrip(tmp_path release = json.loads(bundle_path.read_text())["data_releases"]["uk"] manifest = CountryReleaseManifest.model_validate(release) assert manifest.regional_certification is not None - record = manifest.regional_certification.model_dump(mode="json") + record = manifest.regional_certification.model_dump(mode="json", exclude_none=True) assert record["defaulted_inputs"] == DEFAULTED_INPUTS assert record["dropped_columns"] == [] assert record["limitations"] == [VAT_LIMIT] @@ -235,3 +241,45 @@ def digest(bundle): return canonical_sha256(manifest.model_dump(mode="json")) assert digest(before) != digest(after) + + +def test_missing_period_measurements_refuse(tmp_path): + evidence = input_evidence() + del evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns_by_period"] + with pytest.raises(CertificationError): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_budget_target_records_vat_limit_and_both_named_defaults(tmp_path, monkeypatch): + from tests import test_certify_data_release as fixtures + + monkeypatch.setattr(fixtures, "UK_MODEL_VERSION", "2.147.2") + dense = older_dense() + dense["build"]["built_with_model_package"]["version"] = "2.132.6" + dense["compatible_model_packages"][0]["specifier"] = "==2.132.6" + audit = input_evidence(dense) + audit["model_version"] = "2.147.2" + audit["limitations"] = ["The dense cut was calibrated under an older model."] + path = _tmp_bundle(tmp_path) + certify_with_evidence(path, audit, dense) + release = json.loads(path.read_text())["data_releases"]["uk"] + record = release["regional_certification"] + assert record["certified_for_model_version"] == "2.147.2" + assert record["built_with_model_version"] == "2.132.6" + assert VAT_LIMIT in record["limitations"] + assert set(DEFAULTED_INPUTS) <= set(record["defaulted_inputs"]) + assert release["certification"]["compatibility_basis"] == "built_with_model_package" + + +def test_regional_acceptance_never_widens_national_coverage(tmp_path): + national = _microcosm_uk_national_payload() + national["build"]["built_with_model_package"]["version"] = "2.120.0" + national["compatible_model_packages"][0]["specifier"] = "==2.120.0" + with pytest.raises(CertificationError, match="new data build|compatibility claim"): + _certify_uk( + _tmp_bundle(tmp_path), + [_response(national), _response(older_dense())], + regional_manifest_uri=UK_DENSE_URI, + regional_model_mismatch_reason=REASON, + regional_input_surface_evidence=input_evidence(), + ) From b90eada3aecbedbd1a8c0f98a00d0d0f653c3f64 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 08:46:35 -0400 Subject: [PATCH 09/21] Bootstrap regional audits before target-model certification --- scripts/audit_regional_input_surface.py | 2 ++ tests/test_measure_regional_inputs.py | 30 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/scripts/audit_regional_input_surface.py b/scripts/audit_regional_input_surface.py index 635a610da..cbe5cd1a2 100644 --- a/scripts/audit_regional_input_surface.py +++ b/scripts/audit_regional_input_surface.py @@ -13,6 +13,8 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] +# The target model must be auditable before the current bundle certifies it. +os.environ.setdefault("POLICYENGINE_SKIP_COUNTRY_IMPORTS", "1") sys.path.insert(0, str(REPO_ROOT / "src")) from policyengine.provenance.regional_input_surface import ( # noqa: E402 diff --git a/tests/test_measure_regional_inputs.py b/tests/test_measure_regional_inputs.py index 8cf4006af..4bff297b6 100644 --- a/tests/test_measure_regional_inputs.py +++ b/tests/test_measure_regional_inputs.py @@ -343,3 +343,33 @@ def measure(**kwargs): assert captured["artifact_paths"] == {"dense": path} assert captured["limitations"] == [LIMITATION] assert json.loads(output.read_text()) == result.model_dump(mode="json") + + +def test_audit_cli_bootstraps_before_target_model_is_certified(tmp_path): + """The target must be auditable before it matches the current bundle.""" + import subprocess + + script = Path(__file__).parents[1] / "scripts" / "audit_regional_input_surface.py" + code = """ +import importlib.util +import os +import runpy +import sys +os.environ.pop('POLICYENGINE_SKIP_COUNTRY_IMPORTS', None) +original = importlib.util.find_spec +def refuse_country_autoload(name, *args, **kwargs): + if name in {'policyengine_uk', 'policyengine_us'}: + raise RuntimeError('audit triggered automatic country import') + return original(name, *args, **kwargs) +importlib.util.find_spec = refuse_country_autoload +sys.argv = [sys.argv[1], '--help'] +runpy.run_path(sys.argv[0], run_name='__main__') +""" + result = subprocess.run( + [sys.executable, "-c", code, str(script)], + capture_output=True, + text=True, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + assert "--model-version" in result.stdout From aa59b0d704852eab63087d8146f7da667d2178e0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 09:49:56 -0400 Subject: [PATCH 10/21] Record Budget review fix validation and certification handoff --- .job/HANDOFF.md | 61 +++++++++++++++++++++---------------------------- .job/PR_BODY.md | 19 +++++++++++++++ 2 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 .job/PR_BODY.md diff --git a/.job/HANDOFF.md b/.job/HANDOFF.md index d0029679a..e14bc0201 100644 --- a/.job/HANDOFF.md +++ b/.job/HANDOFF.md @@ -1,44 +1,35 @@ -# PR #704 review-fix handoff +# PR #699 Budget review fix -- Draft PR: https://github.com/PolicyEngine/policyengine.py/pull/704 -- Branch: `budget-local-data-manifest`; confirmed reviewed remote head `fd4e099ed3ff768041f3a537e6e38527c07b0bbf`. -- Validated implementation head SHA: `0d7bf1a641d61b8951f47ab12d8d4867f57df140`. -- The following commit updates only this handoff. Its exact final head is recorded in `.job/HEAD` and `.job/local-data-manifest.bundle`. +- Confirmed origin heads: #699 `afee349d2c14de1b121b0b7e4f03b695d03f418c`; #704 `da2ae8fb8e97145093e35366686df99c1e38449e`. +- Merged #704 into `accept-regional-model-mismatch` with merge commit `c1a65198`. No history rewritten. Release order remains main → #704 → #699 → certification. +- Implementation validated through `b90eada3`; the final report commit is recorded in `.job/HEAD`. -## Fixes +Changes: -- Scan every TRO definition of the manifest artifact ID, including direct graph nodes, nested definitions and artifact references; require one distinct valid SHA-256 assertion. Preserve canonical inline/location checks. -- Validate both local and fetched manifest artifacts after revision rewriting. The certified/default artifact must be present and anchored; every overlapping managed artifact must agree with bundled repo/path/revision, repository type and pinned SHA-256. Compare identity fields separately to prevent ambiguous URI concatenation. Primary manifests may omit separately pinned regional artifacts and overlays. -- Share the existing release-relative diagnostics/evidence normalization with certification. Exact-pair model/core/build and fingerprint acceptance logic remains unchanged; unset fetch/cache/offline behavior is preserved for consistent bundles. -- Replace function attributes with a typed callable cache facade. -- Make the documented export conditional on successful download/verification/atomic replacement; remove release coordination from permanent public docs. -- PR body records the invariants and both downstream actions. No other repository was edited. +- Require typed audited evidence bound to the exact regional cut URI, original manifest digest, every regional H5 identity/hash, and exact target model. Recompute the surface comparison and refuse every dropped column. Record every defaulted input by inspecting all stored periods and the country loader's aliases. +- Port Microcosm #1348's comparison, strengthening its older-model proxy to measure actual byte-verified H5 columns. New producer: `scripts/audit_regional_input_surface.py`. The CLI works before the current bundle certifies its target model. Microcosm handoff callers must generate and forward the new evidence JSON. +- Preserve the regional manifest's raw-byte digest. Keep limitations, including the pre-2.135.7 VAT fix/dashboard limit, through Pydantic validation and bundle hashing. +- Expose the full regional record in receipts and TRACE (`@json` preserves JSON-LD semantics). Area receipts stay uncertified and identify their dense build model. National certification never widens. Explicit reversal removes the record, digest, area routes and dense artifact; the national certificate stays identical. +- Regenerate the receipt schema. Dashboard follow-up: refresh vendored `tests/fixtures/w4/execution_receipt.v1.schema.json`. -## Validation +Validation: -**3,322 unique cases passed; five skipped.** +- Tests first demonstrated missing-evidence acceptance, stale evidence/dropped-column failures, missing receipt/TRACE records, lost JSON-LD semantics, and premature audit CLI model initialization. +- **1,248 selected cases passed; two skipped**, across the UK run and successful reruns of three US metadata cases in the environment with the bundled US pin installed. The UK run initially reported 1,245 passes plus those three missing-package failures; all three reruns passed. One unrelated US offline-import integration case was explicitly deselected. +- Covered all changed provenance/certification modules, the existing roundtrip, required reversal cases, and #704's three local-manifest modules. Logs: `.job/logs/uk-final.txt`, `.job/logs/us-metadata-rerun.txt`. +- Expanded 53-module validation reported 3,008 passes and five skips with no failures before stalling for over nine minutes; it was interrupted and is not claimed complete. A subsequent wider pinned-environment attempt was also interrupted during slow package inspection. Initial setup problems (missing packages, inherited Git variables and an unpinned model) were diagnosed; final test subprocesses clear Git overrides. No interrupted test processes remain. +- `make format`, `make lint`, receipt schema generation/equality, `bundle.py check`, and diff checks pass. +- Mypy on all changed source modules: 24 diagnostics, identical to the merged baseline ignoring moved line numbers. Both new source modules have zero errors. Comparison: `.job/logs/mypy-comparison.txt`. +- Every pytest launch checked load1 ≤110 and free disk ≥55,000,000,000 bytes. At most two workers per invocation. Final launches: load1 75.71 / 97.69; available disk 103.7 GB decimal. -- Initial local/TRO-pin run: 88 passed, including two 80-example Hypothesis properties (byte mutation and duplicate placement). -- Broad 56-module provenance/certification/disclosure run: 3,256 passed, five skipped, one failure from the old missing-default error-message assertion. It now refuses earlier in the shared validator; the corrected complete TRO regeneration module passed all 40 cases on rerun. -- Release/bootstrap-focused run: 189 passed, including 59 cases outside the broad selection and two new default-anchor cases. -- Final affected manifest/certification run: 280 passed, including all 82 local-manifest cases, all 12 TRO-pin cases and four new repo/path-boundary cases. No failures remain in pytest's failure cache. -- `make format`, `make lint`, focused `mypy --follow-imports=silent src/policyengine/provenance/manifest.py`, `git diff --check`, and generated receipt schema checks passed. Documentation shell block passes `bash -n`. -- Every pytest invocation checked load1 <=110 and available disk >=55,000,000,000 bytes; no pytest workers. Observed load1 9.04–19.02; available disk 144.3–146.9 GB decimal. Tests used Python 3.13.9. -- Detailed logs and selected modules remain under `.job/`. No private manifests or microdata were committed. +Actual Budget pair: -## Downstream actions +- National `microcosm-uk-2024-25-national-20261023T075039Z-a133f37a` (2.147.2), dense `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe` (2.132.6). +- The exact model-inventory comparison has no dropped names and 16 added pure inputs, including both `salary_sacrifice_returned_to_income` and `local_bus_uncapped_single_fare`; the complete list is in certification guidance. +- The private dense H5 was unavailable (no HF credential; HTTP 401). This inventory is not a byte audit, and this job does not certify the actual pair. The certification follow-up must run the new audit against the real H5 and record its complete default list. -- Dashboard: refresh vendored `tests/fixtures/w4/execution_receipt.v1.schema.json` when upgrading. -- Scorecard: `pipeline/uk_replay_variants.py:730` retains the setting in offline isolation, while `:750` requires exact runtime bundle equality. Clear `POLICYENGINE_*_DATA_MANIFEST_FILE` during isolation. -- Leave the PR draft; do not merge. No versions, country pins, data or certified releases changed. +Delivery: -## Workspace Git metadata and delivery - -The protected workspace `.git` points into the caller's repository. All mutable Git metadata is instead inside this workspace: - -```bash -GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git status -GIT_DIR="$PWD/.job/git-copy/.git" GIT_WORK_TREE="$PWD" git log -4 --oneline -``` - -The original pointer and caller checkout were not changed. `.job/local-data-manifest.bundle` is portable relative to main `485dc23302ec4abc6a5f7df5c6c03b8525f64c4d`. The single final push is attempted after committing this handoff; its outcome is recorded in `.job/PUSH_RESULT.txt`. `.job/HEAD` contains the final branch SHA. No history was rewritten and nothing was pushed to main or master. +- All writes stayed in the assigned workspace. The protected `.git` points into the caller's checkout; mutable Git metadata is isolated in `.job/repo.git`. Use `source .job/env` before Git commands here. The caller checkout was not written. +- Portable commits: `.job/budget-fix-699.bundle`; final head: `.job/HEAD`. The single push attempt's output is saved in `.job/PUSH_RESULT.txt`. No main/master push; leave #699 draft. +- GitHub blocked the PR body update because the connector requires approval and approval policy is `never`. The replacement body, including the invariants, is saved in `.job/PR_BODY.md`; no PR metadata change was made. diff --git a/.job/PR_BODY.md b/.job/PR_BODY.md new file mode 100644 index 000000000..1b3c5fa33 --- /dev/null +++ b/.job/PR_BODY.md @@ -0,0 +1,19 @@ +Require audited evidence for a UK regional release served on an accepted model mismatch. + +This branch stacks #704 (`da2ae8fb`) into #699 (`afee349d2`) with a merge commit. The intended release order remains main → #704 → #699 → certification. + +Invariants: + +- National certification never widens. The national dataset name/hash, model coverage and core checks remain separate from the regional exception. +- An accepted area mismatch requires a reviewer reason and audited evidence bound to the exact dense cut URI, original manifest bytes, artifact identity/SHA-256, and exact target model package/version. Certification recomputes the comparison and refuses every dropped column. There is no dropped-column override. +- Every defaulted input is recorded across every stored period, accounting for the country loader's input aliases. Free-text limitations and the dense cut's pre-fix VAT limitation survive Pydantic validation and are hashed with the bundle. +- TRACE and execution receipts expose the full regional identity, acceptance, reason, input audit and limitations. TRACE preserves the record as a JSON-LD JSON literal; area receipts remain uncertified and identify their actual build model. +- Re-certifying with `--drop-regional-release` removes the exception, regional manifest digest, area routes and dense artifact without changing the national certificate. + +The comparison is ported from Microcosm #1348's `tools/certify_uk_cut_in_policyengine_py.py` and strengthened to inspect byte-verified H5 entity-table columns by period. `scripts/audit_regional_input_surface.py` produces the required JSON; `certify-data --regional-input-surface-evidence` consumes it with the accepted reason. The original regional manifest's raw-byte digest is retained. + +Budget route: national `microcosm-uk-2024-25-national-20261023T075039Z-a133f37a` (2.147.2) and dense `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe` (built on 2.132.6). The exact model-inventory comparison yields 16 added pure inputs, including `salary_sacrifice_returned_to_income` and `local_bus_uncapped_single_fare`; the complete list is documented. Actual dense-byte evidence must be generated by certification with authenticated access to the H5. This implementation job has no HF credential and does not certify that pair. + +Required downstream refresh: update the dashboard's vendored `tests/fixtures/w4/execution_receipt.v1.schema.json` from this repository's regenerated schema. + +Validation: 1,248 selected cases passed and two skipped across the UK run and three successful US metadata reruns. One unrelated US offline-import integration case was explicitly deselected. Formatting, lint, generated-schema equality and bundle checks pass. Mypy retains the same 24 baseline diagnostics, with zero errors in the new modules. Expanded validation was interrupted after 3,008 reported passes and five skips; it is not claimed complete. Details are recorded in `.job/HANDOFF.md`. Regression tests demonstrated failures before implementation. Leave this PR draft. From 878392cf5a42d7b4916ae7b311c84aa30d383d38 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 13:19:21 -0400 Subject: [PATCH 11/21] Reject foreign regional models and preserve empty mismatch reasons --- changelog.d/699.fixed.md | 2 + docs/engineering/skills/data-certification.md | 9 ++++ scripts/bundle.py | 2 +- src/policyengine/provenance/certification.py | 27 +++++++--- .../provenance/regional_input_surface.py | 6 +++ tests/test_certify_data_release.py | 53 +++++++++++++++++++ tests/test_regional_input_evidence.py | 30 +++++++++++ 7 files changed, 120 insertions(+), 9 deletions(-) diff --git a/changelog.d/699.fixed.md b/changelog.d/699.fixed.md index 55eb27701..55fc07874 100644 --- a/changelog.d/699.fixed.md +++ b/changelog.d/699.fixed.md @@ -1 +1,3 @@ Require a byte-bound input-surface audit when accepting a UK regional model mismatch, refuse dropped columns, and record defaulted inputs and limitations in the bundle, TRACE and execution receipts. Preserve the regional manifest digest and area build model while keeping area receipts uncertified; dropping the regional release removes its exception record. + +Refuse regional releases that explicitly name a different model package, and forward empty CLI reasons so certification rejects them. Document that the UK loader silently skips unknown columns. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 91c2dc2ec..e4805591e 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -112,6 +112,15 @@ Microcosm national release with one of: column**. It records every defaulted input and the free-text limitations in the validated `regional_certification`; the bundle hash binds that record. + The policyengine-uk 2.147.2 dataset loaders silently skip columns absent + from the model's variables: both the + [generic dataset loader](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1113-L1127) + and the + [UK multi-year loader](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1186-L1196) + use `continue` without warning. A removed or renamed input can therefore + disappear silently at runtime; the audit must catch that column loss before + certification. + The national certificate never widens. Area status remains `uncertified`: the certificate still covers the national dataset only (#685). TRACE and execution receipts expose the regional identity, build model, acceptance, diff --git a/scripts/bundle.py b/scripts/bundle.py index 125635cd6..9dd4040a4 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -48,7 +48,7 @@ def _certify_data(args: argparse.Namespace) -> int: argv.extend(["--regional-release-kind", args.regional_release_kind]) if args.drop_regional_release: argv.append("--drop-regional-release") - if args.accept_regional_model_mismatch: + if args.accept_regional_model_mismatch is not None: argv.extend( ["--accept-regional-model-mismatch", args.accept_regional_model_mismatch] ) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index e773ba900..5ddec8e47 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -425,6 +425,23 @@ def model_compatibility_basis( ) +def _regional_build_model_version( + regional_manifest: DataReleaseManifest, model_package: str +) -> Optional[str]: + """Return the recorded regional build version; refuse a different package.""" + built = ( + regional_manifest.build.built_with_model_package + if regional_manifest.build is not None + else None + ) + if built is not None and built.name != model_package: + raise CertificationError( + f"Regional release was built with {built.name}, not {model_package}; " + "regional releases must use the target model package." + ) + return built.version if built is not None else None + + def accepted_regional_mismatch( regional_manifest: DataReleaseManifest, model_package: str, @@ -449,6 +466,7 @@ def accepted_regional_mismatch( raise CertificationError( "--accept-regional-model-mismatch needs a reason a reviewer can read." ) + built_with = _regional_build_model_version(regional_manifest, model_package) if evidence is None: raise CertificationError( "An accepted regional model mismatch requires audited input-surface " @@ -542,14 +560,6 @@ def accepted_regional_mismatch( "Dropped regional input columns are refused: " + ", ".join(comparison["dropped_columns"]) ) - built = ( - regional_manifest.build.built_with_model_package - if (regional_manifest.build is not None) - else None - ) - built_with = ( - built.version if built is not None and built.name == model_package else None - ) limitations = list(audit.limitations) try: predates_vat_fix = built_with is not None and Version(built_with) < Version( @@ -1399,6 +1409,7 @@ def certify( regional_manifest, regional_manifest_sha256, regional_uri_parts = ( fetch_release_manifest(regional_manifest_uri, token=token) ) + _regional_build_model_version(regional_manifest, model_package) # The regional release runs on the same model and core, so it must # cover the pinned versions by its own build or claims. A qualified # pair names the primary release only, so it does not extend here. diff --git a/src/policyengine/provenance/regional_input_surface.py b/src/policyengine/provenance/regional_input_surface.py index d156d64c0..460c53337 100644 --- a/src/policyengine/provenance/regional_input_surface.py +++ b/src/policyengine/provenance/regional_input_surface.py @@ -5,6 +5,12 @@ artifact and period. Country-model input moves are read from that installed model's constructor, so an alias cannot make a supplied input look defaulted. No household values are included in the resulting audit. + +policyengine-uk 2.147.2 silently skips unknown columns in both dataset loaders; +the audit must detect that loss before certification. See the ``continue`` +branches in ``build_from_dataset`` and ``build_from_multi_year_dataset``: +https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1113-L1127 +https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1186-L1196 """ from __future__ import annotations diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index b00893fca..2b9e00975 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1575,6 +1575,59 @@ def fake_main(argv): == "regional-audit.json" ) + def test__given_bundle_wrapper_empty_reason__then_certifier_refuses_it( + self, monkeypatch, tmp_path + ): + bundle_script = _load_bundle_script(monkeypatch) + spec = importlib.util.spec_from_file_location( + "certify_data_release", + Path(__file__).resolve().parents[1] / "scripts" / "certify_data_release.py", + ) + certifier = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(certifier) + monkeypatch.setitem(sys.modules, "certify_data_release", certifier) + bundle_path = _tmp_bundle(tmp_path) + original_bundle = bundle_path.read_bytes() + captured = {} + + def certify_temporary_bundle(**kwargs): + captured.update(kwargs) + kwargs["bundle_path"] = bundle_path + return certify_data_release(**kwargs) + + monkeypatch.setattr(certifier, "certify_data_release", certify_temporary_bundle) + with ( + patch( + "policyengine.provenance.certification.requests.get", + side_effect=[ + _response(_microcosm_uk_national_payload()), + _response(self._older_dense()), + ], + ), + pytest.raises(CertificationError, match="needs a reason"), + ): + bundle_script.main( + [ + "certify-data", + "--country", + "uk", + "--manifest-uri", + UK_NATIONAL_URI, + "--model-version", + UK_MODEL_VERSION, + "--regional-manifest-uri", + UK_DENSE_URI, + "--accept-regional-model-mismatch", + "", + "--no-generate", + "--no-changelog", + "--skip-artifact-check", + ] + ) + assert captured["regional_model_mismatch_reason"] == "" + assert bundle_path.read_bytes() == original_bundle + def test__given_pinned_core__then_the_regional_release_must_cover_it( self, tmp_path ): diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py index 5efdd99a0..71f08401f 100644 --- a/tests/test_regional_input_evidence.py +++ b/tests/test_regional_input_evidence.py @@ -92,6 +92,36 @@ def test_accepted_mismatch_without_audited_evidence_refuses(tmp_path): ) +@pytest.mark.parametrize("model_package", ["policyengine-us", "another-model"]) +@pytest.mark.parametrize("accept_mismatch", [False, True]) +def test_regional_release_built_with_another_model_package_refuses( + tmp_path, model_package, accept_mismatch +): + dense = older_dense() + dense["build"]["built_with_model_package"]["name"] = model_package + if not accept_mismatch: + dense["compatible_model_packages"] = [ + {"name": "policyengine-uk", "specifier": f"=={UK_MODEL_VERSION}"} + ] + bundle_path = _tmp_bundle(tmp_path) + before = bundle_path.read_bytes() + + with pytest.raises( + CertificationError, + match=f"built with {model_package}, not policyengine-uk", + ): + if accept_mismatch: + certify_with_evidence(bundle_path, input_evidence(dense), dense) + else: + _certify_uk( + bundle_path, + [_response(_microcosm_uk_national_payload()), _response(dense)], + regional_manifest_uri=UK_DENSE_URI, + ) + + assert bundle_path.read_bytes() == before + + @pytest.mark.parametrize( ("field", "wrong"), [ From c125eae17b5d861ee63c0e540385857e0e2e1d2d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 23 Oct 2026 13:19:54 -0400 Subject: [PATCH 12/21] Record owner-findings fixes and reservation-limited validation --- .job/HANDOFF.md | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/.job/HANDOFF.md b/.job/HANDOFF.md index e14bc0201..77b4a5c57 100644 --- a/.job/HANDOFF.md +++ b/.job/HANDOFF.md @@ -1,35 +1,23 @@ -# PR #699 Budget review fix +# PR #699 owner findings — Budget Monday re-cut -- Confirmed origin heads: #699 `afee349d2c14de1b121b0b7e4f03b695d03f418c`; #704 `da2ae8fb8e97145093e35366686df99c1e38449e`. -- Merged #704 into `accept-regional-model-mismatch` with merge commit `c1a65198`. No history rewritten. Release order remains main → #704 → #699 → certification. -- Implementation validated through `b90eada3`; the final report commit is recorded in `.job/HEAD`. +- Confirmed GitHub head before editing: `aa59b0d704852eab63087d8146f7da667d2178e0`, branch `accept-regional-model-mismatch`, canonical repository, draft PR. +- Implementation head SHA: `878392cf5a42d7b4916ae7b311c84aa30d383d38`. This handoff is a separate report-only commit; its final delivery SHA is saved in `.job/HEAD`. Changes: -- Require typed audited evidence bound to the exact regional cut URI, original manifest digest, every regional H5 identity/hash, and exact target model. Recompute the surface comparison and refuse every dropped column. Record every defaulted input by inspecting all stored periods and the country loader's aliases. -- Port Microcosm #1348's comparison, strengthening its older-model proxy to measure actual byte-verified H5 columns. New producer: `scripts/audit_regional_input_surface.py`. The CLI works before the current bundle certifies its target model. Microcosm handoff callers must generate and forward the new evidence JSON. -- Preserve the regional manifest's raw-byte digest. Keep limitations, including the pre-2.135.7 VAT fix/dashboard limit, through Pydantic validation and bundle hashing. -- Expose the full regional record in receipts and TRACE (`@json` preserves JSON-LD semantics). Area receipts stay uncertified and identify their dense build model. National certification never widens. Explicit reversal removes the record, digest, area routes and dense artifact; the national certificate stays identical. -- Regenerate the receipt schema. Dashboard follow-up: refresh vendored `tests/fixtures/w4/execution_receipt.v1.schema.json`. +1. Refuse any explicitly different regional `built_with_model_package.name` before checking publisher claims or accepting a version mismatch. Missing build metadata retains its previous behavior. Four regression cases cover `policyengine-us` and another package, with audited mismatch acceptance or a UK compatibility claim; each checks that refusal leaves the bundle unchanged. +2. Forward `--accept-regional-model-mismatch` whenever the value `is not None`. One regression follows the real wrapper and certifier CLI into certification, confirms the empty string is forwarded and refused, and checks that the bundle is unchanged. +3. Document silent unknown-column skipping in both the certification skill and input-audit module docstring. Inspected policyengine-uk 2.147.2, immutable commit `fe0b99a36bd309a722b19937345e34290cb6dd73`: [`build_from_dataset`, lines 1116–1117](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1113-L1127) and [`build_from_multi_year_dataset`, lines 1194–1195](https://github.com/PolicyEngine/policyengine-uk/blob/fe0b99a36bd309a722b19937345e34290cb6dd73/policyengine_uk/simulation.py#L1186-L1196) use `continue` without warning. The old core-warning claim was already absent from the approved head; the documentation now states the actual UK behavior explicitly. Validation: -- Tests first demonstrated missing-evidence acceptance, stale evidence/dropped-column failures, missing receipt/TRACE records, lost JSON-LD semantics, and premature audit CLI model initialization. -- **1,248 selected cases passed; two skipped**, across the UK run and successful reruns of three US metadata cases in the environment with the bundled US pin installed. The UK run initially reported 1,245 passes plus those three missing-package failures; all three reruns passed. One unrelated US offline-import integration case was explicitly deselected. -- Covered all changed provenance/certification modules, the existing roundtrip, required reversal cases, and #704's three local-manifest modules. Logs: `.job/logs/uk-final.txt`, `.job/logs/us-metadata-rerun.txt`. -- Expanded 53-module validation reported 3,008 passes and five skips with no failures before stalling for over nine minutes; it was interrupted and is not claimed complete. A subsequent wider pinned-environment attempt was also interrupted during slow package inspection. Initial setup problems (missing packages, inherited Git variables and an unpinned model) were diagnosed; final test subprocesses clear Git overrides. No interrupted test processes remain. -- `make format`, `make lint`, receipt schema generation/equality, `bundle.py check`, and diff checks pass. -- Mypy on all changed source modules: 24 diagnostics, identical to the merged baseline ignoring moved line numbers. Both new source modules have zero errors. Comparison: `.job/logs/mypy-comparison.txt`. -- Every pytest launch checked load1 ≤110 and free disk ≥55,000,000,000 bytes. At most two workers per invocation. Final launches: load1 75.71 / 97.69; available disk 103.7 GB decimal. - -Actual Budget pair: - -- National `microcosm-uk-2024-25-national-20261023T075039Z-a133f37a` (2.147.2), dense `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe` (2.132.6). -- The exact model-inventory comparison has no dropped names and 16 added pure inputs, including both `salary_sacrifice_returned_to_income` and `local_bus_uncapped_single_fare`; the complete list is in certification guidance. -- The private dense H5 was unavailable (no HF credential; HTTP 401). This inventory is not a byte audit, and this job does not certify the actual pair. The certification follow-up must run the new audit against the real H5 and record its complete default list. +- `python3 ~/reviews/fleet-ops/tools/reservation.py allowed uk-hub` refused: `held back by route-a-release-rerun-d1811-2026-10-23 (reservation.py status)`. +- **0 pytest cases executed; 0 passed, 0 failed. Five new regression cases await CI.** No collection, full suite, country-model run, or audit was launched. Rely on CI as requested. Focused follow-up, once permitted: `pytest tests/test_certify_data_release.py tests/test_regional_input_evidence.py -n 2`. +- Ruff lint and formatting checks passed for all five changed Python files: the bundle wrapper, certification, input audit, and both test modules. `git diff --check` passed. +- Independent source/diff review found no blockers. No data release was certified. Delivery: -- All writes stayed in the assigned workspace. The protected `.git` points into the caller's checkout; mutable Git metadata is isolated in `.job/repo.git`. Use `source .job/env` before Git commands here. The caller checkout was not written. -- Portable commits: `.job/budget-fix-699.bundle`; final head: `.job/HEAD`. The single push attempt's output is saved in `.job/PUSH_RESULT.txt`. No main/master push; leave #699 draft. -- GitHub blocked the PR body update because the connector requires approval and approval policy is `never`. The replacement body, including the invariants, is saved in `.job/PR_BODY.md`; no PR metadata change was made. +- All writes stay in this assigned workspace; mutable Git metadata is isolated in `.job/repo.git`. Run `source .job/env` before Git commands here. The protected `.git` still points to the caller's read-only metadata. +- Two added commits: the fixes above and this report. No history rewrite. The final SHA is in `.job/HEAD`; portable new commits are in `.job/budget-fix-699-r3.bundle`. +- One normal push to `origin HEAD:accept-regional-model-mismatch` is attempted after the report commit. Its outcome is saved in `.job/PUSH_RESULT.txt` and reported to the caller. No push to main/master. Leave #699 draft. From 89a8f760b58f98716f0c5bdfe7a56d2348d333bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:50:55 +0000 Subject: [PATCH 13/21] Expect bundle.py's usage error for an empty mismatch reason scripts/bundle.py reports a ValueError from a subcommand through parser.error, so the certifier's "needs a reason" refusal reaches the caller as SystemExit(2) with that refusal as its context. The test now expects that, and still checks that the empty reason was forwarded, nothing was printed to stdout and the bundle is unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/test_certify_data_release.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 2b9e00975..14cd116ef 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1576,7 +1576,7 @@ def fake_main(argv): ) def test__given_bundle_wrapper_empty_reason__then_certifier_refuses_it( - self, monkeypatch, tmp_path + self, monkeypatch, tmp_path, capsys ): bundle_script = _load_bundle_script(monkeypatch) spec = importlib.util.spec_from_file_location( @@ -1605,7 +1605,7 @@ def certify_temporary_bundle(**kwargs): _response(self._older_dense()), ], ), - pytest.raises(CertificationError, match="needs a reason"), + pytest.raises(SystemExit) as error, ): bundle_script.main( [ @@ -1625,6 +1625,11 @@ def certify_temporary_bundle(**kwargs): "--skip-artifact-check", ] ) + # bundle.py reports the certifier's refusal as a usage error. + assert error.value.code == 2 + assert isinstance(error.value.__context__, CertificationError) + assert "needs a reason" in str(error.value.__context__) + assert capsys.readouterr().out == "" assert captured["regional_model_mismatch_reason"] == "" assert bundle_path.read_bytes() == original_bundle From c1810b08779f575967e551005de3b5864772cbec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:09:14 +0000 Subject: [PATCH 14/21] Audit a regional cut relative to its build model The input-surface audit counted every stored column the target model does not define as dropped, so it refused every Microcosm dense cut on its own columns (area codes, build and source ids, clone indices, SPI donor fields), even against the model the cut was built with. It now compares the stored columns with the build model the regional manifest records as well as the target, as Microcosm #1348's comparison does: - dropped: stored columns the build model defines and the target does not. These still refuse, with no override. - never modelled: stored columns neither model defines, recorded only. - defaulted: target inputs the file does not store, after the target's aliases, that were not inputs of the build model. The build model's variables and inputs come from a throwaway `uv run --no-project --with policyengine-uk==V --with policyengine-core==C` environment, through an injectable runner, without the caller's uv project settings. The probe reports the versions it imported, and any other resolution refuses, as does a manifest that records no build model or core. The evidence moves to schema version 2, so version 1 files refuse. It adds the build model and core versions, the build model's full variable and input lists, and the never-modelled columns, so certification can recompute all three lists exactly. Certification binds both build versions to the regional manifest, recomputes the lists, refuses drops, records never_modelled_columns in regional_certification and names their count in its warning. It matches the .h5 suffix case-insensitively, as the audit does. The execution receipt schema is regenerated. Co-Authored-By: Claude Opus 5.5 --- scripts/audit_regional_input_surface.py | 17 +- scripts/certify_data_release.py | 8 +- .../schemas/execution_receipt.v1.schema.json | 52 ++- src/policyengine/provenance/certification.py | 58 +++- src/policyengine/provenance/manifest.py | 27 +- .../provenance/regional_input_surface.py | 224 ++++++++++++- tests/test_certify_data_release.py | 5 + tests/test_execution_receipt.py | 23 +- tests/test_measure_regional_inputs.py | 297 +++++++++++++++--- tests/test_regional_input_evidence.py | 152 ++++++++- 10 files changed, 748 insertions(+), 115 deletions(-) diff --git a/scripts/audit_regional_input_surface.py b/scripts/audit_regional_input_surface.py index cbe5cd1a2..1ce94de05 100644 --- a/scripts/audit_regional_input_surface.py +++ b/scripts/audit_regional_input_surface.py @@ -2,7 +2,13 @@ Run in an environment with the exact requested policyengine-uk version and PyTables installed. Supply the H5 files from the immutable regional manifest; -the audit verifies their hashes without installing models or downloading data. +the audit verifies their hashes and downloads no microdata. The baseline is the +build model and core the regional manifest records: the audit reads their +variables in a throwaway ``uv run --with`` environment, so it needs ``uv`` +and PyPI. Columns the build model defines and the target does not refuse; +columns neither model defines are recorded as never modelled; and target +inputs the files do not store that were not build inputs are recorded as +defaulted. """ from __future__ import annotations @@ -48,7 +54,14 @@ def main(argv: list[str] | None = None) -> int: token=os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN"), ) args.output.write_text(evidence.model_dump_json(indent=2) + "\n") - print(f"Saved {len(evidence.artifacts)} artifact input audits to {args.output}.") + print( + f"Saved {len(evidence.artifacts)} artifact input audits to {args.output}: " + f"policyengine-uk {evidence.model_version} against build model " + f"{evidence.build_model_version} (core {evidence.build_core_version}); " + f"{len(evidence.dropped_columns)} dropped, " + f"{len(evidence.never_modelled_columns)} never-modelled, " + f"{len(evidence.defaulted_inputs)} defaulted." + ) return 0 diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index 7cdc1a5a1..60b0c9b6c 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -123,14 +123,18 @@ def main(argv=None) -> int: "UK only: route regions to a regional release that does not cover " "the pinned model, recording this reviewed reason in the bundle " "instead of refusing. Requires --regional-input-surface-evidence; " - "dropped columns refuse. The core must still be covered." + "dropped columns (defined by the regional build model, not by the " + "pinned model) refuse. The core must still be covered." ), ) parser.add_argument("--no-generate", action="store_true") parser.add_argument( "--regional-input-surface-evidence", type=Path, - help="Audited JSON evidence bound to the regional bytes and exact target model.", + help=( + "Audit JSON from scripts/audit_regional_input_surface.py, bound to " + "the regional bytes, its build model and the exact target model." + ), ) parser.add_argument("--no-changelog", action="store_true") parser.add_argument( diff --git a/src/policyengine/data/schemas/execution_receipt.v1.schema.json b/src/policyengine/data/schemas/execution_receipt.v1.schema.json index f2e467cc3..4dfcb5b6f 100644 --- a/src/policyengine/data/schemas/execution_receipt.v1.schema.json +++ b/src/policyengine/data/schemas/execution_receipt.v1.schema.json @@ -515,6 +515,13 @@ "title": "Limitations", "type": "array" }, + "never_modelled_columns": { + "items": { + "type": "string" + }, + "title": "Never Modelled Columns", + "type": "array" + }, "regional_manifest_sha256": { "pattern": "^[0-9a-f]{64}$", "title": "Regional Manifest Sha256", @@ -533,6 +540,7 @@ "regional_manifest_sha256", "defaulted_inputs", "dropped_columns", + "never_modelled_columns", "limitations", "evidence" ], @@ -612,7 +620,7 @@ }, "RegionalInputSurfaceEvidence": { "additionalProperties": false, - "description": "An operator's audit of actual regional columns against an exact model.\n\nSurfaces are retained so certification recomputes the comparison rather\nthan trusting a declared zero-loss result. The artifact hashes identify\nthe bytes inspected by the auditor, not merely a model's possible inputs.", + "description": "An operator's audit of actual regional columns against two exact models.\n\nThe baseline is the regional release's build model and core, as its\nmanifest records them; the target is the model the bundle certifies. Both\nsurfaces are retained in full so certification recomputes the comparison\nrather than trusting a declared zero-loss result. The artifact hashes\nidentify the bytes inspected by the auditor, not merely a model's possible\ninputs. Schema version 2 made the comparison relative to the build model;\nversion 1 evidence, which counted columns no model defines as dropped, is\nrefused.", "properties": { "artifacts": { "additionalProperties": { @@ -628,6 +636,32 @@ "title": "Audited By", "type": "string" }, + "build_core_version": { + "minLength": 1, + "title": "Build Core Version", + "type": "string" + }, + "build_model_inputs": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Build Model Inputs", + "type": "array" + }, + "build_model_variables": { + "items": { + "type": "string" + }, + "minItems": 1, + "title": "Build Model Variables", + "type": "array" + }, + "build_model_version": { + "minLength": 1, + "title": "Build Model Version", + "type": "string" + }, "defaulted_inputs": { "items": { "type": "string" @@ -691,6 +725,13 @@ "title": "Model Version", "type": "string" }, + "never_modelled_columns": { + "items": { + "type": "string" + }, + "title": "Never Modelled Columns", + "type": "array" + }, "regional_manifest_sha256": { "pattern": "^[0-9a-f]{64}$", "title": "Regional Manifest Sha256", @@ -702,8 +743,8 @@ "type": "string" }, "schema_version": { - "const": 1, - "default": 1, + "const": 2, + "default": 2, "title": "Schema Version", "type": "integer" } @@ -713,10 +754,15 @@ "regional_manifest_sha256", "model_package", "model_version", + "build_model_version", + "build_core_version", "artifacts", "model_variables", "model_inputs", + "build_model_variables", + "build_model_inputs", "dropped_columns", + "never_modelled_columns", "defaulted_inputs", "audited_by", "limitations" diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 5ddec8e47..324016513 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -458,7 +458,14 @@ def accepted_regional_mismatch( covers the national dataset only, and the regional file was built and calibrated under another model version. The record names both versions and the reason so the bundle, its TRO and every consumer can say so. - Returns ``(record, warning)``. + + The evidence must bind the regional cut, its raw manifest bytes, its H5 + bytes, its recorded build model and core, and the exact target model. + Certification recomputes the comparison against the build model: stored + columns the build model defines and the target does not are dropped and + always refuse; stored columns neither model defines are recorded as never + modelled; and target inputs the file does not store that were not build + inputs are recorded as defaulted. Returns ``(record, warning)``. """ reason = reason.strip() @@ -488,10 +495,25 @@ def accepted_regional_mismatch( "Regional input-surface evidence must bind the exact regional cut, " "raw manifest bytes and target model package/version." ) + built_with_core, _ = _core_claims(regional_manifest) + if built_with is None or built_with_core is None: + raise CertificationError( + "The regional release records no build model and core versions, " + "so an accepted mismatch has no baseline to audit against." + ) + if (audit.build_model_version, audit.build_core_version) != ( + built_with, + built_with_core, + ): + raise CertificationError( + "Regional input-surface evidence must be measured against the " + f"regional release's build model, {model_package} {built_with} with " + f"policyengine-core {built_with_core}." + ) expected_artifacts = { name: artifact for name, artifact in regional_manifest.artifacts.items() - if artifact.kind == "microdata" and artifact.path.endswith(".h5") + if artifact.kind == "microdata" and Path(artifact.path).suffix.lower() == ".h5" } if not expected_artifacts or set(audit.artifacts) != set(expected_artifacts): raise CertificationError( @@ -543,6 +565,8 @@ def accepted_regional_mismatch( audit.model_variables, audit.model_inputs, audit.input_aliases, + build_model_variables=audit.build_model_variables, + build_model_inputs=audit.build_model_inputs, ) except ValueError as exc: raise CertificationError( @@ -550,6 +574,7 @@ def accepted_regional_mismatch( ) from exc if ( audit.dropped_columns != comparison["dropped_columns"] + or audit.never_modelled_columns != comparison["never_modelled_columns"] or audit.defaulted_inputs != comparison["defaulted_inputs"] ): raise CertificationError( @@ -557,14 +582,15 @@ def accepted_regional_mismatch( ) if comparison["dropped_columns"]: raise CertificationError( - "Dropped regional input columns are refused: " + f"Dropped regional input columns are refused ({model_package} " + f"{built_with} defines them and {model_version} does not): " + ", ".join(comparison["dropped_columns"]) ) limitations = list(audit.limitations) try: - predates_vat_fix = built_with is not None and Version(built_with) < Version( - "2.135.7" - ) <= Version(model_version) + predates_vat_fix = ( + Version(built_with) < Version("2.135.7") <= Version(model_version) + ) except InvalidVersion as exc: raise CertificationError( "Regional model versions must be valid versions." @@ -581,15 +607,17 @@ def accepted_regional_mismatch( "regional_manifest_sha256": regional_manifest_sha256, "defaulted_inputs": comparison["defaulted_inputs"], "dropped_columns": comparison["dropped_columns"], + "never_modelled_columns": comparison["never_modelled_columns"], "limitations": limitations, "evidence": audit.model_dump(mode="json", exclude_none=True), } warning = ( - f"regional release: built with {model_package} " - f"{built_with or 'an unrecorded version'} and routed under " - f"{model_version} on an accepted mismatch ({reason}); area runs are " - f"uncertified; audited dropped columns: none; defaulted inputs: " - f"{', '.join(comparison['defaulted_inputs']) or 'none'}. " + f"regional release: built with {model_package} {built_with} and routed " + f"under {model_version} on an accepted mismatch ({reason}); area runs " + "are uncertified; audited dropped columns: none; never-modelled " + "columns (stored, defined by neither model): " + f"{len(comparison['never_modelled_columns'])}; inputs defaulted since " + f"the build model: {', '.join(comparison['defaulted_inputs']) or 'none'}. " + " ".join(limitations) ) return record, warning @@ -1613,9 +1641,11 @@ def certify_data_release( ``regional_manifest_uri`` is refused unless ``drop_regional_release``. ``regional_model_mismatch_reason`` (UK only) requires - ``regional_input_surface_evidence`` audited against the exact target model - and regional manifest/artifact bytes. Dropped columns are always refused; - every defaulted input and limitation is recorded in + ``regional_input_surface_evidence`` (``scripts/audit_regional_input_surface.py``) + bound to the regional manifest and artifact bytes, the regional build + model and core, and the exact target model. Dropped columns, which the + build model defines and the target does not, are always refused. Every + never-modelled column, defaulted input and limitation is recorded in ``regional_certification`` alongside the reason and both model versions. The core must still be covered; area receipts remain uncertified. """ diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 1e00f5fce..2035a2043 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -226,26 +226,40 @@ class RegionalInputArtifactEvidence(BaseModel): class RegionalInputSurfaceEvidence(BaseModel): - """An operator's audit of actual regional columns against an exact model. - - Surfaces are retained so certification recomputes the comparison rather - than trusting a declared zero-loss result. The artifact hashes identify - the bytes inspected by the auditor, not merely a model's possible inputs. + """An operator's audit of actual regional columns against two exact models. + + The baseline is the regional release's build model and core, as its + manifest records them; the target is the model the bundle certifies. Both + surfaces are retained in full so certification recomputes the comparison + rather than trusting a declared zero-loss result. The artifact hashes + identify the bytes inspected by the auditor, not merely a model's possible + inputs. Schema version 2 made the comparison relative to the build model; + version 1 evidence, which counted columns no model defines as dropped, is + refused. """ model_config = {"extra": "forbid"} - schema_version: Literal[1] = 1 + schema_version: Literal[2] = 2 regional_manifest_uri: str = Field(min_length=1) regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") model_package: str = Field(min_length=1) model_version: str = Field(min_length=1) + build_model_version: str = Field(min_length=1) + build_core_version: str = Field(min_length=1) artifacts: dict[str, RegionalInputArtifactEvidence] = Field(min_length=1) model_variables: list[str] = Field(min_length=1) model_inputs: list[str] = Field(min_length=1) + build_model_variables: list[str] = Field(min_length=1) + build_model_inputs: list[str] = Field(min_length=1) input_aliases: Optional[dict[str, str]] = None dropped_columns: list[str] + """Stored columns the build model defines and the target model does not.""" + never_modelled_columns: list[str] + """Stored columns neither model defines; no model loads them.""" defaulted_inputs: list[str] + """Target inputs the file does not store, after the target's input + aliases, that were not inputs of the build model.""" audited_by: str = Field(min_length=1, pattern=r"\S") limitations: list[str] = Field(min_length=1) @@ -267,6 +281,7 @@ class RegionalCertification(BaseModel): regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") defaulted_inputs: list[str] dropped_columns: list[str] + never_modelled_columns: list[str] limitations: list[str] evidence: RegionalInputSurfaceEvidence diff --git a/src/policyengine/provenance/regional_input_surface.py b/src/policyengine/provenance/regional_input_surface.py index 460c53337..31714012e 100644 --- a/src/policyengine/provenance/regional_input_surface.py +++ b/src/policyengine/provenance/regional_input_surface.py @@ -1,10 +1,26 @@ -"""Audit regional H5 bytes against the exact installed UK country model. +"""Audit regional H5 bytes against their build model and the exact UK target. -The set comparison ports Microcosm PR #1348's handoff comparison, replacing -its older-model inventory proxy with the columns actually stored in every -artifact and period. Country-model input moves are read from that installed -model's constructor, so an alias cannot make a supplied input look defaulted. -No household values are included in the resulting audit. +A local-area release keeps serving area runs after the national bundle moves +to a newer policyengine-uk. The audit compares the columns actually stored in +every artifact and period with two models: the release's build model, which +its manifest records (``build.built_with_model_package`` and +``build.built_with_core_package``), and the installed target model. As in +Microcosm PR #1348's handoff comparison, the build model is the baseline: + +- **dropped**: stored columns the build model defines and the target model + does not. The target's loaders skip them, so they refuse acceptance. +- **never modelled**: stored columns neither model defines, such as + Microcosm's area codes, build and source ids, clone indices and SPI donor + fields. No model ever loaded them, so they are recorded as information. +- **defaulted**: target inputs a period does not store, after the target's + input aliases, that were not inputs of the build model. Area runs default + them where the build model did not. + +Columns neither model defines, and inputs absent under both, are not changes. +The build model is read in a throwaway ``uv run`` environment, so the audit +needs ``uv`` and PyPI. Input moves are read from the installed target model's +constructor, so an alias cannot make a supplied input look defaulted. No +household values are included in the resulting audit. policyengine-uk 2.147.2 silently skips unknown columns in both dataset loaders; the audit must detect that loss before certification. See the ``continue`` @@ -18,18 +34,51 @@ import ast import hashlib import inspect +import json +import os import re +import subprocess import textwrap +from collections.abc import Callable from importlib.metadata import PackageNotFoundError, version from pathlib import Path from typing import Any +from packaging.version import InvalidVersion, Version + from policyengine.provenance.manifest import ( + DataReleaseManifest, RegionalInputArtifactEvidence, RegionalInputSurfaceEvidence, ) UK_ENTITIES = ("person", "benunit", "household") +#: The interpreter of the throwaway environment that reads the build model. +BUILD_MODEL_PYTHON = "3.13" +#: The caller's uv project settings, which must not reach that environment. +BUILD_MODEL_STRIPPED_ENV = frozenset( + {"UV_FROZEN", "UV_PROJECT_ENVIRONMENT", "VIRTUAL_ENV"} +) +#: Runs in the build model's environment: the model API ``_model_surface`` +#: uses, and the package versions it actually imported. +BUILD_MODEL_PROBE = """ +import json +from importlib.metadata import version + +import policyengine_uk + +system = policyengine_uk.CountryTaxBenefitSystem() +print(json.dumps({ + "model_version": version("policyengine-uk"), + "core_version": version("policyengine-core"), + "variables": sorted(system.variables), + "inputs": sorted( + name + for name, variable in system.variables.items() + if variable.is_input_variable() + ), +})) +""" class RegionalInputSurfaceError(ValueError): @@ -53,18 +102,30 @@ def compare_input_surface( model_variables: list[str], model_inputs: list[str], input_aliases: dict[str, str] | None = None, + *, + build_model_variables: list[str], + build_model_inputs: list[str], ) -> dict[str, list[str]]: - """Recompute losses and every default, including defaults in just one year. + """Recompute dropped, never-modelled and defaulted names per period. - Inputs supplied by the model's unconditional ``move_values`` calls use the - destination name after the move. Raw columns still have to be variables - defined by the target model. Every artifact and period must pass: the - union of supplied columns cannot hide a missing input in another period. + ``dropped_columns`` are stored columns the build model defines and the + target model does not. ``never_modelled_columns`` are stored columns + neither model defines. ``defaulted_inputs`` are target inputs a period + does not supply that were not inputs of the build model. Inputs supplied + by the target's unconditional ``move_values`` calls use the destination + name after the move. Every artifact and period must pass: the union of + supplied columns cannot hide a missing input in another period. """ variables = _names(model_variables, "model_variables") inputs = _names(model_inputs, "model_inputs") if not inputs <= variables: raise RegionalInputSurfaceError("model_inputs must be model_variables.") + build_variables = _names(build_model_variables, "build_model_variables") + build_inputs = _names(build_model_inputs, "build_model_inputs") + if not build_inputs <= build_variables: + raise RegionalInputSurfaceError( + "build_model_inputs must be build_model_variables." + ) aliases = input_aliases or {} for donor, target in aliases.items(): if ( @@ -79,6 +140,7 @@ def compare_input_surface( if not artifacts: raise RegionalInputSurfaceError("No regional artifact columns measured.") dropped: set[str] = set() + never_modelled: set[str] = set() defaulted: set[str] = set() for name, artifact in artifacts.items(): columns = _names(artifact.columns, f"{name} columns") @@ -96,15 +158,17 @@ def compare_input_surface( f"{name} columns must equal the union of columns_by_period." ) for supplied in periods: - dropped.update(supplied - variables) + dropped.update((supplied & build_variables) - variables) + never_modelled.update(supplied - build_variables - variables) effective = set(supplied) for donor, target in aliases.items(): if donor in effective: effective.remove(donor) effective.add(target) - defaulted.update(inputs - effective) + defaulted.update((inputs - effective) - build_inputs) return { "dropped_columns": sorted(dropped), + "never_modelled_columns": sorted(never_modelled), "defaulted_inputs": sorted(defaulted), } @@ -186,6 +250,108 @@ def _model_surface(model_version: str) -> tuple[list[str], list[str], dict[str, return variables, inputs, _input_aliases(policyengine_uk.Simulation) +def regional_build_versions(manifest: DataReleaseManifest) -> tuple[str, str]: + """Return the regional release's build model and core: the audit baseline. + + A release that records neither, or records other packages, refuses: there + is no model to tell a dropped column from one no model ever defined. + """ + build = manifest.build + model = build.built_with_model_package if build is not None else None + core = build.built_with_core_package if build is not None else None + if model is None or core is None: + raise RegionalInputSurfaceError( + "The regional manifest records no build model and core versions " + "(build.built_with_model_package and build.built_with_core_package), " + "so the audit has no baseline." + ) + if model.name != "policyengine-uk" or core.name != "policyengine-core": + raise RegionalInputSurfaceError( + f"The regional release was built with {model.name} and {core.name}, " + "not policyengine-uk and policyengine-core." + ) + for package in (model, core): + try: + Version(package.version) + except InvalidVersion as exc: + raise RegionalInputSurfaceError( + f"The regional manifest's {package.name} build version " + f"{package.version!r} is not a version." + ) from exc + return model.version, core.version + + +def _build_model_surface( + model_version: str, + core_version: str, + run: Callable[..., Any] | None = None, +) -> tuple[list[str], list[str]]: + """Read the build model's variables and inputs in a throwaway environment. + + ``uv run --no-project`` installs exactly the two pins from PyPI, without + the caller's uv project settings. The probe reports the versions it + imported, and any other resolution refuses. + """ + run = run if run is not None else subprocess.run + label = f"policyengine-uk {model_version} with policyengine-core {core_version}" + command = [ + "uv", + "run", + "--no-project", + "--python", + BUILD_MODEL_PYTHON, + "--with", + f"policyengine-uk=={model_version}", + "--with", + f"policyengine-core=={core_version}", + "python", + "-I", + "-c", + BUILD_MODEL_PROBE, + ] + env = { + key: value + for key, value in os.environ.items() + if key not in BUILD_MODEL_STRIPPED_ENV + } + try: + result = run(command, capture_output=True, text=True, check=True, env=env) + except subprocess.CalledProcessError as exc: + detail = " | ".join((exc.stderr or "").strip().splitlines()[-5:]) + raise RegionalInputSurfaceError( + f"Cannot read the build model {label}: {detail or exc}" + ) from exc + except OSError as exc: + raise RegionalInputSurfaceError( + f"Cannot run uv to read the build model {label}." + ) from exc + try: + surface = json.loads((result.stdout or "").strip().splitlines()[-1]) + except (IndexError, ValueError) as exc: + raise RegionalInputSurfaceError( + f"The probe of the build model {label} printed no surface." + ) from exc + if not isinstance(surface, dict): + raise RegionalInputSurfaceError( + f"The probe of the build model {label} printed no surface." + ) + resolved = (surface.get("model_version"), surface.get("core_version")) + if resolved != (model_version, core_version): + raise RegionalInputSurfaceError( + f"The build-model probe resolved policyengine-uk {resolved[0]} with " + f"policyengine-core {resolved[1]}, not {label}." + ) + variables, inputs = surface.get("variables"), surface.get("inputs") + if not isinstance(variables, list) or not isinstance(inputs, list): + raise RegionalInputSurfaceError( + f"The probe of the build model {label} printed no variable lists." + ) + return ( + sorted(_names(variables, "build_model_variables")), + sorted(_names(inputs, "build_model_inputs")), + ) + + def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as file: @@ -269,12 +435,18 @@ def measure_regional_input_surface( audited_by: str, limitations: list[str], token: str | None = None, + *, + run: Callable[..., Any] | None = None, ) -> RegionalInputSurfaceEvidence: - """Create an audit from actual H5 columns and an exact installed UK model. + """Audit actual H5 columns against the build model and the exact target. The operator supplies local files named by the immutable regional release. Every microdata H5 must be measured and match its manifest's SHA-256 both - before and after metadata inspection. This reads no unit values for output. + before and after metadata inspection. The installed policyengine-uk must + be ``model_version``. The build model and core named by the manifest are + read in a throwaway ``uv run`` environment through ``run`` + (``subprocess.run`` by default). Dropped columns refuse. This reads no + unit values for output. """ # Imported lazily: the certification gate reuses compare_input_surface. from policyengine.provenance.certification import fetch_release_manifest @@ -290,6 +462,7 @@ def measure_regional_input_surface( manifest, manifest_sha256, parts = fetch_release_manifest( regional_manifest_uri, token=token ) + build_model_version, build_core_version = regional_build_versions(manifest) expected = { name: artifact for name, artifact in manifest.artifacts.items() @@ -301,6 +474,9 @@ def measure_regional_input_surface( f"{sorted(expected)}." ) variables, inputs, aliases = _model_surface(model_version) + build_variables, build_inputs = _build_model_surface( + build_model_version, build_core_version, run=run + ) artifacts = {} for name, artifact in sorted(expected.items()): path = Path(artifact_paths[name]) @@ -330,21 +506,35 @@ def measure_regional_input_surface( ), columns_by_period=periods, ) - comparison = compare_input_surface(artifacts, variables, inputs, aliases) + comparison = compare_input_surface( + artifacts, + variables, + inputs, + aliases, + build_model_variables=build_variables, + build_model_inputs=build_inputs, + ) if comparison["dropped_columns"]: raise RegionalInputSurfaceError( - f"Dropped regional input columns refuse acceptance: {comparison['dropped_columns']}." + f"Columns policyengine-uk {build_model_version} defines and " + f"{model_version} does not would be dropped, which refuses " + f"acceptance: {comparison['dropped_columns']}." ) return RegionalInputSurfaceEvidence( regional_manifest_uri=regional_manifest_uri, regional_manifest_sha256=manifest_sha256, model_package="policyengine-uk", model_version=model_version, + build_model_version=build_model_version, + build_core_version=build_core_version, artifacts=artifacts, model_variables=variables, model_inputs=inputs, + build_model_variables=build_variables, + build_model_inputs=build_inputs, input_aliases=aliases, dropped_columns=comparison["dropped_columns"], + never_modelled_columns=comparison["never_modelled_columns"], defaulted_inputs=comparison["defaulted_inputs"], audited_by=audited_by, limitations=limitations, diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 14cd116ef..855faaa12 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1415,6 +1415,10 @@ def test__given_regional_release_without_a_claim__then_refuses(self, tmp_path): def _older_dense(self) -> dict: dense = _microcosm_uk_dense_payload() dense["build"]["built_with_model_package"]["version"] = "2.120.0" + dense["build"]["built_with_core_package"] = { + "name": "policyengine-core", + "version": "3.33.3", + } dense["compatible_model_packages"] = [ {"name": "policyengine-uk", "specifier": "==2.120.0"} ] @@ -1463,6 +1467,7 @@ def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): "regional_manifest_sha256": evidence["regional_manifest_sha256"], "defaulted_inputs": evidence["defaulted_inputs"], "dropped_columns": [], + "never_modelled_columns": [], "limitations": evidence["limitations"], "evidence": evidence, } diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 6589bff47..311558fae 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -453,6 +453,8 @@ def _bundle_with_regional_acceptance(dataset_file): "local_bus_uncapped_single_fare", "salary_sacrifice_returned_to_income", ] + built = ["employment_income", "household_weight"] + stored = ["constituency_code_oa", *built] limitations = [ "The dense cut's VAT predates the policyengine-uk VAT factor fix " "(#2187, 2.135.7); the dashboard holds a VAT limit." @@ -467,28 +469,29 @@ def _bundle_with_regional_acceptance(dataset_file): "regional_manifest_sha256": "b" * 64, "defaulted_inputs": defaulted, "dropped_columns": [], + "never_modelled_columns": ["constituency_code_oa"], "limitations": limitations, "evidence": { + "schema_version": 2, "model_package": "policyengine-uk", "model_version": INSTALLED_UK, + "build_model_version": "2.132.6", + "build_core_version": "3.33.3", "regional_manifest_uri": manifest_uri, "regional_manifest_sha256": "b" * 64, "artifacts": { "area_file": { **artifact, - "columns": ["employment_income", "household_weight"], - "columns_by_period": { - "2025": ["employment_income", "household_weight"] - }, + "columns": stored, + "columns_by_period": {"2025": stored}, } }, - "model_variables": sorted( - ["employment_income", "household_weight", *defaulted] - ), - "model_inputs": sorted( - ["employment_income", "household_weight", *defaulted] - ), + "model_variables": sorted([*built, *defaulted]), + "model_inputs": sorted([*built, *defaulted]), + "build_model_variables": built, + "build_model_inputs": built, "dropped_columns": [], + "never_modelled_columns": ["constituency_code_oa"], "defaulted_inputs": defaulted, "audited_by": "UK hub review", "limitations": limitations, diff --git a/tests/test_measure_regional_inputs.py b/tests/test_measure_regional_inputs.py index 4bff297b6..094d4f126 100644 --- a/tests/test_measure_regional_inputs.py +++ b/tests/test_measure_regional_inputs.py @@ -1,11 +1,13 @@ -"""Measure the actual dense H5 surface, never an older model's inventory.""" +"""Measure the actual dense H5 surface against its build model and the target.""" from __future__ import annotations import hashlib import importlib +import importlib.metadata import importlib.util import json +import subprocess import sys from pathlib import Path from types import SimpleNamespace @@ -17,11 +19,28 @@ from policyengine.provenance.manifest import RegionalInputArtifactEvidence MODEL_VERSION = "2.147.2" +BUILD_VERSION = "2.132.6" +BUILD_CORE_VERSION = "3.33.3" URI = ( "hf://dataset/policyengine/regional@dense-cut/releases/dense/release_manifest.json" ) LIMITATION = "Reviewed dense VAT limitation; dashboard holds a VAT limit" DEFAULTS = ["local_bus_uncapped_single_fare", "salary_sacrifice_returned_to_income"] +#: An input of both models that the files never store: not a change. +ABSENT_UNDER_BOTH = "legacy_default_input" +#: A build-model variable that the target model no longer defines. +REMOVED = "removed_in_target" +#: A Microcosm column that neither model defines. +AREA_CODE = "constituency_code_oa" +BUILD_INPUTS = [ + "benunit_id", + "employment_income_before_lsr", + "household_weight", + ABSENT_UNDER_BOTH, + REMOVED, +] +BUILD_VARIABLES = sorted([*BUILD_INPUTS, "employment_income"]) +PERSON = ("employment_income",) class FakeSimulation: @@ -46,6 +65,7 @@ def __init__(self): "employment_income_before_lsr", "household_weight", "benunit_id", + ABSENT_UNDER_BOTH, *DEFAULTS, } self.variables = { @@ -54,37 +74,67 @@ def __init__(self): } +class FakeRunner: + """Stands in for ``uv run`` reading the build model, so nothing installs.""" + + def __init__(self, **surface): + self.surface = { + "model_version": BUILD_VERSION, + "core_version": BUILD_CORE_VERSION, + "variables": BUILD_VARIABLES, + "inputs": BUILD_INPUTS, + } | surface + self.calls = [] + + def __call__(self, command, **kwargs): + self.calls.append((command, kwargs)) + stdout = "Installed 40 packages\n" + json.dumps(self.surface) + "\n" + return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr="") + + def _module(): return importlib.import_module("policyengine.provenance.regional_input_surface") -def _h5(path, *, periods=None, fmt="fixed", missing=None): +def _h5(path, *, person=PERSON, periods=None, fmt="fixed", missing=None): + """Write UK entity tables: one year, or each period's person columns.""" with pd.HDFStore(path, mode="w") as store: if periods is None: store.put("time_period", pd.Series([2025])) - periods = {None: True} - for period, has_income in periods.items(): + periods = {None: person} + for period, person_columns in periods.items(): for entity, columns in { - "person": {"employment_income": [1.0]} if has_income else {}, - "benunit": {"benunit_id": [0]}, - "household": {"household_weight": [1.0]}, + "person": person_columns or ("benunit_id",), + "benunit": ("benunit_id",), + "household": ("household_weight",), }.items(): if (period, entity) == missing: continue key = entity if period is None else f"{entity}/{period}" - if not columns: - columns = {"benunit_id": [0]} - store.put(key, pd.DataFrame(columns), format=fmt) + frame = pd.DataFrame({column: [1.0] for column in columns}) + store.put(key, frame, format=fmt) return path -def _response(artifact_paths): +def _response(artifact_paths, build=None): payload = { "schema_version": 1, "data_package": {"name": "microcosm-data", "version": "1"}, "dataset_role": "non_default_local_area", "is_default": False, "default_datasets": {}, + "build": { + "built_with_model_package": { + "name": "policyengine-uk", + "version": BUILD_VERSION, + }, + "built_with_core_package": { + "name": "policyengine-core", + "version": BUILD_CORE_VERSION, + }, + } + if build is None + else build, "artifacts": { name: { "kind": "microdata", @@ -116,7 +166,7 @@ def model(monkeypatch): return module -def _measure(module, paths, response=None, **kwargs): +def _measure(module, paths, response=None, run=None, **kwargs): response = _response(paths) if response is None else response with patch( "policyengine.provenance.certification.requests.get", return_value=response @@ -127,17 +177,34 @@ def _measure(module, paths, response=None, **kwargs): MODEL_VERSION, "Budget operator", [LIMITATION], + run=FakeRunner() if run is None else run, **kwargs, ) +def _artifact(columns_by_period): + return RegionalInputArtifactEvidence( + repo_id="r", + path="dense.h5", + revision="cut", + sha256="a" * 64, + columns=sorted({column for c in columns_by_period.values() for column in c}), + columns_by_period=columns_by_period, + ) + + @pytest.mark.parametrize("fmt", ["fixed", "table"]) def test_actual_h5_columns_and_aliases_produce_bound_audit(tmp_path, model, fmt): paths = {"dense": _h5(tmp_path / "dense.h5", fmt=fmt)} response = _response(paths) result = _measure(model, paths, response) + assert result.schema_version == 2 assert result.model_version == MODEL_VERSION assert result.model_package == "policyengine-uk" + assert result.build_model_version == BUILD_VERSION + assert result.build_core_version == BUILD_CORE_VERSION + assert result.build_model_variables == BUILD_VARIABLES + assert result.build_model_inputs == BUILD_INPUTS assert result.regional_manifest_uri == URI assert ( result.regional_manifest_sha256 == hashlib.sha256(response.content).hexdigest() @@ -150,21 +217,26 @@ def test_actual_h5_columns_and_aliases_produce_bound_audit(tmp_path, model, fmt) assert result.input_aliases == {"employment_income": "employment_income_before_lsr"} assert result.defaulted_inputs == DEFAULTS assert result.dropped_columns == [] + assert result.never_modelled_columns == [] assert result.limitations == [LIMITATION] assert result.audited_by == "Budget operator" def test_a_default_in_one_period_is_not_hidden_by_another_period(tmp_path, model): - paths = {"dense": _h5(tmp_path / "dense.h5", periods={"2025": True, "2026": False})} + paths = { + "dense": _h5( + tmp_path / "dense.h5", + periods={ + "2025": (*PERSON, "salary_sacrifice_returned_to_income"), + "2026": PERSON, + }, + ) + } result = _measure(model, paths) - assert result.defaulted_inputs == [ - "employment_income_before_lsr", - *DEFAULTS, - ] - assert "employment_income" in result.artifacts["dense"].columns_by_period["2025"] - assert ( - "employment_income" not in result.artifacts["dense"].columns_by_period["2026"] - ) + assert result.defaulted_inputs == DEFAULTS + periods = result.artifacts["dense"].columns_by_period + assert "salary_sacrifice_returned_to_income" in periods["2025"] + assert "salary_sacrifice_returned_to_income" not in periods["2026"] @pytest.mark.parametrize("entity", ["person", "benunit", "household"]) @@ -172,7 +244,7 @@ def test_each_period_requires_every_entity(tmp_path, model, entity): paths = { "dense": _h5( tmp_path / "dense.h5", - periods={"2025": True, "2026": True}, + periods={"2025": PERSON, "2026": PERSON}, missing=("2026", entity), ) } @@ -192,17 +264,111 @@ def test_extra_metadata_is_not_mistaken_for_a_model_column(tmp_path, model): path = _h5(tmp_path / "dense.h5") with pd.HDFStore(path, mode="a") as store: store.put("metadata", pd.Series(["producer build notes"])) - assert _measure(model, {"dense": path}).dropped_columns == [] + result = _measure(model, {"dense": path}) + assert result.dropped_columns == [] + assert result.never_modelled_columns == [] -def test_an_actual_dropped_column_blocks_audit(tmp_path, model): - path = _h5(tmp_path / "dense.h5") - with pd.HDFStore(path, mode="a") as store: - store.put("person", pd.DataFrame({"removed_column": [1]})) - with pytest.raises(ValueError, match="[Dd]ropped|removed_column"): +def test_a_column_neither_model_defines_is_recorded_not_refused(tmp_path, model): + path = _h5(tmp_path / "dense.h5", person=(*PERSON, AREA_CODE)) + result = _measure(model, {"dense": path}) + assert result.never_modelled_columns == [AREA_CODE] + assert result.dropped_columns == [] + assert AREA_CODE in result.artifacts["dense"].columns + + +def test_a_column_the_build_model_defines_and_the_target_does_not_blocks_audit( + tmp_path, model +): + path = _h5(tmp_path / "dense.h5", person=(*PERSON, REMOVED)) + with pytest.raises(ValueError, match=f"dropped.*{REMOVED}"): _measure(model, {"dense": path}) +def test_an_input_absent_under_both_models_is_not_reported(tmp_path, model): + paths = {"dense": _h5(tmp_path / "dense.h5")} + assert ABSENT_UNDER_BOTH not in _measure(model, paths).defaulted_inputs + # Had the build model computed it, area runs would now default it. + computed = FakeRunner( + inputs=[name for name in BUILD_INPUTS if name != ABSENT_UNDER_BOTH] + ) + assert ABSENT_UNDER_BOTH in _measure(model, paths, run=computed).defaulted_inputs + + +def test_build_model_is_read_at_its_manifest_pins_without_the_callers_project( + tmp_path, model, monkeypatch +): + for name in ("UV_FROZEN", "UV_PROJECT_ENVIRONMENT", "VIRTUAL_ENV"): + monkeypatch.setenv(name, "from the caller's project") + monkeypatch.setenv("KEPT_SETTING", "1") + run = FakeRunner() + _measure(model, {"dense": _h5(tmp_path / "dense.h5")}, run=run) + ((command, kwargs),) = run.calls + assert command[:-1] == [ + "uv", + "run", + "--no-project", + "--python", + "3.13", + "--with", + f"policyengine-uk=={BUILD_VERSION}", + "--with", + f"policyengine-core=={BUILD_CORE_VERSION}", + "python", + "-I", + "-c", + ] + assert command[-1] == model.BUILD_MODEL_PROBE + assert kwargs["check"] is True + assert not {"UV_FROZEN", "UV_PROJECT_ENVIRONMENT", "VIRTUAL_ENV"} & set( + kwargs["env"] + ) + assert kwargs["env"]["KEPT_SETTING"] == "1" + + +@pytest.mark.parametrize( + "resolved", + [{"model_version": "2.132.7"}, {"core_version": "3.33.4"}], + ids=["model", "core"], +) +def test_a_build_model_resolved_at_other_versions_refuses(tmp_path, model, resolved): + paths = {"dense": _h5(tmp_path / "dense.h5")} + with pytest.raises(ValueError, match="resolved"): + _measure(model, paths, run=FakeRunner(**resolved)) + + +@pytest.mark.parametrize( + "unrecorded", ["built_with_model_package", "built_with_core_package", "build"] +) +def test_an_unrecorded_build_version_leaves_no_baseline_and_refuses( + tmp_path, model, unrecorded +): + paths = {"dense": _h5(tmp_path / "dense.h5")} + response = _response(paths) + payload = json.loads(response.content) + if unrecorded == "build": + del payload["build"] + else: + del payload["build"][unrecorded] + response.content = json.dumps(payload).encode() + run = FakeRunner() + with pytest.raises(ValueError, match="baseline"): + _measure(model, paths, response, run=run) + assert run.calls == [] + + +def test_a_failed_build_model_probe_refuses(tmp_path, model): + paths = {"dense": _h5(tmp_path / "dense.h5")} + + def unresolvable(command, **kwargs): + raise subprocess.CalledProcessError( + 1, command, stderr="No solution found when resolving --with dependencies" + ) + + with pytest.raises(ValueError, match="build model.*No solution"): + _measure(model, paths, run=unresolvable) + + def test_installed_model_must_be_the_requested_exact_version( tmp_path, model, monkeypatch ): @@ -250,30 +416,41 @@ def test_every_routed_h5_requires_exactly_one_supplied_path(tmp_path, model, nam _measure(model, {name: path for name in names}, response) +def test_comparison_is_relative_to_the_build_model(): + result = _module().compare_input_surface( + {"dense": _artifact({"2025": ["weight", "area_code", "removed"]})}, + ["weight", "new_input", "old_input", "was_formula"], + ["weight", "new_input", "old_input", "was_formula"], + build_model_variables=["weight", "old_input", "was_formula", "removed"], + build_model_inputs=["weight", "old_input", "removed"], + ) + assert result == { + "dropped_columns": ["removed"], + "never_modelled_columns": ["area_code"], + "defaulted_inputs": ["new_input", "was_formula"], + } + + def test_comparison_uses_each_artifact_and_every_period(): - module = _module() - artifact = RegionalInputArtifactEvidence( - repo_id="r", - path="dense.h5", - revision="cut", - sha256="a" * 64, - columns=["income", "weight"], - columns_by_period={"2025": ["income", "weight"], "2026": ["weight"]}, + artifact = _artifact( + {"2025": ["income", "weight", "new_input"], "2026": ["income", "weight"]} ) - result = module.compare_input_surface( + result = _module().compare_input_surface( {"dense": artifact}, ["income", "income_reported", "weight", "new_input"], ["income_reported", "weight", "new_input"], {"income": "income_reported"}, + build_model_variables=["income", "income_reported", "weight"], + build_model_inputs=["income_reported", "weight"], ) assert result == { "dropped_columns": [], - "defaulted_inputs": ["income_reported", "new_input"], + "never_modelled_columns": [], + "defaulted_inputs": ["new_input"], } def test_comparison_refuses_a_period_union_that_hides_columns(): - module = _module() artifact = RegionalInputArtifactEvidence( repo_id="r", path="dense.h5", @@ -283,7 +460,24 @@ def test_comparison_refuses_a_period_union_that_hides_columns(): columns_by_period={"2025": ["weight", "removed"]}, ) with pytest.raises(ValueError, match="columns|union|period"): - module.compare_input_surface({"dense": artifact}, ["weight"], ["weight"]) + _module().compare_input_surface( + {"dense": artifact}, + ["weight"], + ["weight"], + build_model_variables=["weight"], + build_model_inputs=["weight"], + ) + + +def test_comparison_refuses_build_inputs_outside_the_build_variables(): + with pytest.raises(ValueError, match="build_model_inputs"): + _module().compare_input_surface( + {"dense": _artifact({"2025": ["weight"]})}, + ["weight"], + ["weight"], + build_model_variables=["weight"], + build_model_inputs=["weight", "unknown"], + ) def test_conditional_alias_cannot_silently_claim_a_supplied_input( @@ -302,7 +496,23 @@ def __init__(self): _measure(model, paths) -def test_audit_cli_writes_pydantic_evidence_json(tmp_path, model, monkeypatch): +def test_the_build_model_probe_reads_the_model_api_the_audit_uses(): + """Run the probe with this interpreter, in place of a uv environment.""" + pytest.importorskip("policyengine_uk") + module = _module() + model_version = importlib.metadata.version("policyengine-uk") + core_version = importlib.metadata.version("policyengine-core") + + def run_here(command, **kwargs): + python = command.index("python") + return subprocess.run([sys.executable, *command[python + 1 :]], **kwargs) + + surface = module._build_model_surface(model_version, core_version, run=run_here) + variables, inputs, _ = module._model_surface(model_version) + assert surface == (variables, inputs) + + +def test_audit_cli_writes_pydantic_evidence_json(tmp_path, model, monkeypatch, capsys): script_path = ( Path(__file__).parents[1] / "scripts" / "audit_regional_input_surface.py" ) @@ -311,7 +521,7 @@ def test_audit_cli_writes_pydantic_evidence_json(tmp_path, model, monkeypatch): ) script = importlib.util.module_from_spec(spec) spec.loader.exec_module(script) - path = _h5(tmp_path / "dense.h5") + path = _h5(tmp_path / "dense.h5", person=(*PERSON, AREA_CODE)) result = _measure(model, {"dense": path}) captured = {} @@ -343,12 +553,11 @@ def measure(**kwargs): assert captured["artifact_paths"] == {"dense": path} assert captured["limitations"] == [LIMITATION] assert json.loads(output.read_text()) == result.model_dump(mode="json") + assert "0 dropped, 1 never-modelled, 2 defaulted" in capsys.readouterr().out def test_audit_cli_bootstraps_before_target_model_is_certified(tmp_path): """The target must be auditable before it matches the current bundle.""" - import subprocess - script = Path(__file__).parents[1] / "scripts" / "audit_regional_input_surface.py" code = """ import importlib.util diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py index 71f08401f..63d7190d2 100644 --- a/tests/test_regional_input_evidence.py +++ b/tests/test_regional_input_evidence.py @@ -23,6 +23,8 @@ "the dense cut's VAT predates the policyengine-uk VAT factor fix " "(#2187, 2.135.7); the dashboard holds a VAT limit" ) +DENSE = "microcosm_uk_2024_25_dense" +BUILD_CORE_VERSION = "3.33.3" DEFAULTED_INPUTS = [ "local_bus_uncapped_single_fare", "salary_sacrifice_returned_to_income", @@ -32,26 +34,36 @@ def older_dense() -> dict: dense = _microcosm_uk_dense_payload() dense["build"]["built_with_model_package"]["version"] = "2.120.0" + dense["build"]["built_with_core_package"] = { + "name": "policyengine-core", + "version": BUILD_CORE_VERSION, + } dense["compatible_model_packages"] = [ {"name": "policyengine-uk", "specifier": "==2.120.0"} ] return dense +def manifest_sha256(dense: dict) -> str: + """The digest of the bytes ``_response`` serves for ``dense``.""" + return hashlib.sha256(json.dumps(dense).encode()).hexdigest() + + def input_evidence(dense: dict | None = None) -> dict: + """An audit whose build model lacks the two inputs the target adds.""" dense = older_dense() if dense is None else dense - name = "microcosm_uk_2024_25_dense" - artifact = dense["artifacts"][name] + artifact = dense["artifacts"][DENSE] + build = dense["build"] return { - "schema_version": 1, + "schema_version": 2, "regional_manifest_uri": UK_DENSE_URI, - "regional_manifest_sha256": hashlib.sha256( - json.dumps(dense).encode() - ).hexdigest(), + "regional_manifest_sha256": manifest_sha256(dense), "model_package": "policyengine-uk", "model_version": UK_MODEL_VERSION, + "build_model_version": build["built_with_model_package"]["version"], + "build_core_version": build["built_with_core_package"]["version"], "artifacts": { - name: { + DENSE: { key: artifact[key] for key in ("repo_id", "path", "revision", "sha256") } | { @@ -61,13 +73,22 @@ def input_evidence(dense: dict | None = None) -> dict: }, "model_variables": ["employment_income", *DEFAULTED_INPUTS], "model_inputs": ["employment_income", *DEFAULTED_INPUTS], + "build_model_variables": ["employment_income"], + "build_model_inputs": ["employment_income"], "dropped_columns": [], + "never_modelled_columns": [], "defaulted_inputs": DEFAULTED_INPUTS.copy(), "audited_by": "Budget regional input-surface review", "limitations": [VAT_LIMIT], } +def store_column(evidence: dict, column: str) -> None: + measured = evidence["artifacts"][DENSE] + measured["columns"] = sorted([*measured["columns"], column]) + measured["columns_by_period"]["2025"] = measured["columns"] + + def certify_with_evidence(bundle_path, evidence: dict, dense: dict | None = None): dense = older_dense() if dense is None else dense return _certify_uk( @@ -149,25 +170,122 @@ def test_evidence_for_another_cut_bytes_or_model_refuses(tmp_path, field, wrong) ) def test_evidence_for_other_artifact_bytes_or_identity_refuses(tmp_path, field, wrong): evidence = input_evidence() - evidence["artifacts"]["microcosm_uk_2024_25_dense"][field] = wrong + evidence["artifacts"][DENSE][field] = wrong with pytest.raises(CertificationError): certify_with_evidence(_tmp_bundle(tmp_path), evidence) def test_a_dropped_column_refuses_even_with_an_acceptance_reason(tmp_path): + """The build model defines the stored column and the target does not.""" evidence = input_evidence() - evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns"].append( - "removed_model_variable" - ) - evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns_by_period"][ - "2025" - ].append("removed_model_variable") + store_column(evidence, "removed_model_variable") + evidence["build_model_variables"].append("removed_model_variable") + evidence["build_model_inputs"].append("removed_model_variable") evidence["dropped_columns"] = ["removed_model_variable"] - with pytest.raises(CertificationError, match="[Dd]ropped|removed_model_variable"): + with pytest.raises(CertificationError, match="[Dd]ropped.*removed_model_variable"): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_a_column_neither_model_defines_is_recorded_not_refused(tmp_path): + evidence = input_evidence() + store_column(evidence, "constituency_code_oa") + evidence["never_modelled_columns"] = ["constituency_code_oa"] + bundle_path = _tmp_bundle(tmp_path) + + result = certify_with_evidence(bundle_path, evidence) + + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + record = release["regional_certification"] + assert record["never_modelled_columns"] == ["constituency_code_oa"] + assert record["dropped_columns"] == [] + assert record["evidence"] == evidence + (warning,) = [w for w in result.warnings if "accepted mismatch" in w] + assert "never-modelled columns (stored, defined by neither model): 1" in warning + + +def test_an_input_absent_under_both_models_is_not_reported(tmp_path): + evidence = input_evidence() + for field in ( + "model_variables", + "model_inputs", + "build_model_variables", + "build_model_inputs", + ): + evidence[field].append("legacy_input") + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, evidence) + record = json.loads(bundle_path.read_text())["data_releases"]["uk"][ + "regional_certification" + ] + assert record["defaulted_inputs"] == DEFAULTED_INPUTS + + evidence["defaulted_inputs"] = sorted([*DEFAULTED_INPUTS, "legacy_input"]) + with pytest.raises(CertificationError, match="disagrees"): certify_with_evidence(_tmp_bundle(tmp_path), evidence) -@pytest.mark.parametrize("field", ["dropped_columns", "defaulted_inputs"]) +@pytest.mark.parametrize( + ("field", "wrong"), + [("build_model_version", "2.120.1"), ("build_core_version", "3.33.4")], +) +def test_evidence_against_another_build_model_refuses(tmp_path, field, wrong): + evidence = input_evidence() + evidence[field] = wrong + with pytest.raises(CertificationError, match="build model"): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +@pytest.mark.parametrize( + "unrecorded", ["built_with_model_package", "built_with_core_package"] +) +def test_a_regional_release_without_build_versions_has_no_baseline( + tmp_path, unrecorded +): + dense = older_dense() + evidence = input_evidence(dense) + del dense["build"][unrecorded] + evidence["regional_manifest_sha256"] = manifest_sha256(dense) + with pytest.raises(CertificationError, match="baseline"): + certify_with_evidence(_tmp_bundle(tmp_path), evidence, dense) + + +def test_version_1_evidence_refuses(tmp_path): + evidence = input_evidence() + evidence["schema_version"] = 1 + with pytest.raises(CertificationError, match="schema_version"): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_an_h5_suffix_in_any_case_must_be_measured_as_the_audit_does(): + from policyengine.provenance.certification import accepted_regional_mismatch + from policyengine.provenance.manifest import DataReleaseManifest + + dense = older_dense() + dense["artifacts"][DENSE]["path"] = "microcosm_uk_2024_25_dense.H5" + evidence = input_evidence(dense) + manifest = DataReleaseManifest.model_validate(dense) + + def accept(evidence): + return accepted_regional_mismatch( + manifest, + "policyengine-uk", + UK_MODEL_VERSION, + REASON, + regional_manifest_uri=UK_DENSE_URI, + regional_manifest_sha256=manifest_sha256(dense), + evidence=evidence, + ) + + record, _ = accept(evidence) + assert set(record["evidence"]["artifacts"]) == {DENSE} + evidence["artifacts"] = {"another_file": evidence["artifacts"][DENSE]} + with pytest.raises(CertificationError, match="every regional microdata"): + accept(evidence) + + +@pytest.mark.parametrize( + "field", ["dropped_columns", "never_modelled_columns", "defaulted_inputs"] +) def test_the_declared_comparison_must_match_the_measured_surfaces(tmp_path, field): evidence = input_evidence() evidence[field] = ["invented_variable"] @@ -275,7 +393,7 @@ def digest(bundle): def test_missing_period_measurements_refuse(tmp_path): evidence = input_evidence() - del evidence["artifacts"]["microcosm_uk_2024_25_dense"]["columns_by_period"] + del evidence["artifacts"][DENSE]["columns_by_period"] with pytest.raises(CertificationError): certify_with_evidence(_tmp_bundle(tmp_path), evidence) From f14690ee149be898ee2847acffd8992f5114db59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:11:05 +0000 Subject: [PATCH 15/21] Document the build-relative regional audit and the dense cut's results The data-certification skill now defines dropped, never-modelled and defaulted names relative to the regional build model, notes that the audit needs uv and PyPI to read that model, and replaces the note that the dense cut was never audited with the audit's results for microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe (built with policyengine-uk 2.132.6): 83 never-modelled columns, no dropped column under 2.147.2 or 2.151.0, 16 new defaults under 2.147.2 and 37 under 2.151.0, with 123 inputs already defaulted under 2.132.6. It says which 2.151.0 defaults belong in the limitations. The changelog fragments describe the build-relative comparison. Co-Authored-By: Claude Opus 5.5 --- changelog.d/699.fixed.md | 2 +- .../accept-regional-model-mismatch.added.md | 2 +- docs/engineering/skills/data-certification.md | 88 +++++++++++++------ 3 files changed, 63 insertions(+), 29 deletions(-) diff --git a/changelog.d/699.fixed.md b/changelog.d/699.fixed.md index 55fc07874..5d9d08cf4 100644 --- a/changelog.d/699.fixed.md +++ b/changelog.d/699.fixed.md @@ -1,3 +1,3 @@ -Require a byte-bound input-surface audit when accepting a UK regional model mismatch, refuse dropped columns, and record defaulted inputs and limitations in the bundle, TRACE and execution receipts. Preserve the regional manifest digest and area build model while keeping area receipts uncertified; dropping the regional release removes its exception record. +Require a byte-bound input-surface audit when accepting a UK regional model mismatch. The audit compares the regional H5's stored columns with the regional release's build model as well as the target model: it refuses columns the build model defines and the target does not, records columns neither model defines as never modelled, and reports as defaulted only target inputs the file does not store that were not build-model inputs. The bundle, TRACE and execution receipts record the result and limitations, preserve the regional manifest digest and area build model, and keep area receipts uncertified; dropping the regional release removes its exception record. Refuse regional releases that explicitly name a different model package, and forward empty CLI reasons so certification rejects them. Document that the UK loader silently skips unknown columns. diff --git a/changelog.d/accept-regional-model-mismatch.added.md b/changelog.d/accept-regional-model-mismatch.added.md index a60f85002..0223b8a78 100644 --- a/changelog.d/accept-regional-model-mismatch.added.md +++ b/changelog.d/accept-regional-model-mismatch.added.md @@ -1 +1 @@ -`certify-data --accept-regional-model-mismatch REASON` (UK only) keeps a local-area release routed when it was built on another policyengine-uk version, and records the reviewed mismatch as `regional_certification` in the bundle instead of refusing; area receipts are uncertified. +`certify-data --accept-regional-model-mismatch REASON --regional-input-surface-evidence AUDIT.json` (UK only) keeps a local-area release routed when it was built on another policyengine-uk version, and records the reviewed mismatch as `regional_certification` in the bundle instead of refusing; area receipts are uncertified. `scripts/audit_regional_input_surface.py` writes the audit, reading the release's build model in a throwaway `uv run` environment. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index e4805591e..669dd0e2d 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -83,7 +83,8 @@ Microcosm national release with one of: cut moves to a new policyengine-uk before a local-area cut on that version exists, a reviewer can accept retaining the older regional route. A reason and a byte-bound input-surface audit are both required. First audit the - actual local H5 with the exact target model installed: + actual local H5 with the exact target model installed. The audit also needs + `uv` and access to PyPI (see below): ```bash .venv/bin/python scripts/audit_regional_input_surface.py \ @@ -102,15 +103,38 @@ Microcosm national release with one of: --model-version 2.147.2 ``` - The comparison is ported from Microcosm #1348's - `tools/certify_uk_cut_in_policyengine_py.py`, and strengthened to inspect the - persisted entity-table columns, including each period and country-loader - aliases. Comparing the old model's input inventory alone cannot prove which - columns the file carries. The audit binds the regional release identity, - original manifest bytes, actual H5 bytes and exact target model version. - Certification refuses missing or mismatched evidence and **any dropped - column**. It records every defaulted input and the free-text limitations in - the validated `regional_certification`; the bundle hash binds that record. + The audit compares the entity-table columns the H5 actually stores, in each + period, with two models: the build model and core the regional manifest + records (`build.built_with_model_package` and + `build.built_with_core_package`), and the target. As in Microcosm #1348's + `tools/certify_uk_cut_in_policyengine_py.py`, the build model is the + baseline: + + - **Dropped columns** are stored columns the build model defines and the + target does not. The target's loaders skip them. Certification refuses + **any dropped column**; there is no override. + - **Never-modelled columns** are stored columns neither model defines: + Microcosm's own area codes (among them `constituency_code_oa` and + `la_code_oa`), build and source ids, clone indices and SPI donor fields. + No model ever loaded them, so they are recorded, not refused. + - **Defaulted inputs** are target inputs the file does not store, after the + target's input aliases, that were not inputs of the build model. Area runs + default them where the build model did not. Inputs absent under both + models are not changes. + + The audit reads the build model's variables and inputs in a throwaway + `uv run --no-project --with policyengine-uk== --with + policyengine-core==` environment, so it needs `uv` and access to + PyPI. A regional manifest that does not record both build versions has no + baseline, and the audit refuses it. The evidence binds the regional release + identity, original manifest bytes, actual H5 bytes, both build versions and + the exact target model version, and keeps both models' full variable and + input lists. Certification recomputes all three lists from them and refuses + missing or mismatched evidence, including schema-version-1 evidence from + before the comparison was relative to the build model. It records the + never-modelled columns, every defaulted input and the free-text limitations + in the validated `regional_certification`; the bundle hash binds that + record. The policyengine-uk 2.147.2 dataset loaders silently skip columns absent from the model's variables: both the @@ -130,23 +154,33 @@ Microcosm national release with one of: The regional core checks still apply. Re-certifying with `--drop-regional-release` removes the regional record, routes and artifact. - For the Budget route, the measured **model-inventory comparison** from - 2.132.6 to 2.147.2 has no dropped names and these 16 new pure inputs: - `additional_residential_property_purchased`, `defined_benefit_pension_share`, - `directly_held_shares`, `early_years_school_start_deferred`, - `is_full_year_category_a_director`, `local_bus_uncapped_single_fare`, - `non_residential_property_purchased`, `non_residential_property_secured_debt`, - `other_residential_property_secured_debt`, - `pension_annual_allowance_taxable_lump_sum_death_benefits`, - `pension_contributions_via_salary_sacrifice_pre_2015`, `pip_dl_points`, - `pip_m_points`, `salary_sacrifice_returned_to_income`, - `unit_and_investment_trusts`, and `war_widows_pension_reported`. - Both `salary_sacrifice_returned_to_income` and - `local_bus_uncapped_single_fare` default in that comparison. This is not an - audit of cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`: - its private H5 was unavailable to this implementation job without - authentication. Certification must use the complete list measured from the - actual cut by the audit above, including any further missing stored inputs. + For the Budget route, the audit of dense cut + `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`, built with + policyengine-uk 2.132.6 and policyengine-core 3.33.3, measured in #699: + + - **83 never-modelled columns**, and **no dropped column** under either + 2.147.2 or 2.151.0. The first version of this audit counted those 83 as + dropped and refused the cut, even against its own build model. + - **16 new defaults under 2.147.2:** + `additional_residential_property_purchased`, + `defined_benefit_pension_share`, `directly_held_shares`, + `early_years_school_start_deferred`, `is_full_year_category_a_director`, + `local_bus_uncapped_single_fare`, `non_residential_property_purchased`, + `non_residential_property_secured_debt`, + `other_residential_property_secured_debt`, + `pension_annual_allowance_taxable_lump_sum_death_benefits`, + `pension_contributions_via_salary_sacrifice_pre_2015`, `pip_dl_points`, + `pip_m_points`, `salary_sacrifice_returned_to_income`, + `unit_and_investment_trusts`, and `war_widows_pension_reported`. + - **37 new defaults under 2.151.0:** those 16 and 21 more. The 21 include + the gambling duty inputs, liquid and solid fuel consumption, and bonus + sacrifice (`pension_contributions_via_bonus_sacrifice`), so area + breakdowns of those measures use defaults on 2.151.0. Record that as a + `--limitation` if the route is used on 2.151.0. + - Another **123 inputs already defaulted under 2.132.6**. They are absent + under both models, so they are not changes; the first version listed + them too, giving 139 defaults under 2.147.2 and 160 under 2.151.0. + Record the pre-fix VAT limitation shown above for this dense cut. After the receipt schema changes, refresh the Budget dashboard's vendored From 1a93f7e70c1c6205f200190e16d68c1a533ca665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:56:28 +0000 Subject: [PATCH 16/21] Re-read both models' surfaces at certification, and pin the probe Review of #699 (Vahid Ahmadi, findings 1 and 4). Certification recomputed the dropped, never-modelled and defaulted lists from the surfaces declared in the evidence, so editing the file could hide a drop: leaving a stored column out of build_model_variables, adding it to model_variables, or adding the new defaults to build_model_inputs were all accepted. verify_model_surfaces now reads the target model in process, which certifying an accepted mismatch therefore needs installed at the evidence's model_version, and the build model again through the uv probe, and refuses any list or input alias that differs. Only the stored columns rest on the auditor's word, bound to each H5's SHA-256. The probe resolves reproducibly: uv run --no-config --default-index https://pypi.org/simple, with --exclude-newer set to the regional manifest's build.built_at in UTC, rounded up to the second, and without any UV_* setting except the cache, Python-install and connection ones. Co-Authored-By: Claude Opus 5.5 --- src/policyengine/provenance/certification.py | 28 +++- .../provenance/regional_input_surface.py | 140 ++++++++++++++++-- tests/test_certify_data_release.py | 18 ++- tests/test_measure_regional_inputs.py | 132 ++++++++++++++++- tests/test_regional_input_evidence.py | 119 ++++++++++++++- 5 files changed, 402 insertions(+), 35 deletions(-) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 324016513..f8ab47872 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -461,11 +461,14 @@ def accepted_regional_mismatch( The evidence must bind the regional cut, its raw manifest bytes, its H5 bytes, its recorded build model and core, and the exact target model. - Certification recomputes the comparison against the build model: stored - columns the build model defines and the target does not are dropped and - always refuse; stored columns neither model defines are recorded as never - modelled; and target inputs the file does not store that were not build - inputs are recorded as defaulted. Returns ``(record, warning)``. + Both models' surfaces in it are re-read from the models (the target must + be installed); the stored columns are the auditor's, bound to the H5 + bytes. Certification recomputes the comparison against the build model: + stored columns the build model defines and the target does not are + dropped and always refuse; stored columns neither model defines are + recorded as never modelled; and target inputs the file does not store + that were not build inputs are recorded as defaulted. Returns + ``(record, warning)``. """ reason = reason.strip() @@ -557,8 +560,21 @@ def accepted_regional_mismatch( ) if not all(item.strip() for item in audit.limitations): raise CertificationError("Regional evidence limitations must not be blank.") - from policyengine.provenance.regional_input_surface import compare_input_surface + from policyengine.provenance.regional_input_surface import ( + compare_input_surface, + verify_model_surfaces, + ) + try: + # The lists are re-read from the models themselves, so an edited + # evidence file cannot hide a drop; only the stored columns rest on + # the auditor's word, bound to each H5's SHA-256. + verify_model_surfaces(audit, regional_manifest) + except ValueError as exc: + raise CertificationError( + "Certifying an accepted regional mismatch re-reads both models' " + f"surfaces: {exc}" + ) from exc try: comparison = compare_input_surface( audit.artifacts, diff --git a/src/policyengine/provenance/regional_input_surface.py b/src/policyengine/provenance/regional_input_surface.py index 31714012e..fe94bac22 100644 --- a/src/policyengine/provenance/regional_input_surface.py +++ b/src/policyengine/provenance/regional_input_surface.py @@ -17,10 +17,13 @@ them where the build model did not. Columns neither model defines, and inputs absent under both, are not changes. -The build model is read in a throwaway ``uv run`` environment, so the audit -needs ``uv`` and PyPI. Input moves are read from the installed target model's -constructor, so an alias cannot make a supplied input look defaulted. No -household values are included in the resulting audit. +The build model is read in a throwaway ``uv run`` environment, from PyPI as it +was when the release was built, so the audit needs ``uv`` and PyPI. Input +moves are read from the installed target model's constructor, so an alias +cannot make a supplied input look defaulted. Certification reads both models +again (``verify_model_surfaces``); only the stored columns rest on the +auditor's word, bound to each H5's SHA-256. No household values are included +in the resulting audit. policyengine-uk 2.147.2 silently skips unknown columns in both dataset loaders; the audit must detect that loss before certification. See the ``continue`` @@ -40,6 +43,7 @@ import subprocess import textwrap from collections.abc import Callable +from datetime import UTC, datetime, timedelta from importlib.metadata import PackageNotFoundError, version from pathlib import Path from typing import Any @@ -55,9 +59,24 @@ UK_ENTITIES = ("person", "benunit", "household") #: The interpreter of the throwaway environment that reads the build model. BUILD_MODEL_PYTHON = "3.13" -#: The caller's uv project settings, which must not reach that environment. -BUILD_MODEL_STRIPPED_ENV = frozenset( - {"UV_FROZEN", "UV_PROJECT_ENVIRONMENT", "VIRTUAL_ENV"} +#: The only index that environment resolves from. +BUILD_MODEL_INDEX = "https://pypi.org/simple" +#: uv settings that environment keeps: where uv caches and how it connects, +#: none of which changes what resolves. Every other ``UV_*`` setting (indexes, +#: constraints, overrides, the caller's project) and ``VIRTUAL_ENV`` are +#: removed. +BUILD_MODEL_KEPT_UV_SETTINGS = frozenset( + { + "UV_CACHE_DIR", + "UV_CONCURRENT_DOWNLOADS", + "UV_CONCURRENT_INSTALLS", + "UV_HTTP_TIMEOUT", + "UV_LINK_MODE", + "UV_NATIVE_TLS", + "UV_NO_PROGRESS", + "UV_PYTHON_CACHE_DIR", + "UV_PYTHON_INSTALL_DIR", + } ) #: Runs in the build model's environment: the model API ``_model_surface`` #: uses, and the package versions it actually imported. @@ -281,16 +300,43 @@ def regional_build_versions(manifest: DataReleaseManifest) -> tuple[str, str]: return model.version, core.version +def regional_build_cutoff(manifest: DataReleaseManifest) -> str | None: + """Return when the regional release was built, as a UTC ``--exclude-newer``. + + The build model is read with the packages PyPI had then, so its other + dependencies resolve as they did at the build however they move on. + Rounded up to the second; ``None`` when the manifest records no + ``build.built_at``. + """ + built_at = manifest.build.built_at if manifest.build is not None else None + if built_at is None: + return None + try: + moment = datetime.fromisoformat(built_at) + except ValueError as exc: + raise RegionalInputSurfaceError( + f"The regional manifest's build.built_at {built_at!r} is not a timestamp." + ) from exc + if moment.tzinfo is None: + moment = moment.replace(tzinfo=UTC) + moment = moment.astimezone(UTC) + if moment.microsecond: + moment = moment.replace(microsecond=0) + timedelta(seconds=1) + return moment.strftime("%Y-%m-%dT%H:%M:%SZ") + + def _build_model_surface( model_version: str, core_version: str, run: Callable[..., Any] | None = None, + exclude_newer: str | None = None, ) -> tuple[list[str], list[str]]: """Read the build model's variables and inputs in a throwaway environment. - ``uv run --no-project`` installs exactly the two pins from PyPI, without - the caller's uv project settings. The probe reports the versions it - imported, and any other resolution refuses. + ``uv run --no-project --no-config`` installs exactly the two pins from + PyPI, with packages uploaded before ``exclude_newer`` when it is given, + and without the caller's uv settings or project. The probe reports the + versions it imported, and any other resolution refuses. """ run = run if run is not None else subprocess.run label = f"policyengine-uk {model_version} with policyengine-core {core_version}" @@ -298,6 +344,10 @@ def _build_model_surface( "uv", "run", "--no-project", + "--no-config", + "--default-index", + BUILD_MODEL_INDEX, + *(["--exclude-newer", exclude_newer] if exclude_newer else []), "--python", BUILD_MODEL_PYTHON, "--with", @@ -312,7 +362,8 @@ def _build_model_surface( env = { key: value for key, value in os.environ.items() - if key not in BUILD_MODEL_STRIPPED_ENV + if key != "VIRTUAL_ENV" + and (not key.startswith("UV_") or key in BUILD_MODEL_KEPT_UV_SETTINGS) } try: result = run(command, capture_output=True, text=True, check=True, env=env) @@ -352,6 +403,65 @@ def _build_model_surface( ) +def _surface_difference(claimed: list[str], actual: list[str]) -> str: + extra = sorted(set(claimed) - set(actual)) + missing = sorted(set(actual) - set(claimed)) + shown = [f"+{name}" for name in extra[:5]] + [f"-{name}" for name in missing[:5]] + return ", ".join(shown) or "order or duplicates" + + +def verify_model_surfaces( + evidence: RegionalInputSurfaceEvidence, + regional_manifest: DataReleaseManifest, + *, + run: Callable[..., Any] | None = None, +) -> None: + """Refuse evidence whose model surfaces are not the models' own. + + Certification calls this, so editing the evidence cannot hide a dropped + column. The target model must be installed at ``evidence.model_version``: + its variables, inputs and input aliases are read in process, as the audit + reads them. The build model is read again in a throwaway ``uv run`` + environment, as the audit reads it. The stored columns cannot be re-read + without the H5: the auditor attests them, and the evidence binds them to + each H5's SHA-256. + """ + variables, inputs, aliases = _model_surface(evidence.model_version) + target = f"policyengine-uk {evidence.model_version}" + for field, claimed, actual in ( + ("model_variables", evidence.model_variables, variables), + ("model_inputs", evidence.model_inputs, inputs), + ): + if claimed != actual: + raise RegionalInputSurfaceError( + f"The evidence's {field} are not {target}'s " + f"({_surface_difference(claimed, actual)})." + ) + if (evidence.input_aliases or {}) != aliases: + raise RegionalInputSurfaceError( + f"The evidence's input_aliases are not {target}'s." + ) + build_variables, build_inputs = _build_model_surface( + evidence.build_model_version, + evidence.build_core_version, + run=run, + exclude_newer=regional_build_cutoff(regional_manifest), + ) + build = ( + f"policyengine-uk {evidence.build_model_version} with policyengine-core " + f"{evidence.build_core_version}" + ) + for field, claimed, actual in ( + ("build_model_variables", evidence.build_model_variables, build_variables), + ("build_model_inputs", evidence.build_model_inputs, build_inputs), + ): + if claimed != actual: + raise RegionalInputSurfaceError( + f"The evidence's {field} are not {build}'s " + f"({_surface_difference(claimed, actual)})." + ) + + def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as file: @@ -445,8 +555,9 @@ def measure_regional_input_surface( before and after metadata inspection. The installed policyengine-uk must be ``model_version``. The build model and core named by the manifest are read in a throwaway ``uv run`` environment through ``run`` - (``subprocess.run`` by default). Dropped columns refuse. This reads no - unit values for output. + (``subprocess.run`` by default), resolving as of the manifest's + ``build.built_at``. Dropped columns refuse. This reads no unit values for + output. """ # Imported lazily: the certification gate reuses compare_input_surface. from policyengine.provenance.certification import fetch_release_manifest @@ -463,6 +574,7 @@ def measure_regional_input_surface( regional_manifest_uri, token=token ) build_model_version, build_core_version = regional_build_versions(manifest) + build_cutoff = regional_build_cutoff(manifest) expected = { name: artifact for name, artifact in manifest.artifacts.items() @@ -475,7 +587,7 @@ def measure_regional_input_surface( ) variables, inputs, aliases = _model_surface(model_version) build_variables, build_inputs = _build_model_surface( - build_model_version, build_core_version, run=run + build_model_version, build_core_version, run=run, exclude_newer=build_cutoff ) artifacts = {} for name, artifact in sorted(expected.items()): diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 855faaa12..5e1804a83 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1437,11 +1437,17 @@ def test__given_older_dense_cut__then_refuses_without_an_accepted_mismatch( regional_manifest_uri=UK_DENSE_URI, ) - def test__given_accepted_mismatch__then_routes_and_records_it(self, tmp_path): + def test__given_accepted_mismatch__then_routes_and_records_it( + self, tmp_path, monkeypatch + ): bundle_path = _tmp_bundle(tmp_path) reason = "area receipts are uncertified; the dense cut lags one release" - from tests.test_regional_input_evidence import input_evidence + from tests.test_regional_input_evidence import ( + fake_model_surfaces, + input_evidence, + ) + fake_model_surfaces(monkeypatch) evidence = input_evidence(self._older_dense()) result = _certify_uk( @@ -1510,10 +1516,14 @@ def test__given_an_empty_reason__then_refuses(self, tmp_path, reason): ) def test__given_an_accepted_mismatch__then_the_core_must_still_be_covered( - self, tmp_path + self, tmp_path, monkeypatch ): - from tests.test_regional_input_evidence import input_evidence + from tests.test_regional_input_evidence import ( + fake_model_surfaces, + input_evidence, + ) + fake_model_surfaces(monkeypatch) national = _microcosm_uk_national_payload() national["compatible_core_packages"] = [ {"name": "policyengine-core", "specifier": "==3.33.0"} diff --git a/tests/test_measure_regional_inputs.py b/tests/test_measure_regional_inputs.py index 094d4f126..fcef5651a 100644 --- a/tests/test_measure_regional_inputs.py +++ b/tests/test_measure_regional_inputs.py @@ -21,6 +21,9 @@ MODEL_VERSION = "2.147.2" BUILD_VERSION = "2.132.6" BUILD_CORE_VERSION = "3.33.3" +BUILT_AT = "2026-10-19T02:07:51.099627+00:00" +#: BUILT_AT in UTC, rounded up to the second: the probe's --exclude-newer. +BUILD_CUTOFF = "2026-10-19T02:07:52Z" URI = ( "hf://dataset/policyengine/regional@dense-cut/releases/dense/release_manifest.json" ) @@ -124,6 +127,7 @@ def _response(artifact_paths, build=None): "is_default": False, "default_datasets": {}, "build": { + "built_at": BUILT_AT, "built_with_model_package": { "name": "policyengine-uk", "version": BUILD_VERSION, @@ -295,11 +299,21 @@ def test_an_input_absent_under_both_models_is_not_reported(tmp_path, model): assert ABSENT_UNDER_BOTH in _measure(model, paths, run=computed).defaulted_inputs -def test_build_model_is_read_at_its_manifest_pins_without_the_callers_project( +def test_build_model_is_read_at_its_manifest_pins_without_the_callers_settings( tmp_path, model, monkeypatch ): - for name in ("UV_FROZEN", "UV_PROJECT_ENVIRONMENT", "VIRTUAL_ENV"): - monkeypatch.setenv(name, "from the caller's project") + for name in ( + "UV_FROZEN", + "UV_PROJECT_ENVIRONMENT", + "VIRTUAL_ENV", + "UV_INDEX_URL", + "UV_EXTRA_INDEX_URL", + "UV_CONSTRAINT", + "UV_OVERRIDE", + "UV_CONFIG_FILE", + ): + monkeypatch.setenv(name, "from the caller") + monkeypatch.setenv("UV_CACHE_DIR", "/kept/cache") monkeypatch.setenv("KEPT_SETTING", "1") run = FakeRunner() _measure(model, {"dense": _h5(tmp_path / "dense.h5")}, run=run) @@ -308,6 +322,11 @@ def test_build_model_is_read_at_its_manifest_pins_without_the_callers_project( "uv", "run", "--no-project", + "--no-config", + "--default-index", + "https://pypi.org/simple", + "--exclude-newer", + BUILD_CUTOFF, "--python", "3.13", "--with", @@ -320,12 +339,51 @@ def test_build_model_is_read_at_its_manifest_pins_without_the_callers_project( ] assert command[-1] == model.BUILD_MODEL_PROBE assert kwargs["check"] is True - assert not {"UV_FROZEN", "UV_PROJECT_ENVIRONMENT", "VIRTUAL_ENV"} & set( - kwargs["env"] - ) + assert "VIRTUAL_ENV" not in kwargs["env"] + assert {name for name in kwargs["env"] if name.startswith("UV_")} == { + "UV_CACHE_DIR" + } assert kwargs["env"]["KEPT_SETTING"] == "1" +@pytest.mark.parametrize( + ("built_at", "cutoff"), + [ + (BUILT_AT, BUILD_CUTOFF), + ("2026-10-19T03:07:51+01:00", "2026-10-19T02:07:51Z"), + ("2026-10-19T02:07:51", "2026-10-19T02:07:51Z"), + (None, None), + ], +) +def test_the_build_model_resolves_as_of_the_release_build_in_utc( + tmp_path, model, built_at, cutoff +): + paths = {"dense": _h5(tmp_path / "dense.h5")} + response = _response(paths) + payload = json.loads(response.content) + payload["build"]["built_at"] = built_at + response.content = json.dumps(payload).encode() + run = FakeRunner() + _measure(model, paths, response, run=run) + ((command, _),) = run.calls + if cutoff is None: + assert "--exclude-newer" not in command + else: + assert command[command.index("--exclude-newer") + 1] == cutoff + + +def test_an_unreadable_build_time_refuses_before_any_model_is_read(tmp_path, model): + paths = {"dense": _h5(tmp_path / "dense.h5")} + response = _response(paths) + payload = json.loads(response.content) + payload["build"]["built_at"] = "the day before the Budget" + response.content = json.dumps(payload).encode() + run = FakeRunner() + with pytest.raises(ValueError, match="built_at"): + _measure(model, paths, response, run=run) + assert run.calls == [] + + @pytest.mark.parametrize( "resolved", [{"model_version": "2.132.7"}, {"core_version": "3.33.4"}], @@ -496,6 +554,68 @@ def __init__(self): _measure(model, paths) +def _manifest_with_build_time(): + from policyengine.provenance.manifest import DataReleaseManifest + + return DataReleaseManifest.model_validate( + { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "1"}, + "build": {"built_at": BUILT_AT}, + } + ) + + +def test_certification_rereads_both_models_and_accepts_their_own_surfaces( + tmp_path, model +): + evidence = _measure(model, {"dense": _h5(tmp_path / "dense.h5")}) + run = FakeRunner() + model.verify_model_surfaces(evidence, _manifest_with_build_time(), run=run) + ((command, _),) = run.calls + assert f"policyengine-uk=={BUILD_VERSION}" in command + assert command[command.index("--exclude-newer") + 1] == BUILD_CUTOFF + + +@pytest.mark.parametrize( + ("field", "edit"), + [ + # A stored column the build model defines, claimed as never modelled. + ("build_model_variables", lambda names: [n for n in names if n != REMOVED]), + # The same column claimed as a target variable, so not dropped. + ("model_variables", lambda names: sorted([*names, REMOVED])), + # New defaults claimed as build inputs, so not defaulted. + ("build_model_inputs", lambda names: sorted([*names, *DEFAULTS])), + ("model_inputs", lambda names: names[1:]), + ], +) +def test_an_edited_model_surface_refuses_at_certification(tmp_path, model, field, edit): + evidence = _measure(model, {"dense": _h5(tmp_path / "dense.h5")}) + edited = evidence.model_copy(update={field: edit(getattr(evidence, field))}) + with pytest.raises(ValueError, match=field): + model.verify_model_surfaces( + edited, _manifest_with_build_time(), run=FakeRunner() + ) + + +def test_edited_input_aliases_refuse_at_certification(tmp_path, model): + evidence = _measure(model, {"dense": _h5(tmp_path / "dense.h5")}) + edited = evidence.model_copy(update={"input_aliases": {}}) + with pytest.raises(ValueError, match="input_aliases"): + model.verify_model_surfaces( + edited, _manifest_with_build_time(), run=FakeRunner() + ) + + +def test_certification_needs_the_target_model_installed(tmp_path, model, monkeypatch): + evidence = _measure(model, {"dense": _h5(tmp_path / "dense.h5")}) + monkeypatch.setattr(model, "version", lambda name: BUILD_VERSION) + with pytest.raises(ValueError, match=f"{BUILD_VERSION}.*{MODEL_VERSION}"): + model.verify_model_surfaces( + evidence, _manifest_with_build_time(), run=FakeRunner() + ) + + def test_the_build_model_probe_reads_the_model_api_the_audit_uses(): """Run the probe with this interpreter, in place of a uv environment.""" pytest.importorskip("policyengine_uk") diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py index 63d7190d2..1929a56a6 100644 --- a/tests/test_regional_input_evidence.py +++ b/tests/test_regional_input_evidence.py @@ -89,6 +89,44 @@ def store_column(evidence: dict, column: str) -> None: measured["columns_by_period"]["2025"] = measured["columns"] +def fake_model_surfaces(monkeypatch) -> dict: + """Stand in for the two models certification re-reads, installing neither. + + The surfaces match ``input_evidence``. Returns them, for a test to change + what a model reports, and the reads made. + """ + from policyengine.provenance import regional_input_surface + + surfaces: dict = { + "target": { + "variables": ["employment_income", *DEFAULTED_INPUTS], + "inputs": ["employment_income", *DEFAULTED_INPUTS], + "aliases": {}, + }, + "build": {"variables": ["employment_income"], "inputs": ["employment_income"]}, + "reads": [], + } + + def target_surface(model_version): + surfaces["reads"].append(("target", model_version)) + target = surfaces["target"] + return sorted(target["variables"]), sorted(target["inputs"]), target["aliases"] + + def build_surface(model_version, core_version, run=None, exclude_newer=None): + surfaces["reads"].append(("build", model_version, core_version, exclude_newer)) + build = surfaces["build"] + return sorted(build["variables"]), sorted(build["inputs"]) + + monkeypatch.setattr(regional_input_surface, "_model_surface", target_surface) + monkeypatch.setattr(regional_input_surface, "_build_model_surface", build_surface) + return surfaces + + +@pytest.fixture(autouse=True) +def model_surfaces(monkeypatch): + return fake_model_surfaces(monkeypatch) + + def certify_with_evidence(bundle_path, evidence: dict, dense: dict | None = None): dense = older_dense() if dense is None else dense return _certify_uk( @@ -175,17 +213,85 @@ def test_evidence_for_other_artifact_bytes_or_identity_refuses(tmp_path, field, certify_with_evidence(_tmp_bundle(tmp_path), evidence) -def test_a_dropped_column_refuses_even_with_an_acceptance_reason(tmp_path): +def test_a_dropped_column_refuses_even_with_an_acceptance_reason( + tmp_path, model_surfaces +): """The build model defines the stored column and the target does not.""" evidence = input_evidence() store_column(evidence, "removed_model_variable") - evidence["build_model_variables"].append("removed_model_variable") - evidence["build_model_inputs"].append("removed_model_variable") + for field, surface in ( + ("build_model_variables", "variables"), + ("build_model_inputs", "inputs"), + ): + evidence[field] = sorted([*evidence[field], "removed_model_variable"]) + model_surfaces["build"][surface].append("removed_model_variable") evidence["dropped_columns"] = ["removed_model_variable"] with pytest.raises(CertificationError, match="[Dd]ropped.*removed_model_variable"): certify_with_evidence(_tmp_bundle(tmp_path), evidence) +@pytest.mark.parametrize( + ("claim", "refused_field"), + [ + ("never_modelled", "build_model_variables"), + ("a_target_variable", "model_variables"), + ], +) +def test_an_edited_evidence_file_cannot_hide_a_dropped_column( + tmp_path, model_surfaces, claim, refused_field +): + """The lists in the file are checked against the models, not just each other.""" + model_surfaces["build"]["variables"].append("removed_model_variable") + evidence = input_evidence() + store_column(evidence, "removed_model_variable") + if claim == "never_modelled": + evidence["never_modelled_columns"] = ["removed_model_variable"] + else: + for field in ("build_model_variables", "model_variables"): + evidence[field] = sorted([*evidence[field], "removed_model_variable"]) + bundle_path = _tmp_bundle(tmp_path) + before = bundle_path.read_bytes() + with pytest.raises(CertificationError, match=f"re-reads.*{refused_field}"): + certify_with_evidence(bundle_path, evidence) + assert bundle_path.read_bytes() == before + + +def test_an_edited_evidence_file_cannot_hide_a_default(tmp_path): + evidence = input_evidence() + for field in ("build_model_variables", "build_model_inputs"): + evidence[field] = sorted([*evidence[field], *DEFAULTED_INPUTS]) + evidence["defaulted_inputs"] = [] + with pytest.raises(CertificationError, match="re-reads.*build_model"): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +def test_certification_reads_the_build_model_as_of_the_regional_build( + tmp_path, model_surfaces +): + dense = older_dense() + dense["build"]["built_at"] = "2026-10-19T02:07:51.099627+00:00" + certify_with_evidence(_tmp_bundle(tmp_path), input_evidence(dense), dense) + assert model_surfaces["reads"] == [ + ("target", UK_MODEL_VERSION), + ("build", "2.120.0", BUILD_CORE_VERSION, "2026-10-19T02:07:52Z"), + ] + + +def test_certification_refuses_without_the_target_model_installed( + tmp_path, monkeypatch +): + from policyengine.provenance import regional_input_surface + + def not_installed(model_version): + raise regional_input_surface.RegionalInputSurfaceError( + f"Installed policyengine-uk 2.132.6 differs from requested {model_version}." + ) + + monkeypatch.setattr(regional_input_surface, "_model_surface", not_installed) + with pytest.raises(CertificationError, match="re-reads.*2.132.6 differs"): + certify_with_evidence(_tmp_bundle(tmp_path), input_evidence()) + + def test_a_column_neither_model_defines_is_recorded_not_refused(tmp_path): evidence = input_evidence() store_column(evidence, "constituency_code_oa") @@ -203,7 +309,7 @@ def test_a_column_neither_model_defines_is_recorded_not_refused(tmp_path): assert "never-modelled columns (stored, defined by neither model): 1" in warning -def test_an_input_absent_under_both_models_is_not_reported(tmp_path): +def test_an_input_absent_under_both_models_is_not_reported(tmp_path, model_surfaces): evidence = input_evidence() for field in ( "model_variables", @@ -211,7 +317,10 @@ def test_an_input_absent_under_both_models_is_not_reported(tmp_path): "build_model_variables", "build_model_inputs", ): - evidence[field].append("legacy_input") + evidence[field] = sorted([*evidence[field], "legacy_input"]) + for model in ("target", "build"): + for surface in ("variables", "inputs"): + model_surfaces[model][surface].append("legacy_input") bundle_path = _tmp_bundle(tmp_path) certify_with_evidence(bundle_path, evidence) record = json.loads(bundle_path.read_text())["data_releases"]["uk"][ From 57c493c58ca8ebfba9ac5d561c3c3e3faa7f5f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:58:00 +0000 Subject: [PATCH 17/21] Name the defaulted inputs in the regional limitations Review of #699 (Vahid Ahmadi, finding 2). The inputs area runs default reached regional_certification.defaulted_inputs, TRACE, receipts and the certify warning, but not the limitations unless the operator typed them. Certification now appends one generated limitation naming them, as it appends the VAT one, unless the audit already carries the same text, and none when nothing defaults. The warning gives the count and joins the limitations with semicolons. Co-Authored-By: Claude Opus 5.5 --- src/policyengine/provenance/certification.py | 31 ++++++++++-- tests/test_certify_data_release.py | 13 ++++- tests/test_regional_input_evidence.py | 50 +++++++++++++++++++- 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index f8ab47872..379af66e0 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -425,6 +425,23 @@ def model_compatibility_basis( ) +def regional_defaults_limitation( + model_package: str, model_version: str, built_with: str, defaulted: list[str] +) -> str: + """The limitation certification records for inputs area runs default. + + Names every target input the regional file does not store that was not + an input of its build model, so a reader of the limitations alone knows + which area results rest on defaults. + """ + return ( + f"area runs on {model_package} {model_version} default {len(defaulted)} " + "inputs that the regional file does not store and its build model, " + f"{model_package} {built_with}, did not take as inputs, so area results " + f"for measures that read them use those defaults: {', '.join(defaulted)}" + ) + + def _regional_build_model_version( regional_manifest: DataReleaseManifest, model_package: str ) -> Optional[str]: @@ -467,8 +484,8 @@ def accepted_regional_mismatch( stored columns the build model defines and the target does not are dropped and always refuse; stored columns neither model defines are recorded as never modelled; and target inputs the file does not store - that were not build inputs are recorded as defaulted. Returns - ``(record, warning)``. + that were not build inputs are recorded as defaulted, and named in a + limitation. Returns ``(record, warning)``. """ reason = reason.strip() @@ -613,6 +630,12 @@ def accepted_regional_mismatch( ) from exc if predates_vat_fix and REGIONAL_VAT_LIMITATION not in limitations: limitations.append(REGIONAL_VAT_LIMITATION) + if comparison["defaulted_inputs"]: + defaults = regional_defaults_limitation( + model_package, model_version, built_with, comparison["defaulted_inputs"] + ) + if defaults not in limitations: + limitations.append(defaults) record = { "compatibility_basis": REGIONAL_BASIS_ACCEPTED_MISMATCH, "certified_for_model_version": model_version, @@ -633,8 +656,8 @@ def accepted_regional_mismatch( "are uncertified; audited dropped columns: none; never-modelled " "columns (stored, defined by neither model): " f"{len(comparison['never_modelled_columns'])}; inputs defaulted since " - f"the build model: {', '.join(comparison['defaulted_inputs']) or 'none'}. " - + " ".join(limitations) + f"the build model: {len(comparison['defaulted_inputs'])}. Limitations: " + + "; ".join(limitations) ) return record, warning diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 5e1804a83..14c73dd99 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1442,6 +1442,9 @@ def test__given_accepted_mismatch__then_routes_and_records_it( ): bundle_path = _tmp_bundle(tmp_path) reason = "area receipts are uncertified; the dense cut lags one release" + from policyengine.provenance.certification import ( + regional_defaults_limitation, + ) from tests.test_regional_input_evidence import ( fake_model_surfaces, input_evidence, @@ -1474,7 +1477,15 @@ def test__given_accepted_mismatch__then_routes_and_records_it( "defaulted_inputs": evidence["defaulted_inputs"], "dropped_columns": [], "never_modelled_columns": [], - "limitations": evidence["limitations"], + "limitations": [ + *evidence["limitations"], + regional_defaults_limitation( + "policyengine-uk", + UK_MODEL_VERSION, + "2.120.0", + evidence["defaulted_inputs"], + ), + ], "evidence": evidence, } # The national certificate itself is untouched by the regional record. diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py index 1929a56a6..dbd1ca03e 100644 --- a/tests/test_regional_input_evidence.py +++ b/tests/test_regional_input_evidence.py @@ -29,6 +29,14 @@ "local_bus_uncapped_single_fare", "salary_sacrifice_returned_to_income", ] +#: What certification adds to the limitations for input_evidence's defaults. +DEFAULTS_LIMIT = ( + f"area runs on policyengine-uk {UK_MODEL_VERSION} default 2 inputs that the " + "regional file does not store and its build model, policyengine-uk 2.120.0, " + "did not take as inputs, so area results for measures that read them use " + "those defaults: local_bus_uncapped_single_fare, " + "salary_sacrifice_returned_to_income" +) def older_dense() -> dict: @@ -420,7 +428,7 @@ def test_every_default_and_limitation_survives_validation_and_roundtrip(tmp_path record = manifest.regional_certification.model_dump(mode="json", exclude_none=True) assert record["defaulted_inputs"] == DEFAULTED_INPUTS assert record["dropped_columns"] == [] - assert record["limitations"] == [VAT_LIMIT] + assert record["limitations"] == [VAT_LIMIT, DEFAULTS_LIMIT] assert record["evidence"] == evidence assert record["area_receipts"] == "uncertified" assert record["accepted_reason"] == REASON @@ -433,6 +441,46 @@ def test_every_default_and_limitation_survives_validation_and_roundtrip(tmp_path assert release["certification"]["compatibility_basis"] == "built_with_model_package" +def test_certification_names_the_defaulted_inputs_in_the_limitations(tmp_path): + """Review of #699: a reader of the limitations alone sees the defaults.""" + evidence = input_evidence() + bundle_path = _tmp_bundle(tmp_path) + result = certify_with_evidence(bundle_path, evidence) + record = json.loads(bundle_path.read_text())["data_releases"]["uk"][ + "regional_certification" + ] + assert record["limitations"] == [VAT_LIMIT, DEFAULTS_LIMIT] + assert record["evidence"]["limitations"] == [VAT_LIMIT] + (warning,) = [w for w in result.warnings if "accepted mismatch" in w] + assert "inputs defaulted since the build model: 2." in warning + assert DEFAULTS_LIMIT in warning + + +def test_the_defaults_limitation_is_not_repeated(tmp_path): + evidence = input_evidence() + evidence["limitations"].append(DEFAULTS_LIMIT) + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, evidence) + record = json.loads(bundle_path.read_text())["data_releases"]["uk"][ + "regional_certification" + ] + assert record["limitations"] == [VAT_LIMIT, DEFAULTS_LIMIT] + + +def test_no_defaults_limitation_when_nothing_defaults(tmp_path, model_surfaces): + for surface in ("variables", "inputs"): + model_surfaces["target"][surface] = ["employment_income"] + evidence = input_evidence() + evidence["model_variables"] = evidence["model_inputs"] = ["employment_income"] + evidence["defaulted_inputs"] = [] + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, evidence) + record = json.loads(bundle_path.read_text())["data_releases"]["uk"][ + "regional_certification" + ] + assert record["limitations"] == [VAT_LIMIT] + + def test_evidence_digest_binds_original_manifest_bytes(tmp_path): dense = older_dense() response = _response(dense) From cb0da7f2a9d124f23be4378aee8f06d8f893029d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:03:03 +0000 Subject: [PATCH 18/21] Put a digest-bound summary of the regional exception on area receipts only Review of #699 (Vahid Ahmadi, finding 3). Every UK receipt carried the whole regional_certification record, evidence included: both models' full variable and input lists, roughly 86 KB on 2.132.6 alone, and all of it in execution_identity_sha256. A certified national receipt gains nothing from that, since certificate_sha256 already digests the data_releases.uk record that holds it. Now only an area run, a receipt whose data is the artifact the accepted exception covers, carries regional_certification, as a RegionalCertificationIdentity: the record's model versions, reason, audited lists and limitations, and record_sha256, the RFC 8785 digest of the whole record, evidence included. A certified receipt cannot carry one. verify_regional_certification checks an area receipt's summary and data identity against the manifest the run used, as verify_certificate does for a certificate; verify_certificate no longer compares a regional record. The published schema and its rules say so. The field is new in this pull request, so this does not change a released v1 field. Co-Authored-By: Claude Opus 5.5 --- .../schemas/execution_receipt.v1.schema.json | 258 ++---------------- src/policyengine/provenance/__init__.py | 3 + src/policyengine/provenance/receipt.py | 200 ++++++++++++-- tests/test_execution_receipt.py | 102 +++++-- 4 files changed, 274 insertions(+), 289 deletions(-) diff --git a/src/policyengine/data/schemas/execution_receipt.v1.schema.json b/src/policyengine/data/schemas/execution_receipt.v1.schema.json index 4dfcb5b6f..2bacf34b6 100644 --- a/src/policyengine/data/schemas/execution_receipt.v1.schema.json +++ b/src/policyengine/data/schemas/execution_receipt.v1.schema.json @@ -457,8 +457,9 @@ "title": "ReformIdentity", "type": "object" }, - "RegionalCertification": { - "description": "A reviewer's acceptance of a regional release on another model version.\n\nWritten only by ``certify-data --accept-regional-model-mismatch``: the\nregional release neither was built with nor claims the certified model,\nand area runs on it are uncertified.", + "RegionalCertificationIdentity": { + "additionalProperties": false, + "description": "The accepted regional model mismatch an area run used, bound by digest.\n\nIt summarises the bundle's ``regional_certification`` record. The record's\nevidence (both models' variable and input lists and the measured columns)\nstays in the bundle and its TRACE TRO; ``record_sha256`` binds it.", "properties": { "accepted_reason": { "title": "Accepted Reason", @@ -466,7 +467,6 @@ }, "area_receipts": { "const": "uncertified", - "default": "uncertified", "title": "Area Receipts", "type": "string" }, @@ -480,9 +480,11 @@ } ], "default": null, + "description": "The model the regional release was built with.", "title": "Built With Model Version" }, "certified_for_model_version": { + "description": "The model the national certificate covers and area runs use.", "title": "Certified For Model Version", "type": "string" }, @@ -492,6 +494,7 @@ "type": "string" }, "defaulted_inputs": { + "description": "Inputs area runs default that the regional file does not store and its build model did not take as inputs.", "items": { "type": "string" }, @@ -505,9 +508,6 @@ "title": "Dropped Columns", "type": "array" }, - "evidence": { - "$ref": "#/$defs/RegionalInputSurfaceEvidence" - }, "limitations": { "items": { "type": "string" @@ -516,12 +516,19 @@ "type": "array" }, "never_modelled_columns": { + "description": "Stored columns neither model defines.", "items": { "type": "string" }, "title": "Never Modelled Columns", "type": "array" }, + "record_sha256": { + "description": "RFC 8785 SHA-256 of the bundle manifest's data_releases..regional_certification record, its evidence included (see regional_certification_identity()).", + "pattern": "^[0-9a-f]{64}$", + "title": "Record Sha256", + "type": "string" + }, "regional_manifest_sha256": { "pattern": "^[0-9a-f]{64}$", "title": "Regional Manifest Sha256", @@ -536,238 +543,16 @@ "compatibility_basis", "certified_for_model_version", "accepted_reason", + "area_receipts", "regional_manifest_uri", "regional_manifest_sha256", "defaulted_inputs", "dropped_columns", "never_modelled_columns", "limitations", - "evidence" - ], - "title": "RegionalCertification", - "type": "object" - }, - "RegionalInputArtifactEvidence": { - "additionalProperties": false, - "description": "Measured columns of one byte-verified regional H5 artifact.", - "properties": { - "columns": { - "items": { - "type": "string" - }, - "minItems": 1, - "title": "Columns", - "type": "array" - }, - "columns_by_period": { - "additionalProperties": { - "items": { - "type": "string" - }, - "type": "array" - }, - "minProperties": 1, - "title": "Columns By Period", - "type": "object" - }, - "path": { - "minLength": 1, - "title": "Path", - "type": "string" - }, - "repo_id": { - "minLength": 1, - "title": "Repo Id", - "type": "string" - }, - "repo_type": { - "anyOf": [ - { - "enum": [ - "model", - "dataset" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Repo Type" - }, - "revision": { - "minLength": 1, - "title": "Revision", - "type": "string" - }, - "sha256": { - "pattern": "^[0-9a-f]{64}$", - "title": "Sha256", - "type": "string" - } - }, - "required": [ - "repo_id", - "path", - "revision", - "sha256", - "columns", - "columns_by_period" - ], - "title": "RegionalInputArtifactEvidence", - "type": "object" - }, - "RegionalInputSurfaceEvidence": { - "additionalProperties": false, - "description": "An operator's audit of actual regional columns against two exact models.\n\nThe baseline is the regional release's build model and core, as its\nmanifest records them; the target is the model the bundle certifies. Both\nsurfaces are retained in full so certification recomputes the comparison\nrather than trusting a declared zero-loss result. The artifact hashes\nidentify the bytes inspected by the auditor, not merely a model's possible\ninputs. Schema version 2 made the comparison relative to the build model;\nversion 1 evidence, which counted columns no model defines as dropped, is\nrefused.", - "properties": { - "artifacts": { - "additionalProperties": { - "$ref": "#/$defs/RegionalInputArtifactEvidence" - }, - "minProperties": 1, - "title": "Artifacts", - "type": "object" - }, - "audited_by": { - "minLength": 1, - "pattern": "\\S", - "title": "Audited By", - "type": "string" - }, - "build_core_version": { - "minLength": 1, - "title": "Build Core Version", - "type": "string" - }, - "build_model_inputs": { - "items": { - "type": "string" - }, - "minItems": 1, - "title": "Build Model Inputs", - "type": "array" - }, - "build_model_variables": { - "items": { - "type": "string" - }, - "minItems": 1, - "title": "Build Model Variables", - "type": "array" - }, - "build_model_version": { - "minLength": 1, - "title": "Build Model Version", - "type": "string" - }, - "defaulted_inputs": { - "items": { - "type": "string" - }, - "title": "Defaulted Inputs", - "type": "array" - }, - "dropped_columns": { - "items": { - "type": "string" - }, - "title": "Dropped Columns", - "type": "array" - }, - "input_aliases": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Input Aliases" - }, - "limitations": { - "items": { - "type": "string" - }, - "minItems": 1, - "title": "Limitations", - "type": "array" - }, - "model_inputs": { - "items": { - "type": "string" - }, - "minItems": 1, - "title": "Model Inputs", - "type": "array" - }, - "model_package": { - "minLength": 1, - "title": "Model Package", - "type": "string" - }, - "model_variables": { - "items": { - "type": "string" - }, - "minItems": 1, - "title": "Model Variables", - "type": "array" - }, - "model_version": { - "minLength": 1, - "title": "Model Version", - "type": "string" - }, - "never_modelled_columns": { - "items": { - "type": "string" - }, - "title": "Never Modelled Columns", - "type": "array" - }, - "regional_manifest_sha256": { - "pattern": "^[0-9a-f]{64}$", - "title": "Regional Manifest Sha256", - "type": "string" - }, - "regional_manifest_uri": { - "minLength": 1, - "title": "Regional Manifest Uri", - "type": "string" - }, - "schema_version": { - "const": 2, - "default": 2, - "title": "Schema Version", - "type": "integer" - } - }, - "required": [ - "regional_manifest_uri", - "regional_manifest_sha256", - "model_package", - "model_version", - "build_model_version", - "build_core_version", - "artifacts", - "model_variables", - "model_inputs", - "build_model_variables", - "build_model_inputs", - "dropped_columns", - "never_modelled_columns", - "defaulted_inputs", - "audited_by", - "limitations" + "record_sha256" ], - "title": "RegionalInputSurfaceEvidence", + "title": "RegionalCertificationIdentity", "type": "object" }, "RuntimeContext": { @@ -971,14 +756,14 @@ "regional_certification": { "anyOf": [ { - "$ref": "#/$defs/RegionalCertification" + "$ref": "#/$defs/RegionalCertificationIdentity" }, { "type": "null" } ], "default": null, - "description": "The bundle's accepted regional model mismatch, including its exact regional manifest and artifact identity, measured input surface, acceptance reason and limitations. This records an exception for uncertified area runs and does not widen the national certificate." + "description": "Present only on a run of the dataset the bundle routes area regions to on an accepted model mismatch (an area run), which is uncertified: the exception's model versions, reason, audited lists and limitations, and the digest of its full record, evidence included. It does not widen the national certificate." }, "runtime": { "anyOf": [ @@ -1039,7 +824,8 @@ "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?, regional_certification?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "outputs_sha256": "the canonical output payload the receipt binds", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", - "reform.reform_sha256": "RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {\"$nonfinite\": \"Infinity\"} or {\"$nonfinite\": \"-Infinity\"}, and a payload may not use the key $nonfinite itself" + "reform.reform_sha256": "RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {\"$nonfinite\": \"Infinity\"} or {\"$nonfinite\": \"-Infinity\"}, and a payload may not use the key $nonfinite itself", + "regional_certification.record_sha256": "the bundle manifest's data_releases..regional_certification record, its evidence included" }, "x-policyengine-rules": [ "years is sorted ascending with no duplicates, and each year is written as an integer (2027, not 2027.0).", @@ -1053,6 +839,8 @@ "reform/baseline kind current_law has no representation or reform_sha256 and binding content; kind reform has both.", "wheel_sha256 and wheel_sha256_evidence are present together or not at all.", "certification_status certified requires each of wrapper, model and core to have install_origin absent or archive, or install_origin directory or vcs with wheel_sha256_evidence installed_record_verified; an editable install is never certified.", - "A consumer trusts a certificate only after checking it against the data_releases. record of the published policyengine manifest for certificate.bundle_version (verify_certificate()): certificate_sha256 is that record's digest, every other member of certificate except certificate_source, qualification_receipt_sha256 and tro_composition_fingerprint equals what the record states, and data equals the data identity the record determines. certificate_source is the producer's statement and proves nothing." + "A consumer trusts a certificate only after checking it against the data_releases. record of the published policyengine manifest for certificate.bundle_version (verify_certificate()): certificate_sha256 is that record's digest, every other member of certificate except certificate_source, qualification_receipt_sha256 and tro_composition_fingerprint equals what the record states, and data equals the data identity the record determines. certificate_source is the producer's statement and proves nothing.", + "regional_certification is present only when data is the artifact that the bundle's accepted regional model mismatch (data_releases..regional_certification) covers: an area run, which is uncertified. A certified receipt carries none.", + "A consumer checks regional_certification against the data_releases..regional_certification record of the manifest the run used (verify_regional_certification()): record_sha256 is that record's digest, every other member equals what the record states, and data is the regional artifact the record covers." ] } diff --git a/src/policyengine/provenance/__init__.py b/src/policyengine/provenance/__init__.py index deb2a8bc7..b9eea8d3c 100644 --- a/src/policyengine/provenance/__init__.py +++ b/src/policyengine/provenance/__init__.py @@ -108,6 +108,9 @@ from .receipt import ( verify_certificate as verify_certificate, ) +from .receipt import ( + verify_regional_certification as verify_regional_certification, +) from .trace import ( build_simulation_trace_tro as build_simulation_trace_tro, ) diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index 2492e706f..e8192c582 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -56,6 +56,10 @@ - A certificate is trusted only after :func:`verify_certificate` has checked it, and the receipt's data identity, against the published bundle manifest it names. ``certificate_source`` is the producer's statement, not evidence. +- An area run on a regional release the bundle routes on an accepted model + mismatch is uncertified and carries a summary of that exception with the + digest of its full record; :func:`verify_regional_certification` checks it + against the manifest the run used. Adding an optional field is a compatible v1 change. Removing or redefining a field, or changing what a digest covers, requires schema v2. @@ -147,6 +151,7 @@ "PackageIdentity", "ReceiptError", "ReformIdentity", + "RegionalCertificationIdentity", "RuntimeContext", "WrapperIdentity", "build_execution_receipt", @@ -157,10 +162,12 @@ "fiscal_year_label", "installed_package_identity", "reform_identity", + "regional_certification_identity", "scenario_reform_identity", "strict_json_value", "uk_fiscal_year_convention", "verify_certificate", + "verify_regional_certification", ] RECEIPT_SCHEMA_VERSION: Literal[1] = 1 @@ -219,6 +226,8 @@ "wheel_sha256 and wheel_sha256_evidence are present together or not at all.", "certification_status certified requires each of wrapper, model and core to have install_origin absent or archive, or install_origin directory or vcs with wheel_sha256_evidence installed_record_verified; an editable install is never certified.", "A consumer trusts a certificate only after checking it against the data_releases. record of the published policyengine manifest for certificate.bundle_version (verify_certificate()): certificate_sha256 is that record's digest, every other member of certificate except certificate_source, qualification_receipt_sha256 and tro_composition_fingerprint equals what the record states, and data equals the data identity the record determines. certificate_source is the producer's statement and proves nothing.", + "regional_certification is present only when data is the artifact that the bundle's accepted regional model mismatch (data_releases..regional_certification) covers: an area run, which is uncertified. A certified receipt carries none.", + "A consumer checks regional_certification against the data_releases..regional_certification record of the manifest the run used (verify_regional_certification()): record_sha256 is that record's digest, every other member equals what the record states, and data is the regional artifact the record covers.", ) #: How each digest is computed. @@ -227,6 +236,7 @@ "execution_identity_sha256": "{schema_version, country_id, wrapper, model, core, data, certification_status, certificate?, regional_certification?} where certificate keeps only bundle_version, country_id, certificate_sha256, compatibility_basis, certified_for_model_version, certified_for_core_version, certified_dataset_sha256 and certified_by", "receipt_sha256": "the whole receipt without runtime and without reform.name and baseline.name", "certificate.certificate_sha256": "the bundle manifest's data_releases. record", + "regional_certification.record_sha256": "the bundle manifest's data_releases..regional_certification record, its evidence included", "reform.reform_sha256": 'RFC 8785 over {representation, payload} where payload is the reform exactly as given in that representation, nulls kept (a null in a reform can mean something, unlike an absent receipt field); for policyengine-uk.scenario_composition/v1 the payload is {applied_before_data_load, application: parameter_changes_then_modifiers, members: [{representation, reform_sha256, binding}, ...]} with members in application order; an infinite number in a payload is the object {"$nonfinite": "Infinity"} or {"$nonfinite": "-Infinity"}, and a payload may not use the key $nonfinite itself', "outputs_sha256": "the canonical output payload the receipt binds", } @@ -399,6 +409,46 @@ class CertificateIdentity(_Contract): ) +class RegionalCertificationIdentity(_Contract): + """The accepted regional model mismatch an area run used, bound by digest. + + It summarises the bundle's ``regional_certification`` record. The record's + evidence (both models' variable and input lists and the measured columns) + stays in the bundle and its TRACE TRO; ``record_sha256`` binds it. + """ + + compatibility_basis: Literal["accepted_model_mismatch"] + certified_for_model_version: str = Field( + description="The model the national certificate covers and area runs use." + ) + built_with_model_version: Optional[str] = Field( + default=None, description="The model the regional release was built with." + ) + accepted_reason: str + area_receipts: Literal["uncertified"] + regional_manifest_uri: str + regional_manifest_sha256: str = Field(pattern=SHA256_PATTERN) + defaulted_inputs: list[str] = Field( + description=( + "Inputs area runs default that the regional file does not store " + "and its build model did not take as inputs." + ) + ) + dropped_columns: list[str] + never_modelled_columns: list[str] = Field( + description="Stored columns neither model defines." + ) + limitations: list[str] + record_sha256: str = Field( + pattern=SHA256_PATTERN, + description=( + "RFC 8785 SHA-256 of the bundle manifest's " + "data_releases..regional_certification record, its evidence " + "included (see regional_certification_identity())." + ), + ) + + class ReformIdentity(_Contract): """A reform or baseline, bound by the digest of its canonical form.""" @@ -530,13 +580,14 @@ class ExecutionReceipt(_Contract): data: DataIdentity certification_status: Literal["certified", "uncertified"] certificate: Optional[CertificateIdentity] = None - regional_certification: Optional[RegionalCertification] = Field( + regional_certification: Optional[RegionalCertificationIdentity] = Field( default=None, description=( - "The bundle's accepted regional model mismatch, including its exact " - "regional manifest and artifact identity, measured input surface, " - "acceptance reason and limitations. This records an exception for " - "uncertified area runs and does not widen the national certificate." + "Present only on a run of the dataset the bundle routes area regions " + "to on an accepted model mismatch (an area run), which is " + "uncertified: the exception's model versions, reason, audited lists " + "and limitations, and the digest of its full record, evidence " + "included. It does not widen the national certificate." ), ) reform: ReformIdentity @@ -593,6 +644,11 @@ def _identity_rules(self) -> ExecutionReceipt: if self.certificate is not None: raise ValueError("An uncertified receipt carries no certificate.") return self + if self.regional_certification is not None: + raise ValueError( + "A certified receipt carries no regional_certification: the " + "accepted regional exception covers uncertified area runs only." + ) certificate = self.certificate if certificate is None: raise ValueError("A certified receipt needs a certificate.") @@ -645,8 +701,8 @@ def execution_identity(self) -> dict[str, Any]: if key in payload } if self.regional_certification is not None: - identity["regional_certification"] = self.regional_certification.model_dump( - mode="json", exclude_none=True + identity["regional_certification"] = ( + self.regional_certification.canonical_payload() ) return identity @@ -862,6 +918,24 @@ def certificate_sha256(certification_record: Mapping[str, Any]) -> str: return content_sha256(certification_record) +def regional_certification_identity( + regional_record: Mapping[str, Any], +) -> RegionalCertificationIdentity: + """What an area receipt carries of a bundle's ``regional_certification``. + + Every member of the record except its evidence, and the digest of the + whole record, evidence included. + """ + + summary = RegionalCertification.model_validate(regional_record).model_dump( + mode="json", exclude_none=True + ) + del summary["evidence"] + return RegionalCertificationIdentity( + **summary, record_sha256=content_sha256(regional_record) + ) + + def reform_identity( payload: Optional[Union[Mapping[str, Any], Sequence[Any]]], representation: Optional[ReformRepresentation] = None, @@ -3792,7 +3866,23 @@ def _data_identity( dataset_name: str, dataset_sha256: str, ) -> DataIdentity: - """The data identity of ``dataset_name`` with those bytes, from the bundle. + """The data identity of ``dataset_name`` with those bytes, from the bundle.""" + + identity, _ = _resolved_data( + country_id, record, bundle, dataset_name, dataset_sha256 + ) + return identity + + +def _resolved_data( + country_id: str, + record: Mapping[str, Any], + bundle: Mapping[str, Any], + dataset_name: str, + dataset_sha256: str, +) -> tuple[DataIdentity, bool]: + """The data identity of ``dataset_name`` with those bytes, from the bundle, + and whether they are the artifact the accepted regional exception covers. The dataset is resolved exactly as :func:`policyengine.provenance.dataset_materialization.materialize_dataset` @@ -3855,7 +3945,7 @@ def _data_identity( data_package = record["data_package"] certified_artifact = record.get("certified_data_artifact") or {} certification = record.get("certification") or {} - return DataIdentity( + identity = DataIdentity( data_package=spec.data_package_name, release_version=data_package["version"] if release else spec.revision, build_id=( @@ -3881,6 +3971,7 @@ def _data_identity( dataset_revision=spec.revision, dataset_sha256=dataset_sha256, ) + return identity, regional def _build( @@ -3930,7 +4021,9 @@ def identity(name: str) -> dict[str, Any]: dataset_name = dataset or record["default_dataset"] actual_sha256 = sha256_file(dataset_path) - data = _data_identity(country_id, record, bundle, dataset_name, actual_sha256) + data, regional = _resolved_data( + country_id, record, bundle, dataset_name, actual_sha256 + ) runtime_basis = None if source == "packaged" and ( @@ -3984,7 +4077,13 @@ def identity(name: str) -> dict[str, Any]: data=data, certification_status=status, certificate=certificate, - regional_certification=record.get("regional_certification"), + # National receipts need no copy: certificate_sha256 digests the + # record that holds the regional exception. + regional_certification=( + regional_certification_identity(record["regional_certification"]) + if regional + else None + ), reform=reform, baseline=baseline, fiscal_year=uk_fiscal_year_convention(), @@ -4032,25 +4131,6 @@ def verify_certificate( if not isinstance(record, Mapping): return [f"the bundle has no {receipt.country_id} data release"] problems = [] - regional_record = record.get("regional_certification") - try: - expected_regional = ( - RegionalCertification.model_validate(regional_record).model_dump( - mode="json", exclude_none=True - ) - if regional_record is not None - else None - ) - except ValueError as exc: - problems.append(f"the bundle cannot determine regional_certification: {exc}") - else: - claimed_regional = ( - receipt.regional_certification.model_dump(mode="json", exclude_none=True) - if receipt.regional_certification is not None - else None - ) - if claimed_regional != expected_regional: - problems.append("regional_certification differs from the bundle's record") for package in (receipt.wrapper, receipt.model, receipt.core): if not _package_can_be_certified(package): problems.append( @@ -4098,6 +4178,66 @@ def verify_certificate( return problems +def verify_regional_certification( + receipt: ExecutionReceipt, bundle: Mapping[str, Any] +) -> list[str]: + """Problems with an area receipt's regional exception against a bundle. + + ``bundle`` is the policyengine manifest the run used. The summary is + rebuilt from that manifest's ``data_releases..regional_certification`` + record exactly as :func:`build_execution_receipt` builds it and compared + member by member, ``record_sha256`` included, and the receipt's ``data`` + must be the regional artifact that record covers, with the identity the + bundle determines. An empty list means the receipt names exactly the + bundle's accepted regional exception; the area run stays uncertified. + """ + + claimed = receipt.regional_certification + if claimed is None: + return ["the receipt carries no regional_certification"] + record = (bundle.get("data_releases") or {}).get(receipt.country_id) + regional_record = ( + record.get("regional_certification") if isinstance(record, Mapping) else None + ) + if not isinstance(record, Mapping) or not isinstance(regional_record, Mapping): + return [ + f"the bundle's {receipt.country_id} data release has no " + "regional_certification" + ] + try: + expected = regional_certification_identity(regional_record) + data, regional = _resolved_data( + receipt.country_id, + record, + bundle, + receipt.data.dataset, + receipt.data.dataset_sha256, + ) + except (KeyError, ValueError) as exc: + return [f"the bundle cannot determine regional_certification: {exc}"] + problems = [] + claimed_payload = claimed.canonical_payload() + expected_payload = expected.canonical_payload() + for field in sorted(set(claimed_payload) | set(expected_payload)): + if claimed_payload.get(field) != expected_payload.get(field): + problems.append( + f"regional_certification.{field} differs from the bundle's record" + ) + if not regional: + problems.append( + f"data.dataset {receipt.data.dataset!r} is not the artifact the " + "bundle's regional exception covers" + ) + claimed_data = receipt.data.model_dump() + for field, value in data.model_dump().items(): + if claimed_data[field] != value: + problems.append( + f"data.{field} is {claimed_data[field]!r}, " + f"but the bundle determines {value!r}" + ) + return problems + + def build_execution_receipt_for_microsimulation( microsimulation: Any, *, diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 311558fae..c290c362d 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -500,39 +500,78 @@ def _bundle_with_regional_acceptance(dataset_file): return bundle -@pytest.mark.parametrize("area", [False, True], ids=["national", "area"]) -def test_receipt_exposes_regional_acceptance_without_widening_certification( - dataset_file, schema_validator, area +def test_an_area_receipt_carries_a_digest_bound_summary_of_the_exception( + dataset_file, schema_validator ): - from policyengine.provenance.manifest import RegionalCertification + from policyengine.provenance.receipt import regional_certification_identity bundle = _bundle_with_regional_acceptance(dataset_file) - record = bundle["data_releases"]["uk"] + record = bundle["data_releases"]["uk"]["regional_certification"] - receipt = _build(dataset_file, bundle, dataset="area_file" if area else None) + receipt = _build(dataset_file, bundle, dataset="area_file") payload = json.loads(receipt.model_dump_json(exclude_none=True)) - assert payload["regional_certification"] == RegionalCertification.model_validate( - record["regional_certification"] - ).model_dump(mode="json", exclude_none=True) - assert receipt.certification_status == ("uncertified" if area else "certified") - assert (receipt.certificate is None) is area - assert receipt.data.build_model_version == ( - "2.132.6" if area else record["certification"]["built_with_model_version"] - ) + summary = payload["regional_certification"] + assert summary == regional_certification_identity(record).canonical_payload() + assert summary["record_sha256"] == content_sha256(record) + assert summary["never_modelled_columns"] == ["constituency_code_oa"] + assert summary["limitations"] == record["limitations"] + # The evidence's model surfaces stay in the bundle and TRACE. + assert "evidence" not in summary + assert "model_variables" not in json.dumps(payload) + assert receipt.certification_status == "uncertified" + assert receipt.certificate is None + assert receipt.data.build_model_version == "2.132.6" schema_validator.validate(payload) restored = ExecutionReceipt.model_validate_json(receipt.model_dump_json()) assert restored.regional_certification == receipt.regional_certification assert restored.receipt_sha256 == receipt.receipt_sha256 -def test_regional_limitations_are_bound_to_receipt_identity(dataset_file): +def test_a_national_receipt_leaves_the_exception_to_its_certificate(dataset_file): + """Review of #699: certificate_sha256 already digests the regional record.""" + bundle = _bundle_with_regional_acceptance(dataset_file) + record = bundle["data_releases"]["uk"] + + receipt = _build(dataset_file, bundle) + + assert receipt.certification_status == "certified" + assert "regional_certification" not in receipt.canonical_payload() + assert receipt.certificate.certificate_sha256 == certificate_sha256(record) + assert verify_certificate(receipt, bundle) == [] + changed = copy.deepcopy(bundle) + changed["data_releases"]["uk"]["regional_certification"]["limitations"].append( + "Additional accepted area limitation." + ) + assert any( + "certificate_sha256" in problem + for problem in verify_certificate(receipt, changed) + ) + + +def test_a_certified_receipt_cannot_carry_the_regional_exception(dataset_file): + bundle = _bundle_with_regional_acceptance(dataset_file) + area = _build(dataset_file, bundle, dataset="area_file") + payload = _build(dataset_file, bundle).model_dump(mode="json") + payload["regional_certification"] = area.regional_certification.model_dump( + mode="json" + ) + with pytest.raises(ValidationError, match="regional_certification"): + ExecutionReceipt.model_validate(payload) + + +@pytest.mark.parametrize("change", ["limitations", "evidence"]) +def test_the_regional_record_is_bound_to_area_receipt_identity(dataset_file, change): bundle = _bundle_with_regional_acceptance(dataset_file) receipt = _build(dataset_file, bundle, dataset="area_file") changed_bundle = copy.deepcopy(bundle) record = changed_bundle["data_releases"]["uk"]["regional_certification"] - record["limitations"].append("Additional accepted area limitation.") - record["evidence"]["limitations"].append("Additional accepted area limitation.") + if change == "limitations": + record["limitations"].append("Additional accepted area limitation.") + record["evidence"]["limitations"].append("Additional accepted area limitation.") + else: + # Not in the summary, but in the record its digest covers. + record["evidence"]["audited_by"] = "Another reviewer" changed = _build(dataset_file, changed_bundle, dataset="area_file") @@ -540,17 +579,32 @@ def test_regional_limitations_are_bound_to_receipt_identity(dataset_file): assert changed.receipt_sha256 != receipt.receipt_sha256 -def test_certificate_verification_checks_the_exposed_regional_record(dataset_file): +def test_regional_verification_checks_an_area_receipt_against_the_bundle( + dataset_file, +): + from policyengine.provenance.receipt import verify_regional_certification + bundle = _bundle_with_regional_acceptance(dataset_file) - receipt = _build(dataset_file, bundle) + receipt = _build(dataset_file, bundle, dataset="area_file") + assert verify_regional_certification(receipt, bundle) == [] + payload = receipt.model_dump(mode="json") payload["regional_certification"]["accepted_reason"] = "A different acceptance." - changed = ExecutionReceipt.model_validate(payload) + claimed = ExecutionReceipt.model_validate(payload) + assert verify_regional_certification(claimed, bundle) == [ + "regional_certification.accepted_reason differs from the bundle's record" + ] - assert any( - "regional_certification" in issue - for issue in verify_certificate(changed, bundle) - ) + edited = copy.deepcopy(bundle) + edited["data_releases"]["uk"]["regional_certification"]["evidence"][ + "audited_by" + ] = "Another reviewer" + assert verify_regional_certification(receipt, edited) == [ + "regional_certification.record_sha256 differs from the bundle's record" + ] + assert verify_regional_certification(_build(dataset_file, bundle), bundle) == [ + "the receipt carries no regional_certification" + ] def test_trace_exposes_regional_acceptance_separately_from_national(dataset_file): From 444982be85ffcf5d17965854058d8b2ba9946893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:05:23 +0000 Subject: [PATCH 19/21] Refuse invisible reasons, and require the evidence's schema_version Review of #699 (Vahid Ahmadi, findings 5 and 6). A reason made only of a zero-width space passed certification, because str.strip() keeps it. A \S pattern keeps it too, as it keeps U+FEFF, U+2060 and soft hyphens, so the mismatch reason, the auditor and every limitation now need one printable character that is not whitespace, in the audit and in certification. RegionalCertification.accepted_reason also takes the \S pattern audited_by has, so a hand-edited bundle with a blank reason does not validate. schema_version is now required, so evidence without it refuses rather than reading as version 2. Co-Authored-By: Claude Opus 5.5 --- src/policyengine/provenance/certification.py | 12 ++++++-- src/policyengine/provenance/manifest.py | 4 +-- .../provenance/regional_input_surface.py | 17 +++++++++-- tests/test_certify_data_release.py | 3 +- tests/test_measure_regional_inputs.py | 18 ++++++++++++ tests/test_regional_input_evidence.py | 28 +++++++++++++++++-- 6 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 379af66e0..5ea7436cd 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -488,8 +488,10 @@ def accepted_regional_mismatch( limitation. Returns ``(record, warning)``. """ + from policyengine.provenance.regional_input_surface import readable + reason = reason.strip() - if not reason: + if not readable(reason): raise CertificationError( "--accept-regional-model-mismatch needs a reason a reviewer can read." ) @@ -575,8 +577,12 @@ def accepted_regional_mismatch( raise CertificationError( "Evidence model inputs must belong to its variable surface." ) - if not all(item.strip() for item in audit.limitations): - raise CertificationError("Regional evidence limitations must not be blank.") + if not readable(audit.audited_by) or not all( + readable(item) for item in audit.limitations + ): + raise CertificationError( + "Regional evidence needs an auditor and limitations a reader can see." + ) from policyengine.provenance.regional_input_surface import ( compare_input_surface, verify_model_surfaces, diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 2035a2043..bf4345ee0 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -240,7 +240,7 @@ class RegionalInputSurfaceEvidence(BaseModel): model_config = {"extra": "forbid"} - schema_version: Literal[2] = 2 + schema_version: Literal[2] regional_manifest_uri: str = Field(min_length=1) regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") model_package: str = Field(min_length=1) @@ -275,7 +275,7 @@ class RegionalCertification(BaseModel): compatibility_basis: Literal["accepted_model_mismatch"] certified_for_model_version: str built_with_model_version: Optional[str] = None - accepted_reason: str + accepted_reason: str = Field(min_length=1, pattern=r"\S") area_receipts: Literal["uncertified"] = "uncertified" regional_manifest_uri: str regional_manifest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") diff --git a/src/policyengine/provenance/regional_input_surface.py b/src/policyengine/provenance/regional_input_surface.py index fe94bac22..676941cb4 100644 --- a/src/policyengine/provenance/regional_input_surface.py +++ b/src/policyengine/provenance/regional_input_surface.py @@ -104,6 +104,16 @@ class RegionalInputSurfaceError(ValueError): """A complete, byte-bound regional surface cannot be established.""" +def readable(text: str) -> bool: + """Whether ``text`` shows a reader anything. + + ``str.strip`` and a ``\\S`` pattern both accept text made only of + invisible format characters, such as a zero-width space, so this needs + one printable character that is not whitespace. + """ + return any(char.isprintable() and not char.isspace() for char in text) + + def _names(values: list[str], label: str) -> set[str]: if not values or any( not isinstance(value, str) or not value.strip() or value != value.strip() @@ -563,12 +573,12 @@ def measure_regional_input_surface( from policyengine.provenance.certification import fetch_release_manifest if ( - not audited_by.strip() + not readable(audited_by) or not limitations - or any(not item.strip() for item in limitations) + or not all(readable(item) for item in limitations) ): raise RegionalInputSurfaceError( - "An auditor and nonblank limitations are required." + "An auditor and limitations a reader can see are required." ) manifest, manifest_sha256, parts = fetch_release_manifest( regional_manifest_uri, token=token @@ -633,6 +643,7 @@ def measure_regional_input_surface( f"acceptance: {comparison['dropped_columns']}." ) return RegionalInputSurfaceEvidence( + schema_version=2, regional_manifest_uri=regional_manifest_uri, regional_manifest_sha256=manifest_sha256, model_package="policyengine-uk", diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 14c73dd99..7d2e2c93a 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1513,7 +1513,8 @@ def test__given_a_covering_regional_release__then_an_accepted_mismatch_refuses( regional_model_mismatch_reason="not needed", ) - @pytest.mark.parametrize("reason", ["", " "]) + # A zero-width space is not whitespace, so strip() alone kept it. + @pytest.mark.parametrize("reason", ["", " ", "\t\n", "\u200b", "\ufeff\u2060"]) def test__given_an_empty_reason__then_refuses(self, tmp_path, reason): with pytest.raises(CertificationError, match="needs a reason"): _certify_uk( diff --git a/tests/test_measure_regional_inputs.py b/tests/test_measure_regional_inputs.py index fcef5651a..379a6197e 100644 --- a/tests/test_measure_regional_inputs.py +++ b/tests/test_measure_regional_inputs.py @@ -427,6 +427,24 @@ def unresolvable(command, **kwargs): _measure(model, paths, run=unresolvable) +@pytest.mark.parametrize( + ("audited_by", "limitations"), + [("\u200b", [LIMITATION]), ("Budget operator", ["\u200b"]), (" ", [LIMITATION])], +) +def test_an_invisible_auditor_or_limitation_refuses( + tmp_path, model, audited_by, limitations +): + paths = {"dense": _h5(tmp_path / "dense.h5")} + with patch( + "policyengine.provenance.certification.requests.get", + return_value=_response(paths), + ): + with pytest.raises(ValueError, match="a reader can see"): + model.measure_regional_input_surface( + URI, paths, MODEL_VERSION, audited_by, limitations, run=FakeRunner() + ) + + def test_installed_model_must_be_the_requested_exact_version( tmp_path, model, monkeypatch ): diff --git a/tests/test_regional_input_evidence.py b/tests/test_regional_input_evidence.py index dbd1ca03e..af8e3bf2a 100644 --- a/tests/test_regional_input_evidence.py +++ b/tests/test_regional_input_evidence.py @@ -366,13 +366,37 @@ def test_a_regional_release_without_build_versions_has_no_baseline( certify_with_evidence(_tmp_bundle(tmp_path), evidence, dense) -def test_version_1_evidence_refuses(tmp_path): +@pytest.mark.parametrize("version", [1, None], ids=["version_1", "unversioned"]) +def test_evidence_must_declare_schema_version_2(tmp_path, version): evidence = input_evidence() - evidence["schema_version"] = 1 + if version is None: + del evidence["schema_version"] + else: + evidence["schema_version"] = version with pytest.raises(CertificationError, match="schema_version"): certify_with_evidence(_tmp_bundle(tmp_path), evidence) +@pytest.mark.parametrize("field", ["audited_by", "limitations"]) +def test_invisible_auditor_or_limitations_refuse(tmp_path, field): + evidence = input_evidence() + evidence[field] = "\u200b" if field == "audited_by" else ["\u200b"] + with pytest.raises(CertificationError, match="a reader can see"): + certify_with_evidence(_tmp_bundle(tmp_path), evidence) + + +@pytest.mark.parametrize("reason", [" ", "\t\n"]) +def test_a_bundle_record_with_a_blank_reason_does_not_validate(tmp_path, reason): + from pydantic import ValidationError + + bundle_path = _tmp_bundle(tmp_path) + certify_with_evidence(bundle_path, input_evidence()) + release = json.loads(bundle_path.read_text())["data_releases"]["uk"] + release["regional_certification"]["accepted_reason"] = reason + with pytest.raises(ValidationError, match="accepted_reason"): + CountryReleaseManifest.model_validate(release) + + def test_an_h5_suffix_in_any_case_must_be_measured_as_the_audit_does(): from policyengine.provenance.certification import accepted_regional_mismatch from policyengine.provenance.manifest import DataReleaseManifest From f0870aa79c4f94e58fe954e61eb4df27e1f5cc11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:08:53 +0000 Subject: [PATCH 20/21] Document the certification re-read, area receipts and generated limitation The data-certification skill says certification re-reads both models (the target installed in process, the build model with uv) and that only the stored columns rest on the auditor's word; gives the probe's pinned resolution; says certification names the defaulted inputs in the limitations; and describes the area receipt's summary and record digest. docs/receipts.md documents regional_certification and verify_regional_certification. The audit script's docstring and the changelog fragments follow. Co-Authored-By: Claude Opus 5.5 --- changelog.d/699.fixed.md | 4 +- .../accept-regional-model-mismatch.added.md | 2 +- docs/engineering/skills/data-certification.md | 51 ++++++++++++------- docs/receipts.md | 6 ++- scripts/audit_regional_input_surface.py | 11 ++-- 5 files changed, 47 insertions(+), 27 deletions(-) diff --git a/changelog.d/699.fixed.md b/changelog.d/699.fixed.md index 5d9d08cf4..08497a7f0 100644 --- a/changelog.d/699.fixed.md +++ b/changelog.d/699.fixed.md @@ -1,3 +1,3 @@ -Require a byte-bound input-surface audit when accepting a UK regional model mismatch. The audit compares the regional H5's stored columns with the regional release's build model as well as the target model: it refuses columns the build model defines and the target does not, records columns neither model defines as never modelled, and reports as defaulted only target inputs the file does not store that were not build-model inputs. The bundle, TRACE and execution receipts record the result and limitations, preserve the regional manifest digest and area build model, and keep area receipts uncertified; dropping the regional release removes its exception record. +Require a byte-bound input-surface audit when accepting a UK regional model mismatch. The audit compares the regional H5's stored columns with the regional release's build model as well as the target model: it refuses columns the build model defines and the target does not, records columns neither model defines as never modelled, and reports as defaulted only target inputs the file does not store that were not build-model inputs. Certification reads both models again rather than trusting the evidence's lists, and adds a limitation naming the defaulted inputs. The bundle and TRACE record the result and limitations and preserve the regional manifest digest and area build model; an area run's execution receipt carries a summary bound to the record's digest and stays uncertified. Dropping the regional release removes its exception record. -Refuse regional releases that explicitly name a different model package, and forward empty CLI reasons so certification rejects them. Document that the UK loader silently skips unknown columns. +Refuse regional releases that explicitly name a different model package, and forward empty CLI reasons so certification rejects them, as it rejects reasons with no visible character. Document that the UK loader silently skips unknown columns. diff --git a/changelog.d/accept-regional-model-mismatch.added.md b/changelog.d/accept-regional-model-mismatch.added.md index 0223b8a78..22626b70a 100644 --- a/changelog.d/accept-regional-model-mismatch.added.md +++ b/changelog.d/accept-regional-model-mismatch.added.md @@ -1 +1 @@ -`certify-data --accept-regional-model-mismatch REASON --regional-input-surface-evidence AUDIT.json` (UK only) keeps a local-area release routed when it was built on another policyengine-uk version, and records the reviewed mismatch as `regional_certification` in the bundle instead of refusing; area receipts are uncertified. `scripts/audit_regional_input_surface.py` writes the audit, reading the release's build model in a throwaway `uv run` environment. +`certify-data --accept-regional-model-mismatch REASON --regional-input-surface-evidence AUDIT.json` (UK only) keeps a local-area release routed when it was built on another policyengine-uk version, and records the reviewed mismatch as `regional_certification` in the bundle instead of refusing; area receipts are uncertified. `scripts/audit_regional_input_surface.py` writes the audit, reading the release's build model in a throwaway `uv run` environment, and `verify_regional_certification` checks an area receipt's record of the exception. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 669dd0e2d..fd54ffc34 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -123,18 +123,29 @@ Microcosm national release with one of: models are not changes. The audit reads the build model's variables and inputs in a throwaway - `uv run --no-project --with policyengine-uk== --with + `uv run --no-project --no-config --default-index https://pypi.org/simple + --exclude-newer --with policyengine-uk== --with policyengine-core==` environment, so it needs `uv` and access to - PyPI. A regional manifest that does not record both build versions has no - baseline, and the audit refuses it. The evidence binds the regional release - identity, original manifest bytes, actual H5 bytes, both build versions and - the exact target model version, and keeps both models' full variable and - input lists. Certification recomputes all three lists from them and refuses - missing or mismatched evidence, including schema-version-1 evidence from - before the comparison was relative to the build model. It records the - never-modelled columns, every defaulted input and the free-text limitations - in the validated `regional_certification`; the bundle hash binds that - record. + PyPI. `--exclude-newer` is the manifest's build time, so the build model's + other dependencies resolve as they did then, and every `UV_*` setting + except the cache, Python-install and connection ones is removed. A regional + manifest that does not record both build versions has no baseline, and the + audit refuses it. The evidence binds the regional release identity, + original manifest bytes, actual H5 bytes, both build versions and the exact + target model version, and keeps both models' full variable and input lists. + + Certification does not trust those lists. It reads the target model in + process, so the environment that runs `certify-data` must have the target + policyengine-uk installed, as the audit's does, and reads the build model + again with `uv`; any list or input alias that differs refuses. Only the + stored columns rest on the auditor's word: they cannot be re-read without + the H5, and the evidence binds them to its SHA-256. Certification then + recomputes all three lists and refuses missing or mismatched evidence, + including evidence without `schema_version: 2`, from before the comparison + was relative to the build model. It records the never-modelled columns, + every defaulted input and the free-text limitations in the validated + `regional_certification`, and adds one limitation naming the defaulted + inputs; the bundle hash binds that record. The policyengine-uk 2.147.2 dataset loaders silently skip columns absent from the model's variables: both the @@ -146,11 +157,15 @@ Microcosm national release with one of: certification. The national certificate never widens. Area status remains `uncertified`: - the certificate still covers the national dataset only (#685). TRACE and - execution receipts expose the regional identity, build model, acceptance, - reason, input-surface result and limitations separately from the national - certification. The flag remains UK only; an empty reason, a release that - already covers the target model, or a missing regional release refuses. + the certificate still covers the national dataset only (#685). TRACE + carries the whole regional record, evidence included, separately from the + national certification. An area run's execution receipt carries a summary + of it (both model versions, the reason, the three lists and the + limitations) and `record_sha256`, the digest of the whole record; check it + with `verify_regional_certification`. A national receipt carries none: its + `certificate_sha256` already digests the record. The flag remains UK only; + an empty or invisible reason, a release that already covers the target + model, or a missing regional release refuses. The regional core checks still apply. Re-certifying with `--drop-regional-release` removes the regional record, routes and artifact. @@ -175,8 +190,8 @@ Microcosm national release with one of: - **37 new defaults under 2.151.0:** those 16 and 21 more. The 21 include the gambling duty inputs, liquid and solid fuel consumption, and bonus sacrifice (`pension_contributions_via_bonus_sacrifice`), so area - breakdowns of those measures use defaults on 2.151.0. Record that as a - `--limitation` if the route is used on 2.151.0. + breakdowns of those measures use defaults on 2.151.0. Certification + names every defaulted input in the limitations. - Another **123 inputs already defaulted under 2.132.6**. They are absent under both models, so they are not changes; the first version listed them too, giving 139 defaults under 2.147.2 and 160 under 2.151.0. diff --git a/docs/receipts.md b/docs/receipts.md index 48686f6e8..c39bcdede 100644 --- a/docs/receipts.md +++ b/docs/receipts.md @@ -11,6 +11,7 @@ Schema v1 covers UK results and is defined by `policyengine.provenance.receipt.E | `wrapper`, `model`, `core` | The `policyengine`, `policyengine-uk` and `policyengine-core` versions that actually ran, read from the installed distributions rather than from the bundle manifest. `install_origin` records a PEP 610 direct URL as `archive`, `directory`, `editable` or `vcs`; it is absent for an ordinary index install with no `direct_url.json`. `wheel_sha256` appears only when the receipt builder verified it. One way is `wheel_paths=` (evidence `installed_record_verified`): the installed files, and any bytecode cache the interpreter would load instead of a source file, are checked against that wheel's RECORD. The other is the installer's recorded archive hash (`installer_archive_hash`), taken only when the install came from a `.whl` URL; a source archive's hash is never recorded as a wheel hash. | | `data` | The data release and build, and the model version the data was built with (`build_model_version`, distinct from `model.version`). Also the dataset's logical name, URI, revision and the SHA-256 of the bytes that were simulated. The dataset is resolved exactly as `materialize_dataset` resolves it (overlays, per-dataset repository, package and revision, and the release manifest revision when an artifact has none of its own), and the bytes must be the ones the bundle pins for it: a receipt never gives an artifact's name to other bytes. | | `certification_status`, `certificate` | `certified` only when the bundle certifies all of: the installed model and core versions, that dataset name and those bytes, and the running `policyengine` version. The UK certificate names one dataset, the national default, so a run on any other dataset, the local-area file that constituency and local-authority regions load included, gets an uncertified receipt. Each package must also have an absent or `archive` install origin, or a `directory` or `vcs` origin with `installed_record_verified` wheel evidence. A directory or VCS install without that evidence produces an `uncertified` receipt, which carries no certificate. An editable install is never certified, whatever the evidence: its wheel holds a pointer to a source tree (a `.pth` file or an import hook) rather than the source, so checking the installed files against it does not bind the code that ran. `certificate_source` records whether the builder read the certificate from the installed bundle (`packaged`) or from a manifest the caller passed (`caller_supplied`). It is the producer's statement, not evidence: trust a certificate only after `verify_certificate` (below). With the packaged bundle, the composition fingerprint of its TRACE TRO is recorded as well. | +| `regional_certification` | Present only on an area run: a run on the regional release that the bundle routes constituency and local-authority regions to on an accepted model mismatch (`data_releases.uk.regional_certification`). Such a run is uncertified. The member summarises that exception: the model the release was built with and the one area runs use, the reviewer's reason, the dropped, never-modelled and defaulted lists, and the limitations. `record_sha256` binds the whole record, its evidence included; the evidence's model surfaces stay in the bundle and its TRACE TRO. A national receipt carries none, because its `certificate_sha256` already digests the record. Check it with `verify_regional_certification` (below). | | `reform`, `baseline` | Both are required. Each is `current_law` or a digest of the reform bound to its representation (see below). | | `fiscal_year` | What a year label means: year 2027 is the fiscal year from 6 April 2027 to 5 April 2028, shown as `2027-28`. v1 allows only this convention. | | `years` | The reported years, sorted and unique. | @@ -149,6 +150,8 @@ A reader who holds the members' identities can recompute the composition's diges To check a certificate, call `verify_certificate(receipt, bundle)` with the policyengine manifest for `certificate.bundle_version`. It rebuilds the certificate from that manifest's certification record, the same way the builder does. Then it compares every identity member: the record digest, the basis, the certified model, core and dataset digest, the certifier and the bundle version. It also checks that the receipt's whole `data` identity is the certified dataset with the certified bytes. An empty list means the bundle certifies exactly what the receipt claims, and through the receipt's own rules that pins `model.version`, `core.version` and `wrapper.version` too. `certificate_source`, `qualification_receipt_sha256`, `tro_composition_fingerprint` and wheel hashes are not determined by the bundle, so it does not check them. +To check an area receipt's regional exception, call `verify_regional_certification(receipt, bundle)` with the policyengine manifest the run used. It rebuilds the summary from that manifest's `regional_certification` record, the same way the builder does, and compares every member, `record_sha256` included. It also checks that the receipt's `data` is the regional artifact that record covers, with the identity the bundle determines. An empty list means the receipt names exactly the bundle's exception; the area run stays uncertified. + Pass the release artifact that was loaded as `dataset_path`. A derived per-year file such as `enhanced_frs_2024_25_year_2026.h5` has different bytes from the artifact the bundle pins, so a receipt for it is refused. ## Digests @@ -195,7 +198,8 @@ Consumers that validate with the JSON Schema rather than the Python model must a - a `current_law` reform has no digest, while any other reform has both a representation and a digest; - a wheel digest always comes with its evidence; - a certified receipt has no `editable` package, and no `directory` or `vcs` package without `installed_record_verified` wheel evidence; -- a certificate is trusted only after it, and the receipt's `data`, have been checked against the published manifest it names (`verify_certificate`). +- a certificate is trusted only after it, and the receipt's `data`, have been checked against the published manifest it names (`verify_certificate`); +- `regional_certification` appears only on an uncertified area run, and is checked against the manifest the run used (`verify_regional_certification`). The schema itself fixes the package names (`policyengine`, `policyengine-uk`, `policyengine-core`) and the UK fiscal-year convention, which starts on 6 April. diff --git a/scripts/audit_regional_input_surface.py b/scripts/audit_regional_input_surface.py index 1ce94de05..1c18cb377 100644 --- a/scripts/audit_regional_input_surface.py +++ b/scripts/audit_regional_input_surface.py @@ -4,11 +4,12 @@ PyTables installed. Supply the H5 files from the immutable regional manifest; the audit verifies their hashes and downloads no microdata. The baseline is the build model and core the regional manifest records: the audit reads their -variables in a throwaway ``uv run --with`` environment, so it needs ``uv`` -and PyPI. Columns the build model defines and the target does not refuse; -columns neither model defines are recorded as never modelled; and target -inputs the files do not store that were not build inputs are recorded as -defaulted. +variables in a throwaway ``uv run --with`` environment, resolved from PyPI as +of the manifest's build time, so it needs ``uv`` and PyPI. Certification reads +both models again. Columns the build model defines and the target does not +refuse; columns neither model defines are recorded as never modelled; and +target inputs the files do not store that were not build inputs are recorded +as defaulted. """ from __future__ import annotations From 76519416d1c1b6b977a008fef8d11b06e5b140ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:54:11 +0000 Subject: [PATCH 21/21] Clear the runner's uv settings in the probe-environment test The test asserted that UV_CACHE_DIR was the only uv setting the build-model probe kept, but CI's setup-uv also sets UV_PYTHON_INSTALL_DIR, which the probe keeps on purpose, so Test (3.14) failed. The test now removes every UV_* variable before setting its own. Co-Authored-By: Claude Opus 5.5 --- tests/test_measure_regional_inputs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_measure_regional_inputs.py b/tests/test_measure_regional_inputs.py index 379a6197e..814c0dab9 100644 --- a/tests/test_measure_regional_inputs.py +++ b/tests/test_measure_regional_inputs.py @@ -7,6 +7,7 @@ import importlib.metadata import importlib.util import json +import os import subprocess import sys from pathlib import Path @@ -302,6 +303,9 @@ def test_an_input_absent_under_both_models_is_not_reported(tmp_path, model): def test_build_model_is_read_at_its_manifest_pins_without_the_callers_settings( tmp_path, model, monkeypatch ): + # Start from no uv settings: CI's setup-uv sets kept ones of its own. + for name in [name for name in os.environ if name.startswith("UV_")]: + monkeypatch.delenv(name) for name in ( "UV_FROZEN", "UV_PROJECT_ENVIRONMENT",