diff --git a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261023T075039Z-a133f37a.changed.md b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261023T075039Z-a133f37a.changed.md new file mode 100644 index 00000000..6d22aa77 --- /dev/null +++ b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261023T075039Z-a133f37a.changed.md @@ -0,0 +1 @@ +Certify the UK populace data release `microcosm-uk-2024-25-national-20261023T075039Z-a133f37a` (microcosm_uk_2024_25, policyengine-uk 2.147.2) into the PolicyEngine bundle manifest. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 23c5d813..b90de0af 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -245,6 +245,36 @@ differencing. withholds a standard reform's figures, so the gate cannot recompute that reform's verdict. Accepted by María Juaristi in #685. +### Same-line re-cut exception + +The Budget coordinator's **23 October 2026 decision under d1682 and d1344** +permits a same-line re-cut exception for the Budget week: the data must stay on +the same producer line, and the model must be pinned to the cut's build-time +version. For Saturday's national cut +`microcosm-uk-2024-25-national-20261023T075039Z-a133f37a`, that is +`policyengine-uk==2.147.2` (Microcosm build `829c1bc26`, on main via +[microcosm#1338](https://github.com/PolicyEngine/microcosm/pull/1338)). +Certify national-only with `--drop-regional-release`, stacked on +policyengine.py#704 (`budget-local-data-manifest`, `da2ae8fb` or a reviewed later +head). Constituency and local-authority outputs are absent until Monday's cut; +María Juaristi (`juaristi22`) can override this routing decision. + +The exception's evidence package is: all release-cut gates; the UC floor; +snapshots; the #574 reform contract and Budget-inputs gates on the exact cut, +with the coverage used by #685; the strict check; and a same-line 2×2 +attribution against the previously certified `6cba5d71` national cut on pe-uk +2.132.6. For FY2026-27 through FY2030-31, compare both data cuts under both +model versions, separating model effects on fixed data from data effects on a +fixed model. Map moves above the aggregate-impact report's thresholds to +pe-uk PRs or Microcosm changes, label unmapped moves **unattributed**, and list +limitations. Record incompatible cells as **not computable** with their error +and use the computable cells. Record scorecard replay as **unavailable** until +the Microcosm replay fix lands; unavailable is not a pass. + +This scoped decision does not inherit #685's cross-data-line comparison +exception or accept a new UC-floor gap, missing input or failed gate. Any such +acceptance needs an explicit decision and evidence in the certification PR. + ## Release runbooks Concrete, fill-in-the-id runbooks that replay a specific certification live diff --git a/pyproject.toml b/pyproject.toml index 41a98159..e3770f0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,12 +42,12 @@ graph = [ models = [ "policyengine-core==3.33.3", "policyengine-us==2.2.1", - "policyengine-uk==2.132.6", + "policyengine-uk==2.147.2", "spm-calculator==1.0.0", ] uk = [ "policyengine-core==3.33.3", - "policyengine-uk==2.132.6", + "policyengine-uk==2.147.2", ] us = [ "policyengine-core==3.33.3", @@ -72,7 +72,7 @@ dev = [ "rfc8785>=0.1.4", "policyengine-core==3.33.3", "policyengine-us==2.2.1", - "policyengine-uk==2.132.6", + "policyengine-uk==2.147.2", "spm-calculator==1.0.0", ] diff --git a/src/policyengine/data/bundle/manifest.json b/src/policyengine/data/bundle/manifest.json index 0c21eb0d..e4e720d6 100644 --- a/src/policyengine/data/bundle/manifest.json +++ b/src/policyengine/data/bundle/manifest.json @@ -16,31 +16,32 @@ }, "data_releases": { "uk": { - "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "build_id": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", "bundle_id": "uk-6.10.1", "certification": { - "built_with_model_version": "2.132.6", + "built_with_model_version": "2.147.2", "certified_by": "policyengine.py bundle certification", - "certified_for_model_version": "2.132.6", + "certified_for_core_version": "3.33.3", + "certified_for_model_version": "2.147.2", "compatibility_basis": "built_with_model_package", - "core_binding": "bundle_pin", - "data_build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + "core_binding": "certified", + "data_build_id": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a" }, "certified_data_artifact": { - "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "build_id": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", "data_package": { "name": "microcosm-data", "version": "0.1.0" }, "dataset": "microcosm_uk_2024_25", - "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece", - "uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + "sha256": "64f83900ca3a1242fb017585234844ce89f743d32e41cd2b6558057777f914aa", + "uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261023T075039Z-a133f37a" }, "country_id": "uk", "data_package": { "name": "microcosm-data", "release_manifest_path": "releases/microcosm-uk-2024-25-national/release_manifest.json", - "release_manifest_revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "release_manifest_revision": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", "repo_id": "policyengine/populace-uk-private", "repo_type": "dataset", "version": "0.1.0" @@ -50,86 +51,70 @@ "calibration_diagnostics": { "path": "releases/microcosm-uk-2024-25-national/calibration_diagnostics.json", "repo_id": "policyengine/populace-uk-private", - "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", - "sha256": "34978e2bfde3f2641769c9c3a8f60455c59ecae5a50524b359d74388d7edc888" + "revision": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", + "sha256": "0c975ad04a2d1c0207c3be480d68f645c3a1b24f786c3eef76d6e984dfea0637" }, "microcosm_uk_2024_25": { "path": "microcosm_uk_2024_25.h5", "repo_id": "policyengine/populace-uk-private", - "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", - "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece" + "revision": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", + "sha256": "64f83900ca3a1242fb017585234844ce89f743d32e41cd2b6558057777f914aa" }, "microcosm_uk_2024_25_calibration": { "path": "microcosm_uk_2024_25_calibration.npz", "repo_id": "policyengine/populace-uk-private", - "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", - "sha256": "104a8253bf1aebc91147c9bbb40e2603d1025317ba8c6b991383d5e4d91852c2" - }, - "microcosm_uk_2024_25_dense": { - "path": "microcosm_uk_2024_25_dense.h5", - "repo_id": "policyengine/populace-uk-private", - "repo_type": "dataset", - "revision": "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe", - "sha256": "8e10e130d0260e0b52e2e29902f0502ebdff76a14a3e57d2c876bf4cd8db5872" + "revision": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", + "sha256": "074872730ad5678cdb4e483d0141ece912be65d6bc032d01bf756f2c3868f2a7" }, "release_certification": { "path": "releases/microcosm-uk-2024-25-national/release_certification.json", "repo_id": "policyengine/populace-uk-private", - "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", - "sha256": "6a68cc93595fc5a7ad451d75a902523c25659de27014f50431dd5e19af5bd959" + "revision": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", + "sha256": "633661c40492645236c0d286576c6206d1fe87609ac7ce90234f625397b9260c" }, "release_cut_gates": { "path": "releases/microcosm-uk-2024-25-national/release_cut_gates.json", "repo_id": "policyengine/populace-uk-private", - "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", - "sha256": "c91c1364d9c3be98493039d4e1dd8c2256881a434c1f124dd2ed0701bfaf7751" + "revision": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", + "sha256": "19f9b8bde4cafcfd179607541c5c8fa4247c97858a4ace7a1838bbacd223bc17" }, "score_vs_enhanced_frs": { "path": "releases/microcosm-uk-2024-25-national/score_vs_enhanced_frs.json", "repo_id": "policyengine/populace-uk-private", - "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", - "sha256": "ba8a675f36e7c836bf5521e5933e21bb689442734b7340e05cabd3e0cc55323d" + "revision": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", + "sha256": "cb4dcde88bf016e0646cdc46a9ab9adf9973c77b16a0cb9e1a25987d57b212ab" }, "spine_gates": { "path": "releases/microcosm-uk-2024-25-national/spine_gates.json", "repo_id": "policyengine/populace-uk-private", - "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", - "sha256": "6dbba976dd1304c07da531382cecb11c9cb69512de0ee65126b71f032a0f37d6" + "revision": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", + "sha256": "7959f2a19b6535e67f8d9cb2c37ee68f027892ca10d9d8422be18a8261798f34" }, "terminal_gates": { "path": "releases/microcosm-uk-2024-25-national/terminal_gates.json", "repo_id": "policyengine/populace-uk-private", - "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", - "sha256": "e29439d5ce0ea098d32579284ac06f26f086feba2fb8d12d1e88ac9a54840808" + "revision": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", + "sha256": "dac50ba94464ca4e2b494fd8ffcd04e47da6dd6e49e9f53bdb7384de7983097f" } }, "default_dataset": "microcosm_uk_2024_25", - "default_dataset_uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "default_dataset_uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", "model_package": { "name": "policyengine-uk", - "sha256": "cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354", - "version": "2.132.6", - "wheel_url": "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl" + "sha256": "4b48ab1008b00610abc38192927ac9d4bb22037b6bbd9578926c2899e639f972", + "version": "2.147.2", + "wheel_url": "https://files.pythonhosted.org/packages/96/dd/54ddc9226756877e630a3f9fcda4dbf2ae142635e96a564cab3a062ced62/policyengine_uk-2.147.2-py3-none-any.whl" }, "policyengine_version": "6.10.1", "region_datasets": { - "constituency": { - "path_template": "microcosm_uk_2024_25_dense.h5" - }, - "local_authority": { - "path_template": "microcosm_uk_2024_25_dense.h5" - }, "national": { "path_template": "microcosm_uk_2024_25.h5" } }, - "regional_release_kind": "local_area", - "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", - "regional_source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", - "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", + "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261023T075039Z-a133f37a/releases/microcosm-uk-2024-25-national/release_manifest.json", "schema_version": 1, - "source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", - "version": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + "source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-national-20261023T075039Z-a133f37a/releases/microcosm-uk-2024-25-national/release_manifest.json", + "version": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a" }, "us": { "build_id": "populace-us-2024-spm-20260915", @@ -363,10 +348,10 @@ "policyengine-uk": { "country": "uk", "import_name": "policyengine_uk", - "install_requirement": "policyengine-uk==2.132.6", + "install_requirement": "policyengine-uk==2.147.2", "name": "policyengine-uk", "role": "country_model", - "version": "2.132.6" + "version": "2.147.2" }, "policyengine-us": { "country": "us", diff --git a/src/policyengine/data/bundle/uk.trace.tro.jsonld b/src/policyengine/data/bundle/uk.trace.tro.jsonld index 5030ad7a..2322b406 100644 --- a/src/policyengine/data/bundle/uk.trace.tro.jsonld +++ b/src/policyengine/data/bundle/uk.trace.tro.jsonld @@ -17,7 +17,7 @@ "schema:name": "PolicyEngine", "schema:url": "https://policyengine.org" }, - "schema:dateCreated": "2026-10-19T10:17:15.545356+00:00", + "schema:dateCreated": "2026-10-23T09:57:14.344348+00:00", "schema:description": "TRACE TRO for certified runtime bundle uk-6.10.1 covering the bundle manifest, the certified dataset artifact, the country model wheel, and the country data release manifest when it is available.", "schema:name": "policyengine uk certified bundle TRO", "trov:createdWith": { @@ -45,7 +45,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/data_release_manifest" }, - "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261023T075039Z-a133f37a/releases/microcosm-uk-2024-25-national/release_manifest.json" }, { "@id": "arrangement/1/location/dataset", @@ -53,7 +53,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/dataset" }, - "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/microcosm_uk_2024_25.h5" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261023T075039Z-a133f37a/microcosm_uk_2024_25.h5" }, { "@id": "arrangement/1/location/model_wheel", @@ -61,7 +61,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/model_wheel" }, - "trov:hasLocation": "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl" + "trov:hasLocation": "https://files.pythonhosted.org/packages/96/dd/54ddc9226756877e630a3f9fcda4dbf2ae142635e96a564cab3a062ced62/policyengine_uk-2.147.2-py3-none-any.whl" } ] } @@ -75,51 +75,52 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for uk", "trov:mimeType": "application/json", - "trov:sha256": "095203eb2b45580c7afae0a3c89201f98a4426e4a1e76719cfdc28615ed9e5ca" + "trov:sha256": "d28213451af6d538a7c742b8ad5182e00b25e784ab68638e8428adc400012f01" }, { "@id": "composition/1/artifact/data_release_manifest", "@type": "trov:ResearchArtifact", "schema:name": "microcosm-data release manifest 0.1.0", "trov:mimeType": "application/json", - "trov:sha256": "49f9fa820db2e6544ff5cdceeb2d868a504843a6a1c1c06b37188621e700066f" + "trov:sha256": "63089003fa140b729e9daf24fb430e926f7cb5bd142e3cf6bfa106115aae6aa2" }, { "@id": "composition/1/artifact/dataset", "@type": "trov:ResearchArtifact", "schema:name": "microcosm_uk_2024_25", "trov:mimeType": "application/x-hdf5", - "trov:sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece" + "trov:sha256": "64f83900ca3a1242fb017585234844ce89f743d32e41cd2b6558057777f914aa" }, { "@id": "composition/1/artifact/model_wheel", "@type": "trov:ResearchArtifact", - "schema:name": "policyengine-uk==2.132.6 wheel", + "schema:name": "policyengine-uk==2.147.2 wheel", "trov:mimeType": "application/zip", - "trov:sha256": "cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354" + "trov:sha256": "4b48ab1008b00610abc38192927ac9d4bb22037b6bbd9578926c2899e639f972" } ], "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "0ade76b548f319f98c7ce9426be439bd7c9897fa78d18fe574aab607528282cb" + "trov:sha256": "f164ec51bf8a7a347b437ecf9c461591b6f35a4c5050135352cfc96d2c2f51b8" } }, "trov:hasPerformance": { "@id": "trp/1", "@type": "trov:TransparentResearchPerformance", - "pe:builtWithModelVersion": "2.132.6", + "pe:builtWithModelVersion": "2.147.2", "pe:certifiedBy": "policyengine.py bundle certification", - "pe:certifiedForModelVersion": "2.132.6", + "pe:certifiedForCoreVersion": "3.33.3", + "pe:certifiedForModelVersion": "2.147.2", "pe:compatibilityBasis": "built_with_model_package", - "pe:coreBinding": "bundle_pin", - "pe:dataBuildId": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "pe:coreBinding": "certified", + "pe:dataBuildId": "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a", "pe:emittedIn": "repository-bundle", - "rdfs:comment": "Certification of build microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71 for policyengine-uk 2.132.6.", + "rdfs:comment": "Certification of build microcosm-uk-2024-25-national-20261023T075039Z-a133f37a for policyengine-uk 2.147.2.", "trov:accessedArrangement": { "@id": "arrangement/1" }, - "trov:startedAtTime": "2026-10-19T10:17:15.545356+00:00", + "trov:startedAtTime": "2026-10-23T09:57:14.344348+00:00", "trov:wasConductedBy": { "@id": "trs" } diff --git a/src/policyengine/data/bundle/us.trace.tro.jsonld b/src/policyengine/data/bundle/us.trace.tro.jsonld index 59b518ca..1820877b 100644 --- a/src/policyengine/data/bundle/us.trace.tro.jsonld +++ b/src/policyengine/data/bundle/us.trace.tro.jsonld @@ -74,7 +74,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "095203eb2b45580c7afae0a3c89201f98a4426e4a1e76719cfdc28615ed9e5ca" + "trov:sha256": "d28213451af6d538a7c742b8ad5182e00b25e784ab68638e8428adc400012f01" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -101,7 +101,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "85adc682b3d9684102287c08b7f06bd101b0e00fb5dd57692bb79956669371b4" + "trov:sha256": "37c146a215f35786a31a7f88d09817941507baf867c92b33fc2ff99b2b14c685" } }, "trov:hasPerformance": { diff --git a/tests/fixtures/household_calculator_snapshots/uk_couple_two_kids.json b/tests/fixtures/household_calculator_snapshots/uk_couple_two_kids.json index 28fa19e2..2d539470 100644 --- a/tests/fixtures/household_calculator_snapshots/uk_couple_two_kids.json +++ b/tests/fixtures/household_calculator_snapshots/uk_couple_two_kids.json @@ -13,14 +13,14 @@ "household.equiv_hbai_household_net_income": 52510.29, "household.fuel_duty": 0.0, "household.hbai_household_net_income": 73514.4, - "household.household_benefits": 9441.78, + "household.household_benefits": 9872.8, "household.household_count_people": 4.0, - "household.household_gross_income": 99441.78, + "household.household_gross_income": 99872.8, "household.household_id": 0.0, "household.household_income_decile": 10.0, "household.household_market_income": 90000.0, - "household.household_net_income": 80438.73, - "household.household_tax": 19003.05, + "household.household_net_income": 80869.8, + "household.household_tax": 19003.0, "household.household_wealth_decile": 10.0, "household.household_weight": 1.0, "household.in_poverty_ahc": 0.0, diff --git a/tests/fixtures/household_calculator_snapshots/uk_model_surface.json b/tests/fixtures/household_calculator_snapshots/uk_model_surface.json index 25270d57..c926e1f2 100644 --- a/tests/fixtures/household_calculator_snapshots/uk_model_surface.json +++ b/tests/fixtures/household_calculator_snapshots/uk_model_surface.json @@ -5,7 +5,7 @@ "has_income_tax": true, "has_region_registry": true, "model_package_name": "policyengine-uk", - "num_parameters_bucketed_100s": 32, - "num_variables_bucketed_100s": 11, + "num_parameters_bucketed_100s": 42, + "num_variables_bucketed_100s": 12, "region_registry_country": "uk" } diff --git a/tests/fixtures/household_calculator_snapshots/uk_single_adult_employment_income.json b/tests/fixtures/household_calculator_snapshots/uk_single_adult_employment_income.json index ab0db499..a96a4e09 100644 --- a/tests/fixtures/household_calculator_snapshots/uk_single_adult_employment_income.json +++ b/tests/fixtures/household_calculator_snapshots/uk_single_adult_employment_income.json @@ -19,8 +19,8 @@ "household.household_id": 0.0, "household.household_income_decile": 10.0, "household.household_market_income": 30000.0, - "household.household_net_income": 24939.55, - "household.household_tax": 5060.45, + "household.household_net_income": 24939.6, + "household.household_tax": 5060.4, "household.household_wealth_decile": 10.0, "household.household_weight": 1.0, "household.in_poverty_ahc": 0.0, diff --git a/tests/fixtures/household_calculator_snapshots/uk_single_adult_no_income.json b/tests/fixtures/household_calculator_snapshots/uk_single_adult_no_income.json index 391af449..2dba151c 100644 --- a/tests/fixtures/household_calculator_snapshots/uk_single_adult_no_income.json +++ b/tests/fixtures/household_calculator_snapshots/uk_single_adult_no_income.json @@ -19,8 +19,8 @@ "household.household_id": 0.0, "household.household_income_decile": 10.0, "household.household_market_income": 0.0, - "household.household_net_income": 4918.75, - "household.household_tax": 180.04, + "household.household_net_income": 4918.8, + "household.household_tax": 180.0, "household.household_wealth_decile": 10.0, "household.household_weight": 1.0, "household.in_poverty_ahc": 1.0, diff --git a/tests/fixtures/household_calculator_snapshots/uk_single_parent_one_child.json b/tests/fixtures/household_calculator_snapshots/uk_single_parent_one_child.json index c0d339da..3c4e5025 100644 --- a/tests/fixtures/household_calculator_snapshots/uk_single_parent_one_child.json +++ b/tests/fixtures/household_calculator_snapshots/uk_single_parent_one_child.json @@ -19,8 +19,8 @@ "household.household_id": 0.0, "household.household_income_decile": 10.0, "household.household_market_income": 25000.0, - "household.household_net_income": 24342.45, - "household.household_tax": 3660.45, + "household.household_net_income": 24342.5, + "household.household_tax": 3660.4, "household.household_wealth_decile": 10.0, "household.household_weight": 1.0, "household.in_poverty_ahc": 0.0, diff --git a/tests/test_models.py b/tests/test_models.py index 682925a0..4c38e3a2 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -43,13 +43,13 @@ def test_has_release_manifest_metadata(self): assert uk_latest.release_manifest is not None assert uk_latest.release_manifest.country_id == "uk" assert uk_latest.model_package.name == "policyengine-uk" - assert uk_latest.model_package.version == "2.132.6" + assert uk_latest.model_package.version == "2.147.2" assert uk_latest.data_package.name == "microcosm-data" assert uk_latest.data_package.version == "0.1.0" assert ( uk_latest.default_dataset_uri == "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" - "@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + "@microcosm-uk-2024-25-national-20261023T075039Z-a133f37a" ) def test_has_hundreds_of_parameters(self): diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 006070e2..1ffe0dfd 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -73,11 +73,11 @@ "hf://policyengine/populace-us/populace_us_2024_acs_local.h5" f"@{US_LOCAL_AREA_RELEASE_ID}" ) -UK_MODEL_VERSION = "2.132.6" -UK_BUILT_WITH_MODEL_VERSION = "2.132.6" +UK_MODEL_VERSION = "2.147.2" +UK_BUILT_WITH_MODEL_VERSION = "2.147.2" UK_DATA_RELEASE_VERSION = "0.1.0" -UK_DATA_RELEASE_ID = "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" -UK_DATA_RELEASE_REVISION = "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" +UK_DATA_RELEASE_ID = "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a" +UK_DATA_RELEASE_REVISION = "microcosm-uk-2024-25-national-20261023T075039Z-a133f37a" UK_DATA_RELEASE_PATH = "releases/microcosm-uk-2024-25-national/release_manifest.json" UK_LOCAL_AREA_RELEASE_ID = "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe" # The prior certified default (policyengine-uk-data 1.58.1), kept resolvable @@ -336,22 +336,31 @@ def test__given_uk_dataset_name__then_resolves_to_versioned_hf_url(self): assert resolved == UK_CERTIFIED_DATASET_URI - def test__given_uk_manifest__then_area_regions_route_to_local_release(self): + def test__given_uk_manifest__then_area_regions_follow_the_regional_release( + self, + ): manifest = get_release_manifest("uk") - - assert manifest.regional_release_kind == "local_area" - assert { + region_paths = { region_type: template.path_template for region_type, template in manifest.region_datasets.items() - } == { - "national": "microcosm_uk_2024_25.h5", - "constituency": "microcosm_uk_2024_25_dense.h5", - "local_authority": "microcosm_uk_2024_25_dense.h5", } - assert ( - resolve_dataset_reference("uk", "microcosm_uk_2024_25_dense") - == UK_LOCAL_AREA_DATASET_URI - ) + + if manifest.regional_release_kind == "local_area": + assert region_paths == { + "national": "microcosm_uk_2024_25.h5", + "constituency": "microcosm_uk_2024_25_dense.h5", + "local_authority": "microcosm_uk_2024_25_dense.h5", + } + assert ( + resolve_dataset_reference("uk", "microcosm_uk_2024_25_dense") + == UK_LOCAL_AREA_DATASET_URI + ) + else: + # A national-only cut (--drop-regional-release) routes no area + # regions and bundles no local-area dataset. + assert manifest.regional_release_kind is None + assert region_paths == {"national": "microcosm_uk_2024_25.h5"} + assert "microcosm_uk_2024_25_dense" not in manifest.datasets def test__given_uk_legacy_dataset_names__then_resolves_bundled_aliases(self): assert ( diff --git a/tests/test_uk_geography_disclosure.py b/tests/test_uk_geography_disclosure.py index 07334692..e3e4cb6e 100644 --- a/tests/test_uk_geography_disclosure.py +++ b/tests/test_uk_geography_disclosure.py @@ -467,7 +467,10 @@ def test_the_adapter_is_exported_from_outputs(): def test_a_missing_area_column_names_the_dataset_that_carries_it(): - from policyengine.provenance.manifest import uk_area_dataset_hint + from policyengine.provenance.manifest import ( + get_release_manifest, + uk_area_dataset_hint, + ) baseline, reform = _households() for frame in (baseline, reform): @@ -475,7 +478,41 @@ def test_a_missing_area_column_names_the_dataset_that_carries_it(): with pytest.raises(ValueError, match="no household an area") as error: _compute(baseline, reform, levels=("constituency",)) hint = uk_area_dataset_hint("constituency") - assert "microcosm_uk_2024_25_dense" in hint + if "constituency" in get_release_manifest("uk").region_datasets: + assert "microcosm_uk_2024_25_dense" in hint + assert hint in str(error.value) + else: + # A national-only cut routes no area regions, so there is no + # dataset to name. + assert hint == "" + assert "certified bundle routes" not in str(error.value) + + +def test_a_routed_area_names_its_dataset_in_the_error(monkeypatch): + """Whatever the packaged routing, a routed area's error names its dataset.""" + from policyengine.provenance import manifest + + record = manifest.get_release_manifest("uk") + data = record.model_dump() + data["region_datasets"]["constituency"] = { + "path_template": "microcosm_uk_2024_25_dense.h5" + } + data["datasets"]["microcosm_uk_2024_25_dense"] = { + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe", + } + routed = type(record).model_validate(data) + monkeypatch.setattr(manifest, "get_release_manifest", lambda country: routed) + + baseline, reform = _households() + for frame in (baseline, reform): + del frame["constituency_code_oa"] + with pytest.raises(ValueError, match="no household an area") as error: + _compute(baseline, reform, levels=("constituency",)) + hint = manifest.uk_area_dataset_hint("constituency") + assert "'microcosm_uk_2024_25_dense'" in hint assert hint in str(error.value) diff --git a/tests/test_uk_regions.py b/tests/test_uk_regions.py index 29c6b27e..82ede9df 100644 --- a/tests/test_uk_regions.py +++ b/tests/test_uk_regions.py @@ -15,6 +15,7 @@ ) from policyengine.countries.uk.regions import ( UK_COUNTRIES, + area_regions_routed_to_local_release, build_uk_region_registry, uk_region_registry, ) @@ -24,13 +25,19 @@ # release its constituency and local-authority regions are routed to. UK_CERTIFIED_NATIONAL_URI = ( "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" - "@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + "@microcosm-uk-2024-25-national-20261023T075039Z-a133f37a" ) UK_CERTIFIED_LOCAL_AREA_URI = ( "hf://policyengine/populace-uk-private/microcosm_uk_2024_25_dense.h5" "@microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe" ) UK_AREA_REGION_COUNT = 650 + 361 +# Whether the packaged bundle routes constituency and local-authority regions +# to a local-area release. A national-only cut (--drop-regional-release) +# routes none, so the default registry holds only the national region and the +# four countries, and area lookups find nothing. +ROUTED_TO_LOCAL_RELEASE = area_regions_routed_to_local_release() +ROUTED_AREA_REGION_COUNT = UK_AREA_REGION_COUNT if ROUTED_TO_LOCAL_RELEASE else 0 class TestUKCountries: @@ -180,65 +187,88 @@ def test__given_northern_ireland_region__then_filters_from_national(self): assert ni.requires_filter assert ni.scoping_strategy.variable_value == "NORTHERN_IRELAND" - def test__given_uk_national__then_children_are_countries_and_areas(self): + def test__given_uk_national__then_children_are_countries_and_routed_areas( + self, + ): """Given: UK national region When: Getting its children - Then: The 4 countries, 650 constituencies and 361 local authorities + Then: The 4 countries, and the 650 constituencies and 361 local + authorities only when the bundle routes them to a local-area release """ # When uk_children = uk_region_registry.get_children("uk") # Then - assert Counter(c.region_type for c in uk_children) == { - "country": 4, - "constituency": 650, - "local_authority": 361, - } + expected = {"country": 4} + if ROUTED_TO_LOCAL_RELEASE: + expected |= {"constituency": 650, "local_authority": 361} + assert Counter(c.region_type for c in uk_children) == expected def test__given_uk_registry__then_national_and_area_regions_have_datasets( self, ): - """Given: UK region registry routed to a Microcosm local-area release + """Given: UK region registry When: Getting dataset regions - Then: National loads the national dataset, area regions the local one + Then: National loads the national dataset; area regions load the + local-area release only when the bundle routes them to one """ # When dataset_regions = uk_region_registry.get_dataset_regions() # Then - assert len(dataset_regions) == 1 + UK_AREA_REGION_COUNT + assert len(dataset_regions) == 1 + ROUTED_AREA_REGION_COUNT paths = {r.region_type: set() for r in dataset_regions} for region in dataset_regions: paths[region.region_type].add(region.dataset_path) - assert paths == { - "national": {UK_CERTIFIED_NATIONAL_URI}, - "constituency": {UK_CERTIFIED_LOCAL_AREA_URI}, - "local_authority": {UK_CERTIFIED_LOCAL_AREA_URI}, - } + expected = {"national": {UK_CERTIFIED_NATIONAL_URI}} + if ROUTED_TO_LOCAL_RELEASE: + expected |= { + "constituency": {UK_CERTIFIED_LOCAL_AREA_URI}, + "local_authority": {UK_CERTIFIED_LOCAL_AREA_URI}, + } + assert paths == expected def test__given_uk_registry__then_countries_and_areas_filter(self): """Given: UK region registry When: Getting filter regions - Then: The 4 countries and every area region require a filter + Then: The 4 countries, and every routed area region, require a filter """ # When filter_regions = uk_region_registry.get_filter_regions() # Then - assert len(filter_regions) == 4 + UK_AREA_REGION_COUNT - assert {r.region_type for r in filter_regions} == { - "country", - "constituency", - "local_authority", - } - - def test__given_default_registry__then_has_1016_regions(self): - """Given: Default UK registry routed to a Microcosm local-area release + assert len(filter_regions) == 4 + ROUTED_AREA_REGION_COUNT + expected = {"country"} + if ROUTED_TO_LOCAL_RELEASE: + expected |= {"constituency", "local_authority"} + assert {r.region_type for r in filter_regions} == expected + + def test__given_default_registry__then_counts_the_routed_regions(self): + """Given: Default UK registry When: Counting regions - Then: 1 national + 4 countries + 650 constituencies + 361 LAs = 1,016 + Then: 1 national + 4 countries, plus 650 constituencies + 361 LAs + (1,016 in all) when the bundle routes them to a local-area release + """ + # Then + assert len(uk_region_registry) == 5 + ROUTED_AREA_REGION_COUNT + + def test__given_default_registry__then_area_lookups_follow_the_routing(self): + """Given: Default UK registry + When: Looking up a constituency and a local authority + Then: Both load the local-area release when the bundle routes them to + one; a national-only bundle has neither """ + # When + constituency = uk_region_registry.get("constituency/E14001063") + local_authority = uk_region_registry.get("local_authority/N09000011") + # Then - assert len(uk_region_registry) == 5 + UK_AREA_REGION_COUNT + if ROUTED_TO_LOCAL_RELEASE: + assert constituency.dataset_path == UK_CERTIFIED_LOCAL_AREA_URI + assert local_authority.dataset_path == UK_CERTIFIED_LOCAL_AREA_URI + else: + assert constituency is None + assert local_authority is None class TestUKRegionRegistryBuilder: diff --git a/uv.lock b/uv.lock index 7009bd4c..8f3e6ed9 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.11, <3.15" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -197,7 +197,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser" }, + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -1776,7 +1776,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -1894,9 +1894,9 @@ requires-dist = [ { name = "policyengine-core", marker = "extra == 'models'", specifier = "==3.33.3" }, { name = "policyengine-core", marker = "extra == 'uk'", specifier = "==3.33.3" }, { name = "policyengine-core", marker = "extra == 'us'", specifier = "==3.33.3" }, - { name = "policyengine-uk", marker = "extra == 'dev'", specifier = "==2.132.6" }, - { name = "policyengine-uk", marker = "extra == 'models'", specifier = "==2.132.6" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = "==2.132.6" }, + { name = "policyengine-uk", marker = "extra == 'dev'", specifier = "==2.147.2" }, + { name = "policyengine-uk", marker = "extra == 'models'", specifier = "==2.147.2" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = "==2.147.2" }, { name = "policyengine-us", marker = "extra == 'dev'", specifier = "==2.2.1" }, { name = "policyengine-us", marker = "extra == 'models'", specifier = "==2.2.1" }, { name = "policyengine-us", marker = "extra == 'us'", specifier = "==2.2.1" }, @@ -1946,7 +1946,7 @@ wheels = [ [[package]] name = "policyengine-uk" -version = "2.132.6" +version = "2.147.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1954,9 +1954,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3f/88/bc06328a739b335d4ef5f2d07fd36cc6a3a5e0f6ec1b2c774a14e77e43a3/policyengine_uk-2.132.6.tar.gz", hash = "sha256:f9ce261d9881eee32fab96341c7080cd0c86717b40785b5cdaff44b07262a771", size = 2560464, upload-time = "2026-10-15T10:08:14.263Z" } +sdist = { url = "https://files.pythonhosted.org/packages/89/16/14fe43a9e0323eb29e2877e7860356d7ae682dd89575973adaa4f25a3e33/policyengine_uk-2.147.2.tar.gz", hash = "sha256:7a718b03cb09e21d7e00cfe655b9a2082af5edba747471b5aec20e651e1cc180", size = 3191436, upload-time = "2026-10-22T18:40:49.878Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl", hash = "sha256:cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354", size = 3882707, upload-time = "2026-10-15T10:08:11.572Z" }, + { url = "https://files.pythonhosted.org/packages/96/dd/54ddc9226756877e630a3f9fcda4dbf2ae142635e96a564cab3a062ced62/policyengine_uk-2.147.2-py3-none-any.whl", hash = "sha256:4b48ab1008b00610abc38192927ac9d4bb22037b6bbd9578926c2899e639f972", size = 4705225, upload-time = "2026-10-22T18:40:47.418Z" }, ] [[package]]