From a66e883c4a08d8f111af4ed2fa0e66b2331d7f5f Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 17:50:32 +0100 Subject: [PATCH 01/26] Certify the Microcosm UK national and local-area pair with policyengine-uk 2.132.6 scripts/bundle.py certify-data on the 19 October national cut (microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71), with its local-area cut (microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe) as the regional release, at policyengine-uk 2.132.6 and policyengine-core 3.33.3, the versions both cuts were built with. Constituency and local-authority regions route to the local-area file and join the default UK registry. The enhanced FRS 2024-25 datasets stay loadable by name through dataset_overlays at 1.58.1. uv.lock was re-resolved with uv 0.12.14 (only policyengine-core and policyengine-uk move), and both TRO sidecars were regenerated. The tests that pinned the enhanced FRS, and the UK model-surface snapshot, now pin this pair; the receipt test helper also points a certificate's named core at the installed core. Co-Authored-By: Claude Opus 5.5 --- ...ional-20261019T083323Z-6cba5d71.changed.md | 1 + pyproject.toml | 14 +- src/policyengine/data/bundle/manifest.json | 187 +++++++++++------- .../data/bundle/uk.trace.tro.jsonld | 38 ++-- .../data/bundle/us.trace.tro.jsonld | 4 +- .../uk_model_surface.json | 2 +- tests/test_bundle.py | 22 ++- tests/test_bundle_metadata.py | 10 +- tests/test_dataset_runtime.py | 8 +- tests/test_execution_receipt.py | 33 +++- tests/test_models.py | 10 +- tests/test_release_manifests.py | 85 +++++--- tests/test_uk_regions.py | 90 ++++++--- uv.lock | 26 +-- 14 files changed, 341 insertions(+), 189 deletions(-) create mode 100644 changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md diff --git a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md new file mode 100644 index 000000000..8f9319421 --- /dev/null +++ b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md @@ -0,0 +1 @@ +Switch the UK default dataset from the Enhanced FRS 2024-25 (policyengine-uk-data 1.58.1) to Microcosm UK 2024-25 (`microcosm_uk_2024_25`, cut `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71`), certified with policyengine-uk 2.132.6 and policyengine-core 3.33.3, the versions it was built with. Constituency and local-authority regions now load the Microcosm local-area release (`microcosm_uk_2024_25_dense`, cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`) and are in the default UK region registry. The Enhanced FRS 2024-25 datasets stay loadable by name. UK results change with the data. diff --git a/pyproject.toml b/pyproject.toml index d7bc0a456..f56599608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,17 +40,17 @@ graph = [ "networkx>=3.0", ] models = [ - "policyengine-core==3.33.0", + "policyengine-core==3.33.3", "policyengine-us==2.2.1", - "policyengine-uk==2.130.1", + "policyengine-uk==2.132.6", "spm-calculator==1.0.0", ] uk = [ - "policyengine-core==3.33.0", - "policyengine-uk==2.130.1", + "policyengine-core==3.33.3", + "policyengine-uk==2.132.6", ] us = [ - "policyengine-core==3.33.0", + "policyengine-core==3.33.3", "policyengine-us==2.2.1", "spm-calculator==1.0.0", ] @@ -70,9 +70,9 @@ dev = [ "pytest-cov>=5.0.0", "hypothesis>=6.100.0", "rfc8785>=0.1.4", - "policyengine-core==3.33.0", + "policyengine-core==3.33.3", "policyengine-us==2.2.1", - "policyengine-uk==2.130.1", + "policyengine-uk==2.132.6", "spm-calculator==1.0.0", ] diff --git a/src/policyengine/data/bundle/manifest.json b/src/policyengine/data/bundle/manifest.json index cc56a3ebc..8a2f3fca5 100644 --- a/src/policyengine/data/bundle/manifest.json +++ b/src/policyengine/data/bundle/manifest.json @@ -16,99 +16,120 @@ }, "data_releases": { "uk": { - "build_id": "policyengine-uk-data-1.58.1", + "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", "bundle_id": "uk-6.9.0", "certification": { - "built_with_model_git_sha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", - "built_with_model_version": "2.122.2", + "built_with_model_version": "2.132.6", "certified_by": "policyengine.py bundle certification", - "certified_for_model_version": "2.130.1", - "compatibility_basis": "legacy_compatible_model_package", - "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", - "data_build_id": "policyengine-uk-data-1.58.1" + "certified_for_core_version": "3.33.3", + "certified_for_model_version": "2.132.6", + "compatibility_basis": "built_with_model_package", + "data_build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" }, "certified_data_artifact": { - "build_id": "policyengine-uk-data-1.58.1", + "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", "data_package": { - "name": "policyengine-uk-data", - "version": "1.58.1" + "name": "microcosm-data", + "version": "0.1.0" }, - "dataset": "enhanced_frs_2024_25", - "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", - "uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1" + "dataset": "microcosm_uk_2024_25", + "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece", + "uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" }, "country_id": "uk", "data_package": { - "name": "policyengine-uk-data", - "release_manifest_path": "releases/1.58.1/release_manifest.json", - "release_manifest_revision": "89be6b73fc1e6eeba6dae2f9400676d8510bcd51", - "repo_id": "policyengine/policyengine-uk-data-private", - "repo_type": "model", - "version": "1.58.1" + "name": "microcosm-data", + "release_manifest_path": "releases/microcosm-uk-2024-25-national/release_manifest.json", + "release_manifest_revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "version": "0.1.0" }, "data_producer": "populace", "datasets": { - "enhanced_frs_2024_25": { - "path": "enhanced_frs_2024_25.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + "calibration_diagnostics": { + "path": "releases/microcosm-uk-2024-25-national/calibration_diagnostics.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "34978e2bfde3f2641769c9c3a8f60455c59ecae5a50524b359d74388d7edc888" + }, + "microcosm_uk_2024_25": { + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece" + }, + "microcosm_uk_2024_25_calibration": { + "path": "microcosm_uk_2024_25_calibration.npz", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "104a8253bf1aebc91147c9bbb40e2603d1025317ba8c6b991383d5e4d91852c2" }, - "enhanced_frs_2024_25_tiny": { - "path": "enhanced_frs_2024_25_tiny.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "d5d2bde80f1f028fec98391ea31bb5a0cf8449e3867fc3c1bfe23693b9faecf8" + "microcosm_uk_2024_25_dense": { + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe", + "sha256": "8e10e130d0260e0b52e2e29902f0502ebdff76a14a3e57d2c876bf4cd8db5872" }, - "frs_2024_25": { - "path": "frs_2024_25.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "97af00b2cd2739957f40dcc4de8e0383a6a91d47952c90d4b4921a6007bd2021" + "release_certification": { + "path": "releases/microcosm-uk-2024-25-national/release_certification.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "6a68cc93595fc5a7ad451d75a902523c25659de27014f50431dd5e19af5bd959" }, - "frs_2024_25_tiny": { - "path": "frs_2024_25_tiny.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "e4d7662cd3f053620224b7ceff17266783f09407002a72633b043b844cec3f5a" + "release_cut_gates": { + "path": "releases/microcosm-uk-2024-25-national/release_cut_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "c91c1364d9c3be98493039d4e1dd8c2256881a434c1f124dd2ed0701bfaf7751" }, - "local_authority_weights": { - "path": "local_authority_weights.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "2b7a2f7f9e7081a394b37cb84e909fd3c01d22e421808141684e7c3be59f2089" + "score_vs_enhanced_frs": { + "path": "releases/microcosm-uk-2024-25-national/score_vs_enhanced_frs.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "ba8a675f36e7c836bf5521e5933e21bb689442734b7340e05cabd3e0cc55323d" }, - "local_geography_weights": { - "path": "local_geography_weights.csv.gz", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "97c6e3542543baf737a713bb918ebc3d3dbc3f3765be9bf3414375c322d0e3d1" + "spine_gates": { + "path": "releases/microcosm-uk-2024-25-national/spine_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "6dbba976dd1304c07da531382cecb11c9cb69512de0ee65126b71f032a0f37d6" }, - "parliamentary_constituency_weights": { - "path": "parliamentary_constituency_weights.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "8c696e4d65a22033deb4c4f69a767fdb46e0a4f5580f36b7fb47ee4722629b11" + "terminal_gates": { + "path": "releases/microcosm-uk-2024-25-national/terminal_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "e29439d5ce0ea098d32579284ac06f26f086feba2fb8d12d1e88ac9a54840808" } }, - "default_dataset": "enhanced_frs_2024_25", - "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1", + "default_dataset": "microcosm_uk_2024_25", + "default_dataset_uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", "model_package": { "name": "policyengine-uk", - "sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873", - "version": "2.130.1", - "wheel_url": "https://files.pythonhosted.org/packages/1a/21/88e8addba0eabd9a7d3a29fa5293e141feb405baf0432209387f9a1f467a/policyengine_uk-2.130.1-py3-none-any.whl" + "sha256": "cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354", + "version": "2.132.6", + "wheel_url": "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl" }, "policyengine_version": "6.9.0", "region_datasets": { + "constituency": { + "path_template": "microcosm_uk_2024_25_dense.h5" + }, + "local_authority": { + "path_template": "microcosm_uk_2024_25_dense.h5" + }, "national": { - "path_template": "enhanced_frs_2024_25.h5" + "path_template": "microcosm_uk_2024_25.h5" } }, - "release_manifest_uri": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json", + "regional_release_kind": "local_area", + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", + "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", "schema_version": 1, - "source_manifest_uri": "hf://model/policyengine/policyengine-uk-data-private@89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json", - "version": "policyengine-uk-data-1.58.1" + "source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", + "version": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" }, "us": { "build_id": "populace-us-2024-spm-20260915", @@ -247,6 +268,22 @@ "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" }, + "enhanced_frs_2024_25": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + }, + "enhanced_frs_2024_25_tiny": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "d5d2bde80f1f028fec98391ea31bb5a0cf8449e3867fc3c1bfe23693b9faecf8" + }, "frs_2023_24": { "data_package_name": "policyengine-uk-data", "path": "frs_2023_24.h5", @@ -255,6 +292,22 @@ "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", "sha256": "df26d4d7af9d164aa2d064181b39290292d2f62bb26fee6126fc095fc06da292" }, + "frs_2024_25": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "97af00b2cd2739957f40dcc4de8e0383a6a91d47952c90d4b4921a6007bd2021" + }, + "frs_2024_25_tiny": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "e4d7662cd3f053620224b7ceff17266783f09407002a72633b043b844cec3f5a" + }, "populace_uk_2023": { "data_package_name": "populace-data", "path": "populace_uk_2023.h5", @@ -302,18 +355,18 @@ }, "policyengine-core": { "import_name": "policyengine_core", - "install_requirement": "policyengine-core==3.33.0", + "install_requirement": "policyengine-core==3.33.3", "name": "policyengine-core", "role": "runtime_dependency", - "version": "3.33.0" + "version": "3.33.3" }, "policyengine-uk": { "country": "uk", "import_name": "policyengine_uk", - "install_requirement": "policyengine-uk==2.130.1", + "install_requirement": "policyengine-uk==2.132.6", "name": "policyengine-uk", "role": "country_model", - "version": "2.130.1" + "version": "2.132.6" }, "policyengine-us": { "country": "us", diff --git a/src/policyengine/data/bundle/uk.trace.tro.jsonld b/src/policyengine/data/bundle/uk.trace.tro.jsonld index 66d99811a..c1dbd1475 100644 --- a/src/policyengine/data/bundle/uk.trace.tro.jsonld +++ b/src/policyengine/data/bundle/uk.trace.tro.jsonld @@ -17,7 +17,7 @@ "schema:name": "PolicyEngine", "schema:url": "https://policyengine.org" }, - "schema:dateCreated": "2026-10-14T07:43:49.920588Z", + "schema:dateCreated": "2026-10-19T10:17:15.545356+00:00", "schema:description": "TRACE TRO for certified runtime bundle uk-6.9.0 covering the bundle manifest, the certified dataset artifact, the country model wheel, and the country data release manifest when it is available.", "schema:name": "policyengine uk certified bundle TRO", "trov:createdWith": { @@ -45,7 +45,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/data_release_manifest" }, - "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json" }, { "@id": "arrangement/1/location/dataset", @@ -53,7 +53,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/dataset" }, - "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/1.58.1/enhanced_frs_2024_25.h5" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/microcosm_uk_2024_25.h5" }, { "@id": "arrangement/1/location/model_wheel", @@ -61,7 +61,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/model_wheel" }, - "trov:hasLocation": "https://files.pythonhosted.org/packages/1a/21/88e8addba0eabd9a7d3a29fa5293e141feb405baf0432209387f9a1f467a/policyengine_uk-2.130.1-py3-none-any.whl" + "trov:hasLocation": "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl" } ] } @@ -75,52 +75,50 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for uk", "trov:mimeType": "application/json", - "trov:sha256": "c32e4566725879cda855ec3852e951ee9fda7146e288d5baf05ae628ae49acdd" + "trov:sha256": "ef8235c112d4aee8436a90d5a562d3459ce05ed46a391b0daaa5693d7518acc7" }, { "@id": "composition/1/artifact/data_release_manifest", "@type": "trov:ResearchArtifact", - "schema:name": "policyengine-uk-data release manifest 1.58.1", + "schema:name": "microcosm-data release manifest 0.1.0", "trov:mimeType": "application/json", - "trov:sha256": "545797bf231cd3681e64b677065d921304e9a32f1e46e757a940a412b4136c17" + "trov:sha256": "49f9fa820db2e6544ff5cdceeb2d868a504843a6a1c1c06b37188621e700066f" }, { "@id": "composition/1/artifact/dataset", "@type": "trov:ResearchArtifact", - "schema:name": "enhanced_frs_2024_25", + "schema:name": "microcosm_uk_2024_25", "trov:mimeType": "application/x-hdf5", - "trov:sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + "trov:sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece" }, { "@id": "composition/1/artifact/model_wheel", "@type": "trov:ResearchArtifact", - "schema:name": "policyengine-uk==2.130.1 wheel", + "schema:name": "policyengine-uk==2.132.6 wheel", "trov:mimeType": "application/zip", - "trov:sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873" + "trov:sha256": "cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354" } ], "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "e9b6a021cef492d7e045394a0d814345793fd141e40d9c715723c8cb49963b69" + "trov:sha256": "a94a4050211eb02ab9c22629da54c887355c28015f40301122462882691511d8" } }, "trov:hasPerformance": { "@id": "trp/1", "@type": "trov:TransparentResearchPerformance", - "pe:builtWithModelGitSha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", - "pe:builtWithModelVersion": "2.122.2", + "pe:builtWithModelVersion": "2.132.6", "pe:certifiedBy": "policyengine.py bundle certification", - "pe:certifiedForModelVersion": "2.130.1", - "pe:compatibilityBasis": "legacy_compatible_model_package", - "pe:dataBuildFingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", - "pe:dataBuildId": "policyengine-uk-data-1.58.1", + "pe:certifiedForModelVersion": "2.132.6", + "pe:compatibilityBasis": "built_with_model_package", + "pe:dataBuildId": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", "pe:emittedIn": "repository-bundle", - "rdfs:comment": "Certification of build policyengine-uk-data-1.58.1 for policyengine-uk 2.130.1.", + "rdfs:comment": "Certification of build microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71 for policyengine-uk 2.132.6.", "trov:accessedArrangement": { "@id": "arrangement/1" }, - "trov:startedAtTime": "2026-10-14T07:43:49.920588Z", + "trov:startedAtTime": "2026-10-19T10:17:15.545356+00:00", "trov:wasConductedBy": { "@id": "trs" } diff --git a/src/policyengine/data/bundle/us.trace.tro.jsonld b/src/policyengine/data/bundle/us.trace.tro.jsonld index 0b5e8cf35..8df5f80b8 100644 --- a/src/policyengine/data/bundle/us.trace.tro.jsonld +++ b/src/policyengine/data/bundle/us.trace.tro.jsonld @@ -74,7 +74,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "c32e4566725879cda855ec3852e951ee9fda7146e288d5baf05ae628ae49acdd" + "trov:sha256": "ef8235c112d4aee8436a90d5a562d3459ce05ed46a391b0daaa5693d7518acc7" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -101,7 +101,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "86eb651ac706bcb1858e4b6b995f56f5e8fb868c9ae00ce25cee0207528012c5" + "trov:sha256": "6856a00e80cd939b8aac6d5b8c22110f81668cb621172cbe32565f84f9f8f918" } }, "trov:hasPerformance": { diff --git a/tests/fixtures/household_calculator_snapshots/uk_model_surface.json b/tests/fixtures/household_calculator_snapshots/uk_model_surface.json index e3d367c18..25270d576 100644 --- a/tests/fixtures/household_calculator_snapshots/uk_model_surface.json +++ b/tests/fixtures/household_calculator_snapshots/uk_model_surface.json @@ -1,6 +1,6 @@ { "country_id": "uk", - "data_package_name": "policyengine-uk-data", + "data_package_name": "microcosm-data", "has_employment_income": true, "has_income_tax": true, "has_region_registry": true, diff --git a/tests/test_bundle.py b/tests/test_bundle.py index b30db02ea..e04fb0eb2 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -36,16 +36,15 @@ def test_bundle_manifest_exposes_data_releases(): assert manifest["bundle_version"] == manifest["policyengine_version"] assert manifest["data_releases"]["us"]["data_producer"] == "populace" assert manifest["data_releases"]["us"]["version"].startswith("populace-us-2024-") - # UK certified default reverted to the policyengine-uk-data enhanced FRS per - # the UC caseload postmortem (policyengine-uk-data discussion #464); the - # "populace" producer label names the strict verification contract, not the - # producing repo. + # The UK certified default is a Microcosm national release; the "populace" + # producer label names the strict verification contract, not the producing + # repo. assert manifest["data_releases"]["uk"]["data_producer"] == "populace" assert manifest["data_releases"]["uk"]["version"].startswith( - "policyengine-uk-data-" + "microcosm-uk-2024-25-national-" ) assert manifest["data_releases"]["uk"]["default_dataset_uri"].startswith( - "hf://policyengine/policyengine-uk-data-private/enhanced_frs_" + "hf://policyengine/populace-uk-private/microcosm_uk_" ) @@ -89,11 +88,14 @@ def test_selected_dataset_plan_uses_certified_release_metadata(tmp_path): bundle.get_current_bundle(), ["uk"], data_dir=tmp_path ) + # Only the national default: area regions' local-area dataset is not + # installed with the bundle. + assert [entry_plan.dataset for entry_plan, _ in entries] == ["microcosm_uk_2024_25"] plan, release = entries[0] assert plan.country_id == "uk" - assert plan.data_package_name == "policyengine-uk-data" - assert plan.repo_type == "model" - assert plan.destination == tmp_path / "enhanced_frs_2024_25.h5" + assert plan.data_package_name == "microcosm-data" + assert plan.repo_type == "dataset" + assert plan.destination == tmp_path / "microcosm_uk_2024_25.h5" assert plan.sha256 == release["datasets"][plan.dataset]["sha256"] @@ -202,7 +204,7 @@ def fake_materialize(plan): ) assert [plan.country_id for plan in calls] == ["uk"] - assert result["datasets"][0]["data_package_name"] == "policyengine-uk-data" + assert result["datasets"][0]["data_package_name"] == "microcosm-data" assert result["datasets"][0]["installed_sha256"] == calls[0].sha256 receipt = bundle.read_receipt(tmp_path) assert receipt is not None diff --git a/tests/test_bundle_metadata.py b/tests/test_bundle_metadata.py index 85dd2d016..cdcda8fd3 100644 --- a/tests/test_bundle_metadata.py +++ b/tests/test_bundle_metadata.py @@ -66,12 +66,12 @@ def test_bundle_manifest_carries_uk_data_release(): release = bundle.get_current_bundle()["data_releases"]["uk"] assert release["data_producer"] == "populace" - assert release["data_package"]["name"] == "policyengine-uk-data" - assert release["data_package"]["repo_type"] == "model" - assert release["default_dataset"] == "enhanced_frs_2024_25" - assert release["version"].startswith("policyengine-uk-data-") + assert release["data_package"]["name"] == "microcosm-data" + assert release["data_package"]["repo_type"] == "dataset" + assert release["default_dataset"] == "microcosm_uk_2024_25" + assert release["version"].startswith("microcosm-uk-2024-25-national-") assert release["default_dataset_uri"].startswith( - "hf://policyengine/policyengine-uk-data-private/" + "hf://policyengine/populace-uk-private/" ) diff --git a/tests/test_dataset_runtime.py b/tests/test_dataset_runtime.py index 76cab79b6..44083c06d 100644 --- a/tests/test_dataset_runtime.py +++ b/tests/test_dataset_runtime.py @@ -143,15 +143,15 @@ def test_uk_load_datasets_defaults_to_certified_bundle_dataset(monkeypatch): ) resolve = Mock( return_value=( - "hf://policyengine/policyengine-uk-data-private/" - "enhanced_frs_2024_25.h5@1.56.16" + "hf://policyengine/populace-uk-private/" + "microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-cut" ) ) monkeypatch.setattr(uk_datasets, "resolve_dataset_reference", resolve) assert uk_datasets.load_datasets(years=[]) == {} - resolve.assert_called_once_with("uk", "enhanced_frs_2024_25") + resolve.assert_called_once_with("uk", "microcosm_uk_2024_25") def test_uk_ensure_datasets_defaults_to_certified_bundle_dataset(monkeypatch): @@ -165,7 +165,7 @@ def test_uk_ensure_datasets_defaults_to_certified_bundle_dataset(monkeypatch): assert uk_datasets.ensure_datasets(years=[]) == {} load.assert_called_once_with( - datasets=["enhanced_frs_2024_25"], + datasets=["microcosm_uk_2024_25"], years=[], data_folder="./data", ) diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 33e9b5e58..b5be39e15 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -132,6 +132,8 @@ def _bundle_for( record["certified_data_artifact"]["sha256"] = digest record["datasets"][record["default_dataset"]]["sha256"] = digest record["certification"]["certified_for_model_version"] = model_version + if "certified_for_core_version" in record["certification"]: + record["certification"]["certified_for_core_version"] = core_version bundle["packages"]["policyengine-core"]["version"] = core_version bundle["policyengine_version"] = INSTALLED_WRAPPER record["policyengine_version"] = INSTALLED_WRAPPER @@ -303,10 +305,15 @@ def test_any_uncovered_component_makes_the_receipt_uncertified( elif change == "dataset name": # Same bytes registered under another certified dataset name. record = bundle["data_releases"]["uk"] - record["datasets"]["frs_2024_25"]["sha256"] = record["certified_data_artifact"][ + other = next( + name + for name, reference in record["datasets"].items() + if name != record["default_dataset"] and reference["path"].endswith(".h5") + ) + record["datasets"][other]["sha256"] = record["certified_data_artifact"][ "sha256" ] - kwargs["dataset"] = "frs_2024_25" + kwargs["dataset"] = other receipt = _build(path, bundle, **kwargs) assert receipt.certification_status == "uncertified" assert receipt.certificate is None @@ -548,6 +555,16 @@ def test_a_forged_receipt_with_the_genuine_record_digest_does_not_verify( assert any("data.dataset_sha256" in p for p in problems) +#: A valid compatibility basis other than the committed certificate's. +_OTHER_BASIS = next( + basis + for basis in ("built_with_model_package", "legacy_compatible_model_package") + if basis + != json.loads(BUNDLE_PATH.read_text())["data_releases"]["uk"]["certification"][ + "compatibility_basis" + ] +) + #: Consistent edits a forger could make, each of which the model accepts. FORGERIES = { "bundle version": { @@ -567,7 +584,7 @@ def test_a_forged_receipt_with_the_genuine_record_digest_does_not_verify( "certificate__certified_dataset_sha256": "e" * 64, }, "record digest": {"certificate__certificate_sha256": "e" * 64}, - "basis": {"certificate__compatibility_basis": "built_with_model_package"}, + "basis": {"certificate__compatibility_basis": _OTHER_BASIS}, "certifier": {"certificate__certified_by": "someone else"}, "dataset name": {"data__dataset": "frs_2024_25"}, "data release": {"data__release_version": "9.9.9"}, @@ -621,6 +638,14 @@ def test_every_consistent_forgery_is_reported(tmp_path_factory, names): assert verify_certificate(_forge(genuine, **changes), bundle) +def _certify_core(bundle: dict, version: str) -> None: + """Change the core version a bundle certifies, wherever it states it.""" + certification = bundle["data_releases"]["uk"]["certification"] + if "certified_for_core_version" in certification: + certification["certified_for_core_version"] = version + bundle["packages"]["policyengine-core"].update(version=version) + + @pytest.mark.parametrize( "edit", [ @@ -633,7 +658,7 @@ def test_every_consistent_forgery_is_reported(tmp_path_factory, names): ), lambda b: b["data_releases"]["uk"].update(policyengine_version="0.0.0"), lambda b: b.update(policyengine_version="0.0.0"), - lambda b: b["packages"]["policyengine-core"].update(version="0.0.1"), + lambda b: _certify_core(b, "0.0.1"), ], ids=[ "certifier", diff --git a/tests/test_models.py b/tests/test_models.py index c2ad28752..682925a09 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -43,13 +43,13 @@ def test_has_release_manifest_metadata(self): assert uk_latest.release_manifest is not None assert uk_latest.release_manifest.country_id == "uk" assert uk_latest.model_package.name == "policyengine-uk" - assert uk_latest.model_package.version == "2.130.1" - assert uk_latest.data_package.name == "policyengine-uk-data" - assert uk_latest.data_package.version == "1.58.1" + assert uk_latest.model_package.version == "2.132.6" + assert uk_latest.data_package.name == "microcosm-data" + assert uk_latest.data_package.version == "0.1.0" assert ( uk_latest.default_dataset_uri - == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5" - "@1.58.1" + == "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" + "@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" ) def test_has_hundreds_of_parameters(self): diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 2586c593d..265dd4df1 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -73,12 +73,21 @@ "hf://policyengine/populace-us/populace_us_2024_acs_local.h5" f"@{US_LOCAL_AREA_RELEASE_ID}" ) -UK_MODEL_VERSION = "2.130.1" -UK_BUILT_WITH_MODEL_VERSION = "2.122.2" -UK_DATA_RELEASE_VERSION = "1.58.1" -UK_DATA_RELEASE_ID = "policyengine-uk-data-1.58.1" -UK_DATA_RELEASE_REVISION = "89be6b73fc1e6eeba6dae2f9400676d8510bcd51" -UK_DATA_RELEASE_PATH = "releases/1.58.1/release_manifest.json" +UK_MODEL_VERSION = "2.132.6" +UK_BUILT_WITH_MODEL_VERSION = "2.132.6" +UK_DATA_RELEASE_VERSION = "0.1.0" +UK_DATA_RELEASE_ID = "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" +UK_DATA_RELEASE_REVISION = "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" +UK_DATA_RELEASE_PATH = "releases/microcosm-uk-2024-25-national/release_manifest.json" +UK_LOCAL_AREA_RELEASE_ID = "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe" +# The prior certified default (policyengine-uk-data 1.58.1), kept resolvable +# via dataset_overlays. +UK_EFRS_DATASET_URI = ( + "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1" +) +UK_EFRS_FRS_DATASET_URI = ( + "hf://policyengine/policyengine-uk-data-private/frs_2024_25.h5@1.58.1" +) # The prior certified default, kept resolvable via dataset_overlays. UK_POPULACE_RELEASE_ID = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" UK_POPULACE_DATASET_URI = ( @@ -87,7 +96,12 @@ ) UK_CERTIFICATION_SOURCE = "policyengine.py bundle certification" UK_CERTIFIED_DATASET_URI = ( - "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1" + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" + f"@{UK_DATA_RELEASE_ID}" +) +UK_LOCAL_AREA_DATASET_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25_dense.h5" + f"@{UK_LOCAL_AREA_RELEASE_ID}" ) @@ -226,25 +240,20 @@ def test__given_uk_manifest__then_has_pinned_model_and_data_packages(self): assert manifest.policyengine_version == POLICYENGINE_VERSION assert manifest.model_package.name == "policyengine-uk" assert manifest.model_package.version == UK_MODEL_VERSION - assert manifest.data_package.name == "policyengine-uk-data" + assert manifest.data_package.name == "microcosm-data" assert manifest.data_package.version == UK_DATA_RELEASE_VERSION - assert manifest.data_package.repo_id == ( - "policyengine/policyengine-uk-data-private" - ) + assert manifest.data_package.repo_id == "policyengine/populace-uk-private" assert manifest.data_package.release_manifest_path == UK_DATA_RELEASE_PATH assert ( manifest.data_package.release_manifest_revision == UK_DATA_RELEASE_REVISION ) assert manifest.certified_data_artifact is not None assert manifest.certified_data_artifact.build_id == UK_DATA_RELEASE_ID - assert manifest.certified_data_artifact.dataset == "enhanced_frs_2024_25" + assert manifest.certified_data_artifact.dataset == "microcosm_uk_2024_25" assert manifest.certified_data_artifact.uri == UK_CERTIFIED_DATASET_URI assert manifest.certification is not None assert manifest.certification.data_build_id == UK_DATA_RELEASE_ID - assert ( - manifest.certification.compatibility_basis - == "legacy_compatible_model_package" - ) + assert manifest.certification.compatibility_basis == "built_with_model_package" assert manifest.certification.certified_by == UK_CERTIFICATION_SOURCE assert ( manifest.certification.built_with_model_version @@ -277,10 +286,27 @@ def test__given_dataset_explicit_revision__then_resolves_to_that_revision(self): ) def test__given_uk_dataset_name__then_resolves_to_versioned_hf_url(self): - resolved = resolve_dataset_reference("uk", "enhanced_frs_2024_25") + resolved = resolve_dataset_reference("uk", "microcosm_uk_2024_25") assert resolved == UK_CERTIFIED_DATASET_URI + def test__given_uk_manifest__then_area_regions_route_to_local_release(self): + manifest = get_release_manifest("uk") + + assert manifest.regional_release_kind == "local_area" + assert { + region_type: template.path_template + for region_type, template in manifest.region_datasets.items() + } == { + "national": "microcosm_uk_2024_25.h5", + "constituency": "microcosm_uk_2024_25_dense.h5", + "local_authority": "microcosm_uk_2024_25_dense.h5", + } + assert ( + resolve_dataset_reference("uk", "microcosm_uk_2024_25_dense") + == UK_LOCAL_AREA_DATASET_URI + ) + def test__given_uk_legacy_dataset_names__then_resolves_bundled_aliases(self): assert ( resolve_dataset_reference("uk", "frs_2023_24") == UK_LEGACY_FRS_DATASET_URI @@ -293,6 +319,11 @@ def test__given_uk_legacy_dataset_names__then_resolves_bundled_aliases(self): resolve_dataset_reference("uk", "populace_uk_2023") == UK_POPULACE_DATASET_URI ) + assert ( + resolve_dataset_reference("uk", "enhanced_frs_2024_25") + == UK_EFRS_DATASET_URI + ) + assert resolve_dataset_reference("uk", "frs_2024_25") == UK_EFRS_FRS_DATASET_URI def test__given_explicit_url__then_resolution_is_noop(self): url = "hf://policyengine/policyengine-us-data/cps_2023.h5@1.73.0" @@ -915,11 +946,11 @@ def test__given_manifest_certification__then_release_bundle_exposes_it(self): bundle = model_version.release_bundle assert bundle["bundle_id"] == f"uk-{POLICYENGINE_VERSION}" - assert bundle["default_dataset"] == "enhanced_frs_2024_25" + assert bundle["default_dataset"] == "microcosm_uk_2024_25" assert bundle["default_dataset_uri"] == manifest.default_dataset_uri assert bundle["certified_data_build_id"] == UK_DATA_RELEASE_ID assert bundle["data_build_model_version"] == UK_BUILT_WITH_MODEL_VERSION - assert bundle["compatibility_basis"] == "legacy_compatible_model_package" + assert bundle["compatibility_basis"] == "built_with_model_package" assert bundle["certified_by"] == UK_CERTIFICATION_SOURCE def test__given_runtime_certification__then_release_bundle_prefers_runtime_value( @@ -1099,30 +1130,30 @@ def test__given_uk_managed_dataset_name__then_resolves_within_bundle(self): "materialize_dataset", return_value=_dataset_source( "uk", - "enhanced_frs_2024_25", - "/tmp/enhanced_frs_2024_25.h5", + "microcosm_uk_2024_25", + "/tmp/microcosm_uk_2024_25.h5", ), ): microsim = uk_model.managed_microsimulation( - dataset="enhanced_frs_2024_25" + dataset="microcosm_uk_2024_25" ) dataset = mock_microsimulation.call_args.kwargs["dataset"] - assert dataset == "/tmp/enhanced_frs_2024_25.h5" + assert dataset == "/tmp/microcosm_uk_2024_25.h5" assert ( microsim.policyengine_bundle["policyengine_version"] == POLICYENGINE_VERSION ) - assert microsim.policyengine_bundle["runtime_dataset"] == "enhanced_frs_2024_25" + assert microsim.policyengine_bundle["runtime_dataset"] == "microcosm_uk_2024_25" assert microsim.policyengine_bundle["runtime_dataset_uri"] == ( UK_CERTIFIED_DATASET_URI ) dataset_source = microsim.policyengine_bundle["runtime_dataset_source"] - assert dataset_source == "/tmp/enhanced_frs_2024_25.h5" + assert dataset_source == "/tmp/microcosm_uk_2024_25.h5" assert ( microsim.policyengine_bundle["runtime_dataset_sha256"] - == get_release_manifest("uk").datasets["enhanced_frs_2024_25"].sha256 + == get_release_manifest("uk").datasets["microcosm_uk_2024_25"].sha256 ) - assert microsim.policyengine_bundle["runtime_dataset_repo_type"] == "model" + assert microsim.policyengine_bundle["runtime_dataset_repo_type"] == "dataset" def test__given_uk_unmanaged_dataset_uri__then_source_is_not_rewritten(self): dataset = "hf://policyengine/policyengine-uk-data-private/frs_2022_23.h5@1.40.4" diff --git a/tests/test_uk_regions.py b/tests/test_uk_regions.py index 77d250bcb..29c6b27eb 100644 --- a/tests/test_uk_regions.py +++ b/tests/test_uk_regions.py @@ -1,5 +1,6 @@ """Tests for UK region definitions.""" +from collections import Counter from unittest.mock import patch import pandas as pd @@ -19,6 +20,18 @@ ) from policyengine.provenance.manifest import CountryReleaseManifest +# The certified bundle's national dataset, and the Microcosm local-area +# release its constituency and local-authority regions are routed to. +UK_CERTIFIED_NATIONAL_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" + "@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" +) +UK_CERTIFIED_LOCAL_AREA_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25_dense.h5" + "@microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe" +) +UK_AREA_REGION_COUNT = 650 + 361 + class TestUKCountries: """Tests for UK country definitions.""" @@ -78,11 +91,7 @@ def test__given_uk_registry__then_has_national_region(self): assert national.code == "uk" assert national.label == "United Kingdom" assert national.region_type == "national" - assert ( - national.dataset_path - == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5" - "@1.58.1" - ) + assert national.dataset_path == UK_CERTIFIED_NATIONAL_URI assert not national.requires_filter def test__given_uk_registry__then_has_four_country_regions(self): @@ -171,49 +180,65 @@ def test__given_northern_ireland_region__then_filters_from_national(self): assert ni.requires_filter assert ni.scoping_strategy.variable_value == "NORTHERN_IRELAND" - def test__given_uk_national__then_children_are_countries(self): + def test__given_uk_national__then_children_are_countries_and_areas(self): """Given: UK national region When: Getting its children - Then: All children are country regions + Then: The 4 countries, 650 constituencies and 361 local authorities """ # When uk_children = uk_region_registry.get_children("uk") # Then - assert len(uk_children) == 4 - assert all(c.region_type == "country" for c in uk_children) + assert Counter(c.region_type for c in uk_children) == { + "country": 4, + "constituency": 650, + "local_authority": 361, + } - def test__given_uk_registry__then_only_national_has_dataset(self): - """Given: UK region registry + def test__given_uk_registry__then_national_and_area_regions_have_datasets( + self, + ): + """Given: UK region registry routed to a Microcosm local-area release When: Getting dataset regions - Then: Only national has dedicated dataset + Then: National loads the national dataset, area regions the local one """ # When dataset_regions = uk_region_registry.get_dataset_regions() # Then - assert len(dataset_regions) == 1 - assert dataset_regions[0].code == "uk" + assert len(dataset_regions) == 1 + UK_AREA_REGION_COUNT + paths = {r.region_type: set() for r in dataset_regions} + for region in dataset_regions: + paths[region.region_type].add(region.dataset_path) + assert paths == { + "national": {UK_CERTIFIED_NATIONAL_URI}, + "constituency": {UK_CERTIFIED_LOCAL_AREA_URI}, + "local_authority": {UK_CERTIFIED_LOCAL_AREA_URI}, + } - def test__given_uk_registry__then_filter_regions_are_countries(self): + def test__given_uk_registry__then_countries_and_areas_filter(self): """Given: UK region registry When: Getting filter regions - Then: All 4 countries require filter + Then: The 4 countries and every area region require a filter """ # When filter_regions = uk_region_registry.get_filter_regions() # Then - assert len(filter_regions) == 4 - assert all(r.region_type == "country" for r in filter_regions) + assert len(filter_regions) == 4 + UK_AREA_REGION_COUNT + assert {r.region_type for r in filter_regions} == { + "country", + "constituency", + "local_authority", + } - def test__given_default_registry__then_has_5_regions(self): - """Given: Default UK registry + def test__given_default_registry__then_has_1016_regions(self): + """Given: Default UK registry routed to a Microcosm local-area release When: Counting regions - Then: Contains 1 national + 4 countries = 5 + Then: 1 national + 4 countries + 650 constituencies + 361 LAs = 1,016 """ # Then - assert len(uk_region_registry) == 5 + assert len(uk_region_registry) == 5 + UK_AREA_REGION_COUNT class TestUKRegionRegistryBuilder: @@ -373,19 +398,36 @@ def _uk_routed_manifest(kind: str) -> CountryReleaseManifest: ) +def _uk_unrouted_manifest() -> CountryReleaseManifest: + """A certified national bundle that routes no area regions.""" + manifest = _uk_routed_manifest("local_area") + return manifest.model_copy( + update={ + "region_datasets": {"national": manifest.region_datasets["national"]}, + "regional_release_kind": None, + } + ) + + class TestUKAreaRegionRouting: """Area regions load the dataset certification routes them to.""" def test__given_unrouted_bundle__then_area_regions_filter_the_national_dataset( - self, + self, monkeypatch ): + monkeypatch.setattr( + "policyengine.provenance.manifest.get_release_manifest", + lambda country_id: _uk_unrouted_manifest(), + ) + + default = build_uk_region_registry() registry = build_uk_region_registry( include_constituencies=True, include_local_authorities=True ) assert registry.get("constituency/E14001063").dataset_path is None assert registry.get("local_authority/N09000011").dataset_path is None - assert len(uk_region_registry.get_by_type("constituency")) == 0 + assert len(default) == 5 def test__given_national_first_bundle__then_area_regions_load_the_efrs( self, monkeypatch diff --git a/uv.lock b/uv.lock index a5807ea08..38ce23003 100644 --- a/uv.lock +++ b/uv.lock @@ -1881,13 +1881,13 @@ requires-dist = [ { name = "pandas", specifier = ">=2.0.0" }, { name = "plotly", marker = "extra == 'dev'", specifier = ">=5.0.0" }, { name = "plotly", marker = "extra == 'plotting'", specifier = ">=5.0.0" }, - { name = "policyengine-core", marker = "extra == 'dev'", specifier = "==3.33.0" }, - { name = "policyengine-core", marker = "extra == 'models'", specifier = "==3.33.0" }, - { name = "policyengine-core", marker = "extra == 'uk'", specifier = "==3.33.0" }, - { name = "policyengine-core", marker = "extra == 'us'", specifier = "==3.33.0" }, - { name = "policyengine-uk", marker = "extra == 'dev'", specifier = "==2.130.1" }, - { name = "policyengine-uk", marker = "extra == 'models'", specifier = "==2.130.1" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = "==2.130.1" }, + { name = "policyengine-core", marker = "extra == 'dev'", specifier = "==3.33.3" }, + { name = "policyengine-core", marker = "extra == 'models'", specifier = "==3.33.3" }, + { name = "policyengine-core", marker = "extra == 'uk'", specifier = "==3.33.3" }, + { name = "policyengine-core", marker = "extra == 'us'", specifier = "==3.33.3" }, + { name = "policyengine-uk", marker = "extra == 'dev'", specifier = "==2.132.6" }, + { name = "policyengine-uk", marker = "extra == 'models'", specifier = "==2.132.6" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = "==2.132.6" }, { name = "policyengine-us", marker = "extra == 'dev'", specifier = "==2.2.1" }, { name = "policyengine-us", marker = "extra == 'models'", specifier = "==2.2.1" }, { name = "policyengine-us", marker = "extra == 'us'", specifier = "==2.2.1" }, @@ -1909,7 +1909,7 @@ provides-extras = ["plotting", "graph", "models", "uk", "us", "dev"] [[package]] name = "policyengine-core" -version = "3.33.0" +version = "3.33.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -1930,14 +1930,14 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/90/d7/70936abfab724399ea3d8db04bbd5808339ee1534819bf25ab46d00c8d32/policyengine_core-3.33.0.tar.gz", hash = "sha256:5c923828dc84a7a1ff8f6f407f7a704fe067246dd277ede04c15f409804b9c19", size = 674785, upload-time = "2026-10-10T09:43:12.12Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/af/e8449e8a488468e5ce28b6b9066a2372e05bc7971ad9a441eb4e6efbb740/policyengine_core-3.33.3.tar.gz", hash = "sha256:7da78ae9cc7b1d6833dc30651a0a4775edd2d5e205a73bbafcc0b15c988c880e", size = 693203, upload-time = "2026-10-12T20:53:17.299Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/02/190e06717785ff974120a9136c6c5408a00beaa8cddd792f557a6bf974c4/policyengine_core-3.33.0-py3-none-any.whl", hash = "sha256:2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789", size = 299495, upload-time = "2026-10-10T09:43:10.318Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/253cc99d853e36996f264138f4f8a2c662047da3eda8e37ddf39959397fc/policyengine_core-3.33.3-py3-none-any.whl", hash = "sha256:5847d9938412095fe9848c2ce535838a7dc8fac9238d8fd9e6c9d0a0648e0888", size = 301905, upload-time = "2026-10-12T20:53:15.706Z" }, ] [[package]] name = "policyengine-uk" -version = "2.130.1" +version = "2.132.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1945,9 +1945,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/27/82a97f5ec344d4be34db17bd26fe35077f1fe22f15c99818514d704262e9/policyengine_uk-2.130.1.tar.gz", hash = "sha256:1ce0a924a847b4e3e8ea8ba9702a84b267a0490c84a55ef94da6ed2efd965b34", size = 2496922, upload-time = "2026-10-14T01:58:47.232Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/88/bc06328a739b335d4ef5f2d07fd36cc6a3a5e0f6ec1b2c774a14e77e43a3/policyengine_uk-2.132.6.tar.gz", hash = "sha256:f9ce261d9881eee32fab96341c7080cd0c86717b40785b5cdaff44b07262a771", size = 2560464, upload-time = "2026-10-15T10:08:14.263Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/21/88e8addba0eabd9a7d3a29fa5293e141feb405baf0432209387f9a1f467a/policyengine_uk-2.130.1-py3-none-any.whl", hash = "sha256:f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873", size = 3804651, upload-time = "2026-10-14T01:58:44.937Z" }, + { url = "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl", hash = "sha256:cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354", size = 3882707, upload-time = "2026-10-15T10:08:11.572Z" }, ] [[package]] From 0dc28b48baaf48b0d708ea3bbed1b91d1e12c826 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 17:50:32 +0100 Subject: [PATCH 02/26] Describe the Microcosm UK default in the docs Co-Authored-By: Claude Opus 5.5 --- docs/countries.md | 2 +- docs/microsim.md | 23 +++++++++++++---------- docs/regions.md | 10 +++++++--- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/docs/countries.md b/docs/countries.md index 4e2227a58..256ac9077 100644 --- a/docs/countries.md +++ b/docs/countries.md @@ -33,7 +33,7 @@ Override in any output with `income_variable=`. | | Dataset | |---|---| | US | Populace US 2024 (`populace_us_2024.h5`) | -| UK | Enhanced FRS 2024-25 (`enhanced_frs_2024_25.h5`, policyengine-uk-data) | +| UK | Microcosm UK 2024-25 (`microcosm_uk_2024_25.h5`); constituency and local-authority regions use its local-area release (`microcosm_uk_2024_25_dense.h5`) | ## State / regional breakdown diff --git a/docs/microsim.md b/docs/microsim.md index 14d5c6c1a..b0aaf8988 100644 --- a/docs/microsim.md +++ b/docs/microsim.md @@ -136,9 +136,11 @@ dataset = datasets["populace_us_2024_2026"] The default US dataset is **Populace US 2024** — a Populace-built dataset calibrated to IRS, CMS, SNAP, Census, and other administrative totals. The -current UK certified default is **Enhanced FRS 2024–25**, supplied by -`policyengine-uk-data`. **Populace UK 2023** remains available as a named, -non-default bundle dataset. +current UK certified default is **Microcosm UK 2024–25**, a Microcosm build +calibrated to HMRC, DWP, OBR and other administrative totals; constituency and +local-authority regions use its local-area release. **Enhanced FRS 2024–25** +(`enhanced_frs_2024_25`, policyengine-uk-data 1.58.1) and **Populace UK 2023** +remain available as named, non-default bundle datasets. PolicyEngine.py obtains the repository type, immutable revision, and SHA-256 from the installed release bundle. An existing file in the configured data @@ -199,9 +201,10 @@ ca = Simulation( ### UK private data and raw h5 access UK population data uses licensed Family Resources Survey inputs. The default -UK release bundle points to the private -`policyengine/policyengine-uk-data-private` Hugging Face repository. Set -`HUGGING_FACE_TOKEN` to a token from a Hugging Face account with access: +UK release bundle points to the private `policyengine/populace-uk-private` +Hugging Face repository, and the Enhanced FRS datasets to +`policyengine/policyengine-uk-data-private`. Set `HUGGING_FACE_TOKEN` to a +token from a Hugging Face account with access: ```bash export HUGGING_FACE_TOKEN=hf_... @@ -216,11 +219,11 @@ import policyengine as pe from policyengine.core import Simulation datasets = pe.uk.ensure_datasets( - datasets=["enhanced_frs_2024_25"], + datasets=["microcosm_uk_2024_25"], years=[2026], data_folder="./data", ) -dataset = datasets["enhanced_frs_2024_25_2026"] +dataset = datasets["microcosm_uk_2024_25_2026"] simulation = Simulation( dataset=dataset, @@ -237,7 +240,7 @@ from policyengine.provenance import materialize_dataset result = materialize_dataset( "uk", - "enhanced_frs_2024_25", + "microcosm_uk_2024_25", ) print(result.path) @@ -261,7 +264,7 @@ projected year's tables to policyengine-uk as observed data would make the State Pension follow CPI instead. Each projected year file therefore keeps its data year, `dataset.data_year` -(2024 for Enhanced FRS 2024–25), and that year's tables, +(2024 for Microcosm UK 2024–25), and that year's tables, `dataset.data_year_data`. `Simulation.run()` projects the data year's tables forward as policyengine-uk does, and uses the file's own tables for the simulated year. A run of a year file then gives the same result, record by diff --git a/docs/regions.md b/docs/regions.md index 242c9bed8..5c94ab969 100644 --- a/docs/regions.md +++ b/docs/regions.md @@ -102,12 +102,16 @@ when nothing is routed: | `regional_release_kind` | Area regions load | In the default registry | | --- | --- | --- | -| none (today) | the national dataset | no | +| none | the national dataset | no | | `area_coded_national` | another release's area-coded national dataset (the enhanced FRS), while the default is a national file without area codes | no | | `local_area` | a certified local-area release | yes | -Pass `include_constituencies=True` and `include_local_authorities=True` to -`build_uk_region_registry` to include them regardless. +The certified bundle routes both to the Microcosm local-area release +(`local_area`, `microcosm_uk_2024_25_dense.h5`), so the default registry has +all 650 constituencies and 361 local authorities. Pass +`include_constituencies=True` and `include_local_authorities=True` to +`build_uk_region_registry` to include them regardless of routing, or `False` +to leave them out. ```python from policyengine.countries.uk.regions import build_uk_region_registry From e6e852e97dcaa59e2f1f6830f5d2c0b44bb6fd15 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 18:32:14 +0100 Subject: [PATCH 03/26] Qualify uk-data candidates against the last uk-data-line bundle in tests The qualification tests build their candidates on the policyengine-uk-data line and read the packaged bundle, which is now certified on the Microcosm line, so every such candidate became other_line. They now read a frozen copy of the last bundle main certified on the uk-data line (2591af3e), including through the candidate CLI's BUNDLE_PATH. The constant-rewrite test and the real-CLI replay test still read the packaged bundle: the constant files pin it, and the rewrite now takes it to a successor cut on its own line. Co-Authored-By: Claude Opus 5.5 --- .../uk_qualification/uk_data_line_bundle.json | 339 ++++++++++++++++++ tests/qualification_fixtures.py | 6 + tests/test_uk_qualification_pipeline.py | 45 ++- tests/test_uk_qualification_workflow.py | 3 + 4 files changed, 380 insertions(+), 13 deletions(-) create mode 100644 tests/fixtures/uk_qualification/uk_data_line_bundle.json diff --git a/tests/fixtures/uk_qualification/uk_data_line_bundle.json b/tests/fixtures/uk_qualification/uk_data_line_bundle.json new file mode 100644 index 000000000..c0e4ee8c3 --- /dev/null +++ b/tests/fixtures/uk_qualification/uk_data_line_bundle.json @@ -0,0 +1,339 @@ +{ + "bundle_version": "6.8.0", + "citation": { + "publisher": "PolicyEngine", + "title": "PolicyEngine bundle 6.8.0", + "type": "software-bundle", + "version": "6.8.0" + }, + "countries": { + "uk": { + "model_package": "policyengine-uk" + }, + "us": { + "model_package": "policyengine-us" + } + }, + "data_releases": { + "uk": { + "build_id": "policyengine-uk-data-1.58.1", + "bundle_id": "uk-6.8.0", + "certification": { + "built_with_model_git_sha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", + "built_with_model_version": "2.122.2", + "certified_by": "policyengine.py bundle certification", + "certified_for_model_version": "2.130.1", + "compatibility_basis": "legacy_compatible_model_package", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "data_build_id": "policyengine-uk-data-1.58.1" + }, + "certified_data_artifact": { + "build_id": "policyengine-uk-data-1.58.1", + "data_package": { + "name": "policyengine-uk-data", + "version": "1.58.1" + }, + "dataset": "enhanced_frs_2024_25", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1" + }, + "country_id": "uk", + "data_package": { + "name": "policyengine-uk-data", + "release_manifest_path": "releases/1.58.1/release_manifest.json", + "release_manifest_revision": "89be6b73fc1e6eeba6dae2f9400676d8510bcd51", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "version": "1.58.1" + }, + "data_producer": "populace", + "datasets": { + "enhanced_frs_2024_25": { + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + }, + "enhanced_frs_2024_25_tiny": { + "path": "enhanced_frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "d5d2bde80f1f028fec98391ea31bb5a0cf8449e3867fc3c1bfe23693b9faecf8" + }, + "frs_2024_25": { + "path": "frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "97af00b2cd2739957f40dcc4de8e0383a6a91d47952c90d4b4921a6007bd2021" + }, + "frs_2024_25_tiny": { + "path": "frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "e4d7662cd3f053620224b7ceff17266783f09407002a72633b043b844cec3f5a" + }, + "local_authority_weights": { + "path": "local_authority_weights.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "2b7a2f7f9e7081a394b37cb84e909fd3c01d22e421808141684e7c3be59f2089" + }, + "local_geography_weights": { + "path": "local_geography_weights.csv.gz", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "97c6e3542543baf737a713bb918ebc3d3dbc3f3765be9bf3414375c322d0e3d1" + }, + "parliamentary_constituency_weights": { + "path": "parliamentary_constituency_weights.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "8c696e4d65a22033deb4c4f69a767fdb46e0a4f5580f36b7fb47ee4722629b11" + } + }, + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1", + "model_package": { + "name": "policyengine-uk", + "sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873", + "version": "2.130.1", + "wheel_url": "https://files.pythonhosted.org/packages/1a/21/88e8addba0eabd9a7d3a29fa5293e141feb405baf0432209387f9a1f467a/policyengine_uk-2.130.1-py3-none-any.whl" + }, + "policyengine_version": "6.8.0", + "region_datasets": { + "national": { + "path_template": "enhanced_frs_2024_25.h5" + } + }, + "release_manifest_uri": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json", + "schema_version": 1, + "source_manifest_uri": "hf://model/policyengine/policyengine-uk-data-private@89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json", + "version": "policyengine-uk-data-1.58.1" + }, + "us": { + "build_id": "populace-us-2024-spm-20260915", + "bundle_id": "us-6.8.0", + "certification": { + "built_with_model_version": "2.2.1", + "certified_by": "policyengine.py bundle certification", + "certified_for_model_version": "2.2.1", + "compatibility_basis": "built_with_model_package", + "data_build_id": "populace-us-2024-spm-20260915" + }, + "certified_data_artifact": { + "build_id": "populace-us-2024-spm-20260915", + "data_package": { + "name": "microcosm-data", + "version": "0.1.0" + }, + "dataset": "populace_us_2024", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915" + }, + "country_id": "us", + "data_package": { + "name": "microcosm-data", + "release_manifest_path": "releases/populace-us-2024-spm-20260915/release_manifest.json", + "release_manifest_revision": "populace-us-2024-spm-20260915", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "version": "0.1.0" + }, + "data_producer": "populace", + "datasets": { + "build_manifest": { + "path": "releases/populace-us-2024-spm-20260915/build_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "f3fd734867324bca7619f5f083a8115e32104bea3948d48723b345cfd39fa03b" + }, + "calibration_diagnostics": { + "path": "releases/populace-us-2024-spm-20260915/calibration_diagnostics.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "870449b44e86b13b25bcea1a57f0e7af37f4d4db18be815eea3acdf9fe6eb40e" + }, + "parent_build_manifest": { + "path": "releases/populace-us-2024-spm-20260915/parent_build_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "1990e8fd37ce66f499b8e6600c5896701bb07015be132a12de64e321a2edc67e" + }, + "parent_release_manifest": { + "path": "releases/populace-us-2024-spm-20260915/parent_release_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "dd949ba3c4c7a56aff8442c6db5a031d2c84c3da59f5d14832f468c358604506" + }, + "populace_us_2024": { + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" + }, + "populace_us_2024_acs_local": { + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "revision": "populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z", + "sha256": "3ce781c440a7f933b1d952ff4e9d9f65f4698c64b94d2fad165af0c662838d6c" + }, + "source_enrichment": { + "path": "releases/populace-us-2024-spm-20260915/source_enrichment.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "85329907b2b9c4299855b0970b1f24ede5819d3b16c312eb7400b4203bd71203" + }, + "source_enrichment_compatibility": { + "path": "releases/populace-us-2024-spm-20260915/source_enrichment_compatibility.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "63109acffd486453c99c476dece6b0154cc5388e5a430830a30744a2249444d1" + }, + "source_spm_independence_provenance": { + "path": "releases/populace-us-2024-spm-20260915/source_spm_independence_provenance.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "265e844178ed3856d68c1638820f3089e0e1e1363aa4677e33bebb37cbd5bff3" + }, + "source_spm_person_independence": { + "path": "releases/populace-us-2024-spm-20260915/source_spm_person_independence.csv", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "22b5968d90fecfeef7614583e493fe10cc16bda8b5be82e6f49a5bc2102d3ce5" + }, + "us_source_coverage": { + "path": "releases/populace-us-2024-spm-20260915/us_source_coverage.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "6406c8686c292015a5bc7265a42402f89935f9395a8b63510efdd4d9562e7ff5" + } + }, + "default_dataset": "populace_us_2024", + "default_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915", + "model_package": { + "name": "policyengine-us", + "sha256": "0993a6c73fcdfbe171a796aca9d302741bd8e00ab83388ad81c15a08740318c6", + "version": "2.2.1", + "wheel_url": "https://files.pythonhosted.org/packages/5e/09/86ac3968d2a18e28fce3ba1186fc21728e9f299b96a806d6f204f1e0aaed/policyengine_us-2.2.1-py3-none-any.whl" + }, + "policyengine_version": "6.8.0", + "region_datasets": { + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, + "national": { + "path_template": "populace_us_2024.h5" + }, + "state": { + "path_template": "populace_us_2024_acs_local.h5" + } + }, + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/releases/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/releases/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/release_manifest.json", + "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", + "schema_version": 1, + "source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", + "version": "populace-us-2024-spm-20260915" + } + }, + "dataset_overlays": { + "uk": { + "enhanced_frs_2023_24": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2023_24.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" + }, + "frs_2023_24": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2023_24.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "sha256": "df26d4d7af9d164aa2d064181b39290292d2f62bb26fee6126fc095fc06da292" + }, + "populace_uk_2023": { + "data_package_name": "populace-data", + "path": "populace_uk_2023.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" + } + } + }, + "extras": { + "models": [ + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "spm-calculator" + ], + "uk": [ + "policyengine-core", + "policyengine-uk" + ], + "us": [ + "policyengine-core", + "policyengine-us", + "spm-calculator" + ] + }, + "measurements": { + "spm": { + "as_of": null, + "county_vintage": "2020", + "forecast_content_sha256": "3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173", + "geography_id": null, + "geography_kind": "county", + "scenario": "ce_trend" + } + }, + "packages": { + "policyengine": { + "import_name": "policyengine", + "install_requirement": "policyengine==6.8.0", + "name": "policyengine", + "role": "bundle_carrier", + "version": "6.8.0" + }, + "policyengine-core": { + "import_name": "policyengine_core", + "install_requirement": "policyengine-core==3.33.0", + "name": "policyengine-core", + "role": "runtime_dependency", + "version": "3.33.0" + }, + "policyengine-uk": { + "country": "uk", + "import_name": "policyengine_uk", + "install_requirement": "policyengine-uk==2.130.1", + "name": "policyengine-uk", + "role": "country_model", + "version": "2.130.1" + }, + "policyengine-us": { + "country": "us", + "import_name": "policyengine_us", + "install_requirement": "policyengine-us==2.2.1", + "name": "policyengine-us", + "role": "country_model", + "version": "2.2.1" + }, + "spm-calculator": { + "country": "us", + "import_name": "spm_calculator", + "install_requirement": "spm-calculator==1.0.0", + "name": "spm-calculator", + "role": "runtime_dependency", + "sha256": "e354937a5e1a4045d4966ed594a528d8b02866fabaac9bb5672017004b627305", + "version": "1.0.0", + "wheel_url": "https://files.pythonhosted.org/packages/19/a0/c484f69a0ebf88a9b9fd0ac28176f8df66550f46714f1b0c5ec0b360824e/spm_calculator-1.0.0-py3-none-any.whl" + } + }, + "policyengine_version": "6.8.0", + "schema_version": 2 +} diff --git a/tests/qualification_fixtures.py b/tests/qualification_fixtures.py index bf74fc960..91c4fceda 100644 --- a/tests/qualification_fixtures.py +++ b/tests/qualification_fixtures.py @@ -20,6 +20,12 @@ ) FIXTURES = Path(__file__).parent / "fixtures" / "uk_replay" +#: The last packaged bundle certified on the policyengine-uk-data line (main at +#: 2591af3e, before Microcosm UK became the default). ``make_pair`` builds +#: candidates on that line, so qualification tests run against this bundle. +UK_DATA_LINE_BUNDLE_PATH = ( + Path(__file__).parent / "fixtures" / "uk_qualification" / "uk_data_line_bundle.json" +) VENDORED_SCHEMAS = ( Path(__file__).resolve().parents[1] / "src" diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 5d94bf5b4..92696e177 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -49,6 +49,7 @@ from tests.qualification_fixtures import ( HARNESS, ISSUED, + UK_DATA_LINE_BUNDLE_PATH, VENDORED_SCHEMAS, gates, golden, @@ -61,9 +62,13 @@ ) REPO_ROOT = Path(__file__).resolve().parents[1] -BUNDLE = json.loads( +#: The bundle this checkout packages. +PACKAGED_BUNDLE = json.loads( (REPO_ROOT / "src/policyengine/data/bundle/manifest.json").read_text() ) +#: The candidates these tests build are on the policyengine-uk-data line, so +#: they are qualified against the last bundle certified on that line. +BUNDLE = json.loads(UK_DATA_LINE_BUNDLE_PATH.read_text()) def _wheel(name, version, digest): @@ -158,6 +163,7 @@ def test_resolve_refuses_a_pair_other_than_the_one_requested( ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH pair = make_pair(model="2.200.0") resolved = cli.ResolvedCandidate( pair, @@ -479,6 +485,7 @@ def test_requalify_names_the_target_only_while_its_baseline_is_stale( ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH current = _promotion_receipt(pair=_promotable_pair_and_branch()) stale = current.model_copy( update={ @@ -522,13 +529,16 @@ def test_incomplete_pairs_requalify_on_a_new_base_only(tmp_path): ) -def _new_record(old): +SUCCESSOR_CUT = "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa" + + +def _successor_record(old): + """The next certificate on the packaged bundle's own data line.""" new = json.loads(json.dumps(old)) - new["model_package"]["version"] = "2.126.8" - new["certification"]["built_with_model_version"] = "2.126.0" - new["certification"]["data_build_id"] = "policyengine-uk-data-1.59.0" - new["data_package"]["version"] = "1.59.0" - new["data_package"]["release_manifest_revision"] = "1" * 40 + new["model_package"]["version"] = "2.140.0" + new["certification"]["built_with_model_version"] = "2.140.0" + new["certification"]["data_build_id"] = SUCCESSOR_CUT + new["data_package"]["release_manifest_revision"] = SUCCESSOR_CUT return new @@ -540,8 +550,10 @@ def test_constant_rewrite_leaves_no_old_values(tmp_path): ): (tmp_path / relative).parent.mkdir(parents=True, exist_ok=True) shutil.copy(REPO_ROOT / relative, tmp_path / relative) - old = BUNDLE["data_releases"]["uk"] - new = _new_record(old) + # The constant files pin the packaged certificate, so promotion rewrites + # them from it. + old = PACKAGED_BUNDLE["data_releases"]["uk"] + new = _successor_record(old) replacements = certified_constant_replacements(old, new) changed = rewrite_certified_constants(tmp_path, replacements) assert set(changed) == { @@ -554,9 +566,8 @@ def test_constant_rewrite_leaves_no_old_values(tmp_path): for before, after in replacements: assert f'"{before}"' not in text and f'@{before}"' not in text manifests = (tmp_path / "tests/test_release_manifests.py").read_text() - assert 'UK_MODEL_VERSION = "2.126.8"' in manifests - assert 'UK_DATA_RELEASE_PATH = "releases/1.59.0/release_manifest.json"' in manifests - assert 'UK_DATA_RELEASE_REVISION = "' + "1" * 40 + '"' in manifests + assert 'UK_MODEL_VERSION = "2.140.0"' in manifests + assert f'UK_DATA_RELEASE_REVISION = "{SUCCESSOR_CUT}"' in manifests assert rewrite_certified_constants(tmp_path, replacements) == [] @@ -762,7 +773,7 @@ def test_the_real_cli_generates_replay_requests_despite_country_import_suppressi candidate = request["candidate"] assert candidate["policyengine"]["source"]["git"]["commit"] == HARNESS.git_commit bundle = candidate["expected_release_bundle"] - record = BUNDLE["data_releases"]["uk"] + record = PACKAGED_BUNDLE["data_releases"]["uk"] assert bundle["model_version"] == record["model_package"]["version"] assert bundle["data_version"] == record["data_package"]["version"] assert bundle["default_dataset"] == record["default_dataset"] @@ -1383,6 +1394,7 @@ def test_promotion_makes_main_exactly_the_qualified_commit( ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH calls = [] target = { @@ -1514,6 +1526,7 @@ def _sweep_cli( ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH calls = [] listing = [ { @@ -1665,6 +1678,7 @@ def test_a_candidate_listing_at_its_limit_is_refused(monkeypatch, listed, refuse ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH listing = [ { "number": number, @@ -1890,6 +1904,7 @@ def test_only_pull_requests_to_main_are_candidates(monkeypatch): ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH calls = [] listing = [ { @@ -2721,6 +2736,7 @@ def test_forced_stage_recovery_binds_the_terminal_receipt_it_supersedes( ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH monkeypatch.setattr(cli, "utc_now", lambda: now) assert cli.main(["recoverable", "--ledger", str(tmp_path)]) == 0 assert json.loads(capsys.readouterr().out) == payload @@ -2801,6 +2817,7 @@ def test_resolution_preserves_authorization_to_recover_forced_work( ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH pair = make_pair(model="2.200.0") terminal = make_receipt( pair=pair, @@ -2926,6 +2943,7 @@ def test_recovery_requires_a_confirmed_failed_owner( ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH calls = [] def run(argv, **kwargs): @@ -2959,6 +2977,7 @@ def test_catchup_finds_abandoned_stages_without_replay_requests( ) cli = importlib.util.module_from_spec(spec) spec.loader.exec_module(cli) + cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH pair = make_pair() attempt = attempt_directory(tmp_path, "uk", pair.pair_id, "a" * 12) payload = {"pair": pair.canonical_payload()} diff --git a/tests/test_uk_qualification_workflow.py b/tests/test_uk_qualification_workflow.py index 10f1c2a8b..4022c3054 100644 --- a/tests/test_uk_qualification_workflow.py +++ b/tests/test_uk_qualification_workflow.py @@ -21,6 +21,7 @@ ) from tests.qualification_fixtures import ( HARNESS, + UK_DATA_LINE_BUNDLE_PATH, gates, graphql_base_changes_output, make_pair, @@ -40,6 +41,8 @@ def _cli(): ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + # These tests build candidates on the policyengine-uk-data line. + module.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH return module From 271dd5dc69b87a8a51054cb729ce320135b3d905 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 21:20:29 +0100 Subject: [PATCH 04/26] Build the released-bundle replay request for a dataset-repo release The UK data package is now a Hugging Face dataset repository, and Microcosm release manifests record no artifact sizes. The release consumer addressed every repository as a model repository, and the request needs the dataset's size, so the post-release replay request could not be built, silently, as its step continues on error. released_bundle_request now passes the repository type to hf_file and reads a missing size through hf_size; release_consumers.py builds the datasets/ URL and reads X-Linked-Size from a HEAD request. The release job warns when the request cannot be built. The request test runs on both the uk-data model repository and the packaged dataset repository. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/push.yaml | 5 + scripts/release_consumers.py | 43 +++++- .../provenance/uk_qualification.py | 31 ++++- tests/test_uk_qualification_pipeline.py | 122 +++++++++++++++--- 4 files changed, 175 insertions(+), 26 deletions(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index fe77ff440..3d09625a1 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -316,6 +316,11 @@ jobs: VERSION: ${{ needs.Publish.outputs.version }} HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} run: python scripts/release_consumers.py scorecard-request --version "$VERSION" > "$RUNNER_TEMP/scorecard-request.json" + - name: Warn that no released-bundle replay was requested + if: steps.released.outcome == 'failure' || steps.scorecard-request.outcome == 'failure' + env: + VERSION: ${{ needs.Publish.outputs.version }} + run: echo "::warning title=No released-bundle replay::The scorecard's replay request for policyengine $VERSION could not be built; see the steps above." - name: Generate GitHub App token for the scorecard if: steps.scorecard-request.outcome == 'success' id: scorecard-token diff --git a/scripts/release_consumers.py b/scripts/release_consumers.py index 57ab70cf8..45d63fbbe 100644 --- a/scripts/release_consumers.py +++ b/scripts/release_consumers.py @@ -52,17 +52,53 @@ def _pypi(path: str) -> dict: return response.json() -def _hf_file(repo_id: str, revision: str, path: str) -> bytes: +HF_REPO_PREFIXES = {"model": "", "dataset": "datasets/", "space": "spaces/"} + + +def _hf_url(repo_id: str, revision: str, path: str, repo_type: str) -> str: + prefix = HF_REPO_PREFIXES[repo_type] + return f"https://huggingface.co/{prefix}{repo_id}/resolve/{revision}/{path}" + + +def _hf_headers() -> dict: token = os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN") + return {"Authorization": f"Bearer {token}"} if token else {} + + +def _hf_file(repo_id: str, revision: str, path: str, repo_type: str = "model") -> bytes: response = requests.get( - f"https://huggingface.co/{repo_id}/resolve/{revision}/{path}", - headers={"Authorization": f"Bearer {token}"} if token else {}, + _hf_url(repo_id, revision, path, repo_type), + headers=_hf_headers(), timeout=TIMEOUT, ) response.raise_for_status() return response.content +def _hf_size(repo_id: str, revision: str, path: str, repo_type: str = "model") -> int: + """A file's size from Hugging Face's response headers, without downloading it. + + A large-file pointer answers with a redirect that carries + ``X-Linked-Size``; a small file answers directly with its length. + """ + + response = requests.head( + _hf_url(repo_id, revision, path, repo_type), + headers=_hf_headers(), + timeout=TIMEOUT, + allow_redirects=False, + ) + response.raise_for_status() + size = response.headers.get("X-Linked-Size") + if size is None and response.status_code == 200: + size = response.headers.get("Content-Length") + if not size: + raise SystemExit( + f"Hugging Face reports no size for {repo_id}/{path}@{revision}." + ) + return int(size) + + def cmd_payload(args: argparse.Namespace) -> int: payload = release_dispatch_payload( _bundle(), version=args.version, fetch_receipt=fetch_ledger_receipt @@ -126,6 +162,7 @@ def cmd_scorecard_request(args: argparse.Namespace) -> int: ).hexdigest(), release_bundle=release_bundle, hf_file=_hf_file, + hf_size=_hf_size, pypi_project=_pypi, requested_by={ "repository": repository, diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index cb1c9988c..16f8488b2 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -1757,18 +1757,26 @@ def released_bundle_request( wheel_sha256: str, packaged_manifest_sha256: str, release_bundle: Mapping[str, Optional[str]], - hf_file: Callable[[str, str, str], bytes], + hf_file: Callable[[str, str, str, str], bytes], pypi_project: Callable[[str], Mapping[str, Any]], requested_by: Mapping[str, Any], + hf_size: Optional[Callable[[str, str, str, str], int]] = None, ) -> ReplayQualificationRequest: - """``released_bundle`` replay request for a published policyengine release.""" + """``released_bundle`` replay request for a published policyengine release. + + ``hf_file`` and ``hf_size`` take a repository id, revision, path and + repository type. The dataset's size comes from the release manifest, or + from ``hf_size`` when the manifest records none, as Microcosm's do not. + """ record = bundle["data_releases"]["uk"] data_package = record["data_package"] + repo_type = data_package.get("repo_type", "model") manifest_bytes = hf_file( data_package["repo_id"], data_package["release_manifest_revision"], data_package["release_manifest_path"], + repo_type, ) manifest = DataReleaseManifest.model_validate_json(manifest_bytes) dataset = record["default_dataset"] @@ -1778,6 +1786,18 @@ def released_bundle_request( raise ValueError( "The release manifest's dataset hash differs from the bundle's." ) + data_repo_id = reference.get("repo_id") or data_package["repo_id"] + data_repo_type = reference.get("repo_type") or repo_type + size_bytes = artifact.size_bytes + if size_bytes is None: + if hf_size is None: + raise ValueError( + f"The release manifest records no size for {dataset!r}; pass " + "hf_size to read it from Hugging Face." + ) + size_bytes = hf_size( + data_repo_id, reference["revision"], reference["path"], data_repo_type + ) core_version = bundle["packages"][CORE_PACKAGE]["version"] core_wheel = _wheel( pypi_project(CORE_PACKAGE)["releases"][core_version], CORE_PACKAGE, core_version @@ -1805,13 +1825,12 @@ def released_bundle_request( "build_id": record["certified_data_artifact"].get("build_id") or record["build_id"], "dataset": dataset, - "repo_id": reference.get("repo_id") or data_package["repo_id"], - "repo_type": reference.get("repo_type") - or data_package.get("repo_type", "model"), + "repo_id": data_repo_id, + "repo_type": data_repo_type, "revision": reference["revision"], "path": reference["path"], "sha256": reference["sha256"], - "size_bytes": artifact.size_bytes, + "size_bytes": size_bytes, "release_manifest_sha256": hashlib.sha256( manifest_bytes ).hexdigest(), diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 92696e177..2d71f6bae 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -17,6 +17,7 @@ from __future__ import annotations import hashlib +import importlib.util import json import os import shutil @@ -24,6 +25,7 @@ import sys from pathlib import Path from types import SimpleNamespace +from unittest.mock import MagicMock import jsonschema import pytest @@ -3310,39 +3312,57 @@ def test_release_payload_names_the_qualification_receipt(): } -def test_released_bundle_request_is_valid_for_the_scorecard(): +@pytest.mark.parametrize( + ("bundle", "manifest_size"), + [(BUNDLE, 128369110), (PACKAGED_BUNDLE, None)], + ids=["uk-data model repo", "packaged dataset repo without sizes"], +) +def test_released_bundle_request_is_valid_for_the_scorecard(bundle, manifest_size): from policyengine.provenance.uk_qualification import released_bundle_request - record = BUNDLE["data_releases"]["uk"] + record = bundle["data_releases"]["uk"] dataset = record["default_dataset"] + reference = record["datasets"][dataset] + artifact = { + "kind": "microdata", + "path": reference["path"], + "repo_id": reference.get("repo_id") or record["data_package"]["repo_id"], + "revision": reference["revision"], + "sha256": reference["sha256"], + } + if manifest_size is not None: + artifact["size_bytes"] = manifest_size manifest = { "schema_version": 1, "data_package": { - "name": "policyengine-uk-data", + "name": record["data_package"]["name"], "version": record["data_package"]["version"], }, "default_datasets": {"national": dataset}, - "artifacts": { - dataset: { - "kind": "microdata", - "path": record["datasets"][dataset]["path"], - "repo_id": record["datasets"][dataset]["repo_id"], - "revision": record["datasets"][dataset]["revision"], - "sha256": record["datasets"][dataset]["sha256"], - "size_bytes": 128369110, - } - }, + "artifacts": {dataset: artifact}, } - core = BUNDLE["packages"]["policyengine-core"]["version"] + repo_type = record["data_package"]["repo_type"] + fetched = [] + + def hf_file(repo, rev, path, kind): + fetched.append(("file", repo, kind)) + return json.dumps(manifest).encode() + + def hf_size(repo, rev, path, kind): + fetched.append(("size", repo, kind)) + return 321798995 + + core = bundle["packages"]["policyengine-core"]["version"] request = released_bundle_request( - BUNDLE, - version=BUNDLE["policyengine_version"], + bundle, + version=bundle["policyengine_version"], wheel_sha256=sha("7"), packaged_manifest_sha256=sha("8"), release_bundle=golden("qualification_request_golden")["candidate"][ "expected_release_bundle" ], - hf_file=lambda repo, rev, path: json.dumps(manifest).encode(), + hf_file=hf_file, + hf_size=hf_size, pypi_project=lambda name: { "releases": {core: [_wheel("policyengine-core", core, sha("6"))]} }, @@ -3360,6 +3380,74 @@ def test_released_bundle_request_is_valid_for_the_scorecard(): assert request.document()["candidate"]["policyengine"]["source"] == { "pypi": {"wheel_sha256": sha("7")} } + data = request.document()["candidate"]["data"] + assert data["repo_type"] == repo_type + # The manifest's size when it records one, else the file's own size. + assert data["size_bytes"] == manifest_size or ( + manifest_size is None and data["size_bytes"] == 321798995 + ) + assert fetched[0] == ("file", record["data_package"]["repo_id"], repo_type) + assert [item[0] for item in fetched] == ( + ["file"] if manifest_size is not None else ["file", "size"] + ) + + +def test_released_bundle_request_needs_a_size_source(): + from policyengine.provenance.uk_qualification import released_bundle_request + + record = PACKAGED_BUNDLE["data_releases"]["uk"] + dataset = record["default_dataset"] + reference = record["datasets"][dataset] + manifest = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "default_datasets": {"national": dataset}, + "artifacts": { + dataset: { + "kind": "microdata", + "path": reference["path"], + "repo_id": reference["repo_id"], + "revision": reference["revision"], + "sha256": reference["sha256"], + } + }, + } + with pytest.raises(ValueError, match="records no size"): + released_bundle_request( + PACKAGED_BUNDLE, + version=PACKAGED_BUNDLE["policyengine_version"], + wheel_sha256=sha("7"), + packaged_manifest_sha256=sha("8"), + release_bundle={}, + hf_file=lambda *args: json.dumps(manifest).encode(), + pypi_project=lambda name: {}, + requested_by={}, + ) + + +def test_release_consumers_address_dataset_repositories(monkeypatch): + monkeypatch.setenv("POLICYENGINE_SKIP_COUNTRY_IMPORTS", "1") + spec = importlib.util.spec_from_file_location( + "release_consumers_for_test", REPO_ROOT / "scripts" / "release_consumers.py" + ) + consumers = importlib.util.module_from_spec(spec) + spec.loader.exec_module(consumers) + + assert consumers._hf_url("org/data", "v1", "a.h5", "dataset") == ( + "https://huggingface.co/datasets/org/data/resolve/v1/a.h5" + ) + assert consumers._hf_url("org/model", "v1", "a.h5", "model") == ( + "https://huggingface.co/org/model/resolve/v1/a.h5" + ) + + def head(url, **kwargs): + assert kwargs["allow_redirects"] is False + response = MagicMock(status_code=302, headers={"X-Linked-Size": "321"}) + response.raise_for_status = lambda: None + return response + + monkeypatch.setattr(consumers.requests, "head", head) + assert consumers._hf_size("org/data", "v1", "a.h5", "dataset") == 321 def test_expected_release_bundle_uses_the_scorecard_certification_environment( From 22a2c49494b82d892ce6040124a6daa17ef91662 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 21:23:05 +0100 Subject: [PATCH 05/26] Remove the overlays a certification supersedes, and check overlays Overlays may not shadow a certified dataset name, but certify-data rewrote data_releases without looking at dataset_overlays. A rollback to policyengine-uk-data, or a national-first route carrying the enhanced FRS, would certify names the bundle keeps as overlays, and then every release-manifest read, and so import policyengine, would fail. Certification now removes the overlays the new release certifies and says so in its warnings, and bundle.py check refuses an overlay that shadows a certified name. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/data-certification.md | 6 +++ scripts/bundle.py | 7 +-- scripts/generate_bundle_artifacts.py | 1 + .../provenance/bundle_validation.py | 20 +++++++ src/policyengine/provenance/certification.py | 17 ++++++ tests/test_bundle_metadata.py | 18 +++++++ tests/test_certify_data_release.py | 53 ++++++++++++++++++- 7 files changed, 117 insertions(+), 5 deletions(-) diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 42eec89be..9f0058fdf 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -92,6 +92,12 @@ a run without `--regional-manifest-uri` is refused; pass the current regional release again (or its replacement), or `--drop-regional-release` to remove the routing on purpose. The two flags cannot be combined. +An overlay may never shadow a certified dataset name, so certification removes +any `dataset_overlays.` entry whose name the new release certifies. A +rollback to policyengine-uk-data with `--drop-regional-release`, for example, +removes the enhanced FRS overlays. `bundle.py check` refuses a bundle with an +overlay that shadows a certified name. + ### US For US Populace certification, certify the Populace release manifest directly: diff --git a/scripts/bundle.py b/scripts/bundle.py index 42179f4fe..53b74a26a 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -90,11 +90,12 @@ def _check(args: argparse.Namespace) -> int: generate, load_bundle_manifest, validate_bundle_measurements, + validate_dataset_overlays, ) - validate_bundle_measurements( - load_bundle_manifest(), require_published_spm=args.published_spm - ) + bundle = load_bundle_manifest() + validate_bundle_measurements(bundle, require_published_spm=args.published_spm) + validate_dataset_overlays(bundle) result = generate(check=True) if not args.include_tros: return result diff --git a/scripts/generate_bundle_artifacts.py b/scripts/generate_bundle_artifacts.py index cdbfaea79..e941bb856 100644 --- a/scripts/generate_bundle_artifacts.py +++ b/scripts/generate_bundle_artifacts.py @@ -37,6 +37,7 @@ _VALIDATION_MODULE = importlib.util.module_from_spec(_VALIDATION_SPEC) _VALIDATION_SPEC.loader.exec_module(_VALIDATION_MODULE) validate_bundle_measurements = _VALIDATION_MODULE.validate_bundle_measurements +validate_dataset_overlays = _VALIDATION_MODULE.validate_dataset_overlays OPTIONAL_DEPENDENCIES_HEADER = "[project.optional-dependencies]" NEXT_SECTION_PATTERN = re.compile(r"\n\[tool\.setuptools\]", re.MULTILINE) diff --git a/src/policyengine/provenance/bundle_validation.py b/src/policyengine/provenance/bundle_validation.py index 51324de80..150a6929d 100644 --- a/src/policyengine/provenance/bundle_validation.py +++ b/src/policyengine/provenance/bundle_validation.py @@ -24,6 +24,26 @@ def _is_normalized_version(value: Any) -> bool: ) +def validate_dataset_overlays(bundle: Mapping[str, Any]) -> None: + """Refuse an overlay that shadows a certified dataset name. + + The runtime refuses such a bundle on every release-manifest read, so + ``import policyengine`` would fail; maintenance checks refuse it first. + """ + overlays = bundle.get("dataset_overlays") or {} + for country, entries in overlays.items(): + release = (bundle.get("data_releases") or {}).get(country) or {} + certified = set(release.get("datasets") or {}) + if release.get("default_dataset"): + certified.add(release["default_dataset"]) + shadowed = sorted(set(entries or {}) & certified) + if shadowed: + raise ValueError( + f"dataset_overlays.{country} shadows certified datasets: " + f"{', '.join(shadowed)}. Remove those overlays." + ) + + def validate_spm_configuration(configuration: Any) -> None: """Validate JSON defaults using only the standard library. diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 31327f576..73fcb1b3f 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -1372,6 +1372,23 @@ def certify_data_release( qualified_pair_id=qualified_pair_id, ) bundle.setdefault("data_releases", {})[country] = data_release + # Overlays may never shadow a certified name, so a release that certifies + # a name the bundle keeps as an overlay (a rollback to the release it came + # from, or the dataset a national-first route carries) supersedes it. + overlays = (bundle.get("dataset_overlays") or {}).get(country) or {} + superseded = sorted( + name + for name in overlays + if name in data_release["datasets"] or name == data_release["default_dataset"] + ) + for name in superseded: + del overlays[name] + if superseded: + warnings = [ + *warnings, + f"Removed the dataset_overlays.{country} entries this release now " + f"certifies: {', '.join(superseded)}.", + ] bundle.setdefault("countries", {}).setdefault(country, {})["model_package"] = ( model_package ) diff --git a/tests/test_bundle_metadata.py b/tests/test_bundle_metadata.py index cdcda8fd3..937e4f88f 100644 --- a/tests/test_bundle_metadata.py +++ b/tests/test_bundle_metadata.py @@ -143,3 +143,21 @@ def test_export_release_assets_writes_bundle_assets(monkeypatch, tmp_path): "PolicyEngine bundle" in (tmp_path / f"policyengine-bundle-{version}.citation.txt").read_text() ) + + +def test_dataset_overlays_may_not_shadow_certified_names(): + import copy + + import pytest + + from policyengine.provenance.bundle_validation import validate_dataset_overlays + + current = bundle.get_current_bundle() + validate_dataset_overlays(current) + + release = current["data_releases"]["uk"] + for name in (release["default_dataset"], next(iter(release["datasets"]))): + shadowing = copy.deepcopy(current) + shadowing["dataset_overlays"]["uk"][name] = {"path": f"{name}.h5"} + with pytest.raises(ValueError, match=f"shadows certified datasets: {name}"): + validate_dataset_overlays(shadowing) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 74964a50f..4c2c5510e 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -29,6 +29,7 @@ from policyengine.provenance.manifest import ( CountryReleaseManifest, DataReleaseManifest, + _apply_dataset_overlays, ) TAG = "populace-us-2024-aaaaaaa-20260101" @@ -1171,7 +1172,9 @@ def _response(payload: dict) -> MagicMock: return response -def _certify_uk(bundle_path: Path, responses: list, **kwargs): +def _certify_uk( + bundle_path: Path, responses: list, manifest_uri: str = UK_NATIONAL_URI, **kwargs +): with ( patch( "policyengine.provenance.certification.requests.get", @@ -1197,7 +1200,7 @@ def _certify_uk(bundle_path: Path, responses: list, **kwargs): return certify_data_release( country="uk", data_producer="populace", - manifest_uri=UK_NATIONAL_URI, + manifest_uri=manifest_uri, model_version=UK_MODEL_VERSION, bundle_path=bundle_path, **kwargs, @@ -1498,6 +1501,52 @@ def test__given_routed_bundle__then_recertifying_without_regional_refuses( assert "regional_release_kind" not in release assert set(release["region_datasets"]) == {"national"} + def test__given_a_rollback_to_overlaid_names__then_their_overlays_go( + self, tmp_path + ): + """A rollback certifies names the bundle keeps as overlays. + + Overlays may not shadow certified names, so a bundle that kept them + would refuse every release-manifest read, and import with it. + """ + bundle_path = _tmp_bundle(tmp_path) + _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + regional_manifest_uri=UK_DENSE_URI, + ) + bundle = json.loads(bundle_path.read_text()) + kept = {"path": "populace_uk_2023.h5", "repo_id": "x", "revision": "r"} + bundle["dataset_overlays"] = { + "uk": { + "enhanced_frs_2024_25": {"path": "enhanced_frs_2024_25.h5"}, + "frs_2024_25": {"path": "frs_2024_25.h5"}, + "populace_uk_2023": kept, + } + } + bundle_path.write_text(json.dumps(bundle)) + + result = _certify_uk( + bundle_path, + [_response(_efrs_release_payload())], + manifest_uri=EFRS_URI, + drop_regional_release=True, + ) + + bundle = json.loads(bundle_path.read_text()) + assert bundle["data_releases"]["uk"]["default_dataset"] == ( + "enhanced_frs_2024_25" + ) + assert bundle["dataset_overlays"]["uk"] == {"populace_uk_2023": kept} + assert any("enhanced_frs_2024_25, frs_2024_25" in w for w in result.warnings) + manifest = CountryReleaseManifest.model_validate( + _apply_dataset_overlays("uk", bundle["data_releases"]["uk"], bundle) + ) + assert "populace_uk_2023" in manifest.datasets + def test__given_drop_and_a_regional_release__then_refuses_both(self, tmp_path): # Dropping the routing and routing to a release are opposite requests; # neither silently wins. From 7e8df2d3b0188c9d8ee97dcd4095741430c15b18 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 21:26:40 +0100 Subject: [PATCH 06/26] Keep a certificate's named core equal to the bundle's core pin This is the first certificate on main to name its core. Receipts certify the core a certificate names, falling back to the core pin only when it names none, but the strict TRO check compared the pin with the certificate only for a qualified exact pair. So a core-only pin change (update-packages --core, or certifying the other country with --core-version) would pass every check and leave every UK receipt uncertified. The strict check now refuses a core pin that differs from any core a certificate names. New tests cover the check, a pin-only edit against a certificate that names its core, and a certificate that names none. The receipt test for another dataset name no longer depends on the routing. Co-Authored-By: Claude Opus 5.5 --- scripts/generate_trace_tros.py | 14 ++++++++- tests/test_execution_receipt.py | 43 ++++++++++++++++++++++------ tests/test_release_tro_generation.py | 22 ++++++++++++++ 3 files changed, 70 insertions(+), 9 deletions(-) diff --git a/scripts/generate_trace_tros.py b/scripts/generate_trace_tros.py index b34562f82..e01e89e5e 100644 --- a/scripts/generate_trace_tros.py +++ b/scripts/generate_trace_tros.py @@ -138,8 +138,8 @@ def _validate_runtime_model(bundle: dict, country: CountryReleaseManifest) -> No model = country.model_package prefix = f"{country.country_id}: strict TRACE release" certification = country.certification + core = bundle.get("packages", {}).get("policyengine-core", {}) if certification and certification.compatibility_basis == "qualified_exact_pair": - core = bundle.get("packages", {}).get("policyengine-core", {}) if ( not certification.qualification_pair_id or not certification.certified_for_core_version @@ -148,6 +148,18 @@ def _validate_runtime_model(bundle: dict, country: CountryReleaseManifest) -> No raise ValueError( f"{prefix}: runtime core differs from the qualified exact pair" ) + elif ( + certification + and certification.certified_for_core_version + and core.get("version") != certification.certified_for_core_version + ): + # Receipts certify the core the certificate names, not the pin, so a + # core-only pin change would leave every receipt uncertified. + raise ValueError( + f"{prefix}: runtime core {core.get('version')} differs from the " + f"certified core {certification.certified_for_core_version}; " + "re-certify the data release for the new core" + ) if ( model.name != key or component.get("name") != model.name diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index b5be39e15..0c540873d 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -305,21 +305,48 @@ def test_any_uncovered_component_makes_the_receipt_uncertified( elif change == "dataset name": # Same bytes registered under another certified dataset name. record = bundle["data_releases"]["uk"] - other = next( - name - for name, reference in record["datasets"].items() - if name != record["default_dataset"] and reference["path"].endswith(".h5") + record["datasets"]["same_bytes_other_name"] = dict( + record["datasets"][record["default_dataset"]] ) - record["datasets"][other]["sha256"] = record["certified_data_artifact"][ - "sha256" - ] - kwargs["dataset"] = other + kwargs["dataset"] = "same_bytes_other_name" receipt = _build(path, bundle, **kwargs) assert receipt.certification_status == "uncertified" assert receipt.certificate is None assert receipt.model.version == INSTALLED_UK # installed, never the manifest's +def test_a_core_the_certificate_names_decides_over_the_pin(dataset_file): + # bundle.py check refuses a pin that differs from a named core + # (scripts/generate_trace_tros.py), because receipts follow the certificate. + bundle = _bundle_for(dataset_file.read_bytes()) + assert ( + "certified_for_core_version" in bundle["data_releases"]["uk"]["certification"] + ) + bundle["packages"]["policyengine-core"]["version"] = "0.0.1" + + receipt = _build(dataset_file, bundle) + + assert receipt.certification_status == "certified" + assert receipt.certificate.certified_for_core_version == INSTALLED_CORE + + +@pytest.mark.parametrize("pin", [INSTALLED_CORE, "0.0.1"]) +def test_a_certificate_without_a_core_certifies_the_pinned_core(dataset_file, pin): + # Certificates from before --core-version, and receipts from + # policyengine 6.5.0-6.8.0, name no core: the pin is the certified core. + bundle = _bundle_for(dataset_file.read_bytes()) + del bundle["data_releases"]["uk"]["certification"]["certified_for_core_version"] + bundle["packages"]["policyengine-core"]["version"] = pin + + receipt = _build(dataset_file, bundle) + + if pin == INSTALLED_CORE: + assert receipt.certification_status == "certified" + assert receipt.certificate.certified_for_core_version == INSTALLED_CORE + else: + assert receipt.certification_status == "uncertified" + + @pytest.mark.parametrize( "years", [[2027.9], [True], "2027", [], [2027, "2028"], 2027, [np.float64(2027)]], diff --git a/tests/test_release_tro_generation.py b/tests/test_release_tro_generation.py index 69ffd6a82..16aa76166 100644 --- a/tests/test_release_tro_generation.py +++ b/tests/test_release_tro_generation.py @@ -114,6 +114,28 @@ def test_runtime_pin_update_requires_matching_certified_model(release): assert maintenance._check_tros(strict=True) == 0 +def test_runtime_core_must_match_a_core_the_certificate_names(release): + """A core-only pin change must not leave receipts on another core.""" + + path = release[0] / "manifest.json" + payload = json.loads(path.read_bytes()) + # The fixture release was built with, and claims, this core. + certified_core = "3.26.11" + payload["data_releases"]["us"]["certification"]["certified_for_core_version"] = ( + certified_core + ) + payload["packages"]["policyengine-core"]["version"] = certified_core + path.write_text(json.dumps(payload)) + manifest.get_release_manifest.cache_clear() + generator.generated_tros(strict=True) + + payload["packages"]["policyengine-core"]["version"] = "99.0.0" + path.write_text(json.dumps(payload)) + manifest.get_release_manifest.cache_clear() + with pytest.raises(ValueError, match="runtime core 99.0.0 differs"): + generator.generated_tros(strict=True) + + @pytest.mark.parametrize("field", ["name", "version", "sha256", "wheel_url"]) def test_runtime_model_identity_must_match_certification(release, field): path = release[0] / "manifest.json" From 8ea62c12ad937142b46f3c069293be9aed3ec41f Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 21:28:15 +0100 Subject: [PATCH 07/26] Keep or drop the area routing when preparing a candidate prepare_candidate_commit ran certify-data without a regional flag, so on a main routed to a local-area release every forced requalification failed at preparation: certification refuses to drop the routing silently, and no workflow input could pass a flag. A pair on main's data line now keeps main's regional release, which must serve the pair by its own claims. A pair from another line drops the routing, which belongs to the line it would replace, and leaves the test constants alone, since rewriting them by value from another line's certificate writes into unrelated constants. The preparation test now also runs on the packaged, routed bundle, and the runbook says what a forced cross-line run does. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/runbooks/uk-qualification.md | 14 ++++++ .../provenance/uk_qualification.py | 44 ++++++++++++++-- tests/test_uk_qualification_pipeline.py | 50 +++++++++++++++++-- 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/docs/engineering/runbooks/uk-qualification.md b/docs/engineering/runbooks/uk-qualification.md index 41e245ce4..b4f0a0350 100644 --- a/docs/engineering/runbooks/uk-qualification.md +++ b/docs/engineering/runbooks/uk-qualification.md @@ -118,3 +118,17 @@ write here, such as renaming another branch into main to make a run against another base look like the candidate's: main is not protected, so they could push it directly. Treat any such change as an incident, not as something the gate will catch. + +## A pair from another data line + +Promotion compares only pairs on main's data line (its UK certificate's data +package and repository). While Microcosm UK is the default, the +policyengine-uk-data candidates the workflow resolves are `other_line`: they +are recorded and never promoted, and recovery and check refreshes skip them. + +`force` still qualifies such a pair, for evidence. Its candidate commit drops +main's area routing (`--drop-regional-release`), because the routing belongs +to the line the pair would replace, and leaves the test constants that pin +main's certificate as they are, so the candidate's own tests fail. Changing +the data line is a reviewed certification pull request (see +`docs/engineering/skills/data-certification.md`), not a promotion. diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index 16f8488b2..6fd9d936a 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -752,6 +752,33 @@ def _run( ) +def regional_certification_arguments( + bundle: Mapping[str, Any], pair: CandidatePair +) -> list[str]: + """``certify-data``'s regional flags for certifying ``pair`` onto ``bundle``. + + Certification refuses to drop a bundle's regional routing silently. A + pair on main's data line keeps main's regional release, which must then + serve the pair's model and core by its own claims. A pair from another + line (a forced qualification, never promoted automatically) replaces the + line the routing belongs to, so the routing is dropped and the pair's + national dataset serves every region type. + """ + + record = bundle["data_releases"]["uk"] + regional_uri = record.get("regional_source_manifest_uri") + if regional_uri is None: + return [] + if data_line(pair) != certified_data_line(bundle): + return ["--drop-regional-release"] + return [ + "--regional-manifest-uri", + regional_uri, + "--regional-release-kind", + record.get("regional_release_kind") or "local_area", + ] + + def prepare_candidate_commit( repo_root: Path, resolved: ResolvedCandidate, @@ -770,7 +797,9 @@ def prepare_candidate_commit( bundle_path = ( repo_root / "src" / "policyengine" / "data" / "bundle" / "manifest.json" ) - old_record = json.loads(bundle_path.read_text())["data_releases"]["uk"] + old_bundle = json.loads(bundle_path.read_text()) + old_record = old_bundle["data_releases"]["uk"] + same_line = data_line(pair) == certified_data_line(old_bundle) run( [ python, @@ -789,6 +818,7 @@ def prepare_candidate_commit( "--qualified-pair-id", pair.pair_id, "--no-changelog", + *regional_certification_arguments(old_bundle, pair), ], cwd=repo_root, ) @@ -818,8 +848,16 @@ def prepare_candidate_commit( [python, "scripts/bundle.py", "generate", "--include-tros"], cwd=repo_root, ) - constants = rewrite_certified_constants( - repo_root, certified_constant_replacements(old_record, new_record) + # The constant files pin main's line. Rewriting them by value from + # another line's certificate would write that line's values into + # unrelated constants, so a cross-line candidate leaves them, and its + # tests show the change of line. + constants = ( + rewrite_certified_constants( + repo_root, certified_constant_replacements(old_record, new_record) + ) + if same_line + else [] ) basis = new_record["certification"]["compatibility_basis"] basis_note = { diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 2d71f6bae..cf57c602e 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -573,11 +573,18 @@ def test_constant_rewrite_leaves_no_old_values(tmp_path): assert rewrite_certified_constants(tmp_path, replacements) == [] +@pytest.mark.parametrize( + "routed", [False, True], ids=["uk-data main", "main routed to Microcosm"] +) @pytest.mark.parametrize("qualification_only", [False, True]) def test_preparation_generates_new_sidecars_and_verifies_the_exact_pair( - tmp_path, monkeypatch, qualification_only + tmp_path, monkeypatch, qualification_only, routed ): - """Run preparation, certification and real TRO generation without a network.""" + """Run preparation, certification and real TRO generation without a network. + + On a main routed to another line's local-area release, the uk-data + candidate (a forced qualification) drops the routing it cannot keep. + """ from policyengine.provenance import certification, manifest from policyengine.provenance.uk_qualification import prepare_candidate_commit @@ -604,7 +611,7 @@ def test_preparation_generates_new_sidecars_and_verifies_the_exact_pair( pair = resolved.pair bundle_dir = tmp_path / "src/policyengine/data/bundle" bundle_dir.mkdir(parents=True) - source = json.loads(json.dumps(BUNDLE)) + source = json.loads(json.dumps(PACKAGED_BUNDLE if routed else BUNDLE)) source["data_releases"].pop("us") source["packages"].pop("policyengine-us") for packages in source["extras"].values(): @@ -701,6 +708,13 @@ def run(command, *, cwd, env=None): assert (tmp_path / prepared["changelog_fragment"]).is_file() assert maintenance.main(["check", "--include-tros", "--strict-tros"]) == 0 assert calls[-1][2:] == ["check", "--include-tros", "--strict-tros"] + certify = next(command for command in calls if "certify-data" in command) + record = json.loads(bundle_path.read_bytes())["data_releases"]["uk"] + assert ("--drop-regional-release" in certify) == routed + assert "regional_source_manifest_uri" not in record + if routed: + # The constant files pin main's line, not the candidate's. + assert prepared["constant_files"] == [] if qualification_only: changed = json.loads(bundle_path.read_bytes()) changed["packages"]["policyengine-core"]["version"] = "99.0.0" @@ -709,6 +723,36 @@ def run(command, *, cwd, env=None): tros.generated_tros(strict=True) +def test_regional_flags_keep_routing_on_its_line_and_drop_it_across_lines(): + from policyengine.provenance.uk_qualification import ( + regional_certification_arguments, + ) + + uk_data_pair = make_pair() + record = PACKAGED_BUNDLE["data_releases"]["uk"] + microcosm_pair = uk_data_pair.model_copy( + update={ + "data": uk_data_pair.data.model_copy( + update={ + "data_package": record["data_package"]["name"], + "repo_id": record["data_package"]["repo_id"], + } + ) + } + ) + + assert regional_certification_arguments(BUNDLE, uk_data_pair) == [] + assert regional_certification_arguments(PACKAGED_BUNDLE, uk_data_pair) == [ + "--drop-regional-release" + ] + assert regional_certification_arguments(PACKAGED_BUNDLE, microcosm_pair) == [ + "--regional-manifest-uri", + record["regional_source_manifest_uri"], + "--regional-release-kind", + record["regional_release_kind"], + ] + + def test_replay_request_is_valid_for_the_scorecard(): pair = make_pair() release_bundle = dict( From 8d766085df80b4f1cf7c4132f285abef8768a480 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 21:31:05 +0100 Subject: [PATCH 08/26] Skip pairs from another data line in recovery and check refreshes cmd_recoverable had no data-line filter, so an attempt for a uk-data pair abandoned around the switch stayed recoverable: every two-hourly re-dispatch resolved as other_line before recovery and cleared nothing. It now skips an unforced attempt whose pair is off main's data line, and cmd_refresh_pr_checks no longer re-runs CI for such pairs, which promotion never takes. The qualification tests now choose main's bundle in one place: load_candidate_cli in tests/qualification_fixtures.py replaces ten inline loaders, and the ledger concurrency test, which still read the packaged bundle, now uses the uk-data-line bundle its pair belongs to, so its promotion assertion is no longer vacuous. That frozen bundle is now main's 6.9.0 release (5f97905e). Co-Authored-By: Claude Opus 5.5 --- scripts/check_uk_candidate.py | 22 ++- .../uk_qualification/uk_data_line_bundle.json | 20 +-- tests/qualification_fixtures.py | 26 ++- .../test_qualification_ledger_concurrency.py | 14 +- tests/test_uk_qualification_pipeline.py | 162 ++++++++---------- tests/test_uk_qualification_workflow.py | 12 +- 6 files changed, 142 insertions(+), 114 deletions(-) diff --git a/scripts/check_uk_candidate.py b/scripts/check_uk_candidate.py index b82c432c4..852274e52 100644 --- a/scripts/check_uk_candidate.py +++ b/scripts/check_uk_candidate.py @@ -75,6 +75,8 @@ candidate_action, candidate_branch, candidate_pr_body, + certified_data_line, + data_line, expected_release_bundle, finalize_attempt, load_attempt_budget_inputs, @@ -528,6 +530,7 @@ def owner_run_failed(run_id: str) -> bool: def cmd_recoverable(args: argparse.Namespace) -> int: now = utc_now() + line = certified_data_line(json.loads(BUNDLE_PATH.read_text())) for pair_dir in sorted((Path(args.ledger) / "uk").glob("*")): receipt = pair_dir / "qualification_receipt.json" stages = list((pair_dir / "attempts").glob("*/staging.json")) @@ -544,8 +547,20 @@ def cmd_recoverable(args: argparse.Namespace) -> int: attempt, terminal ): continue + candidate = read_json(attempt / "candidate.json") + # An unforced run of a pair from another data line resolves as + # other_line before any recovery, so dispatching it clears nothing. + if not candidate.get("force") and ( + data_line(CandidatePair.model_validate(candidate["pair"])) != line + ): + print( + f"::notice::not recovering {pair_dir.name}: its pair is not on " + "main's data line", + file=sys.stderr, + ) + continue if attempt_is_recoverable(attempt, now=now, owner_is_failed=owner_run_failed): - print(json.dumps(read_json(attempt / "candidate.json"))) + print(json.dumps(candidate)) return 0 @@ -856,7 +871,12 @@ def cmd_refresh_pr_checks(args: argparse.Namespace) -> int: pull_requests = {pr["headRefName"]: pr for pr in _candidate_pull_requests()} now = _now() + line = certified_data_line(json.loads(BUNDLE_PATH.read_text())) for receipt in latest_receipts(Path(args.ledger)): + if data_line(receipt.pair) != line: + # Promotion never takes a pair from another line, so its runs + # are not worth re-running. + continue branch = candidate_branch(receipt.pair) head = receipt.harness.git_commit pull_request = pull_requests.get(branch) diff --git a/tests/fixtures/uk_qualification/uk_data_line_bundle.json b/tests/fixtures/uk_qualification/uk_data_line_bundle.json index c0e4ee8c3..cc56a3ebc 100644 --- a/tests/fixtures/uk_qualification/uk_data_line_bundle.json +++ b/tests/fixtures/uk_qualification/uk_data_line_bundle.json @@ -1,10 +1,10 @@ { - "bundle_version": "6.8.0", + "bundle_version": "6.9.0", "citation": { "publisher": "PolicyEngine", - "title": "PolicyEngine bundle 6.8.0", + "title": "PolicyEngine bundle 6.9.0", "type": "software-bundle", - "version": "6.8.0" + "version": "6.9.0" }, "countries": { "uk": { @@ -17,7 +17,7 @@ "data_releases": { "uk": { "build_id": "policyengine-uk-data-1.58.1", - "bundle_id": "uk-6.8.0", + "bundle_id": "uk-6.9.0", "certification": { "built_with_model_git_sha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", "built_with_model_version": "2.122.2", @@ -99,7 +99,7 @@ "version": "2.130.1", "wheel_url": "https://files.pythonhosted.org/packages/1a/21/88e8addba0eabd9a7d3a29fa5293e141feb405baf0432209387f9a1f467a/policyengine_uk-2.130.1-py3-none-any.whl" }, - "policyengine_version": "6.8.0", + "policyengine_version": "6.9.0", "region_datasets": { "national": { "path_template": "enhanced_frs_2024_25.h5" @@ -112,7 +112,7 @@ }, "us": { "build_id": "populace-us-2024-spm-20260915", - "bundle_id": "us-6.8.0", + "bundle_id": "us-6.9.0", "certification": { "built_with_model_version": "2.2.1", "certified_by": "policyengine.py bundle certification", @@ -217,7 +217,7 @@ "version": "2.2.1", "wheel_url": "https://files.pythonhosted.org/packages/5e/09/86ac3968d2a18e28fce3ba1186fc21728e9f299b96a806d6f204f1e0aaed/policyengine_us-2.2.1-py3-none-any.whl" }, - "policyengine_version": "6.8.0", + "policyengine_version": "6.9.0", "region_datasets": { "congressional_district": { "path_template": "populace_us_2024_acs_local.h5" @@ -295,10 +295,10 @@ "packages": { "policyengine": { "import_name": "policyengine", - "install_requirement": "policyengine==6.8.0", + "install_requirement": "policyengine==6.9.0", "name": "policyengine", "role": "bundle_carrier", - "version": "6.8.0" + "version": "6.9.0" }, "policyengine-core": { "import_name": "policyengine_core", @@ -334,6 +334,6 @@ "wheel_url": "https://files.pythonhosted.org/packages/19/a0/c484f69a0ebf88a9b9fd0ac28176f8df66550f46714f1b0c5ec0b360824e/spm_calculator-1.0.0-py3-none-any.whl" } }, - "policyengine_version": "6.8.0", + "policyengine_version": "6.9.0", "schema_version": 2 } diff --git a/tests/qualification_fixtures.py b/tests/qualification_fixtures.py index 91c4fceda..cf70f42a2 100644 --- a/tests/qualification_fixtures.py +++ b/tests/qualification_fixtures.py @@ -20,12 +20,34 @@ ) FIXTURES = Path(__file__).parent / "fixtures" / "uk_replay" +REPO_ROOT = Path(__file__).resolve().parents[1] #: The last packaged bundle certified on the policyengine-uk-data line (main at -#: 2591af3e, before Microcosm UK became the default). ``make_pair`` builds -#: candidates on that line, so qualification tests run against this bundle. +#: 5f97905e, the 6.9.0 release, before Microcosm UK became the default). +#: ``make_pair`` builds candidates on that line, so qualification tests run +#: against this bundle. UK_DATA_LINE_BUNDLE_PATH = ( Path(__file__).parent / "fixtures" / "uk_qualification" / "uk_data_line_bundle.json" ) +UK_DATA_LINE_BUNDLE = json.loads(UK_DATA_LINE_BUNDLE_PATH.read_text()) + + +def load_candidate_cli( + name: str = "check_uk_candidate_for_test", + bundle_path: Path = UK_DATA_LINE_BUNDLE_PATH, +): + """``scripts/check_uk_candidate.py``, reading ``bundle_path`` as main's bundle.""" + + import importlib.util + + spec = importlib.util.spec_from_file_location( + name, REPO_ROOT / "scripts" / "check_uk_candidate.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.BUNDLE_PATH = bundle_path + return module + + VENDORED_SCHEMAS = ( Path(__file__).resolve().parents[1] / "src" diff --git a/tests/test_qualification_ledger_concurrency.py b/tests/test_qualification_ledger_concurrency.py index 3ca61a03d..95b374168 100644 --- a/tests/test_qualification_ledger_concurrency.py +++ b/tests/test_qualification_ledger_concurrency.py @@ -56,14 +56,20 @@ stage_attempt, write_gate_results, ) -from tests.qualification_fixtures import HARNESS, ISSUED, gates, golden, make_pair +from tests.qualification_fixtures import ( + HARNESS, + ISSUED, + UK_DATA_LINE_BUNDLE, + gates, + golden, + make_pair, +) ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / ".github" / "qualification-ledger.sh" CLI = ROOT / "scripts" / "check_uk_candidate.py" -BASE = json.loads( - (ROOT / "src" / "policyengine" / "data" / "bundle" / "manifest.json").read_text() -) +# PAIR is a policyengine-uk-data pair, so main is a bundle on that line. +BASE = UK_DATA_LINE_BUNDLE PAIR = make_pair( model="2.200.0", data=BASE["data_releases"]["uk"]["data_package"]["version"] ) diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index cf57c602e..ace9455e8 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -51,12 +51,13 @@ from tests.qualification_fixtures import ( HARNESS, ISSUED, - UK_DATA_LINE_BUNDLE_PATH, + UK_DATA_LINE_BUNDLE, VENDORED_SCHEMAS, gates, golden, graphql_base_changes_output, jq, + load_candidate_cli, make_pair, make_receipt, rest_pulls_output, @@ -70,7 +71,7 @@ ) #: The candidates these tests build are on the policyengine-uk-data line, so #: they are qualified against the last bundle certified on that line. -BUNDLE = json.loads(UK_DATA_LINE_BUNDLE_PATH.read_text()) +BUNDLE = UK_DATA_LINE_BUNDLE def _wheel(name, version, digest): @@ -157,15 +158,7 @@ def test_resolve_refuses_a_pair_other_than_the_one_requested( resolve to another (a moved data tag, another core), the run fails and nothing else is qualified.""" - import importlib.util - - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_pair_pin", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_candidate_pair_pin") pair = make_pair(model="2.200.0") resolved = cli.ResolvedCandidate( pair, @@ -479,15 +472,8 @@ def test_requalify_names_the_target_only_while_its_baseline_is_stale( monkeypatch, tmp_path, capsys ): import argparse - import importlib.util - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_requalify", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_candidate_requalify") current = _promotion_receipt(pair=_promotable_pair_and_branch()) stale = current.model_copy( update={ @@ -1425,7 +1411,6 @@ def test_promotion_makes_main_exactly_the_qualified_commit( moves main to it with a push leased on that parent.""" import argparse - import importlib.util checks, runs, at_commit, main, push_code, automatic, promotes, readiness = ( PROMOTION_CASES[case] @@ -1434,13 +1419,7 @@ def test_promotion_makes_main_exactly_the_qualified_commit( branch = candidate_branch(qualified.pair) commit = qualified.harness.git_commit parent = "p" * 40 - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_for_test", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_candidate_for_test") calls = [] target = { @@ -1557,8 +1536,6 @@ def _sweep_cli( ``shared_fails`` fails the REST listing of a branch's pull requests ("listing") or the base-change query ("history").""" - import importlib.util - qualified = _promotion_receipt(pair=_promotable_pair_and_branch()) branch = candidate_branch(qualified.pair) commit = qualified.harness.git_commit @@ -1566,13 +1543,7 @@ def _sweep_cli( if superseded: receipts.append(_promotion_receipt(pair=SUPERSEDED_PAIR)) stale = "candidate/uk/stale-pair" - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_for_test", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_candidate_for_test") calls = [] listing = [ { @@ -1716,15 +1687,7 @@ def test_a_candidate_listing_at_its_limit_is_refused(monkeypatch, listed, refuse listing that reaches it may have missed the target; promotion refuses to go on rather than act on part of the list.""" - import importlib.util - - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_for_test", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_candidate_for_test") listing = [ { "number": number, @@ -1942,15 +1905,7 @@ def test_only_pull_requests_to_main_are_candidates(monkeypatch): branch cannot collapse into one entry that sends readiness changes to the wrong one.""" - import importlib.util - - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_for_test", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_candidate_for_test") calls = [] listing = [ { @@ -2752,8 +2707,6 @@ def _publish_as_attempt(ledger, pair, receipt, key): def test_forced_stage_recovery_binds_the_terminal_receipt_it_supersedes( tmp_path, verdict, main_advanced, staged_base, monkeypatch, capsys ): - import importlib.util - from policyengine.provenance.qualification_ledger import attempt_directory from policyengine.provenance.uk_qualification import stage_attempt @@ -2777,12 +2730,7 @@ def test_forced_stage_recovery_binds_the_terminal_receipt_it_supersedes( candidate_action(pair, BUNDLE, tmp_path, base_commit=base, now=now)[0] == "qualify" ) - spec = importlib.util.spec_from_file_location( - "check_uk_bound_recovery_for_test", REPO_ROOT / "scripts/check_uk_candidate.py" - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_bound_recovery_for_test") monkeypatch.setattr(cli, "utc_now", lambda: now) assert cli.main(["recoverable", "--ledger", str(tmp_path)]) == 0 assert json.loads(capsys.readouterr().out) == payload @@ -2852,18 +2800,10 @@ def test_forced_recovery_preserves_newer_work_on_another_base(tmp_path, protecte def test_resolution_preserves_authorization_to_recover_forced_work( tmp_path, monkeypatch, verdict, main_advanced, abandonment ): - import importlib.util - from policyengine.provenance.qualification_ledger import attempt_directory from policyengine.provenance.uk_qualification import stage_attempt - spec = importlib.util.spec_from_file_location( - "check_uk_resolve_recovery_for_test", - REPO_ROOT / "scripts/check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_resolve_recovery_for_test") pair = make_pair(model="2.200.0") terminal = make_receipt( pair=pair, @@ -2982,14 +2922,7 @@ def test_cross_base_recovery_allocates_new_keys_without_overwriting_evidence(tmp def test_recovery_requires_a_confirmed_failed_owner( monkeypatch, status, conclusion, expected ): - import importlib.util - - spec = importlib.util.spec_from_file_location( - "check_uk_owner_for_test", REPO_ROOT / "scripts/check_uk_candidate.py" - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_owner_for_test") calls = [] def run(argv, **kwargs): @@ -3010,20 +2943,75 @@ def run(argv, **kwargs): assert not cli.owner_run_failed("42") +PACKAGED_BUNDLE_PATH = REPO_ROOT / "src/policyengine/data/bundle/manifest.json" + + +@pytest.mark.parametrize("force", [False, True]) +def test_recovery_skips_unforced_pairs_from_another_data_line( + tmp_path, monkeypatch, capsys, force +): + from policyengine.provenance.qualification_ledger import attempt_directory + from policyengine.provenance.uk_qualification import stage_attempt + + # Main is the packaged bundle; make_pair builds a uk-data pair. + cli = load_candidate_cli("check_uk_recovery_line", PACKAGED_BUNDLE_PATH) + pair = make_pair() + attempt = attempt_directory(tmp_path, "uk", pair.pair_id, "a" * 12) + payload = {"pair": pair.canonical_payload(), "force": force} + stage_attempt(attempt, payload, HARNESS) + write_json( + attempt / "staging.json", + {"sequence": 1, "owner_run_id": "42", "expires_at": "2026-10-14T14:00:00Z"}, + ) + monkeypatch.setattr(cli, "utc_now", lambda: "2026-10-14T11:00:00Z") + monkeypatch.setattr(cli, "owner_run_failed", lambda run_id: run_id == "42") + + assert cli.main(["recoverable", "--ledger", str(tmp_path)]) == 0 + + out = capsys.readouterr() + if force: + assert json.loads(out.out) == payload + else: + assert out.out == "" + assert "not on main's data line" in out.err + + +def test_check_refresh_skips_pairs_from_another_data_line(tmp_path, monkeypatch): + import argparse + + cli = load_candidate_cli("check_uk_refresh_line", PACKAGED_BUNDLE_PATH) + receipt = _promotion_receipt(pair=make_pair(model="2.200.0")) + assert receipt.promotable + branch = candidate_branch(receipt.pair) + monkeypatch.setattr(cli, "latest_receipts", lambda ledger: [receipt]) + monkeypatch.setattr( + cli, + "_candidate_pull_requests", + lambda: [ + { + "number": 1, + "headRefName": branch, + "headRefOid": receipt.harness.git_commit, + } + ], + ) + + def unexpected(*args, **kwargs): + raise AssertionError("a pair from another data line was refreshed") + + monkeypatch.setattr(cli, "_shared_branch_pull_requests", unexpected) + monkeypatch.setattr(cli, "_candidate_runs", unexpected) + + assert cli.cmd_refresh_pr_checks(argparse.Namespace(ledger=str(tmp_path))) == 0 + + def test_catchup_finds_abandoned_stages_without_replay_requests( tmp_path, monkeypatch, capsys ): - import importlib.util - from policyengine.provenance.qualification_ledger import attempt_directory from policyengine.provenance.uk_qualification import stage_attempt - spec = importlib.util.spec_from_file_location( - "check_uk_recovery_for_test", REPO_ROOT / "scripts/check_uk_candidate.py" - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) - cli.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH + cli = load_candidate_cli("check_uk_recovery_for_test") pair = make_pair() attempt = attempt_directory(tmp_path, "uk", pair.pair_id, "a" * 12) payload = {"pair": pair.canonical_payload()} diff --git a/tests/test_uk_qualification_workflow.py b/tests/test_uk_qualification_workflow.py index 4022c3054..1e07a780e 100644 --- a/tests/test_uk_qualification_workflow.py +++ b/tests/test_uk_qualification_workflow.py @@ -3,7 +3,6 @@ from __future__ import annotations import argparse -import importlib.util import json import os import subprocess @@ -21,9 +20,9 @@ ) from tests.qualification_fixtures import ( HARNESS, - UK_DATA_LINE_BUNDLE_PATH, gates, graphql_base_changes_output, + load_candidate_cli, make_pair, make_receipt, rest_pulls_output, @@ -36,14 +35,7 @@ def _cli(): - spec = importlib.util.spec_from_file_location( - "qualification_workflow_cli", REPO_ROOT / "scripts/check_uk_candidate.py" - ) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - # These tests build candidates on the policyengine-uk-data line. - module.BUNDLE_PATH = UK_DATA_LINE_BUNDLE_PATH - return module + return load_candidate_cli("qualification_workflow_cli") def _workflow(): From 5882164abbdb56c004272ca97006142bed6e12d3 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 21:32:37 +0100 Subject: [PATCH 09/26] Give a regional release's datasets their own identity in receipts _data_identity treated every certified dataset entry as part of the primary release, so a receipt for a run on the local-area file named the national cut as its build and manifest revision, beside the local file's own revision. Only an entry at the primary release's revision now takes the release's identity; any other entry is named, like an overlay, by its own revision. Co-Authored-By: Claude Opus 5.5 --- src/policyengine/provenance/receipt.py | 11 ++++++++++- tests/test_execution_receipt.py | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index c1a6c525a..12bcbcdbd 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -3785,7 +3785,16 @@ def _data_identity( f"The file is not the bundle's {dataset_name!r}: it hashes to " f"{dataset_sha256}, but {spec.source_uri} is {spec.sha256}." ) - release = dataset_name in (record.get("datasets") or {}) + datasets = record.get("datasets") or {} + entry = datasets.get(dataset_name) + primary = datasets.get(record.get("default_dataset")) or {} + # A regional release's artifacts are certified beside the primary release + # but are not part of it, so only entries at the primary release's own + # revision take its build and manifest identity. + release = entry is not None and entry.get("revision") in ( + None, + primary.get("revision"), + ) data_package = record["data_package"] certified_artifact = record.get("certified_data_artifact") or {} certification = record.get("certification") or {} diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 0c540873d..de0fccf59 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -410,6 +410,27 @@ def test_overlay_datasets_receive_uncertified_receipts(dataset_file): assert receipt.data.build_id is None +def test_a_regional_release_dataset_keeps_its_own_identity(dataset_file): + """A local-area file is certified beside the national release, not in it.""" + + bundle = _bundle_for(dataset_file.read_bytes()) + record = bundle["data_releases"]["uk"] + record["datasets"]["area_file"] = { + "path": "area_file.h5", + "repo_id": record["data_package"]["repo_id"], + "revision": "regional-release-cut", + "sha256": hashlib.sha256(dataset_file.read_bytes()).hexdigest(), + } + + receipt = _build(dataset_file, bundle, dataset="area_file") + + assert receipt.certification_status == "uncertified" + assert receipt.data.dataset_revision == "regional-release-cut" + assert receipt.data.release_version == "regional-release-cut" + assert receipt.data.build_id is None + assert receipt.data.release_manifest_revision is None + + def test_an_omitted_artifact_revision_is_the_release_manifest_revision(dataset_file): """Round-4 review M5: the receipt names what the materializer downloads.""" From e1ae884b0760cd4d598832b2b6574c326c2e9a12 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 21:34:30 +0100 Subject: [PATCH 10/26] Cache a nested bundle artifact under its release path Materialized bundle artifacts were cached by basename, so in a shared data directory the UK and US calibration_diagnostics.json overwrote each other. A nested artifact now keeps its release path under the data directory; top-level files, every H5 among them, keep their cache path. A path that is absolute or leaves the repository is refused. Co-Authored-By: Claude Opus 5.5 --- .../provenance/dataset_materialization.py | 20 +++++++++++++++++-- tests/test_dataset_materialization.py | 19 ++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/policyengine/provenance/dataset_materialization.py b/src/policyengine/provenance/dataset_materialization.py index 35824357f..2f441aaea 100644 --- a/src/policyengine/provenance/dataset_materialization.py +++ b/src/policyengine/provenance/dataset_materialization.py @@ -5,7 +5,7 @@ import os import tempfile from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Literal, Optional import requests @@ -84,6 +84,22 @@ def metadata_destination(self) -> Path: return Path(f"{self.destination}.metadata.json") +def _bundle_destination(data_dir: Path, path: str) -> Path: + """Where a bundle artifact is cached. + + A top-level file is cached by its name. A nested one keeps its release + path, so two releases' files of one name (each country's + ``calibration_diagnostics.json``) never share a cache entry. + """ + + relative = PurePosixPath(path) + if relative.is_absolute() or ".." in relative.parts: + raise DatasetMaterializationError( + f"Bundle artifact path {path!r} must be relative to its repository." + ) + return data_dir.joinpath(*relative.parts) + + def _resolve_bundle_dataset( country_id: str, dataset: Optional[str] = None, @@ -116,7 +132,7 @@ def _resolve_bundle_dataset( revision=reference.revision or _artifact_revision(country_manifest.data_package), sha256=reference.sha256, - destination=data_dir / Path(reference.path).name, + destination=_bundle_destination(data_dir, reference.path), metadata_sha256=reference.metadata_sha256, ) diff --git a/tests/test_dataset_materialization.py b/tests/test_dataset_materialization.py index 256e004da..b2b9a8267 100644 --- a/tests/test_dataset_materialization.py +++ b/tests/test_dataset_materialization.py @@ -252,3 +252,22 @@ def test_authentication_failure_does_not_retry_repository_type(tmp_path, status_ ) assert len(session.calls) == 1 + + +def test_same_named_artifacts_of_two_releases_never_share_a_cache_path(tmp_path): + from policyengine.provenance.dataset_materialization import ( + DatasetMaterializationError, + _bundle_destination, + _resolve_bundle_dataset, + ) + + uk = _resolve_bundle_dataset("uk", "calibration_diagnostics", data_dir=tmp_path) + us = _resolve_bundle_dataset("us", "calibration_diagnostics", data_dir=tmp_path) + assert uk.destination.name == us.destination.name == "calibration_diagnostics.json" + assert uk.destination != us.destination + # A top-level file keeps its cache path. + default = _resolve_bundle_dataset("uk", data_dir=tmp_path) + assert default.destination == tmp_path / default.path + for path in ("/etc/passwd", "../outside.h5", "releases/../../outside.h5"): + with pytest.raises(DatasetMaterializationError, match="must be relative"): + _bundle_destination(tmp_path, path) From 5f3df72f616c80d1bc4745a3d4bc41c29950e127 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 21:37:30 +0100 Subject: [PATCH 11/26] Name the local-area dataset when an area output lacks its codes The Microcosm national default has no constituency_code_oa or la_code_oa; only the local-area file does, while the enhanced FRS it replaces had both. So constituency and local-authority outputs on the default dataset now raise. Their errors now name the dataset the bundle routes those regions to, and the disclosure and outputs docs say to run those levels on it. Co-Authored-By: Claude Opus 5.5 --- docs/disclosure.md | 4 ++ docs/outputs.md | 5 +++ .../outputs/uk_geography_disclosure.py | 3 ++ .../outputs/uk_geography_impact.py | 8 ++++ src/policyengine/provenance/manifest.py | 37 +++++++++++++++++++ tests/test_constituency_impact.py | 6 ++- tests/test_uk_geography_disclosure.py | 24 ++++++++++++ 7 files changed, 86 insertions(+), 1 deletion(-) diff --git a/docs/disclosure.md b/docs/disclosure.md index b6af7dca0..b66c76969 100644 --- a/docs/disclosure.md +++ b/docs/disclosure.md @@ -22,6 +22,10 @@ impact.findings # what blocks publication, if anything impact.public # the rows that may be published, or None while blocked ``` +For constituency and local-authority levels, `baseline` and `reform` must run +on a dataset whose households carry those codes: the local-area dataset +`microcosm_uk_2024_25_dense`, not the Microcosm national default. + The UK datasets derive from surveys under the UK Data Service End User Licence. The defaults below follow its disclosure rules. You can raise them for a study with stricter conditions; you cannot lower them. diff --git a/docs/outputs.md b/docs/outputs.md index c995e4eca..6c96e2b2d 100644 --- a/docs/outputs.md +++ b/docs/outputs.md @@ -273,6 +273,11 @@ control: each number passes a floor on the survey households behind it and carries a 90% interval. Use it for anything you publish. See [Disclosure control](disclosure.md). +Constituency and local-authority levels need simulations on a dataset whose +households carry those codes. The Microcosm national default does not: run +them on the local-area dataset the bundle routes those regions to, +`microcosm_uk_2024_25_dense`. Regions can come from either. + ```python from policyengine.outputs import compute_uk_geography_disclosure diff --git a/src/policyengine/outputs/uk_geography_disclosure.py b/src/policyengine/outputs/uk_geography_disclosure.py index 3e589f4a6..470557e4a 100644 --- a/src/policyengine/outputs/uk_geography_disclosure.py +++ b/src/policyengine/outputs/uk_geography_disclosure.py @@ -26,6 +26,7 @@ Release, estimate_change, ) +from policyengine.provenance.manifest import uk_area_dataset_hint if TYPE_CHECKING: from policyengine.core.simulation import Simulation @@ -252,9 +253,11 @@ def areas_for(level: str) -> Optional[AreaWeights]: if areas is None: if level in self.levels: columns = UK_DISCLOSURE_LEVEL_COLUMNS[level] + hint = uk_area_dataset_hint(level) raise ValueError( f"The household output gives no household an area in " f"any of {list(columns)} for level {level!r}" + + (f". {hint}" if hint else "") ) continue estimated[level] = areas diff --git a/src/policyengine/outputs/uk_geography_impact.py b/src/policyengine/outputs/uk_geography_impact.py index f6158d722..ded47ed28 100644 --- a/src/policyengine/outputs/uk_geography_impact.py +++ b/src/policyengine/outputs/uk_geography_impact.py @@ -14,8 +14,14 @@ default_local_search_dirs, packaged_uk_area_list_path, ) +from policyengine.provenance.manifest import uk_area_dataset_hint logger = logging.getLogger(__name__) +#: The region type whose dataset carries each longwise area column. +AREA_REGION_TYPES = { + "constituency_code_oa": "constituency", + "la_code_oa": "local_authority", +} def warn_unprotected_uk_geography_output(name: str, level: str) -> None: @@ -142,10 +148,12 @@ def compute_longwise_uk_geography_impacts( "household_weight", ]: if column not in baseline_household.columns: + hint = uk_area_dataset_hint(AREA_REGION_TYPES.get(column, "")) raise ValueError( f"UK geography impacts require baseline household column " f"'{column}'. Re-run the simulation with an output dataset " f"that preserves UK geography passthrough columns." + + (f" {hint}" if hint else "") ) if "household_net_income" not in reform_household.columns: raise ValueError( diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index d43e3f74b..e5acd6e93 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -706,6 +706,43 @@ def resolve_default_datasets(country_id: str) -> list[str]: return list(manifest.datasets.keys()) +def region_dataset_name(country_id: str, region_type: str) -> Optional[str]: + """The certified dataset that ``region_type`` regions are routed to, if any.""" + + manifest = get_release_manifest(country_id) + template = manifest.region_datasets.get(region_type) + if template is None: + return None + return next( + ( + name + for name, reference in manifest.datasets.items() + if reference.path == template.path_template + ), + None, + ) + + +def uk_area_dataset_hint(region_type: str) -> str: + """A sentence naming the dataset whose households carry an area's codes. + + The Microcosm national default has no constituency or local-authority + codes; the local-area release the bundle routes those regions to does. + """ + + try: + name = region_dataset_name("uk", region_type) + except ValueError: + return "" + if name is None: + return "" + return ( + f"The certified bundle routes {region_type} regions to the dataset " + f"{name!r}, whose households carry these codes: run the simulations " + "on that dataset." + ) + + def resolve_region_dataset_path( country_id: str, region_type: str, diff --git a/tests/test_constituency_impact.py b/tests/test_constituency_impact.py index 59820857b..64500c9da 100644 --- a/tests/test_constituency_impact.py +++ b/tests/test_constituency_impact.py @@ -252,12 +252,16 @@ def test_compute_constituency_impacts_requires_longwise_geography_column(): } ) - with pytest.raises(ValueError, match="constituency_code_oa"): + with pytest.raises(ValueError, match="constituency_code_oa") as error: compute_uk_constituency_impacts( baseline, reform, download_missing_assets=False, ) + # The error names the dataset whose households carry the codes. + from policyengine.provenance.manifest import uk_area_dataset_hint + + assert uk_area_dataset_hint("constituency") in str(error.value) def test_constituency_impacts_warn_that_they_are_not_disclosure_controlled(caplog): diff --git a/tests/test_uk_geography_disclosure.py b/tests/test_uk_geography_disclosure.py index d4245ab10..073346926 100644 --- a/tests/test_uk_geography_disclosure.py +++ b/tests/test_uk_geography_disclosure.py @@ -464,3 +464,27 @@ def test_the_release_is_available_only_after_a_run(): def test_the_adapter_is_exported_from_outputs(): assert outputs.UKGeographyDisclosure is UKGeographyDisclosure assert outputs.compute_uk_geography_disclosure is compute_uk_geography_disclosure + + +def test_a_missing_area_column_names_the_dataset_that_carries_it(): + from policyengine.provenance.manifest import uk_area_dataset_hint + + baseline, reform = _households() + for frame in (baseline, reform): + del frame["constituency_code_oa"] + with pytest.raises(ValueError, match="no household an area") as error: + _compute(baseline, reform, levels=("constituency",)) + hint = uk_area_dataset_hint("constituency") + assert "microcosm_uk_2024_25_dense" in hint + assert hint in str(error.value) + + +def test_no_hint_when_nothing_routes_the_area(monkeypatch): + from policyengine.provenance import manifest + + record = manifest.get_release_manifest("uk") + unrouted = record.model_copy( + update={"region_datasets": {"national": record.region_datasets["national"]}} + ) + monkeypatch.setattr(manifest, "get_release_manifest", lambda country: unrouted) + assert manifest.uk_area_dataset_hint("constituency") == "" From 02ec73e65f90e96cc955ef50b0172f0d948eddf9 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 21:37:30 +0100 Subject: [PATCH 12/26] Record the switch's consequences and the UC-floor exception The changelog now says that area outputs need the local-area dataset, that the enhanced FRS weight matrices no longer resolve by name, and which policyengine-uk version import accepts with a token. The data certification skill records the accepted Universal Credit floor exception, so the next certification neither flags it again nor treats the floor as optional. households.md and the README no longer name the old core pin or call the UK data repository a model repository. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 ++++--- ...-25-national-20261019T083323Z-6cba5d71.changed.md | 2 +- docs/engineering/skills/data-certification.md | 12 +++++++++++- docs/households.md | 2 +- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f92796d6d..b7da84050 100644 --- a/README.md +++ b/README.md @@ -70,9 +70,10 @@ print(f"Total UC spending: £{agg.result / 1e9:.1f}bn") For baseline-vs-reform comparisons, see `pe.uk.economic_impact_analysis` and its US counterpart. -UK population data is stored in a private Hugging Face model repository. Set -`HUGGING_FACE_TOKEN` to a token from an account with access before running UK -population examples. To download the raw `.h5` file directly, see +UK population data is stored in private Hugging Face repositories: the +Microcosm default in a dataset repository, and the enhanced FRS in a model +repository. Set `HUGGING_FACE_TOKEN` to a token from an account with access +before running UK population examples. To download the raw `.h5` file directly, see [Microsimulation](docs/microsim.md#uk-private-data-and-raw-h5-access). ## Documentation diff --git a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md index 8f9319421..30b7e7e08 100644 --- a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md +++ b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md @@ -1 +1 @@ -Switch the UK default dataset from the Enhanced FRS 2024-25 (policyengine-uk-data 1.58.1) to Microcosm UK 2024-25 (`microcosm_uk_2024_25`, cut `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71`), certified with policyengine-uk 2.132.6 and policyengine-core 3.33.3, the versions it was built with. Constituency and local-authority regions now load the Microcosm local-area release (`microcosm_uk_2024_25_dense`, cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`) and are in the default UK region registry. The Enhanced FRS 2024-25 datasets stay loadable by name. UK results change with the data. +Switch the UK default dataset from the Enhanced FRS 2024-25 (policyengine-uk-data 1.58.1) to Microcosm UK 2024-25 (`microcosm_uk_2024_25`, cut `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71`), certified with policyengine-uk 2.132.6 and policyengine-core 3.33.3, the versions it was built with. Constituency and local-authority regions now load the Microcosm local-area release (`microcosm_uk_2024_25_dense`, cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`) and are in the default UK region registry. UK results change with the data. The national file has no constituency or local-authority codes, so constituency and local-authority outputs (`compute_uk_geography_disclosure` at those levels, `compute_uk_constituency_impacts`, `compute_uk_local_authority_impacts`) need simulations on `microcosm_uk_2024_25_dense`. The Enhanced FRS 2024-25 datasets stay loadable by name, but its weight matrices (`parliamentary_constituency_weights`, `local_authority_weights`, `local_geography_weights`) no longer resolve. With a Hugging Face token set, importing policyengine with any policyengine-uk other than 2.132.6 raises, as it did before for versions the Enhanced FRS release did not claim. The certified default is below the Universal Credit caseload floor (0.897 of DWP's benchmark), an exception accepted and recorded in the data certification skill. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 9f0058fdf..847c317ec 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -188,7 +188,17 @@ python scripts/check_uk_uc_floor.py --dataset Date: Tue, 20 Oct 2026 23:33:14 +0100 Subject: [PATCH 13/26] Check core drift once, and say what a named core binds The strict check compared the core pin with the certificate's core in two branches, with two messages. One comparison now covers every basis and names the versions and the remedy; the exact-pair branch only checks that the pair names its pair and core. A named core binds the core pin, and Microcosm UK releases claim one exact core, so a core bump now needs Microcosm cuts that claim the new core, or a UK re-certification without --core-version. The skill and docs/bundles.md say so, and the skill's UK commands pass --core-version so the binding survives a re-certification done its way. The receipt tests set the named core they test instead of assuming the packaged certificate has one. Co-Authored-By: Claude Opus 5.5 --- docs/bundles.md | 5 ++++ docs/engineering/skills/data-certification.md | 14 +++++++++-- scripts/generate_trace_tros.py | 25 +++++++++++-------- tests/test_execution_receipt.py | 12 +++++---- tests/test_uk_qualification_pipeline.py | 2 +- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/docs/bundles.md b/docs/bundles.md index dc06c6cad..62ccc8438 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -127,6 +127,11 @@ python scripts/bundle.py update-packages \ --uk 2.91.0 ``` +A data certificate that names its core (`certify-data --core-version`) binds +the core pin: `--core` then fails `check --strict-tros` until that country is +re-certified for the new core. See the data certification skill for the UK +case. + To certify a new data release from a data-producer manifest, run: ```bash diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 847c317ec..0ea7c698f 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -50,7 +50,8 @@ Microcosm national release with one of: python scripts/bundle.py certify-data --country uk --data-producer populace \ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ --regional-manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" \ - --model-version "" + --model-version "" \ + --core-version "" ``` This records `regional_release_kind: local_area`, and the default UK region @@ -65,7 +66,8 @@ Microcosm national release with one of: --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ --regional-manifest-uri "hf://model/policyengine/policyengine-uk-data-private@/releases//release_manifest.json" \ --regional-release-kind area_coded_national \ - --model-version "" + --model-version "" \ + --core-version "" ``` Certification carries that release's national dataset @@ -84,6 +86,14 @@ model, and the pinned core when `--core-version` is given, by its own build or compatibility claims (a `--qualified-pair-id` names the primary release only), and HEAD-checks every artifact it carries. +`--core-version` also names the core in the certificate, and receipts then +certify that core, not the bundle's core pin. So `check --strict-tros` refuses +a core pin that differs from it: a core bump, a US `--core-version` run +included, needs the country re-certified for the new core first. Microcosm UK +releases claim one exact core, so that needs Microcosm cuts that claim the new +core. To move the core before they exist, re-certify UK without +`--core-version`: its receipts then certify the pin, as before October 2026. + ### Re-certifying a routed bundle Certification rewrites `data_releases.` from the manifests it is diff --git a/scripts/generate_trace_tros.py b/scripts/generate_trace_tros.py index e01e89e5e..595124adb 100644 --- a/scripts/generate_trace_tros.py +++ b/scripts/generate_trace_tros.py @@ -139,16 +139,18 @@ def _validate_runtime_model(bundle: dict, country: CountryReleaseManifest) -> No prefix = f"{country.country_id}: strict TRACE release" certification = country.certification core = bundle.get("packages", {}).get("policyengine-core", {}) - if certification and certification.compatibility_basis == "qualified_exact_pair": - if ( - not certification.qualification_pair_id - or not certification.certified_for_core_version - or core.get("version") != certification.certified_for_core_version - ): - raise ValueError( - f"{prefix}: runtime core differs from the qualified exact pair" - ) - elif ( + if ( + certification + and certification.compatibility_basis == "qualified_exact_pair" + and not ( + certification.qualification_pair_id + and certification.certified_for_core_version + ) + ): + raise ValueError( + f"{prefix}: a qualified exact pair must name its pair and core" + ) + if ( certification and certification.certified_for_core_version and core.get("version") != certification.certified_for_core_version @@ -158,7 +160,8 @@ def _validate_runtime_model(bundle: dict, country: CountryReleaseManifest) -> No raise ValueError( f"{prefix}: runtime core {core.get('version')} differs from the " f"certified core {certification.certified_for_core_version}; " - "re-certify the data release for the new core" + "re-certify the data release for the new core, or without " + "--core-version so that receipts certify the pin" ) if ( model.name != key diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index de0fccf59..66fe4d220 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -319,8 +319,8 @@ def test_a_core_the_certificate_names_decides_over_the_pin(dataset_file): # bundle.py check refuses a pin that differs from a named core # (scripts/generate_trace_tros.py), because receipts follow the certificate. bundle = _bundle_for(dataset_file.read_bytes()) - assert ( - "certified_for_core_version" in bundle["data_releases"]["uk"]["certification"] + bundle["data_releases"]["uk"]["certification"]["certified_for_core_version"] = ( + INSTALLED_CORE ) bundle["packages"]["policyengine-core"]["version"] = "0.0.1" @@ -332,10 +332,12 @@ def test_a_core_the_certificate_names_decides_over_the_pin(dataset_file): @pytest.mark.parametrize("pin", [INSTALLED_CORE, "0.0.1"]) def test_a_certificate_without_a_core_certifies_the_pinned_core(dataset_file, pin): - # Certificates from before --core-version, and receipts from - # policyengine 6.5.0-6.8.0, name no core: the pin is the certified core. + # Certificates from before --core-version, and those certified without + # it, name no core: the pin is the certified core. bundle = _bundle_for(dataset_file.read_bytes()) - del bundle["data_releases"]["uk"]["certification"]["certified_for_core_version"] + bundle["data_releases"]["uk"]["certification"].pop( + "certified_for_core_version", None + ) bundle["packages"]["policyengine-core"]["version"] = pin receipt = _build(dataset_file, bundle) diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index ace9455e8..029916cc3 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -705,7 +705,7 @@ def run(command, *, cwd, env=None): changed = json.loads(bundle_path.read_bytes()) changed["packages"]["policyengine-core"]["version"] = "99.0.0" write_json(bundle_path, changed) - with pytest.raises(ValueError, match="runtime core.*qualified exact pair"): + with pytest.raises(ValueError, match="runtime core 99.0.0 differs"): tros.generated_tros(strict=True) From 90172bd18f8738d9eeb39d422c1f4a9175395870 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 23:36:13 +0100 Subject: [PATCH 14/26] State the overlay rule once, and warn when a name stops resolving The rule that an overlay may not shadow a certified name was written three times: at runtime, in the maintenance validator, and in the pruning certification does. shadowed_dataset_overlays in the stdlib validator now states it, and all three use it. A pruned overlay does not come back, so a later certification that stops certifying its name (rolling forward after a rollback, or a ship-together run after a national-first one) dropped the name silently. Certification now warns about every name that stops resolving. The texts that said overlays always survive re-certification now say which ones do, and the skill has a re-add step. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/data-certification.md | 7 +++ docs/release-bundles.md | 7 +-- .../provenance/bundle_validation.py | 21 +++++++-- src/policyengine/provenance/certification.py | 22 +++++++--- src/policyengine/provenance/manifest.py | 43 ++++++++++--------- tests/test_certify_data_release.py | 3 ++ tests/test_release_manifests.py | 5 ++- 7 files changed, 74 insertions(+), 34 deletions(-) diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 0ea7c698f..37d23ca66 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -108,6 +108,13 @@ rollback to policyengine-uk-data with `--drop-regional-release`, for example, removes the enhanced FRS overlays. `bundle.py check` refuses a bundle with an overlay that shadows a certified name. +Removed overlays do not come back by themselves. Certification warns about +every dataset name that stops resolving. After a certification that stops +certifying a name that must stay loadable (rolling forward after a rollback, +or a ship-together run after a national-first one), re-add its overlay from +the previous bundle, as #685 did for `enhanced_frs_2024_25`, `frs_2024_25` and +their `_tiny` files. + ### US For US Populace certification, certify the Populace release manifest directly: diff --git a/docs/release-bundles.md b/docs/release-bundles.md index 927c77d3b..def3658d2 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -237,9 +237,10 @@ sibling `dataset_overlays.{country}` map: `resolve_dataset_reference` and `managed_microsimulation` load them by name at their own pinned, sha-verified revision. Overlays are strictly additive: an overlay may not shadow the certified default or any certified dataset, so -default resolution is untouched. Because certification only rewrites -`data_releases`, overlays survive re-certification without any manual re-add -step. +default resolution is untouched. Certification keeps every overlay except one +whose name the new release certifies itself: it removes that overlay and says +so. If a later release stops certifying the name, re-add the overlay; +certification warns about every name that stops resolving. Datasets used automatically for a region type are not overlays. They are certified into `data_releases.{country}.datasets`, and diff --git a/src/policyengine/provenance/bundle_validation.py b/src/policyengine/provenance/bundle_validation.py index 150a6929d..525b86975 100644 --- a/src/policyengine/provenance/bundle_validation.py +++ b/src/policyengine/provenance/bundle_validation.py @@ -24,6 +24,22 @@ def _is_normalized_version(value: Any) -> bool: ) +def shadowed_dataset_overlays( + release: Mapping[str, Any], overlays: Mapping[str, Any] +) -> list[str]: + """The overlay names a country's certified release itself certifies. + + Overlays are strictly additive: one may never shadow the certified default + or any certified dataset, so it can never alter resolution. The runtime + refuses a shadowing overlay, maintenance checks refuse it first, and + certification removes the overlays a new release supersedes. + """ + certified = set(release.get("datasets") or {}) + if release.get("default_dataset"): + certified.add(release["default_dataset"]) + return sorted(set(overlays or {}) & certified) + + def validate_dataset_overlays(bundle: Mapping[str, Any]) -> None: """Refuse an overlay that shadows a certified dataset name. @@ -33,10 +49,7 @@ def validate_dataset_overlays(bundle: Mapping[str, Any]) -> None: overlays = bundle.get("dataset_overlays") or {} for country, entries in overlays.items(): release = (bundle.get("data_releases") or {}).get(country) or {} - certified = set(release.get("datasets") or {}) - if release.get("default_dataset"): - certified.add(release["default_dataset"]) - shadowed = sorted(set(entries or {}) & certified) + shadowed = shadowed_dataset_overlays(release, entries) if shadowed: raise ValueError( f"dataset_overlays.{country} shadows certified datasets: " diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 73fcb1b3f..a59c84877 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -52,6 +52,7 @@ CONSTITUENCY_ASSET_SPEC, LOCAL_AUTHORITY_ASSET_SPEC, ) +from policyengine.provenance.bundle_validation import shadowed_dataset_overlays from policyengine.provenance.manifest import ( HF_REQUEST_TIMEOUT_SECONDS, DataReleaseArtifact, @@ -1371,16 +1372,13 @@ def certify_data_release( core_version=core_version, qualified_pair_id=qualified_pair_id, ) + previous_release = (bundle.get("data_releases") or {}).get(country) or {} bundle.setdefault("data_releases", {})[country] = data_release # Overlays may never shadow a certified name, so a release that certifies # a name the bundle keeps as an overlay (a rollback to the release it came # from, or the dataset a national-first route carries) supersedes it. overlays = (bundle.get("dataset_overlays") or {}).get(country) or {} - superseded = sorted( - name - for name in overlays - if name in data_release["datasets"] or name == data_release["default_dataset"] - ) + superseded = shadowed_dataset_overlays(data_release, overlays) for name in superseded: del overlays[name] if superseded: @@ -1389,6 +1387,20 @@ def certify_data_release( f"Removed the dataset_overlays.{country} entries this release now " f"certifies: {', '.join(superseded)}.", ] + # A name the previous release certified, and neither the new release nor + # an overlay keeps, stops resolving. Say so, so that it can be re-added. + dropped = sorted( + set(previous_release.get("datasets") or {}) + - set(data_release["datasets"]) + - set(overlays) + ) + if dropped: + warnings = [ + *warnings, + f"These {country.upper()} datasets no longer resolve by name: " + f"{', '.join(dropped)}. Re-add any that must stay loadable under " + f"dataset_overlays.{country}.", + ] bundle.setdefault("countries", {}).setdefault(country, {})["model_package"] = ( model_package ) diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index e5acd6e93..61e324922 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -10,6 +10,8 @@ import requests from pydantic import BaseModel, Field +from policyengine.provenance.bundle_validation import shadowed_dataset_overlays + HF_REQUEST_TIMEOUT_SECONDS = 30 PYPI_REQUEST_TIMEOUT_SECONDS = 30 @@ -315,33 +317,34 @@ def _apply_dataset_overlays( registry of *additional* named datasets that resolution should expose without ever certifying them as the default — for example a staged artifact published in its own immutable release (see - ``docs/release-bundles.md``). Because certification only rewrites - ``data_releases``, overlays survive re-certification untouched. + ``docs/release-bundles.md``). Certification keeps every overlay except + one whose name the new release certifies itself, which it removes and + reports; re-add such an overlay if a later release stops certifying it. - Overlays are strictly additive: an overlay may not shadow the certified - default dataset or any certified dataset entry, so it can never alter - default resolution. + Overlays are strictly additive (``shadowed_dataset_overlays``): an overlay + may not shadow the certified default dataset or any certified dataset + entry, so it can never alter default resolution. """ overlays = (bundle.get(DATASET_OVERLAYS_KEY) or {}).get(country_id) or {} if not overlays: return release_payload - certified_datasets = dict(release_payload.get("datasets") or {}) + shadowed = shadowed_dataset_overlays(release_payload, overlays) default_dataset = release_payload.get("default_dataset") - for overlay_name, overlay_reference in overlays.items(): - if overlay_name == default_dataset: - raise ValueError( - f"Dataset overlay '{overlay_name}' for country '{country_id}' " - "shadows the certified default dataset. Overlays must never " - "change default resolution." - ) - if overlay_name in certified_datasets: - raise ValueError( - f"Dataset overlay '{overlay_name}' for country '{country_id}' " - "collides with a certified dataset entry. Overlays must be " - "additive, not overrides." - ) - certified_datasets[overlay_name] = overlay_reference + if default_dataset in shadowed: + raise ValueError( + f"Dataset overlay '{default_dataset}' for country '{country_id}' " + "shadows the certified default dataset. Overlays must never " + "change default resolution." + ) + if shadowed: + raise ValueError( + f"Dataset overlay '{shadowed[0]}' for country '{country_id}' " + "collides with a certified dataset entry. Overlays must be " + "additive, not overrides." + ) + certified_datasets = dict(release_payload.get("datasets") or {}) + certified_datasets.update(overlays) return {**release_payload, "datasets": certified_datasets} diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 4c2c5510e..bcf6a2296 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1542,6 +1542,9 @@ def test__given_a_rollback_to_overlaid_names__then_their_overlays_go( ) assert bundle["dataset_overlays"]["uk"] == {"populace_uk_2023": kept} assert any("enhanced_frs_2024_25, frs_2024_25" in w for w in result.warnings) + # The Microcosm names stop resolving, and certification says so. + dropped = next(w for w in result.warnings if "no longer resolve" in w) + assert "microcosm_uk_2024_25_dense" in dropped manifest = CountryReleaseManifest.model_validate( _apply_dataset_overlays("uk", bundle["data_releases"]["uk"], bundle) ) diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 265dd4df1..4256e32d7 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -1202,8 +1202,9 @@ class TestDatasetOverlays: ``dataset_overlays`` is the hand-maintained sibling of ``data_releases``. ``certify_data_release`` rewrites ``data_releases.{country}`` wholesale, so - these tests pin the invariant that overlays are additive, survive - re-certification, and can never hijack default resolution. + these tests pin the invariant that overlays are additive, survive a + re-certification that does not certify their names, and can never hijack + default resolution. """ def teardown_method(self): From 741b269575251bd1580966ca02fb89c89562836b Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 23:39:51 +0100 Subject: [PATCH 15/26] Close candidate PRs from another data line, and never recover them The catch-up re-dispatches recoverable attempts without force, so even a forced run of a uk-data pair resolved as other_line and cleared nothing: it was re-dispatched every two hours for good. Recovery now skips every pair from another line, after the recoverability check so that live attempts log nothing. A forced cross-line run opens a candidate pull request that can never be promoted, and its body said merging promotes the pair. Promotion now closes the open pull requests of pairs from another line (the ledger keeps their evidence), and such a PR says it is never promoted. The runbook no longer says an unforced run is recorded. Tests that need a routed Microcosm main now read a frozen copy of the bundle this PR certifies, so a rollback of the packaged bundle does not break them. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/runbooks/uk-qualification.md | 20 +- scripts/check_uk_candidate.py | 41 +- .../provenance/uk_qualification.py | 63 ++- .../microcosm_routed_bundle.json | 392 ++++++++++++++++++ tests/qualification_fixtures.py | 10 + tests/test_uk_qualification_pipeline.py | 57 +-- 6 files changed, 528 insertions(+), 55 deletions(-) create mode 100644 tests/fixtures/uk_qualification/microcosm_routed_bundle.json diff --git a/docs/engineering/runbooks/uk-qualification.md b/docs/engineering/runbooks/uk-qualification.md index b4f0a0350..332f8e1cb 100644 --- a/docs/engineering/runbooks/uk-qualification.md +++ b/docs/engineering/runbooks/uk-qualification.md @@ -123,12 +123,16 @@ something the gate will catch. Promotion compares only pairs on main's data line (its UK certificate's data package and repository). While Microcosm UK is the default, the -policyengine-uk-data candidates the workflow resolves are `other_line`: they -are recorded and never promoted, and recovery and check refreshes skip them. - -`force` still qualifies such a pair, for evidence. Its candidate commit drops -main's area routing (`--drop-regional-release`), because the routing belongs -to the line the pair would replace, and leaves the test constants that pin -main's certificate as they are, so the candidate's own tests fail. Changing -the data line is a reviewed certification pull request (see +policyengine-uk-data candidates the workflow resolves are `other_line`. An +unforced run of one stops there and writes nothing to the ledger. Recovery and +check refreshes skip such pairs, and promotion closes their open pull requests; +the ledger keeps their evidence. + +`force` still qualifies such a pair, for evidence only. Its candidate commit +drops main's area routing (`--drop-regional-release`), because the routing +belongs to the line the pair would replace, and leaves the test constants that +pin main's certificate as they are, so the candidate's own tests fail. Its pull +request says it is never promoted, and promotion closes it. The catch-up does +not recover such a run: re-run it by hand with `force`. Changing the data line +is a reviewed certification pull request (see `docs/engineering/skills/data-certification.md`), not a promotion. diff --git a/scripts/check_uk_candidate.py b/scripts/check_uk_candidate.py index 852274e52..c2fbdef2b 100644 --- a/scripts/check_uk_candidate.py +++ b/scripts/check_uk_candidate.py @@ -466,8 +466,12 @@ def cmd_finalize(args: argparse.Namespace) -> int: attempt, Path(args.ledger), issued_by=issued_by(), completed_at=utc_now() ) payload = read_json(attempt / "candidate.json") + line = certified_data_line(json.loads(BUNDLE_PATH.read_text())) body = candidate_pr_body( - receipt, receipt.pair, ledger_url=ledger_url(receipt.pair_id) + receipt, + receipt.pair, + ledger_url=ledger_url(receipt.pair_id), + on_main_line=data_line(receipt.pair) == line, ) if args.pr_body: Path(args.pr_body).write_text(body) @@ -547,20 +551,22 @@ def cmd_recoverable(args: argparse.Namespace) -> int: attempt, terminal ): continue - candidate = read_json(attempt / "candidate.json") - # An unforced run of a pair from another data line resolves as - # other_line before any recovery, so dispatching it clears nothing. - if not candidate.get("force") and ( - data_line(CandidatePair.model_validate(candidate["pair"])) != line + if not attempt_is_recoverable( + attempt, now=now, owner_is_failed=owner_run_failed ): + continue + candidate = read_json(attempt / "candidate.json") + # The catch-up re-dispatches without force, and a pair from another + # data line then resolves as other_line before any recovery, so the + # dispatch would clear nothing. A forced run is re-run by hand. + if data_line(CandidatePair.model_validate(candidate["pair"])) != line: print( f"::notice::not recovering {pair_dir.name}: its pair is not on " - "main's data line", + "main's data line; re-run it by hand with force", file=sys.stderr, ) continue - if attempt_is_recoverable(attempt, now=now, owner_is_failed=owner_run_failed): - print(json.dumps(candidate)) + print(json.dumps(candidate)) return 0 @@ -950,6 +956,8 @@ def cmd_promote(args: argparse.Namespace) -> int: ) for branch in plan.close_branches: print(f"superseded {branch}") + for branch in plan.close_off_line: + print(f"off main's data line {branch}") if not args.apply: return 0 for branch in plan.close_branches: @@ -966,6 +974,21 @@ def cmd_promote(args: argparse.Namespace) -> int: ], check=False, ) + for branch in plan.close_off_line: + subprocess.run( + [ + "gh", + "pr", + "close", + numbers.get(branch, branch), + "--repo", + LEDGER_REPOSITORY, + "--comment", + "This pair is not on main's data line, so it is never promoted. " + "The ledger keeps this candidate's evidence.", + ], + check=False, + ) # Only the plan's target may be ready, and only while _promote finds it # on current main: any other ready candidate (left by a run that could # not undo it, or by older code) goes back to draft. diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index 6fd9d936a..a1e1ced42 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -14,9 +14,11 @@ ``prepare`` On a checkout of ``main``, certify the pair into the bundle manifest (exact ``==`` claim backed by the pair id when no publisher claim covers - it), regenerate pins, ``uv.lock``, TROs, the certified-version test - constants and household snapshots together, and commit that as the - candidate commit on ``candidate/uk/-``. + it), regenerate pins, ``uv.lock``, TROs and household snapshots + together, and the certified-version test constants for a pair on main's + data line, and commit that as the candidate commit on + ``candidate/uk/-``. A routed bundle keeps its regional + release for a pair on its line and drops it for a pair from another. ``gates`` Run the mandatory local gates on the candidate commit (:mod:`policyengine.provenance.uk_candidate`). @@ -1585,6 +1587,9 @@ class PromotionPlan: #: The target, when its qualification compared against an earlier #: certificate than main's: it needs requalifying before promotion. requalify: Optional[CandidatePair] = None + #: Open candidate branches whose pair is not on main's data line. They are + #: never promoted, so their pull requests are closed. + close_off_line: list[str] = field(default_factory=list) def plan_promotion( @@ -1604,9 +1609,24 @@ def plan_promotion( A branch that moved since is left alone and reported. """ - # Only pairs on main's data line are ordered against it; others are left - # alone (neither promoted nor closed). + # Only pairs on main's data line are ordered against it. A pair from + # another line is never promoted, so its open pull request is closed; the + # ledger keeps its evidence. + receipts = list(receipts) + line = certified_data_line(bundle) + off_line = sorted( + { + candidate_branch(receipt.pair) + for receipt in receipts + if data_line(receipt.pair) != line + } + & set(open_candidate_branches) + ) receipts = _same_line_receipts(receipts, bundle) + + def plan(**fields: Any) -> PromotionPlan: + return PromotionPlan(**fields, close_off_line=off_line) + current = certified_order_key(bundle) target = select_promotion_target(receipts, current) baseline_matches = target is not None and ( @@ -1625,14 +1645,14 @@ def plan_promotion( elif baseline_matches and dominates(target.pair.order_key, key): close.append(branch) if target is None: - return PromotionPlan( + return plan( target=None, ready_branch=None, close_branches=sorted(close), reason=f"no promotable pair advances {current}", ) if not baseline_matches: - return PromotionPlan( + return plan( target=target, ready_branch=None, close_branches=sorted(close), @@ -1647,7 +1667,7 @@ def plan_promotion( qualified_commit = target.harness.git_commit head = open_candidate_branches.get(branch) if head is None: - return PromotionPlan( + return plan( target=target, ready_branch=None, close_branches=sorted(close), @@ -1657,7 +1677,7 @@ def plan_promotion( ), ) if head != qualified_commit: - return PromotionPlan( + return plan( target=target, ready_branch=None, close_branches=sorted(close), @@ -1667,7 +1687,7 @@ def plan_promotion( "nothing is promoted" ), ) - return PromotionPlan( + return plan( target=target, ready_branch=branch, ready_commit=qualified_commit, @@ -1682,8 +1702,13 @@ def candidate_pr_body( *, ledger_url: str, prepared: Optional[Mapping[str, Any]] = None, + on_main_line: bool = True, ) -> str: - """Markdown body for a candidate PR: the pair, the gates and the verdict.""" + """Markdown body for a candidate PR: the pair, the gates and the verdict. + + ``on_main_line`` is false for a pair from another data line than main's + certificate (a forced qualification), which is never promoted. + """ lines = [ f"Automated certification candidate for **policyengine-uk {pair.model.version}**, " @@ -1705,11 +1730,21 @@ def candidate_pr_body( summary = gate.summary.replace("|", "\\|").replace("\n", " ") lines.append(f"| `{gate_id}` | {gate.status} | {summary} |") verdict = receipt.verdict if receipt else "pending" + if on_main_line: + outcome = ( + "Merging this PR promotes the pair. Until cos d1344 is ruled, " + "promotion is a reviewed merge; the certification evidence test " + "refuses an exact-pair certificate without a promotable ledger receipt." + ) + else: + outcome = ( + "This pair is not on main's data line, so it was qualified for " + "evidence only and is never promoted. Its tests fail by design, and " + "promotion closes this PR; the ledger keeps the evidence." + ) lines += [ "", - f"**Verdict: {verdict}.** Merging this PR promotes the pair. Until cos d1344 " - "is ruled, promotion is a reviewed merge; the certification evidence test " - "refuses an exact-pair certificate without a promotable ledger receipt.", + f"**Verdict: {verdict}.** {outcome}", "", "🤖 Generated with [Claude Code](https://claude.com/claude-code)", ] diff --git a/tests/fixtures/uk_qualification/microcosm_routed_bundle.json b/tests/fixtures/uk_qualification/microcosm_routed_bundle.json new file mode 100644 index 000000000..8a2f3fca5 --- /dev/null +++ b/tests/fixtures/uk_qualification/microcosm_routed_bundle.json @@ -0,0 +1,392 @@ +{ + "bundle_version": "6.9.0", + "citation": { + "publisher": "PolicyEngine", + "title": "PolicyEngine bundle 6.9.0", + "type": "software-bundle", + "version": "6.9.0" + }, + "countries": { + "uk": { + "model_package": "policyengine-uk" + }, + "us": { + "model_package": "policyengine-us" + } + }, + "data_releases": { + "uk": { + "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "bundle_id": "uk-6.9.0", + "certification": { + "built_with_model_version": "2.132.6", + "certified_by": "policyengine.py bundle certification", + "certified_for_core_version": "3.33.3", + "certified_for_model_version": "2.132.6", + "compatibility_basis": "built_with_model_package", + "data_build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + }, + "certified_data_artifact": { + "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "data_package": { + "name": "microcosm-data", + "version": "0.1.0" + }, + "dataset": "microcosm_uk_2024_25", + "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece", + "uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + }, + "country_id": "uk", + "data_package": { + "name": "microcosm-data", + "release_manifest_path": "releases/microcosm-uk-2024-25-national/release_manifest.json", + "release_manifest_revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "version": "0.1.0" + }, + "data_producer": "populace", + "datasets": { + "calibration_diagnostics": { + "path": "releases/microcosm-uk-2024-25-national/calibration_diagnostics.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "34978e2bfde3f2641769c9c3a8f60455c59ecae5a50524b359d74388d7edc888" + }, + "microcosm_uk_2024_25": { + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece" + }, + "microcosm_uk_2024_25_calibration": { + "path": "microcosm_uk_2024_25_calibration.npz", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "104a8253bf1aebc91147c9bbb40e2603d1025317ba8c6b991383d5e4d91852c2" + }, + "microcosm_uk_2024_25_dense": { + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe", + "sha256": "8e10e130d0260e0b52e2e29902f0502ebdff76a14a3e57d2c876bf4cd8db5872" + }, + "release_certification": { + "path": "releases/microcosm-uk-2024-25-national/release_certification.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "6a68cc93595fc5a7ad451d75a902523c25659de27014f50431dd5e19af5bd959" + }, + "release_cut_gates": { + "path": "releases/microcosm-uk-2024-25-national/release_cut_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "c91c1364d9c3be98493039d4e1dd8c2256881a434c1f124dd2ed0701bfaf7751" + }, + "score_vs_enhanced_frs": { + "path": "releases/microcosm-uk-2024-25-national/score_vs_enhanced_frs.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "ba8a675f36e7c836bf5521e5933e21bb689442734b7340e05cabd3e0cc55323d" + }, + "spine_gates": { + "path": "releases/microcosm-uk-2024-25-national/spine_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "6dbba976dd1304c07da531382cecb11c9cb69512de0ee65126b71f032a0f37d6" + }, + "terminal_gates": { + "path": "releases/microcosm-uk-2024-25-national/terminal_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "e29439d5ce0ea098d32579284ac06f26f086feba2fb8d12d1e88ac9a54840808" + } + }, + "default_dataset": "microcosm_uk_2024_25", + "default_dataset_uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "model_package": { + "name": "policyengine-uk", + "sha256": "cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354", + "version": "2.132.6", + "wheel_url": "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl" + }, + "policyengine_version": "6.9.0", + "region_datasets": { + "constituency": { + "path_template": "microcosm_uk_2024_25_dense.h5" + }, + "local_authority": { + "path_template": "microcosm_uk_2024_25_dense.h5" + }, + "national": { + "path_template": "microcosm_uk_2024_25.h5" + } + }, + "regional_release_kind": "local_area", + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", + "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", + "schema_version": 1, + "source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", + "version": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + }, + "us": { + "build_id": "populace-us-2024-spm-20260915", + "bundle_id": "us-6.9.0", + "certification": { + "built_with_model_version": "2.2.1", + "certified_by": "policyengine.py bundle certification", + "certified_for_model_version": "2.2.1", + "compatibility_basis": "built_with_model_package", + "data_build_id": "populace-us-2024-spm-20260915" + }, + "certified_data_artifact": { + "build_id": "populace-us-2024-spm-20260915", + "data_package": { + "name": "microcosm-data", + "version": "0.1.0" + }, + "dataset": "populace_us_2024", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915" + }, + "country_id": "us", + "data_package": { + "name": "microcosm-data", + "release_manifest_path": "releases/populace-us-2024-spm-20260915/release_manifest.json", + "release_manifest_revision": "populace-us-2024-spm-20260915", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "version": "0.1.0" + }, + "data_producer": "populace", + "datasets": { + "build_manifest": { + "path": "releases/populace-us-2024-spm-20260915/build_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "f3fd734867324bca7619f5f083a8115e32104bea3948d48723b345cfd39fa03b" + }, + "calibration_diagnostics": { + "path": "releases/populace-us-2024-spm-20260915/calibration_diagnostics.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "870449b44e86b13b25bcea1a57f0e7af37f4d4db18be815eea3acdf9fe6eb40e" + }, + "parent_build_manifest": { + "path": "releases/populace-us-2024-spm-20260915/parent_build_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "1990e8fd37ce66f499b8e6600c5896701bb07015be132a12de64e321a2edc67e" + }, + "parent_release_manifest": { + "path": "releases/populace-us-2024-spm-20260915/parent_release_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "dd949ba3c4c7a56aff8442c6db5a031d2c84c3da59f5d14832f468c358604506" + }, + "populace_us_2024": { + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" + }, + "populace_us_2024_acs_local": { + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "revision": "populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z", + "sha256": "3ce781c440a7f933b1d952ff4e9d9f65f4698c64b94d2fad165af0c662838d6c" + }, + "source_enrichment": { + "path": "releases/populace-us-2024-spm-20260915/source_enrichment.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "85329907b2b9c4299855b0970b1f24ede5819d3b16c312eb7400b4203bd71203" + }, + "source_enrichment_compatibility": { + "path": "releases/populace-us-2024-spm-20260915/source_enrichment_compatibility.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "63109acffd486453c99c476dece6b0154cc5388e5a430830a30744a2249444d1" + }, + "source_spm_independence_provenance": { + "path": "releases/populace-us-2024-spm-20260915/source_spm_independence_provenance.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "265e844178ed3856d68c1638820f3089e0e1e1363aa4677e33bebb37cbd5bff3" + }, + "source_spm_person_independence": { + "path": "releases/populace-us-2024-spm-20260915/source_spm_person_independence.csv", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "22b5968d90fecfeef7614583e493fe10cc16bda8b5be82e6f49a5bc2102d3ce5" + }, + "us_source_coverage": { + "path": "releases/populace-us-2024-spm-20260915/us_source_coverage.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "6406c8686c292015a5bc7265a42402f89935f9395a8b63510efdd4d9562e7ff5" + } + }, + "default_dataset": "populace_us_2024", + "default_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915", + "model_package": { + "name": "policyengine-us", + "sha256": "0993a6c73fcdfbe171a796aca9d302741bd8e00ab83388ad81c15a08740318c6", + "version": "2.2.1", + "wheel_url": "https://files.pythonhosted.org/packages/5e/09/86ac3968d2a18e28fce3ba1186fc21728e9f299b96a806d6f204f1e0aaed/policyengine_us-2.2.1-py3-none-any.whl" + }, + "policyengine_version": "6.9.0", + "region_datasets": { + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, + "national": { + "path_template": "populace_us_2024.h5" + }, + "state": { + "path_template": "populace_us_2024_acs_local.h5" + } + }, + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/releases/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/releases/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/release_manifest.json", + "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", + "schema_version": 1, + "source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", + "version": "populace-us-2024-spm-20260915" + } + }, + "dataset_overlays": { + "uk": { + "enhanced_frs_2023_24": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2023_24.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" + }, + "enhanced_frs_2024_25": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + }, + "enhanced_frs_2024_25_tiny": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "d5d2bde80f1f028fec98391ea31bb5a0cf8449e3867fc3c1bfe23693b9faecf8" + }, + "frs_2023_24": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2023_24.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "sha256": "df26d4d7af9d164aa2d064181b39290292d2f62bb26fee6126fc095fc06da292" + }, + "frs_2024_25": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "97af00b2cd2739957f40dcc4de8e0383a6a91d47952c90d4b4921a6007bd2021" + }, + "frs_2024_25_tiny": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "e4d7662cd3f053620224b7ceff17266783f09407002a72633b043b844cec3f5a" + }, + "populace_uk_2023": { + "data_package_name": "populace-data", + "path": "populace_uk_2023.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" + } + } + }, + "extras": { + "models": [ + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "spm-calculator" + ], + "uk": [ + "policyengine-core", + "policyengine-uk" + ], + "us": [ + "policyengine-core", + "policyengine-us", + "spm-calculator" + ] + }, + "measurements": { + "spm": { + "as_of": null, + "county_vintage": "2020", + "forecast_content_sha256": "3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173", + "geography_id": null, + "geography_kind": "county", + "scenario": "ce_trend" + } + }, + "packages": { + "policyengine": { + "import_name": "policyengine", + "install_requirement": "policyengine==6.9.0", + "name": "policyengine", + "role": "bundle_carrier", + "version": "6.9.0" + }, + "policyengine-core": { + "import_name": "policyengine_core", + "install_requirement": "policyengine-core==3.33.3", + "name": "policyengine-core", + "role": "runtime_dependency", + "version": "3.33.3" + }, + "policyengine-uk": { + "country": "uk", + "import_name": "policyengine_uk", + "install_requirement": "policyengine-uk==2.132.6", + "name": "policyengine-uk", + "role": "country_model", + "version": "2.132.6" + }, + "policyengine-us": { + "country": "us", + "import_name": "policyengine_us", + "install_requirement": "policyengine-us==2.2.1", + "name": "policyengine-us", + "role": "country_model", + "version": "2.2.1" + }, + "spm-calculator": { + "country": "us", + "import_name": "spm_calculator", + "install_requirement": "spm-calculator==1.0.0", + "name": "spm-calculator", + "role": "runtime_dependency", + "sha256": "e354937a5e1a4045d4966ed594a528d8b02866fabaac9bb5672017004b627305", + "version": "1.0.0", + "wheel_url": "https://files.pythonhosted.org/packages/19/a0/c484f69a0ebf88a9b9fd0ac28176f8df66550f46714f1b0c5ec0b360824e/spm_calculator-1.0.0-py3-none-any.whl" + } + }, + "policyengine_version": "6.9.0", + "schema_version": 2 +} diff --git a/tests/qualification_fixtures.py b/tests/qualification_fixtures.py index cf70f42a2..724ab2b0f 100644 --- a/tests/qualification_fixtures.py +++ b/tests/qualification_fixtures.py @@ -29,6 +29,16 @@ Path(__file__).parent / "fixtures" / "uk_qualification" / "uk_data_line_bundle.json" ) UK_DATA_LINE_BUNDLE = json.loads(UK_DATA_LINE_BUNDLE_PATH.read_text()) +#: A main certified on the Microcosm line with its area regions routed to a +#: local-area release: the bundle #685 certified, frozen, so that tests of +#: other-line pairs do not depend on what main certifies now. +MICROCOSM_ROUTED_BUNDLE_PATH = ( + Path(__file__).parent + / "fixtures" + / "uk_qualification" + / "microcosm_routed_bundle.json" +) +MICROCOSM_ROUTED_BUNDLE = json.loads(MICROCOSM_ROUTED_BUNDLE_PATH.read_text()) def load_candidate_cli( diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 029916cc3..209fe2b4b 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -38,6 +38,7 @@ build_replay_request, candidate_action, candidate_branch, + candidate_pr_body, certified_constant_replacements, choose_core_version, core_requirement, @@ -51,6 +52,8 @@ from tests.qualification_fixtures import ( HARNESS, ISSUED, + MICROCOSM_ROUTED_BUNDLE, + MICROCOSM_ROUTED_BUNDLE_PATH, UK_DATA_LINE_BUNDLE, VENDORED_SCHEMAS, gates, @@ -597,7 +600,7 @@ def test_preparation_generates_new_sidecars_and_verifies_the_exact_pair( pair = resolved.pair bundle_dir = tmp_path / "src/policyengine/data/bundle" bundle_dir.mkdir(parents=True) - source = json.loads(json.dumps(PACKAGED_BUNDLE if routed else BUNDLE)) + source = json.loads(json.dumps(MICROCOSM_ROUTED_BUNDLE if routed else BUNDLE)) source["data_releases"].pop("us") source["packages"].pop("policyengine-us") for packages in source["extras"].values(): @@ -715,7 +718,7 @@ def test_regional_flags_keep_routing_on_its_line_and_drop_it_across_lines(): ) uk_data_pair = make_pair() - record = PACKAGED_BUNDLE["data_releases"]["uk"] + record = MICROCOSM_ROUTED_BUNDLE["data_releases"]["uk"] microcosm_pair = uk_data_pair.model_copy( update={ "data": uk_data_pair.data.model_copy( @@ -728,10 +731,12 @@ def test_regional_flags_keep_routing_on_its_line_and_drop_it_across_lines(): ) assert regional_certification_arguments(BUNDLE, uk_data_pair) == [] - assert regional_certification_arguments(PACKAGED_BUNDLE, uk_data_pair) == [ + assert regional_certification_arguments(MICROCOSM_ROUTED_BUNDLE, uk_data_pair) == [ "--drop-regional-release" ] - assert regional_certification_arguments(PACKAGED_BUNDLE, microcosm_pair) == [ + assert regional_certification_arguments( + MICROCOSM_ROUTED_BUNDLE, microcosm_pair + ) == [ "--regional-manifest-uri", record["regional_source_manifest_uri"], "--regional-release-kind", @@ -973,8 +978,8 @@ def test_a_pair_from_another_data_line_is_never_ordered_against_main(tmp_path): """A uk-data release number says nothing about a Microcosm build. With a Microcosm certificate on main, every uk-data pair compared as newer (1.58.x > 0.1.0) and would have been qualified and promoted back. Now - automated qualification skips it unless forced, and promotion neither - readies nor closes it.""" + automated qualification skips it unless forced, and promotion never + readies it and closes its pull request.""" from policyengine.provenance.uk_qualification import requalification_target @@ -994,6 +999,9 @@ def test_a_pair_from_another_data_line_is_never_ordered_against_main(tmp_path): plan = plan_promotion([promotable], bundle, {branch: promotable.harness.git_commit}) assert plan.target is None and plan.ready_branch is None assert branch not in plan.close_branches + assert plan.close_off_line == [branch] + body = candidate_pr_body(promotable, uk_data, ledger_url="u", on_main_line=False) + assert "never promoted" in body and "promotes the pair" not in body assert requalification_target([promotable], bundle, tmp_path, "b" * 40) is None # On main's own data line the same pair still advances. @@ -2943,18 +2951,18 @@ def run(argv, **kwargs): assert not cli.owner_run_failed("42") -PACKAGED_BUNDLE_PATH = REPO_ROOT / "src/policyengine/data/bundle/manifest.json" - - @pytest.mark.parametrize("force", [False, True]) -def test_recovery_skips_unforced_pairs_from_another_data_line( +def test_recovery_skips_pairs_from_another_data_line( tmp_path, monkeypatch, capsys, force ): + """The catch-up re-dispatches without force, so even a forced run of a + pair from another line would resolve as other_line and clear nothing.""" + from policyengine.provenance.qualification_ledger import attempt_directory from policyengine.provenance.uk_qualification import stage_attempt - # Main is the packaged bundle; make_pair builds a uk-data pair. - cli = load_candidate_cli("check_uk_recovery_line", PACKAGED_BUNDLE_PATH) + # Main is a routed Microcosm bundle; make_pair builds a uk-data pair. + cli = load_candidate_cli("check_uk_recovery_line", MICROCOSM_ROUTED_BUNDLE_PATH) pair = make_pair() attempt = attempt_directory(tmp_path, "uk", pair.pair_id, "a" * 12) payload = {"pair": pair.canonical_payload(), "force": force} @@ -2964,22 +2972,23 @@ def test_recovery_skips_unforced_pairs_from_another_data_line( {"sequence": 1, "owner_run_id": "42", "expires_at": "2026-10-14T14:00:00Z"}, ) monkeypatch.setattr(cli, "utc_now", lambda: "2026-10-14T11:00:00Z") - monkeypatch.setattr(cli, "owner_run_failed", lambda run_id: run_id == "42") + # A live attempt is not recoverable, and logs nothing. + monkeypatch.setattr(cli, "owner_run_failed", lambda run_id: False) assert cli.main(["recoverable", "--ledger", str(tmp_path)]) == 0 + assert capsys.readouterr() == ("", "") + monkeypatch.setattr(cli, "owner_run_failed", lambda run_id: run_id == "42") + assert cli.main(["recoverable", "--ledger", str(tmp_path)]) == 0 out = capsys.readouterr() - if force: - assert json.loads(out.out) == payload - else: - assert out.out == "" - assert "not on main's data line" in out.err + assert out.out == "" + assert "not on main's data line" in out.err def test_check_refresh_skips_pairs_from_another_data_line(tmp_path, monkeypatch): import argparse - cli = load_candidate_cli("check_uk_refresh_line", PACKAGED_BUNDLE_PATH) + cli = load_candidate_cli("check_uk_refresh_line", MICROCOSM_ROUTED_BUNDLE_PATH) receipt = _promotion_receipt(pair=make_pair(model="2.200.0")) assert receipt.promotable branch = candidate_branch(receipt.pair) @@ -3346,8 +3355,8 @@ def test_release_payload_names_the_qualification_receipt(): @pytest.mark.parametrize( ("bundle", "manifest_size"), - [(BUNDLE, 128369110), (PACKAGED_BUNDLE, None)], - ids=["uk-data model repo", "packaged dataset repo without sizes"], + [(BUNDLE, 128369110), (MICROCOSM_ROUTED_BUNDLE, None)], + ids=["uk-data model repo", "Microcosm dataset repo without sizes"], ) def test_released_bundle_request_is_valid_for_the_scorecard(bundle, manifest_size): from policyengine.provenance.uk_qualification import released_bundle_request @@ -3427,7 +3436,7 @@ def hf_size(repo, rev, path, kind): def test_released_bundle_request_needs_a_size_source(): from policyengine.provenance.uk_qualification import released_bundle_request - record = PACKAGED_BUNDLE["data_releases"]["uk"] + record = MICROCOSM_ROUTED_BUNDLE["data_releases"]["uk"] dataset = record["default_dataset"] reference = record["datasets"][dataset] manifest = { @@ -3446,8 +3455,8 @@ def test_released_bundle_request_needs_a_size_source(): } with pytest.raises(ValueError, match="records no size"): released_bundle_request( - PACKAGED_BUNDLE, - version=PACKAGED_BUNDLE["policyengine_version"], + MICROCOSM_ROUTED_BUNDLE, + version=MICROCOSM_ROUTED_BUNDLE["policyengine_version"], wheel_sha256=sha("7"), packaged_manifest_sha256=sha("8"), release_bundle={}, From e40d3ba1e2eee18c3f5fc286fb93c61de5b3289c Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 23:41:41 +0100 Subject: [PATCH 16/26] Use the shared Hugging Face helpers in the release consumer release_consumers.py had its own URL and header helpers, which missed HUGGINGFACE_HUB_TOKEN and the shared helper's quoting. It now uses https_dataset_uri and hugging_face_auth_headers. _hf_size reads a small file's Content-Length directly, and follows a redirect that carries no size. The released-bundle request test now asserts the full repository, revision, path and repository type of both Hugging Face calls, and the release-consumer test asserts the URLs, the Authorization header and each HEAD answer the size reader handles. Co-Authored-By: Claude Opus 5.5 --- scripts/release_consumers.py | 39 +++++----- tests/test_uk_qualification_pipeline.py | 95 +++++++++++++++++++------ 2 files changed, 93 insertions(+), 41 deletions(-) diff --git a/scripts/release_consumers.py b/scripts/release_consumers.py index 45d63fbbe..ea5bd662b 100644 --- a/scripts/release_consumers.py +++ b/scripts/release_consumers.py @@ -29,6 +29,10 @@ import requests # noqa: E402 +from policyengine.provenance.manifest import ( # noqa: E402 + https_dataset_uri, + hugging_face_auth_headers, +) from policyengine.provenance.qualification_ledger import ( # noqa: E402 fetch_ledger_receipt, ) @@ -52,23 +56,10 @@ def _pypi(path: str) -> dict: return response.json() -HF_REPO_PREFIXES = {"model": "", "dataset": "datasets/", "space": "spaces/"} - - -def _hf_url(repo_id: str, revision: str, path: str, repo_type: str) -> str: - prefix = HF_REPO_PREFIXES[repo_type] - return f"https://huggingface.co/{prefix}{repo_id}/resolve/{revision}/{path}" - - -def _hf_headers() -> dict: - token = os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN") - return {"Authorization": f"Bearer {token}"} if token else {} - - def _hf_file(repo_id: str, revision: str, path: str, repo_type: str = "model") -> bytes: response = requests.get( - _hf_url(repo_id, revision, path, repo_type), - headers=_hf_headers(), + https_dataset_uri(repo_id, path, revision, repo_type=repo_type), + headers=hugging_face_auth_headers(), timeout=TIMEOUT, ) response.raise_for_status() @@ -78,20 +69,26 @@ def _hf_file(repo_id: str, revision: str, path: str, repo_type: str = "model") - def _hf_size(repo_id: str, revision: str, path: str, repo_type: str = "model") -> int: """A file's size from Hugging Face's response headers, without downloading it. - A large-file pointer answers with a redirect that carries - ``X-Linked-Size``; a small file answers directly with its length. + A large-file pointer answers HEAD with a redirect that carries + ``X-Linked-Size``. A small file answers with its own ``Content-Length``, + or with a redirect that leads to it, which is then followed. """ + url = https_dataset_uri(repo_id, path, revision, repo_type=repo_type) + headers = hugging_face_auth_headers() response = requests.head( - _hf_url(repo_id, revision, path, repo_type), - headers=_hf_headers(), - timeout=TIMEOUT, - allow_redirects=False, + url, headers=headers, timeout=TIMEOUT, allow_redirects=False ) response.raise_for_status() size = response.headers.get("X-Linked-Size") if size is None and response.status_code == 200: size = response.headers.get("Content-Length") + elif size is None: + followed = requests.head( + url, headers=headers, timeout=TIMEOUT, allow_redirects=True + ) + followed.raise_for_status() + size = followed.headers.get("Content-Length") if not size: raise SystemExit( f"Hugging Face reports no size for {repo_id}/{path}@{revision}." diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 209fe2b4b..7d1d7aa4e 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -3382,15 +3382,16 @@ def test_released_bundle_request_is_valid_for_the_scorecard(bundle, manifest_siz "default_datasets": {"national": dataset}, "artifacts": {dataset: artifact}, } - repo_type = record["data_package"]["repo_type"] + data_package = record["data_package"] + repo_type = data_package["repo_type"] fetched = [] - def hf_file(repo, rev, path, kind): - fetched.append(("file", repo, kind)) + def hf_file(*request): + fetched.append(("file", *request)) return json.dumps(manifest).encode() - def hf_size(repo, rev, path, kind): - fetched.append(("size", repo, kind)) + def hf_size(*request): + fetched.append(("size", *request)) return 321798995 core = bundle["packages"]["policyengine-core"]["version"] @@ -3427,9 +3428,24 @@ def hf_size(repo, rev, path, kind): assert data["size_bytes"] == manifest_size or ( manifest_size is None and data["size_bytes"] == 321798995 ) - assert fetched[0] == ("file", record["data_package"]["repo_id"], repo_type) - assert [item[0] for item in fetched] == ( - ["file"] if manifest_size is not None else ["file", "size"] + manifest_request = ( + "file", + data_package["repo_id"], + data_package["release_manifest_revision"], + data_package["release_manifest_path"], + repo_type, + ) + size_request = ( + "size", + reference.get("repo_id") or data_package["repo_id"], + reference["revision"], + reference["path"], + reference.get("repo_type") or repo_type, + ) + assert fetched == ( + [manifest_request] + if manifest_size is not None + else [manifest_request, size_request] ) @@ -3468,27 +3484,66 @@ def test_released_bundle_request_needs_a_size_source(): def test_release_consumers_address_dataset_repositories(monkeypatch): monkeypatch.setenv("POLICYENGINE_SKIP_COUNTRY_IMPORTS", "1") + for name in ("HUGGING_FACE_TOKEN", "HF_TOKEN"): + monkeypatch.delenv(name, raising=False) + # The shared helper reads every token variable the script lists. + monkeypatch.setenv("HUGGINGFACE_HUB_TOKEN", "hub-token") spec = importlib.util.spec_from_file_location( "release_consumers_for_test", REPO_ROOT / "scripts" / "release_consumers.py" ) consumers = importlib.util.module_from_spec(spec) spec.loader.exec_module(consumers) + url = "https://huggingface.co/datasets/org/data/resolve/v1/releases/a%20b.h5" + authorized = {"Authorization": "Bearer hub-token"} + calls = [] - assert consumers._hf_url("org/data", "v1", "a.h5", "dataset") == ( - "https://huggingface.co/datasets/org/data/resolve/v1/a.h5" - ) - assert consumers._hf_url("org/model", "v1", "a.h5", "model") == ( - "https://huggingface.co/org/model/resolve/v1/a.h5" - ) + def response(status, headers, content=b""): + answer = MagicMock(status_code=status, headers=headers, content=content) + answer.raise_for_status = lambda: None + return answer + + def get(target, **kwargs): + calls.append(("get", target, kwargs["headers"])) + return response(200, {}, b"manifest") - def head(url, **kwargs): - assert kwargs["allow_redirects"] is False - response = MagicMock(status_code=302, headers={"X-Linked-Size": "321"}) - response.raise_for_status = lambda: None - return response + answers = [] + def head(target, **kwargs): + calls.append(("head", target, kwargs["headers"], kwargs["allow_redirects"])) + return answers.pop(0) + + monkeypatch.setattr(consumers.requests, "get", get) monkeypatch.setattr(consumers.requests, "head", head) - assert consumers._hf_size("org/data", "v1", "a.h5", "dataset") == 321 + + assert consumers._hf_file("org/data", "v1", "releases/a b.h5", "dataset") == ( + b"manifest" + ) + assert calls.pop() == ("get", url, authorized) + + # A large file's redirect carries its size. + answers.append(response(302, {"X-Linked-Size": "321", "Content-Length": "1152"})) + assert consumers._hf_size("org/data", "v1", "releases/a b.h5", "dataset") == 321 + assert calls == [("head", url, authorized, False)] + calls.clear() + + # A small file answers with its own length, or with a redirect to it. + answers.append(response(200, {"Content-Length": "4288"})) + assert consumers._hf_size("org/data", "v1", "releases/a b.h5", "dataset") == 4288 + answers.extend( + [ + response(307, {"Content-Length": "0"}), + response(200, {"Content-Length": "77"}), + ] + ) + assert consumers._hf_size("org/data", "v1", "releases/a b.h5", "dataset") == 77 + assert calls[-2:] == [ + ("head", url, authorized, False), + ("head", url, authorized, True), + ] + + model_url = "https://huggingface.co/org/model/resolve/v1/a.h5" + consumers._hf_file("org/model", "v1", "a.h5", "model") + assert calls.pop() == ("get", model_url, authorized) def test_expected_release_bundle_uses_the_scorecard_certification_environment( From df5a42118a9e7fa861f03e6031eb5a8357ed187d Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 23:42:53 +0100 Subject: [PATCH 17/26] Refuse Windows-style escapes in bundle artifact cache paths The cache-path guard validated with PurePosixPath but joined with the platform Path, so on Windows a backslash, drive-letter or UNC path passed the check and landed outside the data directory. Repository paths use forward slashes, so the guard now also refuses backslashes, drives and roots. Co-Authored-By: Claude Opus 5.5 --- .../provenance/dataset_materialization.py | 13 +++++++++++-- tests/test_dataset_materialization.py | 11 ++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/policyengine/provenance/dataset_materialization.py b/src/policyengine/provenance/dataset_materialization.py index 2f441aaea..c93f546e6 100644 --- a/src/policyengine/provenance/dataset_materialization.py +++ b/src/policyengine/provenance/dataset_materialization.py @@ -5,7 +5,7 @@ import os import tempfile from dataclasses import dataclass -from pathlib import Path, PurePosixPath +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Literal, Optional import requests @@ -93,7 +93,16 @@ def _bundle_destination(data_dir: Path, path: str) -> Path: """ relative = PurePosixPath(path) - if relative.is_absolute() or ".." in relative.parts: + # Repository paths use forward slashes. A backslash, drive or root would + # make the platform path leave data_dir on Windows. + windows = PureWindowsPath(path) + if ( + relative.is_absolute() + or ".." in relative.parts + or "\\" in path + or windows.drive + or windows.root + ): raise DatasetMaterializationError( f"Bundle artifact path {path!r} must be relative to its repository." ) diff --git a/tests/test_dataset_materialization.py b/tests/test_dataset_materialization.py index b2b9a8267..8eb572f5a 100644 --- a/tests/test_dataset_materialization.py +++ b/tests/test_dataset_materialization.py @@ -268,6 +268,15 @@ def test_same_named_artifacts_of_two_releases_never_share_a_cache_path(tmp_path) # A top-level file keeps its cache path. default = _resolve_bundle_dataset("uk", data_dir=tmp_path) assert default.destination == tmp_path / default.path - for path in ("/etc/passwd", "../outside.h5", "releases/../../outside.h5"): + for path in ( + "/etc/passwd", + "../outside.h5", + "releases/../../outside.h5", + "..\\..\\outside.h5", + "C:\\outside.h5", + "C:outside.h5", + "\\outside.h5", + "\\\\server\\share\\outside.h5", + ): with pytest.raises(DatasetMaterializationError, match="must be relative"): _bundle_destination(tmp_path, path) From 3b2db90737a2e1d16d9c2174a2d70054ba37797e Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 23:42:53 +0100 Subject: [PATCH 18/26] Refuse same-line preparation that main's local-area cut cannot serve Preparation keeps main's regional release for a pair on main's data line, but that release was certified for main's model and core and gets no qualified-pair escape, so certification refused any other model or core deep inside certify-data. Preparation now refuses such a pair up front and says a local-area cut for its versions is needed. Co-Authored-By: Claude Opus 5.5 --- .../provenance/uk_qualification.py | 17 ++++++++++ tests/test_uk_qualification_pipeline.py | 34 +++++++++++++------ 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index a1e1ced42..dc2be994f 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -773,6 +773,23 @@ def regional_certification_arguments( return [] if data_line(pair) != certified_data_line(bundle): return ["--drop-regional-release"] + # Main's regional release was certified for main's model and core, and + # gets no qualified-pair escape. A pair on other versions needs its own + # local-area cut, which a candidate pair does not carry. + certified_core = (record.get("certification") or {}).get( + "certified_for_core_version" + ) or bundle["packages"][CORE_PACKAGE]["version"] + if (pair.model.version, pair.core.version) != ( + record["model_package"]["version"], + certified_core, + ): + raise ValueError( + f"Main routes UK area regions to {regional_uri}, certified for " + f"policyengine-uk {record['model_package']['version']} and " + f"policyengine-core {certified_core}. A candidate on " + f"{pair.model.version} and {pair.core.version} needs a local-area " + "cut for those versions; certify it by hand with that cut." + ) return [ "--regional-manifest-uri", regional_uri, diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 7d1d7aa4e..6553fcac1 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -719,15 +719,23 @@ def test_regional_flags_keep_routing_on_its_line_and_drop_it_across_lines(): uk_data_pair = make_pair() record = MICROCOSM_ROUTED_BUNDLE["data_releases"]["uk"] - microcosm_pair = uk_data_pair.model_copy( - update={ - "data": uk_data_pair.data.model_copy( - update={ - "data_package": record["data_package"]["name"], - "repo_id": record["data_package"]["repo_id"], - } - ) - } + + def microcosm_pair(model, core): + pair = make_pair(model=model, core=core) + return pair.model_copy( + update={ + "data": pair.data.model_copy( + update={ + "data_package": record["data_package"]["name"], + "repo_id": record["data_package"]["repo_id"], + } + ) + } + ) + + certified = ( + record["model_package"]["version"], + record["certification"]["certified_for_core_version"], ) assert regional_certification_arguments(BUNDLE, uk_data_pair) == [] @@ -735,13 +743,19 @@ def test_regional_flags_keep_routing_on_its_line_and_drop_it_across_lines(): "--drop-regional-release" ] assert regional_certification_arguments( - MICROCOSM_ROUTED_BUNDLE, microcosm_pair + MICROCOSM_ROUTED_BUNDLE, microcosm_pair(*certified) ) == [ "--regional-manifest-uri", record["regional_source_manifest_uri"], "--regional-release-kind", record["regional_release_kind"], ] + # Main's local-area cut serves only main's model and core. + for model, core in (("2.140.0", certified[1]), (certified[0], "3.34.0")): + with pytest.raises(ValueError, match="needs a local-area cut"): + regional_certification_arguments( + MICROCOSM_ROUTED_BUNDLE, microcosm_pair(model, core) + ) def test_replay_request_is_valid_for_the_scorecard(): From 2993625ea7dd95aa6ef922a624edc9b99a1da4f1 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 23:43:38 +0100 Subject: [PATCH 19/26] Split the changelog by kind and document uncertified area receipts The weight matrices that no longer resolve by name are a removal, so they move to their own removed fragment, which makes the release a minor one; the switch itself stays a change. The change fragment now says that receipts for constituency and local-authority runs are uncertified, since the certificate covers the national dataset only, that the named core binds the core pin, and that 0.897 is against the floor's 6.7m benchmark, not DWP's 6.2m. regions.md and receipts.md say what certification does and does not cover for the local-area release. Co-Authored-By: Claude Opus 5.5 --- ...k-2024-25-national-20261019T083323Z-6cba5d71.changed.md | 2 +- ...k-2024-25-national-20261019T083323Z-6cba5d71.removed.md | 1 + docs/receipts.md | 2 +- docs/regions.md | 7 ++++++- 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.removed.md diff --git a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md index 30b7e7e08..5e8963431 100644 --- a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md +++ b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md @@ -1 +1 @@ -Switch the UK default dataset from the Enhanced FRS 2024-25 (policyengine-uk-data 1.58.1) to Microcosm UK 2024-25 (`microcosm_uk_2024_25`, cut `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71`), certified with policyengine-uk 2.132.6 and policyengine-core 3.33.3, the versions it was built with. Constituency and local-authority regions now load the Microcosm local-area release (`microcosm_uk_2024_25_dense`, cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`) and are in the default UK region registry. UK results change with the data. The national file has no constituency or local-authority codes, so constituency and local-authority outputs (`compute_uk_geography_disclosure` at those levels, `compute_uk_constituency_impacts`, `compute_uk_local_authority_impacts`) need simulations on `microcosm_uk_2024_25_dense`. The Enhanced FRS 2024-25 datasets stay loadable by name, but its weight matrices (`parliamentary_constituency_weights`, `local_authority_weights`, `local_geography_weights`) no longer resolve. With a Hugging Face token set, importing policyengine with any policyengine-uk other than 2.132.6 raises, as it did before for versions the Enhanced FRS release did not claim. The certified default is below the Universal Credit caseload floor (0.897 of DWP's benchmark), an exception accepted and recorded in the data certification skill. +Switch the UK default dataset from the Enhanced FRS 2024-25 (policyengine-uk-data 1.58.1) to Microcosm UK 2024-25 (`microcosm_uk_2024_25`, cut `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71`), certified with policyengine-uk 2.132.6 and policyengine-core 3.33.3, the versions it was built with. Constituency and local-authority regions now load the Microcosm local-area release (`microcosm_uk_2024_25_dense`, cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`) and are in the default UK region registry. UK results change with the data. The national file has no constituency or local-authority codes, so constituency and local-authority outputs (`compute_uk_geography_disclosure` at those levels, `compute_uk_constituency_impacts`, `compute_uk_local_authority_impacts`) need simulations on `microcosm_uk_2024_25_dense`, and receipts for those runs are uncertified: the certificate covers the national dataset only. The Enhanced FRS 2024-25 datasets stay loadable by name. The certificate names policyengine-core 3.33.3, so the core pin cannot move until UK is re-certified for the new core (see the data certification skill). With a Hugging Face token set, importing policyengine with any policyengine-uk other than 2.132.6 raises, as it did before for versions the Enhanced FRS release did not claim. The certified default is below the Universal Credit caseload floor (0.897 of the floor's 6.7m benchmark, policyengine-uk-data's derivation; Microcosm calibrates to DWP Stat-Xplore's 6.2m), an exception accepted and recorded in the data certification skill. diff --git a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.removed.md b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.removed.md new file mode 100644 index 000000000..17a271aa4 --- /dev/null +++ b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.removed.md @@ -0,0 +1 @@ +The Enhanced FRS weight matrices (`parliamentary_constituency_weights`, `local_authority_weights`, `local_geography_weights`) no longer resolve by name, now that the UK default is Microcosm UK and its area regions use the Microcosm local-area release. diff --git a/docs/receipts.md b/docs/receipts.md index e4bc2ec69..a082f449b 100644 --- a/docs/receipts.md +++ b/docs/receipts.md @@ -10,7 +10,7 @@ Schema v1 covers UK results and is defined by `policyengine.provenance.receipt.E |---|---| | `wrapper`, `model`, `core` | The `policyengine`, `policyengine-uk` and `policyengine-core` versions that actually ran, read from the installed distributions rather than from the bundle manifest. `install_origin` records a PEP 610 direct URL as `archive`, `directory`, `editable` or `vcs`; it is absent for an ordinary index install with no `direct_url.json`. `wheel_sha256` appears only when the receipt builder verified it. One way is `wheel_paths=` (evidence `installed_record_verified`): the installed files, and any bytecode cache the interpreter would load instead of a source file, are checked against that wheel's RECORD. The other is the installer's recorded archive hash (`installer_archive_hash`), taken only when the install came from a `.whl` URL; a source archive's hash is never recorded as a wheel hash. | | `data` | The data release and build, and the model version the data was built with (`build_model_version`, distinct from `model.version`). Also the dataset's logical name, URI, revision and the SHA-256 of the bytes that were simulated. The dataset is resolved exactly as `materialize_dataset` resolves it (overlays, per-dataset repository, package and revision, and the release manifest revision when an artifact has none of its own), and the bytes must be the ones the bundle pins for it: a receipt never gives an artifact's name to other bytes. | -| `certification_status`, `certificate` | `certified` only when the bundle certifies all of: the installed model and core versions, that dataset name and those bytes, and the running `policyengine` version. Each package must also have an absent or `archive` install origin, or a `directory` or `vcs` origin with `installed_record_verified` wheel evidence. A directory or VCS install without that evidence produces an `uncertified` receipt, which carries no certificate. An editable install is never certified, whatever the evidence: its wheel holds a pointer to a source tree (a `.pth` file or an import hook) rather than the source, so checking the installed files against it does not bind the code that ran. `certificate_source` records whether the builder read the certificate from the installed bundle (`packaged`) or from a manifest the caller passed (`caller_supplied`). It is the producer's statement, not evidence: trust a certificate only after `verify_certificate` (below). With the packaged bundle, the composition fingerprint of its TRACE TRO is recorded as well. | +| `certification_status`, `certificate` | `certified` only when the bundle certifies all of: the installed model and core versions, that dataset name and those bytes, and the running `policyengine` version. The UK certificate names one dataset, the national default, so a run on any other dataset, the local-area file that constituency and local-authority regions load included, gets an uncertified receipt. Each package must also have an absent or `archive` install origin, or a `directory` or `vcs` origin with `installed_record_verified` wheel evidence. A directory or VCS install without that evidence produces an `uncertified` receipt, which carries no certificate. An editable install is never certified, whatever the evidence: its wheel holds a pointer to a source tree (a `.pth` file or an import hook) rather than the source, so checking the installed files against it does not bind the code that ran. `certificate_source` records whether the builder read the certificate from the installed bundle (`packaged`) or from a manifest the caller passed (`caller_supplied`). It is the producer's statement, not evidence: trust a certificate only after `verify_certificate` (below). With the packaged bundle, the composition fingerprint of its TRACE TRO is recorded as well. | | `reform`, `baseline` | Both are required. Each is `current_law` or a digest of the reform bound to its representation (see below). | | `fiscal_year` | What a year label means: year 2027 is the fiscal year from 6 April 2027 to 5 April 2028, shown as `2027-28`. v1 allows only this convention. | | `years` | The reported years, sorted and unique. | diff --git a/docs/regions.md b/docs/regions.md index 5c94ab969..7dc32d2a0 100644 --- a/docs/regions.md +++ b/docs/regions.md @@ -104,7 +104,12 @@ when nothing is routed: | --- | --- | --- | | none | the national dataset | no | | `area_coded_national` | another release's area-coded national dataset (the enhanced FRS), while the default is a national file without area codes | no | -| `local_area` | a certified local-area release | yes | +| `local_area` | a local-area release that certification vets and pins | yes | + +Certification checks a local-area release's role, its single H5 file and its +own model and core claims, and pins the file's bytes, but the certificate covers +the national dataset only: receipts for runs on the local-area file are +uncertified. The certified bundle routes both to the Microcosm local-area release (`local_area`, `microcosm_uk_2024_25_dense.h5`), so the default registry has From 6bed93974b3198b4f0a3df63acd08e1d268916a3 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Tue, 20 Oct 2026 23:59:49 +0100 Subject: [PATCH 20/26] Record the comparison-gate acceptance next to the UC-floor exception The metrics review and the scorecard replay measure a candidate against the previous certified pair, which a change of data line makes meaningless, so they were not run for the Microcosm switch. The skill now records that acceptance beside the UC-floor exception. Reform responsiveness checks the candidate against fixed minima, so it was run on the national cut. The UKDS output rule is its own paragraph again. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/data-certification.md | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 37d23ca66..67c2677be 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -207,17 +207,25 @@ The benchmarks and their sources are in `src/policyengine/data/qualification/uk_uc_floor.json`. The receipt holds weighted national totals only. -**Recorded exception (policyengine.py#685, October 2026).** The Microcosm UK -national cut `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71` became -the default with benefit units at 0.897 of the floor's 6.7m benchmark; spend -passed at 0.959. The floor's benchmarks are not the figures Microcosm -calibrates to (DWP Stat-Xplore's 6.2m households on Universal Credit in -payment, and its own OBR spend line). Their definitions are to be settled -when the UK data checks move to the data pipelines. Until then, a later -Microcosm cut with the same gap needs the same explicit acceptance in its pull -request, recorded here. Before quoting any other figure from UK data -in a PR, apply the UKDS output rules: no unit-record values, no cells under -10 unweighted cases, no differencing. +Before quoting any other figure from UK data in a PR, apply the UKDS output +rules: no unit-record values, no cells under 10 unweighted cases, no +differencing. + +**Recorded exceptions (policyengine.py#685, October 2026).** + +- **UC floor.** The Microcosm UK national cut + `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71` became the default + with benefit units at 0.897 of the floor's 6.7m benchmark; spend passed at + 0.959. The floor's benchmarks are not the figures Microcosm calibrates to + (DWP Stat-Xplore's 6.2m households on Universal Credit in payment, and its own + OBR spend line). Their definitions are to be settled when the UK data checks + move to the data pipelines. Until then, a later Microcosm cut with the same + gap needs the same explicit acceptance in its pull request, recorded here. +- **Comparison gates.** The metrics review and the scorecard replay measure a + candidate against the previous certified pair, and a change of data line + moves the totals by design, so they were not run for that switch; its pull + request compares the two lines' weighted totals instead. The other gates, + reform responsiveness included, were run on the national cut. ## Release runbooks From 6ee9157cba427ef95c0c5e3c1530156e2e4d9f44 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Wed, 21 Oct 2026 11:57:20 +0100 Subject: [PATCH 21/26] Let a certificate follow the core pin openly, and re-certify UK that way The bundle has one core pin, shared by both countries. A certificate that names no core follows it, so another country can move the pin, but until now that showed only as a missing field, and nothing compared the pin with the data release's own core claims. A certificate now says how it covers the core: core_binding bundle_pin (the default, no --core-version) or certified (--core-version names and binds it). Certification checks the bundle's pin against each release's build core and claims and warns when it is outside them; the strict TRO check warns the same way later, and refuses only a pin that differs from a named core. The TRO records the binding. UK is re-certified to follow the pin: both Microcosm releases claim 3.33.3, the pin, so it warns about nothing. The check now also shows that the US release claims only 3.32.5, which the pin has been past since before this PR. Co-Authored-By: Claude Opus 5.5 --- docs/bundles.md | 11 ++- docs/engineering/skills/data-certification.md | 41 +++++---- docs/regions.md | 7 +- scripts/generate_trace_tros.py | 52 ++++++++--- src/policyengine/data/bundle/manifest.json | 2 +- .../data/bundle/uk.trace.tro.jsonld | 5 +- .../data/bundle/us.trace.tro.jsonld | 4 +- src/policyengine/provenance/certification.py | 86 ++++++++++++++++--- src/policyengine/provenance/manifest.py | 8 ++ src/policyengine/provenance/trace.py | 6 ++ tests/test_certify_data_release.py | 37 ++++++++ tests/test_release_tro_generation.py | 29 +++++++ tests/test_uk_qualification_pipeline.py | 2 +- 13 files changed, 237 insertions(+), 53 deletions(-) diff --git a/docs/bundles.md b/docs/bundles.md index 62ccc8438..21bcd37a9 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -127,10 +127,13 @@ python scripts/bundle.py update-packages \ --uk 2.91.0 ``` -A data certificate that names its core (`certify-data --core-version`) binds -the core pin: `--core` then fails `check --strict-tros` until that country is -re-certified for the new core. See the data certification skill for the UK -case. +By default a data certificate follows the core pin (`core_binding: +bundle_pin`), so `--core` can move it; `bundle.py check --include-tros +--strict-tros` then warns when a data release does not claim the new core. A +certificate that binds its core (`certify-data --core-version`, +`core_binding: certified`) instead fails that check until the country is +re-certified for the new core. See "The core" in the data certification +skill. To certify a new data release from a data-producer manifest, run: diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 67c2677be..4b974b70b 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -50,8 +50,7 @@ Microcosm national release with one of: python scripts/bundle.py certify-data --country uk --data-producer populace \ --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ --regional-manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases//release_manifest.json" \ - --model-version "" \ - --core-version "" + --model-version "" ``` This records `regional_release_kind: local_area`, and the default UK region @@ -66,8 +65,7 @@ Microcosm national release with one of: --manifest-uri "hf://dataset/policyengine/populace-uk-private@/releases/microcosm-uk-2024-25-national/release_manifest.json" \ --regional-manifest-uri "hf://model/policyengine/policyengine-uk-data-private@/releases//release_manifest.json" \ --regional-release-kind area_coded_national \ - --model-version "" \ - --core-version "" + --model-version "" ``` Certification carries that release's national dataset @@ -82,17 +80,30 @@ Microcosm national release with one of: `dataset_overlays.uk` entry. Either way, certification checks that the regional release serves the pinned -model, and the pinned core when `--core-version` is given, by its own build or -compatibility claims (a `--qualified-pair-id` names the primary release only), -and HEAD-checks every artifact it carries. - -`--core-version` also names the core in the certificate, and receipts then -certify that core, not the bundle's core pin. So `check --strict-tros` refuses -a core pin that differs from it: a core bump, a US `--core-version` run -included, needs the country re-certified for the new core first. Microcosm UK -releases claim one exact core, so that needs Microcosm cuts that claim the new -core. To move the core before they exist, re-certify UK without -`--core-version`: its receipts then certify the pin, as before October 2026. +model by its own build or compatibility claims (a `--qualified-pair-id` names +the primary release only), checks its core claims as for the primary release +(below), and HEAD-checks every artifact it carries. + +### The core + +The bundle has one core pin, `packages.policyengine-core`, shared by every +country. A certificate says how it covers it: + +- **`core_binding: bundle_pin` (the default, no `--core-version`).** The + certificate names no core and certifies whatever core the bundle pins, so + the pin can move with another country without re-certifying this one. + Certification checks the pin against each release's build-time core and + core claims, and warns, without refusing, when the pin is outside them. + `bundle.py check --include-tros --strict-tros` warns the same way when the + pin later moves outside the claims. Receipts then certify the pin without + the release's claim; the warning, the certificate and its TRO say so. +- **`core_binding: certified` (`--core-version`).** The certificate names the + core (`certified_for_core_version`). Certification refuses a core that a + release neither was built with nor claims, and the strict check refuses a + pin that differs from the named core, so moving the core needs a + re-certification first. A qualified exact pair always binds its core. + +Use the default unless a release must not run on any other core. ### Re-certifying a routed bundle diff --git a/docs/regions.md b/docs/regions.md index 7dc32d2a0..7c713bf64 100644 --- a/docs/regions.md +++ b/docs/regions.md @@ -107,9 +107,10 @@ when nothing is routed: | `local_area` | a local-area release that certification vets and pins | yes | Certification checks a local-area release's role, its single H5 file and its -own model and core claims, and pins the file's bytes, but the certificate covers -the national dataset only: receipts for runs on the local-area file are -uncertified. +own model claims, checks its core claims (refusing an unclaimed core with +`--core-version`, warning about one without it), and pins the file's bytes. The +certificate covers the national dataset only: receipts for runs on the +local-area file are uncertified. The certified bundle routes both to the Microcosm local-area release (`local_area`, `microcosm_uk_2024_25_dense.h5`), so the default registry has diff --git a/scripts/generate_trace_tros.py b/scripts/generate_trace_tros.py index 595124adb..656d1868b 100644 --- a/scripts/generate_trace_tros.py +++ b/scripts/generate_trace_tros.py @@ -22,7 +22,10 @@ from jsonschema import Draft202012Validator from release_lock import PYPI, load_toml, validate_registry_artifacts -from policyengine.provenance.certification import validate_release_manifest +from policyengine.provenance.certification import ( + core_claim_warning, + validate_release_manifest, +) from policyengine.provenance.manifest import ( CountryReleaseManifest, DataReleaseManifest, @@ -139,29 +142,42 @@ def _validate_runtime_model(bundle: dict, country: CountryReleaseManifest) -> No prefix = f"{country.country_id}: strict TRACE release" certification = country.certification core = bundle.get("packages", {}).get("policyengine-core", {}) - if ( - certification + exact_pair = ( + certification is not None and certification.compatibility_basis == "qualified_exact_pair" - and not ( - certification.qualification_pair_id - and certification.certified_for_core_version - ) + ) + if exact_pair and not ( + certification.qualification_pair_id and certification.certified_for_core_version ): raise ValueError( f"{prefix}: a qualified exact pair must name its pair and core" ) if ( - certification + certification is not None + and certification.core_binding is not None + and (certification.core_binding == "certified") + != (certification.certified_for_core_version is not None) + ): + raise ValueError( + f"{prefix}: core_binding {certification.core_binding!r} disagrees with " + "certified_for_core_version" + ) + if ( + certification is not None and certification.certified_for_core_version and core.get("version") != certification.certified_for_core_version ): # Receipts certify the core the certificate names, not the pin, so a # core-only pin change would leave every receipt uncertified. + remedy = ( + "requalify the pair for the new core" + if exact_pair + else "re-certify the data release for the new core, or without " + "--core-version so that the certificate follows the core pin" + ) raise ValueError( f"{prefix}: runtime core {core.get('version')} differs from the " - f"certified core {certification.certified_for_core_version}; " - "re-certify the data release for the new core, or without " - "--core-version so that receipts certify the pin" + f"certified core {certification.certified_for_core_version}; {remedy}" ) if ( model.name != key @@ -237,6 +253,20 @@ def generated_tros(*, strict: bool = False) -> list[tuple[Path, bytes]]: ) if strict: _validate_reviewed_inputs(country_manifest, data_release_manifest, tro_path) + certification = country_manifest.certification + core = ( + bundle.get("packages", {}).get("policyengine-core", {}).get("version") + ) + if ( + certification is not None + and certification.certified_for_core_version is None + and core is not None + ): + # The certificate follows the core pin: an unclaimed pin is + # reported, not refused. + warning = core_claim_warning(data_release_manifest, core) + if warning is not None: + print(f"warning: {country_id}: {warning}", file=sys.stderr) tro = build_trace_tro_from_release_bundle( country_manifest, data_release_manifest, diff --git a/src/policyengine/data/bundle/manifest.json b/src/policyengine/data/bundle/manifest.json index 8a2f3fca5..c6a8eff3d 100644 --- a/src/policyengine/data/bundle/manifest.json +++ b/src/policyengine/data/bundle/manifest.json @@ -21,9 +21,9 @@ "certification": { "built_with_model_version": "2.132.6", "certified_by": "policyengine.py bundle certification", - "certified_for_core_version": "3.33.3", "certified_for_model_version": "2.132.6", "compatibility_basis": "built_with_model_package", + "core_binding": "bundle_pin", "data_build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" }, "certified_data_artifact": { diff --git a/src/policyengine/data/bundle/uk.trace.tro.jsonld b/src/policyengine/data/bundle/uk.trace.tro.jsonld index c1dbd1475..4fdaa92b7 100644 --- a/src/policyengine/data/bundle/uk.trace.tro.jsonld +++ b/src/policyengine/data/bundle/uk.trace.tro.jsonld @@ -75,7 +75,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for uk", "trov:mimeType": "application/json", - "trov:sha256": "ef8235c112d4aee8436a90d5a562d3459ce05ed46a391b0daaa5693d7518acc7" + "trov:sha256": "6bbbfdfb433a5d8de4b1110238311338fd0425d92d4b34f06e7f8d3db3001fad" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -102,7 +102,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "a94a4050211eb02ab9c22629da54c887355c28015f40301122462882691511d8" + "trov:sha256": "1cce237175e76a9e5f9dffeb6ec3573dbc78c5facc5a7c55d06886d453641e3a" } }, "trov:hasPerformance": { @@ -112,6 +112,7 @@ "pe:certifiedBy": "policyengine.py bundle certification", "pe:certifiedForModelVersion": "2.132.6", "pe:compatibilityBasis": "built_with_model_package", + "pe:coreBinding": "bundle_pin", "pe:dataBuildId": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", "pe:emittedIn": "repository-bundle", "rdfs:comment": "Certification of build microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71 for policyengine-uk 2.132.6.", diff --git a/src/policyengine/data/bundle/us.trace.tro.jsonld b/src/policyengine/data/bundle/us.trace.tro.jsonld index 8df5f80b8..dc9975833 100644 --- a/src/policyengine/data/bundle/us.trace.tro.jsonld +++ b/src/policyengine/data/bundle/us.trace.tro.jsonld @@ -74,7 +74,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "ef8235c112d4aee8436a90d5a562d3459ce05ed46a391b0daaa5693d7518acc7" + "trov:sha256": "6bbbfdfb433a5d8de4b1110238311338fd0425d92d4b34f06e7f8d3db3001fad" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -101,7 +101,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "6856a00e80cd939b8aac6d5b8c22110f81668cb621172cbe32565f84f9f8f918" + "trov:sha256": "35c18fb640a23dfd78be0933a636c3088973934d7eff61ed2223c020da065d79" } }, "trov:hasPerformance": { diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index a59c84877..dc773e215 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -422,20 +422,8 @@ def _check_core_compatibility( case the certificate's basis must be the qualified exact pair. """ - build = manifest.build - built_with_core = ( - build.built_with_core_package.version - if build is not None and build.built_with_core_package is not None - else None - ) - core_claims = [ - package.specifier - for package in manifest.compatible_core_packages - if package.name == "policyengine-core" - ] - if built_with_core == core_version or any( - _specifier_matches(core_version, specifier) for specifier in core_claims - ): + built_with_core, core_claims = _core_claims(manifest) + if _core_is_claimed(manifest, core_version): return False if qualified_pair_id is not None: warnings.append( @@ -451,6 +439,53 @@ def _check_core_compatibility( ) +def _core_claims(manifest: DataReleaseManifest) -> tuple[Optional[str], list[str]]: + """A release's build-time core and its publisher core claims.""" + + build = manifest.build + built_with_core = ( + build.built_with_core_package.version + if build is not None and build.built_with_core_package is not None + else None + ) + core_claims = [ + package.specifier + for package in manifest.compatible_core_packages + if package.name == "policyengine-core" + ] + return built_with_core, core_claims + + +def _core_is_claimed(manifest: DataReleaseManifest, core_version: str) -> bool: + """Whether a release was built with, or claims, ``core_version``.""" + + built_with_core, core_claims = _core_claims(manifest) + return built_with_core == core_version or any( + _specifier_matches(core_version, specifier) for specifier in core_claims + ) + + +def core_claim_warning( + manifest: DataReleaseManifest, core_version: str +) -> Optional[str]: + """Why ``core_version`` is outside a release's core claims, or None. + + For a certificate that follows the bundle's core pin: the pin is not held + to the release's claims, so a pin outside them is reported, not refused. + """ + + if _core_is_claimed(manifest, core_version): + return None + built_with_core, core_claims = _core_claims(manifest) + return ( + f"the bundle's policyengine-core {core_version} is neither the " + f"build-time core ({built_with_core or 'unrecorded'}) nor in the " + f"release's core claims {core_claims or '(none declared)'}; the " + "certificate names no core, so it follows the core pin, and receipts " + "certify that core without the release's claim" + ) + + def head_artifact(artifact, token: Optional[str] = None) -> bool: # Repo type is not recorded per artifact in the schema; try dataset # first (the publishing convention), then model-style. @@ -1002,6 +1037,9 @@ def build_country_manifest_payload( } if core_version is not None: certification["certified_for_core_version"] = core_version + certification["core_binding"] = "certified" + else: + certification["core_binding"] = "bundle_pin" if qualified_pair_id is not None: certification["qualification_pair_id"] = qualified_pair_id certification["qualification_ledger_url"] = ( @@ -1110,6 +1148,7 @@ def certify( regional_release_kind: RegionalReleaseKind = "local_area", core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, + bundle_core_version: Optional[str] = None, ) -> tuple[dict, str, list[str]]: raise NotImplementedError @@ -1132,6 +1171,7 @@ def certify( regional_release_kind: RegionalReleaseKind = "local_area", core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, + bundle_core_version: Optional[str] = None, ) -> tuple[dict, str, list[str]]: raise CertificationError( "Legacy data-producer certification updates are not implemented in " @@ -1157,10 +1197,13 @@ def certify( regional_release_kind: RegionalReleaseKind = "local_area", core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, + bundle_core_version: Optional[str] = None, ) -> tuple[dict, str, list[str]]: manifest, manifest_sha256, uri_parts = fetch_release_manifest( manifest_uri, token=token ) + primary_manifest = manifest + regional_manifest = None regional_uri_parts = None recorded_release_kind = None regional_artifacts: dict[str, DataReleaseArtifact] = {} @@ -1210,6 +1253,18 @@ def certify( qualified_pair_id=qualified_pair_id, ) warnings = warnings + regional_warnings + if core_version is None and bundle_core_version is not None: + # The certificate follows the bundle's core pin: report, not + # refuse, a pin that a release does not claim. + for label, release in ( + ("data release", primary_manifest), + ("regional release", regional_manifest), + ): + if release is None: + continue + warning = core_claim_warning(release, bundle_core_version) + if warning is not None: + warnings.append(f"{label}: {warning}") for filename in required_supplemental_release_files( country, manifest, uri_parts @@ -1371,6 +1426,9 @@ def certify_data_release( regional_release_kind=regional_release_kind, core_version=core_version, qualified_pair_id=qualified_pair_id, + bundle_core_version=( + (bundle.get("packages") or {}).get("policyengine-core") or {} + ).get("version"), ) previous_release = (bundle.get("data_releases") or {}).get(country) or {} bundle.setdefault("data_releases", {})[country] = data_release diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 61e324922..6480ea128 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -169,6 +169,14 @@ class DataCertification(BaseModel): data_build_fingerprint: Optional[str] = None certified_by: Optional[str] = None certified_for_core_version: Optional[str] = None + #: How the certificate covers policyengine-core. ``certified``: it names + #: the core it was certified for (``certified_for_core_version``), and the + #: bundle's core pin must equal it. ``bundle_pin``: it certifies whatever + #: core the bundle pins, without holding the pin to the data release's + #: core claims; certification and the strict check warn when the pin is + #: outside them. Absent in certificates from before October 2026, which + #: follow the pin the same way. + core_binding: Optional[Literal["certified", "bundle_pin"]] = None qualification_pair_id: Optional[str] = None qualification_ledger_url: Optional[str] = None diff --git a/src/policyengine/provenance/trace.py b/src/policyengine/provenance/trace.py index 350c676f5..bcffda78a 100644 --- a/src/policyengine/provenance/trace.py +++ b/src/policyengine/provenance/trace.py @@ -534,6 +534,12 @@ def _build_bundle_performance( certification.certified_for_model_version ) performance["pe:compatibilityBasis"] = certification.compatibility_basis + if certification.core_binding is not None: + performance["pe:coreBinding"] = certification.core_binding + if certification.certified_for_core_version is not None: + performance["pe:certifiedForCoreVersion"] = ( + certification.certified_for_core_version + ) if certification.built_with_model_version is not None: performance["pe:builtWithModelVersion"] = ( certification.built_with_model_version diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index bcf6a2296..092420f61 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -1447,8 +1447,45 @@ def test__given_pinned_core__then_the_regional_release_must_cover_it( ) release = json.loads(bundle_path.read_text())["data_releases"]["uk"] assert release["certification"]["certified_for_core_version"] == "3.33.0" + assert release["certification"]["core_binding"] == "certified" assert release["regional_release_kind"] == "area_coded_national" + def test__given_no_core_version__then_the_certificate_follows_the_pin_openly( + self, tmp_path + ): + """Without --core-version the certificate follows the bundle's core pin. + + It says so (core_binding bundle_pin), and certification reports a + release that does not claim the pin instead of refusing it. + """ + national = _microcosm_uk_national_payload() + national["compatible_core_packages"] = [ + {"name": "policyengine-core", "specifier": "==3.33.3"} + ] + bundle_path = _tmp_bundle(tmp_path) + bundle = json.loads(bundle_path.read_text()) + bundle["packages"]["policyengine-core"] = { + "name": "policyengine-core", + "version": "3.33.3", + } + bundle_path.write_text(json.dumps(bundle)) + + result = _certify_uk( + bundle_path, + [_response(national), _response(_microcosm_uk_dense_payload())], + regional_manifest_uri=UK_DENSE_URI, + ) + + written = json.loads(bundle_path.read_text()) + certification = written["data_releases"]["uk"]["certification"] + assert certification["core_binding"] == "bundle_pin" + assert "certified_for_core_version" not in certification + assert written["packages"]["policyengine-core"]["version"] == "3.33.3" + # The national release claims the pin; the local-area one claims no core. + unclaimed = [w for w in result.warnings if "follows the core pin" in w] + assert len(unclaimed) == 1 + assert unclaimed[0].startswith("regional release: ") + def test__given_unreachable_regional_artifact__then_refuses(self, tmp_path): with ( patch( diff --git a/tests/test_release_tro_generation.py b/tests/test_release_tro_generation.py index 16aa76166..d0aa9410d 100644 --- a/tests/test_release_tro_generation.py +++ b/tests/test_release_tro_generation.py @@ -136,6 +136,35 @@ def test_runtime_core_must_match_a_core_the_certificate_names(release): generator.generated_tros(strict=True) +@pytest.mark.parametrize(("pin", "warned"), [("3.26.11", False), ("99.0.0", True)]) +def test_a_certificate_that_follows_the_pin_reports_an_unclaimed_pin( + release, capsys, pin, warned +): + """A certificate that names no core follows the core pin: an unclaimed + pin is reported, not refused.""" + + path = release[0] / "manifest.json" + payload = json.loads(path.read_bytes()) + payload["data_releases"]["us"]["certification"]["core_binding"] = "bundle_pin" + payload["packages"]["policyengine-core"]["version"] = pin + path.write_text(json.dumps(payload)) + manifest.get_release_manifest.cache_clear() + + generator.generated_tros(strict=True) + + assert ("follows the core pin" in capsys.readouterr().err) == warned + + +def test_a_core_binding_must_agree_with_the_named_core(release): + path = release[0] / "manifest.json" + payload = json.loads(path.read_bytes()) + payload["data_releases"]["us"]["certification"]["core_binding"] = "certified" + path.write_text(json.dumps(payload)) + manifest.get_release_manifest.cache_clear() + with pytest.raises(ValueError, match="core_binding 'certified' disagrees"): + generator.generated_tros(strict=True) + + @pytest.mark.parametrize("field", ["name", "version", "sha256", "wheel_url"]) def test_runtime_model_identity_must_match_certification(release, field): path = release[0] / "manifest.json" diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 6553fcac1..34bf4139a 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -708,7 +708,7 @@ def run(command, *, cwd, env=None): changed = json.loads(bundle_path.read_bytes()) changed["packages"]["policyengine-core"]["version"] = "99.0.0" write_json(bundle_path, changed) - with pytest.raises(ValueError, match="runtime core 99.0.0 differs"): + with pytest.raises(ValueError, match="runtime core 99.0.0 differs.*requalify"): tros.generated_tros(strict=True) From b25cb782e543f8be9cc38d9c9c62e994817c5694 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Wed, 21 Oct 2026 11:57:35 +0100 Subject: [PATCH 22/26] Close other-line PRs only at their qualified commit; refuse same-line routing - Promotion closed an other-line PR whenever its pair had a receipt, even while a forced re-run was in flight at a newer head, whose verdict step then found no PR. It now closes such a PR only at the commit its receipt qualified. The plan splits receipts by data line once, and the same-line plan returns plain PromotionPlans, so no return can drop the closes. - Promote closed PRs and then tried to return them to draft, which gh refuses. One helper now closes PRs, warns when a close fails, and the draft sweep skips what it closed. - candidate_pr_body's on_main_line has no default, finalize builds the body only when asked for it, and the stale cos d1344 clause is gone. - Preparation now refuses a same-line pair on a routed main outright: the routing would need a local-area cut for the pair's versions, which a candidate does not carry, and the claims check it replaced compared with main's certificate rather than the cut. New tests cover a same-line target beside an other-line PR, promote --apply on a Microcosm main, and finalize for an other-line pair. The frozen routed bundle is refreshed to this PR's certificate. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/certify-uk-release.yaml | 3 +- scripts/check_uk_candidate.py | 91 +++++++----- .../provenance/uk_qualification.py | 107 +++++++------- .../microcosm_routed_bundle.json | 2 +- tests/test_uk_qualification_pipeline.py | 137 ++++++++++++++++-- 5 files changed, 238 insertions(+), 102 deletions(-) diff --git a/.github/workflows/certify-uk-release.yaml b/.github/workflows/certify-uk-release.yaml index a1dd71962..3f27350ee 100644 --- a/.github/workflows/certify-uk-release.yaml +++ b/.github/workflows/certify-uk-release.yaml @@ -7,7 +7,8 @@ # mandatory gates on that exact commit, replayed by policyengine-scorecard, # and recorded in the uk-qualification-ledger branch. Promotion is # serialized: it readies the newest promotable candidate PR that advances -# main, closes superseded ones, and (cos d1344) fast-forwards main to its +# main, closes superseded ones and those of pairs from another data line +# (which are never promoted), and (cos d1344) fast-forwards main to its # qualified commit once every check on it has passed, which succeeds only # while main is still the commit it was built on; otherwise the catch-up # requalifies it on current main. diff --git a/scripts/check_uk_candidate.py b/scripts/check_uk_candidate.py index c2fbdef2b..7920b8c33 100644 --- a/scripts/check_uk_candidate.py +++ b/scripts/check_uk_candidate.py @@ -34,6 +34,7 @@ import subprocess import sys import time +from collections.abc import Sequence from functools import lru_cache from pathlib import Path from typing import Any, Optional @@ -466,14 +467,14 @@ def cmd_finalize(args: argparse.Namespace) -> int: attempt, Path(args.ledger), issued_by=issued_by(), completed_at=utc_now() ) payload = read_json(attempt / "candidate.json") - line = certified_data_line(json.loads(BUNDLE_PATH.read_text())) - body = candidate_pr_body( - receipt, - receipt.pair, - ledger_url=ledger_url(receipt.pair_id), - on_main_line=data_line(receipt.pair) == line, - ) if args.pr_body: + line = certified_data_line(json.loads(BUNDLE_PATH.read_text())) + body = candidate_pr_body( + receipt, + receipt.pair, + ledger_url=ledger_url(receipt.pair_id), + on_main_line=data_line(receipt.pair) == line, + ) Path(args.pr_body).write_text(body) print(f"verdict {receipt.verdict}; receipt {receipt.receipt_sha256}") set_output( @@ -935,6 +936,35 @@ def cmd_refresh_pr_checks(args: argparse.Namespace) -> int: return 0 +def _close_pull_requests(closes: Sequence[tuple[str, str]]) -> set[str]: + """Close each ``(number, comment)``; return the numbers GitHub closed. + + A failed close is reported, and the pull request stays open for the next + run to close. + """ + + closed = set() + for number, comment in closes: + result = subprocess.run( + [ + "gh", + "pr", + "close", + number, + "--repo", + LEDGER_REPOSITORY, + "--comment", + comment, + ], + check=False, + ) + if result.returncode == 0: + closed.add(number) + else: + print(f"::warning::could not close pull request {number}") + return closed + + def cmd_promote(args: argparse.Namespace) -> int: bundle = json.loads(BUNDLE_PATH.read_text()) problems: list[str] = [] @@ -960,42 +990,33 @@ def cmd_promote(args: argparse.Namespace) -> int: print(f"off main's data line {branch}") if not args.apply: return 0 - for branch in plan.close_branches: - subprocess.run( - [ - "gh", - "pr", - "close", - numbers.get(branch, branch), - "--repo", - LEDGER_REPOSITORY, - "--comment", - f"Superseded: {plan.reason}. The ledger keeps this candidate's evidence.", - ], - check=False, - ) - for branch in plan.close_off_line: - subprocess.run( - [ - "gh", - "pr", - "close", - numbers.get(branch, branch), - "--repo", - LEDGER_REPOSITORY, - "--comment", + closed = _close_pull_requests( + [ + ( + numbers[branch], + f"Superseded: {plan.reason}. The ledger keeps this candidate's " + "evidence.", + ) + for branch in plan.close_branches + ] + + [ + ( + numbers[branch], "This pair is not on main's data line, so it is never promoted. " "The ledger keeps this candidate's evidence.", - ], - check=False, - ) + ) + for branch in plan.close_off_line + ] + ) # Only the plan's target may be ready, and only while _promote finds it # on current main: any other ready candidate (left by a run that could # not undo it, or by older code) goes back to draft. returned = [ _set_ready(str(pr["number"]), False) for pr in sorted(pull_requests, key=lambda pr: pr["number"]) - if not pr["isDraft"] and pr["headRefName"] != plan.ready_branch + if not pr["isDraft"] + and pr["headRefName"] != plan.ready_branch + and str(pr["number"]) not in closed ] safe = True if plan.ready_branch is not None and plan.ready_commit is not None: diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index dc2be994f..0ae373212 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -17,8 +17,9 @@ it), regenerate pins, ``uv.lock``, TROs and household snapshots together, and the certified-version test constants for a pair on main's data line, and commit that as the candidate commit on - ``candidate/uk/-``. A routed bundle keeps its regional - release for a pair on its line and drops it for a pair from another. + ``candidate/uk/-``. On a main routed to a local-area release, + a pair from another line drops the routing, and a pair on main's own line + is refused: it needs a local-area cut certified by hand. ``gates`` Run the mandatory local gates on the candidate commit (:mod:`policyengine.provenance.uk_candidate`). @@ -29,7 +30,8 @@ Record the qualification receipt in the ledger. ``promote`` Pick the newest promotable pair that advances ``main`` and mark its - candidate PR ready, closing superseded candidates. + candidate PR ready, closing superseded candidates, and closing the PRs + of pairs from another data line at the commit their receipt qualified. Network, git and GitHub access are injected so every decision can be tested offline. @@ -354,11 +356,22 @@ def data_line(pair: CandidatePair) -> tuple[str, str]: return (pair.data.data_package, pair.data.repo_id) +def split_by_data_line( + receipts: Iterable[QualificationReceipt], bundle: Mapping[str, Any] +) -> tuple[list[QualificationReceipt], list[QualificationReceipt]]: + """``(on main's data line, from another line)``.""" + + line = certified_data_line(bundle) + same, other = [], [] + for receipt in receipts: + (same if data_line(receipt.pair) == line else other).append(receipt) + return same, other + + def _same_line_receipts( receipts: Iterable[QualificationReceipt], bundle: Mapping[str, Any] ) -> list[QualificationReceipt]: - line = certified_data_line(bundle) - return [receipt for receipt in receipts if data_line(receipt.pair) == line] + return split_by_data_line(receipts, bundle)[0] def certified_order_key(bundle: Mapping[str, Any]) -> tuple[str, str, str]: @@ -759,12 +772,13 @@ def regional_certification_arguments( ) -> list[str]: """``certify-data``'s regional flags for certifying ``pair`` onto ``bundle``. - Certification refuses to drop a bundle's regional routing silently. A - pair on main's data line keeps main's regional release, which must then - serve the pair's model and core by its own claims. A pair from another - line (a forced qualification, never promoted automatically) replaces the - line the routing belongs to, so the routing is dropped and the pair's - national dataset serves every region type. + Certification refuses to drop a bundle's regional routing silently. A pair + from another data line (a forced qualification, never promoted) replaces + the line the routing belongs to, so the routing is dropped and the pair's + national dataset serves every region type. A pair on a routed main's own + line is refused: the routing would need a local-area cut for the pair's + versions, which a candidate pair does not carry, and automated + qualification checks no local-area file. """ record = bundle["data_releases"]["uk"] @@ -773,29 +787,12 @@ def regional_certification_arguments( return [] if data_line(pair) != certified_data_line(bundle): return ["--drop-regional-release"] - # Main's regional release was certified for main's model and core, and - # gets no qualified-pair escape. A pair on other versions needs its own - # local-area cut, which a candidate pair does not carry. - certified_core = (record.get("certification") or {}).get( - "certified_for_core_version" - ) or bundle["packages"][CORE_PACKAGE]["version"] - if (pair.model.version, pair.core.version) != ( - record["model_package"]["version"], - certified_core, - ): - raise ValueError( - f"Main routes UK area regions to {regional_uri}, certified for " - f"policyengine-uk {record['model_package']['version']} and " - f"policyengine-core {certified_core}. A candidate on " - f"{pair.model.version} and {pair.core.version} needs a local-area " - "cut for those versions; certify it by hand with that cut." - ) - return [ - "--regional-manifest-uri", - regional_uri, - "--regional-release-kind", - record.get("regional_release_kind") or "local_area", - ] + raise ValueError( + f"Main routes UK area regions to {regional_uri}. Automated " + "qualification cannot carry a local-area cut for a candidate on main's " + "own data line; certify such a pair by hand, with its local-area cut " + "(docs/engineering/skills/data-certification.md)." + ) def prepare_candidate_commit( @@ -1627,23 +1624,28 @@ def plan_promotion( """ # Only pairs on main's data line are ordered against it. A pair from - # another line is never promoted, so its open pull request is closed; the - # ledger keeps its evidence. - receipts = list(receipts) - line = certified_data_line(bundle) - off_line = sorted( + # another line is never promoted, so its pull request is closed, but only + # at the commit its receipt qualified: a newer head is a forced re-run + # still in flight, which its own receipt decides. The ledger keeps the + # evidence. + same_line, off_line = split_by_data_line(receipts, bundle) + plan = _plan_same_line_promotion(same_line, bundle, open_candidate_branches) + plan.close_off_line = sorted( { candidate_branch(receipt.pair) - for receipt in receipts - if data_line(receipt.pair) != line + for receipt in off_line + if open_candidate_branches.get(candidate_branch(receipt.pair)) + == receipt.harness.git_commit } - & set(open_candidate_branches) ) - receipts = _same_line_receipts(receipts, bundle) + return plan - def plan(**fields: Any) -> PromotionPlan: - return PromotionPlan(**fields, close_off_line=off_line) +def _plan_same_line_promotion( + receipts: Sequence[QualificationReceipt], + bundle: Mapping[str, Any], + open_candidate_branches: Mapping[str, str], +) -> PromotionPlan: current = certified_order_key(bundle) target = select_promotion_target(receipts, current) baseline_matches = target is not None and ( @@ -1662,14 +1664,14 @@ def plan(**fields: Any) -> PromotionPlan: elif baseline_matches and dominates(target.pair.order_key, key): close.append(branch) if target is None: - return plan( + return PromotionPlan( target=None, ready_branch=None, close_branches=sorted(close), reason=f"no promotable pair advances {current}", ) if not baseline_matches: - return plan( + return PromotionPlan( target=target, ready_branch=None, close_branches=sorted(close), @@ -1684,7 +1686,7 @@ def plan(**fields: Any) -> PromotionPlan: qualified_commit = target.harness.git_commit head = open_candidate_branches.get(branch) if head is None: - return plan( + return PromotionPlan( target=target, ready_branch=None, close_branches=sorted(close), @@ -1694,7 +1696,7 @@ def plan(**fields: Any) -> PromotionPlan: ), ) if head != qualified_commit: - return plan( + return PromotionPlan( target=target, ready_branch=None, close_branches=sorted(close), @@ -1704,7 +1706,7 @@ def plan(**fields: Any) -> PromotionPlan: "nothing is promoted" ), ) - return plan( + return PromotionPlan( target=target, ready_branch=branch, ready_commit=qualified_commit, @@ -1718,8 +1720,8 @@ def candidate_pr_body( pair: CandidatePair, *, ledger_url: str, + on_main_line: bool, prepared: Optional[Mapping[str, Any]] = None, - on_main_line: bool = True, ) -> str: """Markdown body for a candidate PR: the pair, the gates and the verdict. @@ -1749,8 +1751,7 @@ def candidate_pr_body( verdict = receipt.verdict if receipt else "pending" if on_main_line: outcome = ( - "Merging this PR promotes the pair. Until cos d1344 is ruled, " - "promotion is a reviewed merge; the certification evidence test " + "Merging this PR promotes the pair; the certification evidence test " "refuses an exact-pair certificate without a promotable ledger receipt." ) else: diff --git a/tests/fixtures/uk_qualification/microcosm_routed_bundle.json b/tests/fixtures/uk_qualification/microcosm_routed_bundle.json index 8a2f3fca5..c6a8eff3d 100644 --- a/tests/fixtures/uk_qualification/microcosm_routed_bundle.json +++ b/tests/fixtures/uk_qualification/microcosm_routed_bundle.json @@ -21,9 +21,9 @@ "certification": { "built_with_model_version": "2.132.6", "certified_by": "policyengine.py bundle certification", - "certified_for_core_version": "3.33.3", "certified_for_model_version": "2.132.6", "compatibility_basis": "built_with_model_package", + "core_binding": "bundle_pin", "data_build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" }, "certified_data_artifact": { diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 34bf4139a..93bc0be40 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -735,24 +735,17 @@ def microcosm_pair(model, core): certified = ( record["model_package"]["version"], - record["certification"]["certified_for_core_version"], + MICROCOSM_ROUTED_BUNDLE["packages"]["policyengine-core"]["version"], ) assert regional_certification_arguments(BUNDLE, uk_data_pair) == [] assert regional_certification_arguments(MICROCOSM_ROUTED_BUNDLE, uk_data_pair) == [ "--drop-regional-release" ] - assert regional_certification_arguments( - MICROCOSM_ROUTED_BUNDLE, microcosm_pair(*certified) - ) == [ - "--regional-manifest-uri", - record["regional_source_manifest_uri"], - "--regional-release-kind", - record["regional_release_kind"], - ] - # Main's local-area cut serves only main's model and core. - for model, core in (("2.140.0", certified[1]), (certified[0], "3.34.0")): - with pytest.raises(ValueError, match="needs a local-area cut"): + # A same-line pair needs a local-area cut for its versions, which no + # candidate carries, whatever those versions are. + for model, core in (certified, ("2.140.0", certified[1]), (certified[0], "3.34.0")): + with pytest.raises(ValueError, match="certify such a pair by hand"): regional_certification_arguments( MICROCOSM_ROUTED_BUNDLE, microcosm_pair(model, core) ) @@ -1033,6 +1026,126 @@ def _promotable_pair_and_branch(core=None): return make_pair(model="2.200.0", data=current, core=core) +def _other_line_pair(model="2.900.0"): + """A pair on the Microcosm line, which BUNDLE (a uk-data main) is not.""" + + pair = make_pair(model=model) + return pair.model_copy( + update={ + "data": pair.data.model_copy( + update={ + "data_package": "microcosm-data", + "repo_id": "policyengine/populace-uk-private", + } + ) + } + ) + + +def test_promotion_closes_other_line_prs_only_at_their_qualified_commit(): + target = _promotion_receipt(pair=_promotable_pair_and_branch()) + other = _promotion_receipt(pair=_other_line_pair()) + target_branch = candidate_branch(target.pair) + other_branch = candidate_branch(other.pair) + + plan = plan_promotion( + [target, other], + BUNDLE, + { + target_branch: target.harness.git_commit, + other_branch: other.harness.git_commit, + }, + ) + assert plan.ready_branch == target_branch + assert plan.close_off_line == [other_branch] + assert other_branch not in plan.close_branches + + # A newer head is a forced re-run still in flight: its own receipt decides. + in_flight = plan_promotion( + [target, other], + BUNDLE, + {target_branch: target.harness.git_commit, other_branch: "f" * 40}, + ) + assert in_flight.ready_branch == target_branch + assert in_flight.close_off_line == [] + + +def test_promote_closes_other_line_prs_and_never_drafts_a_closed_one(monkeypatch): + import argparse + + cli = load_candidate_cli("check_uk_promote_line", MICROCOSM_ROUTED_BUNDLE_PATH) + receipt = _promotion_receipt(pair=make_pair(model="2.200.0")) + branch = candidate_branch(receipt.pair) + monkeypatch.setattr(cli, "latest_receipts", lambda ledger, problems=None: [receipt]) + monkeypatch.setattr( + cli, + "_candidate_pull_requests", + lambda: [ + { + "number": 7, + "headRefName": branch, + "headRefOid": receipt.harness.git_commit, + "isDraft": False, + "isCrossRepository": False, + } + ], + ) + commands = [] + monkeypatch.setattr( + cli.subprocess, + "run", + lambda command, **kwargs: commands.append(command) + or subprocess.CompletedProcess(command, 0), + ) + drafted = [] + monkeypatch.setattr( + cli, "_set_ready", lambda number, ready: drafted.append(number) or True + ) + + assert cli.cmd_promote(argparse.Namespace(ledger="ledger", apply=True)) == 0 + assert commands == [ + [ + "gh", + "pr", + "close", + "7", + "--repo", + cli.LEDGER_REPOSITORY, + "--comment", + "This pair is not on main's data line, so it is never promoted. " + "The ledger keeps this candidate's evidence.", + ] + ] + # gh refuses to return a closed pull request to draft. + assert drafted == [] + + +def test_finalize_says_an_other_line_pair_is_never_promoted(monkeypatch, tmp_path): + import argparse + + cli = load_candidate_cli("check_uk_finalize_line", MICROCOSM_ROUTED_BUNDLE_PATH) + receipt = _promotion_receipt(pair=make_pair(model="2.200.0")) + attempt = tmp_path / "attempt" + write_json( + attempt / "candidate.json", + { + "pair": receipt.pair.canonical_payload(), + "branch": candidate_branch(receipt.pair), + }, + ) + monkeypatch.setattr(cli, "finalize_attempt", lambda *args, **kwargs: receipt) + monkeypatch.setattr(cli, "issued_by", lambda: ISSUED) + monkeypatch.setattr(cli, "set_output", lambda **kwargs: None) + body = tmp_path / "body.md" + + namespace = argparse.Namespace( + attempt=str(attempt), ledger=str(tmp_path / "ledger"), pr_body=str(body) + ) + assert cli.cmd_finalize(namespace) == 0 + text = body.read_text() + assert "never promoted" in text and "promotes the pair" not in text + + def test_pairs_that_share_versions_never_share_a_branch(): """Interrupted review of #574: a qualified pair readied the branch of an incomplete pair with the same model and data but another core.""" From 1fda0694724c5dcf08c44fb19dbf5032e42dde19 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Wed, 21 Oct 2026 11:57:45 +0100 Subject: [PATCH 23/26] One overlay refusal; harden the cache path and the size request - The overlay rule had one helper but three messages, one naming only the first shadowed overlay. shadowing_overlays_message is now the one refusal, naming every shadowed overlay, at runtime and in the check. - The cache-path guard looked for a drive only in the first segment, and the join restarts at a later one on Windows. It now refuses a drive or root in any segment and checks that the joined path stays inside the data directory. - requests asks for compressed answers, which carry no Content-Length, so the size HEADs now ask for the identity encoding, as huggingface_hub does. - The request test now has a dataset entry with its own repository, revision and type, and asserts both calls and the whole data block literally. Co-Authored-By: Claude Opus 5.5 --- scripts/release_consumers.py | 4 +- .../provenance/bundle_validation.py | 15 ++- .../provenance/dataset_materialization.py | 18 ++-- src/policyengine/provenance/manifest.py | 18 +--- tests/test_dataset_materialization.py | 3 + tests/test_release_manifests.py | 4 +- tests/test_uk_qualification_pipeline.py | 93 ++++++++++++++++++- 7 files changed, 126 insertions(+), 29 deletions(-) diff --git a/scripts/release_consumers.py b/scripts/release_consumers.py index ea5bd662b..9c96a06a5 100644 --- a/scripts/release_consumers.py +++ b/scripts/release_consumers.py @@ -75,7 +75,9 @@ def _hf_size(repo_id: str, revision: str, path: str, repo_type: str = "model") - """ url = https_dataset_uri(repo_id, path, revision, repo_type=repo_type) - headers = hugging_face_auth_headers() + # A compressed answer has no Content-Length, so ask for the bytes as they + # are, as huggingface_hub does. + headers = {**hugging_face_auth_headers(), "Accept-Encoding": "identity"} response = requests.head( url, headers=headers, timeout=TIMEOUT, allow_redirects=False ) diff --git a/src/policyengine/provenance/bundle_validation.py b/src/policyengine/provenance/bundle_validation.py index 525b86975..51c474481 100644 --- a/src/policyengine/provenance/bundle_validation.py +++ b/src/policyengine/provenance/bundle_validation.py @@ -40,6 +40,16 @@ def shadowed_dataset_overlays( return sorted(set(overlays or {}) & certified) +def shadowing_overlays_message(country: str, shadowed: list[str]) -> str: + """The one refusal for overlays that shadow certified datasets.""" + + return ( + f"dataset_overlays.{country} shadows certified datasets: " + f"{', '.join(shadowed)}. Overlays must be additive and never change " + "resolution: remove those overlays." + ) + + def validate_dataset_overlays(bundle: Mapping[str, Any]) -> None: """Refuse an overlay that shadows a certified dataset name. @@ -51,10 +61,7 @@ def validate_dataset_overlays(bundle: Mapping[str, Any]) -> None: release = (bundle.get("data_releases") or {}).get(country) or {} shadowed = shadowed_dataset_overlays(release, entries) if shadowed: - raise ValueError( - f"dataset_overlays.{country} shadows certified datasets: " - f"{', '.join(shadowed)}. Remove those overlays." - ) + raise ValueError(shadowing_overlays_message(country, shadowed)) def validate_spm_configuration(configuration: Any) -> None: diff --git a/src/policyengine/provenance/dataset_materialization.py b/src/policyengine/provenance/dataset_materialization.py index c93f546e6..28e8f60b9 100644 --- a/src/policyengine/provenance/dataset_materialization.py +++ b/src/policyengine/provenance/dataset_materialization.py @@ -93,20 +93,26 @@ def _bundle_destination(data_dir: Path, path: str) -> Path: """ relative = PurePosixPath(path) - # Repository paths use forward slashes. A backslash, drive or root would - # make the platform path leave data_dir on Windows. - windows = PureWindowsPath(path) + # Repository paths use forward slashes. A backslash, or a drive or root in + # any segment, would make the platform path leave data_dir on Windows. if ( relative.is_absolute() or ".." in relative.parts or "\\" in path - or windows.drive - or windows.root + or any( + PureWindowsPath(part).drive or PureWindowsPath(part).root + for part in relative.parts + ) ): raise DatasetMaterializationError( f"Bundle artifact path {path!r} must be relative to its repository." ) - return data_dir.joinpath(*relative.parts) + destination = data_dir.joinpath(*relative.parts) + if destination == data_dir or not destination.is_relative_to(data_dir): + raise DatasetMaterializationError( + f"Bundle artifact path {path!r} must be relative to its repository." + ) + return destination def _resolve_bundle_dataset( diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 6480ea128..f3b15fb5d 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -10,7 +10,10 @@ import requests from pydantic import BaseModel, Field -from policyengine.provenance.bundle_validation import shadowed_dataset_overlays +from policyengine.provenance.bundle_validation import ( + shadowed_dataset_overlays, + shadowing_overlays_message, +) HF_REQUEST_TIMEOUT_SECONDS = 30 PYPI_REQUEST_TIMEOUT_SECONDS = 30 @@ -338,19 +341,8 @@ def _apply_dataset_overlays( return release_payload shadowed = shadowed_dataset_overlays(release_payload, overlays) - default_dataset = release_payload.get("default_dataset") - if default_dataset in shadowed: - raise ValueError( - f"Dataset overlay '{default_dataset}' for country '{country_id}' " - "shadows the certified default dataset. Overlays must never " - "change default resolution." - ) if shadowed: - raise ValueError( - f"Dataset overlay '{shadowed[0]}' for country '{country_id}' " - "collides with a certified dataset entry. Overlays must be " - "additive, not overrides." - ) + raise ValueError(shadowing_overlays_message(country_id, shadowed)) certified_datasets = dict(release_payload.get("datasets") or {}) certified_datasets.update(overlays) diff --git a/tests/test_dataset_materialization.py b/tests/test_dataset_materialization.py index 8eb572f5a..5180ebd88 100644 --- a/tests/test_dataset_materialization.py +++ b/tests/test_dataset_materialization.py @@ -277,6 +277,9 @@ def test_same_named_artifacts_of_two_releases_never_share_a_cache_path(tmp_path) "C:outside.h5", "\\outside.h5", "\\\\server\\share\\outside.h5", + "releases/D:outside.h5", + "a/b/E:/outside.h5", + "", ): with pytest.raises(DatasetMaterializationError, match="must be relative"): _bundle_destination(tmp_path, path) diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 4256e32d7..aff21577a 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -1283,7 +1283,7 @@ def test__given_overlay_shadowing_default__then_raises(self): try: _apply_dataset_overlays("us", payload, bundle) except ValueError as error: - assert "shadows the certified default" in str(error) + assert "shadows certified datasets: populace_us_2024" in str(error) else: raise AssertionError("Expected overlay shadowing the default to fail") @@ -1302,6 +1302,6 @@ def test__given_overlay_colliding_with_certified_dataset__then_raises(self): try: _apply_dataset_overlays("us", payload, bundle) except ValueError as error: - assert "collides with a certified dataset" in str(error) + assert "shadows certified datasets: demographics" in str(error) else: raise AssertionError("Expected overlay colliding with a dataset to fail") diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 93bc0be40..5a6d2316d 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -3576,6 +3576,91 @@ def hf_size(*request): ) +def test_released_bundle_request_uses_the_datasets_own_location(): + from policyengine.provenance.uk_qualification import released_bundle_request + + bundle = json.loads(json.dumps(MICROCOSM_ROUTED_BUNDLE)) + record = bundle["data_releases"]["uk"] + reference = record["datasets"]["microcosm_uk_2024_25"] + reference.update( + repo_id="policyengine/other-repo", revision="other-revision", repo_type="model" + ) + manifest = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "default_datasets": {"national": "microcosm_uk_2024_25"}, + "artifacts": { + "microcosm_uk_2024_25": { + "kind": "microdata", + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/other-repo", + "revision": "other-revision", + "sha256": reference["sha256"], + } + }, + } + manifest_bytes = json.dumps(manifest).encode() + fetched = [] + + def hf_file(*request): + fetched.append(("file", *request)) + return manifest_bytes + + def hf_size(*request): + fetched.append(("size", *request)) + return 1234 + + core = bundle["packages"]["policyengine-core"]["version"] + request = released_bundle_request( + bundle, + version=bundle["policyengine_version"], + wheel_sha256=sha("7"), + packaged_manifest_sha256=sha("8"), + release_bundle=golden("qualification_request_golden")["candidate"][ + "expected_release_bundle" + ], + hf_file=hf_file, + hf_size=hf_size, + pypi_project=lambda name: { + "releases": {core: [_wheel("policyengine-core", core, sha("6"))]} + }, + requested_by={ + "repository": "PolicyEngine/policyengine.py", + "workflow_run_id": 5, + "commit": "e" * 40, + }, + ) + + cut = "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + assert fetched == [ + ( + "file", + "policyengine/populace-uk-private", + cut, + "releases/microcosm-uk-2024-25-national/release_manifest.json", + "dataset", + ), + ( + "size", + "policyengine/other-repo", + "other-revision", + "microcosm_uk_2024_25.h5", + "model", + ), + ] + assert request.document()["candidate"]["data"] == { + "build_id": cut, + "dataset": "microcosm_uk_2024_25", + "repo_id": "policyengine/other-repo", + "repo_type": "model", + "revision": "other-revision", + "path": "microcosm_uk_2024_25.h5", + "sha256": reference["sha256"], + "size_bytes": 1234, + "release_manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + } + + def test_released_bundle_request_needs_a_size_source(): from policyengine.provenance.uk_qualification import released_bundle_request @@ -3622,6 +3707,8 @@ def test_release_consumers_address_dataset_repositories(monkeypatch): spec.loader.exec_module(consumers) url = "https://huggingface.co/datasets/org/data/resolve/v1/releases/a%20b.h5" authorized = {"Authorization": "Bearer hub-token"} + # A compressed answer carries no Content-Length. + sized = {**authorized, "Accept-Encoding": "identity"} calls = [] def response(status, headers, content=b""): @@ -3650,7 +3737,7 @@ def head(target, **kwargs): # A large file's redirect carries its size. answers.append(response(302, {"X-Linked-Size": "321", "Content-Length": "1152"})) assert consumers._hf_size("org/data", "v1", "releases/a b.h5", "dataset") == 321 - assert calls == [("head", url, authorized, False)] + assert calls == [("head", url, sized, False)] calls.clear() # A small file answers with its own length, or with a redirect to it. @@ -3664,8 +3751,8 @@ def head(target, **kwargs): ) assert consumers._hf_size("org/data", "v1", "releases/a b.h5", "dataset") == 77 assert calls[-2:] == [ - ("head", url, authorized, False), - ("head", url, authorized, True), + ("head", url, sized, False), + ("head", url, sized, True), ] model_url = "https://huggingface.co/org/model/resolve/v1/a.h5" From 0f5e05baf9b7db388d4e99fe4cb06630e67770aa Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Wed, 21 Oct 2026 11:57:45 +0100 Subject: [PATCH 24/26] Record the needs-review acceptances and the switch's new behaviours MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The data certification skill records María Juaristi's acceptance of the two needs-review gates: the missing incapacity_benefit_reported, whose enhanced FRS column carries no values, and the reform the small-cell rule withholds from reform responsiveness. The changelog says how certificates now cover the core, that promotion closes other-line PRs, and that recovery no longer re-dispatches them. Co-Authored-By: Claude Opus 5.5 --- ...k-2024-25-national-20261019T083323Z-6cba5d71.changed.md | 2 +- docs/engineering/skills/data-certification.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md index 5e8963431..1a6ef427f 100644 --- a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md +++ b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md @@ -1 +1 @@ -Switch the UK default dataset from the Enhanced FRS 2024-25 (policyengine-uk-data 1.58.1) to Microcosm UK 2024-25 (`microcosm_uk_2024_25`, cut `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71`), certified with policyengine-uk 2.132.6 and policyengine-core 3.33.3, the versions it was built with. Constituency and local-authority regions now load the Microcosm local-area release (`microcosm_uk_2024_25_dense`, cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`) and are in the default UK region registry. UK results change with the data. The national file has no constituency or local-authority codes, so constituency and local-authority outputs (`compute_uk_geography_disclosure` at those levels, `compute_uk_constituency_impacts`, `compute_uk_local_authority_impacts`) need simulations on `microcosm_uk_2024_25_dense`, and receipts for those runs are uncertified: the certificate covers the national dataset only. The Enhanced FRS 2024-25 datasets stay loadable by name. The certificate names policyengine-core 3.33.3, so the core pin cannot move until UK is re-certified for the new core (see the data certification skill). With a Hugging Face token set, importing policyengine with any policyengine-uk other than 2.132.6 raises, as it did before for versions the Enhanced FRS release did not claim. The certified default is below the Universal Credit caseload floor (0.897 of the floor's 6.7m benchmark, policyengine-uk-data's derivation; Microcosm calibrates to DWP Stat-Xplore's 6.2m), an exception accepted and recorded in the data certification skill. +Switch the UK default dataset from the Enhanced FRS 2024-25 (policyengine-uk-data 1.58.1) to Microcosm UK 2024-25 (`microcosm_uk_2024_25`, cut `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71`), certified with policyengine-uk 2.132.6 and policyengine-core 3.33.3, the versions it was built with. Constituency and local-authority regions now load the Microcosm local-area release (`microcosm_uk_2024_25_dense`, cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`) and are in the default UK region registry. UK results change with the data. The national file has no constituency or local-authority codes, so constituency and local-authority outputs (`compute_uk_geography_disclosure` at those levels, `compute_uk_constituency_impacts`, `compute_uk_local_authority_impacts`) need simulations on `microcosm_uk_2024_25_dense`, and receipts for those runs are uncertified: the certificate covers the national dataset only. The Enhanced FRS 2024-25 datasets stay loadable by name. Data certificates now say how they cover policyengine-core: by default one follows the bundle's core pin (`core_binding: bundle_pin`, as the UK certificate does; the Microcosm releases claim core 3.33.3), and certification and `bundle.py check --include-tros --strict-tros` warn when the pin is outside a release's core claims, as it is for the US release today; `--core-version` binds the core instead (`core_binding: certified`). With a Hugging Face token set, importing policyengine with any policyengine-uk other than 2.132.6 raises, as it did before for versions the Enhanced FRS release did not claim. The certified default is below the Universal Credit caseload floor (0.897 of the floor's 6.7m benchmark, policyengine-uk-data's derivation; Microcosm calibrates to DWP Stat-Xplore's 6.2m), an exception accepted and recorded in the data certification skill. UK candidate promotion now closes the pull requests of pairs from another data line than main's certificate, at the commit their receipt qualified, and recovery no longer re-dispatches attempts for such pairs. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 4b974b70b..23c5d8133 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -237,6 +237,13 @@ differencing. moves the totals by design, so they were not run for that switch; its pull request compares the two lines' weighted totals instead. The other gates, reform responsiveness included, were run on the national cut. +- **Budget-critical inputs: needs review.** `incapacity_benefit_reported` is + missing from the Microcosm file but was a column of the enhanced FRS. The + enhanced FRS column carries no values (Incapacity Benefit is closed), so + both datasets give zero. Accepted by María Juaristi in #685. +- **Reform responsiveness: needs review.** The metrics tool's small-cell rule + withholds a standard reform's figures, so the gate cannot recompute that + reform's verdict. Accepted by María Juaristi in #685. ## Release runbooks From 12cfbc0af973ec67964e5d494aaecb793d9c6db6 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Wed, 21 Oct 2026 11:58:09 +0100 Subject: [PATCH 25/26] Say when promotion closes a cross-line PR, and which ones need a hand Co-Authored-By: Claude Opus 5.5 --- docs/engineering/runbooks/uk-qualification.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/engineering/runbooks/uk-qualification.md b/docs/engineering/runbooks/uk-qualification.md index 332f8e1cb..9392e340c 100644 --- a/docs/engineering/runbooks/uk-qualification.md +++ b/docs/engineering/runbooks/uk-qualification.md @@ -132,7 +132,9 @@ the ledger keeps their evidence. drops main's area routing (`--drop-regional-release`), because the routing belongs to the line the pair would replace, and leaves the test constants that pin main's certificate as they are, so the candidate's own tests fail. Its pull -request says it is never promoted, and promotion closes it. The catch-up does -not recover such a run: re-run it by hand with `force`. Changing the data line +request says it is never promoted, and promotion closes it once its receipt is +published, at the commit that receipt qualified. The catch-up does not recover +such a run: re-run it by hand with `force`. A run whose attempt dies before it +publishes a receipt leaves its pull request open; close that one by hand. Changing the data line is a reviewed certification pull request (see `docs/engineering/skills/data-certification.md`), not a promotion. From 681c7b586ac528e54ef65177997771bae1bad104 Mon Sep 17 00:00:00 2001 From: Maria Juaristi Date: Wed, 21 Oct 2026 12:01:08 +0100 Subject: [PATCH 26/26] Format the promote test with CI's ruff Co-Authored-By: Claude Opus 5.5 --- tests/test_uk_qualification_pipeline.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 5a6d2316d..ea07e2d0f 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -1094,8 +1094,9 @@ def test_promote_closes_other_line_prs_and_never_drafts_a_closed_one(monkeypatch monkeypatch.setattr( cli.subprocess, "run", - lambda command, **kwargs: commands.append(command) - or subprocess.CompletedProcess(command, 0), + lambda command, **kwargs: ( + commands.append(command) or subprocess.CompletedProcess(command, 0) + ), ) drafted = [] monkeypatch.setattr(