From afde5761d39e6c030fd476980e5d28e6d0b5f15d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:42:48 +0400 Subject: [PATCH 1/7] Add durable regional dataset defaults --- scripts/export_bundle_release_assets.py | 4 +- src/policyengine/bundle.py | 9 +- src/policyengine/data/bundle/manifest.json | 10 +- .../data/bundle/uk.trace.tro.jsonld | 4 +- .../data/bundle/us.trace.tro.jsonld | 4 +- src/policyengine/provenance/manifest.py | 119 ++++++++++++++- tests/test_bundle.py | 27 ++++ tests/test_release_manifests.py | 141 +++++++++++++++--- 8 files changed, 284 insertions(+), 34 deletions(-) diff --git a/scripts/export_bundle_release_assets.py b/scripts/export_bundle_release_assets.py index c0ff3a08..0c10721b 100644 --- a/scripts/export_bundle_release_assets.py +++ b/scripts/export_bundle_release_assets.py @@ -8,6 +8,8 @@ from generate_bundle_artifacts import BUNDLE_MANIFEST, REPO_ROOT +from policyengine.provenance.manifest import normalise_bundle_dataset_metadata + def _write_json(dist_dir: Path, name: str, payload: object) -> Path: path = dist_dir / name @@ -26,7 +28,7 @@ def main() -> int: parser.add_argument("--dist-dir", type=Path, default=REPO_ROOT / "dist") args = parser.parse_args() - bundle = json.loads(BUNDLE_MANIFEST.read_text()) + bundle = normalise_bundle_dataset_metadata(json.loads(BUNDLE_MANIFEST.read_text())) version = bundle["bundle_version"] args.dist_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/policyengine/bundle.py b/src/policyengine/bundle.py index cb27430a..69c5610a 100644 --- a/src/policyengine/bundle.py +++ b/src/policyengine/bundle.py @@ -29,7 +29,10 @@ _resolve_bundle_dataset, _reuse_or_download_bundle_files, ) -from policyengine.provenance.manifest import CountryReleaseManifest +from policyengine.provenance.manifest import ( + CountryReleaseManifest, + normalise_bundle_dataset_metadata, +) from policyengine.utils.hashing import sha256_file BUNDLE_MANIFEST_RESOURCE = ("data", "bundle", "manifest.json") @@ -72,6 +75,10 @@ def _normalise_manifest(manifest: Mapping[str, Any]) -> dict[str, Any]: payload.setdefault("packages", {}) payload.setdefault("extras", {}) payload.setdefault("data_releases", _data_releases_from_countries(payload)) + try: + payload = normalise_bundle_dataset_metadata(payload) + except ValueError as exc: + raise BundleError(str(exc)) from exc try: validate_bundle_measurements(payload) except ValueError as exc: diff --git a/src/policyengine/data/bundle/manifest.json b/src/policyengine/data/bundle/manifest.json index 2c67fafe..fbb6c3e5 100644 --- a/src/policyengine/data/bundle/manifest.json +++ b/src/policyengine/data/bundle/manifest.json @@ -255,8 +255,8 @@ "path": "populace_us_2024_acs_local.h5", "repo_id": "policyengine/populace-us", "repo_type": "dataset", - "revision": "populace-us-2024-buildo-acs-local-77e2061-20260724T110908Z", - "sha256": "71763290ded993789af0ad818fd833a6aabb9ca7ff7d14f78315340d8617c6f4" + "revision": "populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z", + "sha256": "769756c31f3ca646d12c272511744dec04c0e68870c6946dd945fbba65b6a7ec" } } }, @@ -330,5 +330,11 @@ } }, "policyengine_version": "6.2.1", + "regional_dataset_defaults": { + "us": { + "congressional_district": "populace_us_2024_acs_local", + "state": "populace_us_2024_acs_local" + } + }, "schema_version": 2 } diff --git a/src/policyengine/data/bundle/uk.trace.tro.jsonld b/src/policyengine/data/bundle/uk.trace.tro.jsonld index 81179410..9c749246 100644 --- a/src/policyengine/data/bundle/uk.trace.tro.jsonld +++ b/src/policyengine/data/bundle/uk.trace.tro.jsonld @@ -75,7 +75,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for uk", "trov:mimeType": "application/json", - "trov:sha256": "0b80f402511f5cec0cfbe018b7f8b42e83b54a7bdfc5c9c210a2f77bb63bd21b" + "trov:sha256": "bf23eb08a005de9496ad7305db9797587e946c24d1d0473c7fde6b7005b50197" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -102,7 +102,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "a318203a1ff8cf141dfe8b138902e445ae1ab7bcce6e3d714e87661a4bb6deaa" + "trov:sha256": "6843d5810a942503c6dd088edcf9ccdb1b10c232fd822199bfb2ac2861825a37" } }, "trov:hasPerformance": { diff --git a/src/policyengine/data/bundle/us.trace.tro.jsonld b/src/policyengine/data/bundle/us.trace.tro.jsonld index 213e1300..3f64d7e9 100644 --- a/src/policyengine/data/bundle/us.trace.tro.jsonld +++ b/src/policyengine/data/bundle/us.trace.tro.jsonld @@ -74,7 +74,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "0b80f402511f5cec0cfbe018b7f8b42e83b54a7bdfc5c9c210a2f77bb63bd21b" + "trov:sha256": "bf23eb08a005de9496ad7305db9797587e946c24d1d0473c7fde6b7005b50197" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -101,7 +101,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "7f7f19f41f2cfd707098768565562f6509b454b48c79bcd0558dab061b5b40a6" + "trov:sha256": "3a2b8b8e6d4d899c093942144e82643b754673f2d65d52e26fad68f520a3b2c3" } }, "trov:hasPerformance": { diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index 99ca72be..a155186d 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -4,7 +4,7 @@ from functools import lru_cache from importlib.resources import files from pathlib import Path -from typing import Literal, Optional +from typing import Any, Literal, Mapping, Optional from urllib.parse import quote import requests @@ -265,6 +265,7 @@ def fetch_pypi_wheel_metadata(name: str, version: str) -> dict[str, Optional[str DATASET_OVERLAYS_KEY = "dataset_overlays" +REGIONAL_DATASET_DEFAULTS_KEY = "regional_dataset_defaults" def _apply_dataset_overlays( @@ -285,8 +286,9 @@ def _apply_dataset_overlays( ``data_releases``, overlays survive re-certification untouched. Overlays are strictly additive: an overlay may not shadow the certified - default dataset or any certified dataset entry, so it can never alter - default resolution. + default dataset or conflict with a certified dataset entry, so it can never + alter default resolution. An identical existing entry is accepted to make + bundle normalization idempotent. """ overlays = (bundle.get(DATASET_OVERLAYS_KEY) or {}).get(country_id) or {} if not overlays: @@ -302,6 +304,8 @@ def _apply_dataset_overlays( "change default resolution." ) if overlay_name in certified_datasets: + if certified_datasets[overlay_name] == overlay_reference: + continue raise ValueError( f"Dataset overlay '{overlay_name}' for country '{country_id}' " "collides with a certified dataset entry. Overlays must be " @@ -312,6 +316,109 @@ def _apply_dataset_overlays( return {**release_payload, "datasets": certified_datasets} +def _apply_regional_dataset_defaults( + country_id: str, + release_payload: dict, + bundle: Mapping[str, Any], +) -> dict: + """Apply durable region-to-dataset selections to one country release. + + Certification replaces ``data_releases.{country}`` as a unit. Regional + defaults therefore live beside ``data_releases`` and refer to logical + dataset names after overlays have been merged. The country-wide default + remains owned by the certified release. + """ + + defaults_by_country = bundle.get(REGIONAL_DATASET_DEFAULTS_KEY) or {} + if not isinstance(defaults_by_country, Mapping): + raise ValueError(f"{REGIONAL_DATASET_DEFAULTS_KEY} must be a mapping.") + regional_defaults = defaults_by_country.get(country_id) or {} + if not isinstance(regional_defaults, Mapping): + raise ValueError( + f"{REGIONAL_DATASET_DEFAULTS_KEY}.{country_id} must be a mapping." + ) + if not regional_defaults: + return release_payload + + datasets = release_payload.get("datasets") or {} + if not isinstance(datasets, Mapping): + raise ValueError(f"Datasets for country '{country_id}' must be a mapping.") + region_datasets = dict(release_payload.get("region_datasets") or {}) + + for region_type, dataset_name in regional_defaults.items(): + if region_type == "national": + raise ValueError( + f"Regional dataset defaults for country '{country_id}' must not " + "override the national certified default." + ) + if not isinstance(dataset_name, str) or dataset_name not in datasets: + raise ValueError( + f"Regional dataset default for country '{country_id}' and region " + f"type '{region_type}' references unknown dataset '{dataset_name}'." + ) + reference = datasets[dataset_name] + if not isinstance(reference, Mapping) or not isinstance( + reference.get("path"), str + ): + raise ValueError( + f"Regional dataset default '{dataset_name}' for country " + f"'{country_id}' has no artifact path." + ) + regional_template = {"path_template": reference["path"]} + certified_template = region_datasets.get(region_type) + if certified_template is not None and certified_template != regional_template: + raise ValueError( + f"Regional dataset default for country '{country_id}' and region " + f"type '{region_type}' conflicts with certified region dataset " + f"template {certified_template!r}." + ) + region_datasets[region_type] = regional_template + + return {**release_payload, "region_datasets": region_datasets} + + +def normalise_bundle_dataset_metadata(bundle: Mapping[str, Any]) -> dict[str, Any]: + """Return a bundle with overlays and regional defaults applied. + + Both the public bundle API and country release loading use this function so + callers observe the same dataset registry and region mapping. + """ + + payload = dict(bundle) + releases = payload.get("data_releases") + if releases is None: + return payload + if not isinstance(releases, Mapping): + raise ValueError("data_releases must be a mapping.") + + defaults_by_country = payload.get(REGIONAL_DATASET_DEFAULTS_KEY) or {} + if not isinstance(defaults_by_country, Mapping): + raise ValueError(f"{REGIONAL_DATASET_DEFAULTS_KEY} must be a mapping.") + unknown_countries = set(defaults_by_country) - set(releases) + if unknown_countries: + raise ValueError( + "Regional dataset defaults reference countries without data releases: " + f"{sorted(unknown_countries)}." + ) + + normalised_releases: dict[str, Any] = {} + for country_id, release_payload in releases.items(): + if not isinstance(country_id, str) or not isinstance(release_payload, dict): + raise ValueError("Each data release must be a country-keyed mapping.") + release_payload = _apply_dataset_overlays( + country_id, + release_payload, + payload, + ) + normalised_releases[country_id] = _apply_regional_dataset_defaults( + country_id, + release_payload, + payload, + ) + payload["data_releases"] = normalised_releases + return payload + + @lru_cache def get_release_manifest(country_id: str) -> CountryReleaseManifest: manifest_path = files("policyengine").joinpath("data", "bundle", "manifest.json") @@ -321,11 +428,11 @@ def get_release_manifest(country_id: str) -> CountryReleaseManifest: source_bytes = manifest_path.read_text().encode() bundle = json.loads(source_bytes) try: - release_payload = bundle["data_releases"][country_id] + release_payload = normalise_bundle_dataset_metadata(bundle)["data_releases"][ + country_id + ] except KeyError as exc: raise ValueError(f"No bundled data release for country '{country_id}'") from exc - - release_payload = _apply_dataset_overlays(country_id, release_payload, bundle) manifest = CountryReleaseManifest.model_validate(release_payload) manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest() return manifest diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 3d090b4e..75cb4ecc 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -8,6 +8,7 @@ from policyengine import bundle from policyengine.cli import main as cli_main from policyengine.provenance.dataset_materialization import MaterializedDataset +from policyengine.provenance.manifest import get_release_manifest def _sha256(payload: bytes) -> str: @@ -48,6 +49,20 @@ def test_bundle_manifest_exposes_data_releases(): ) +def test_bundle_and_country_manifest_share_normalised_dataset_metadata(): + bundle_release = bundle.get_current_bundle()["data_releases"]["us"] + country_release = get_release_manifest("us") + + assert set(bundle_release["datasets"]) == set(country_release.datasets) + assert { + region_type: template["path_template"] + for region_type, template in bundle_release["region_datasets"].items() + } == { + region_type: template.path_template + for region_type, template in country_release.region_datasets.items() + } + + def test_bundle_install_requirements_are_country_scoped(): manifest = bundle.get_current_bundle() @@ -82,6 +97,18 @@ def test_selected_dataset_plan_uses_certified_release_metadata(tmp_path): assert plan.sha256 == release["datasets"][plan.dataset]["sha256"] +def test_selected_us_dataset_plan_installs_only_national_default(tmp_path): + manifest = bundle.get_current_bundle() + + entries = bundle._selected_dataset_plans(manifest, ["us"], data_dir=tmp_path) + + assert len(entries) == 1 + plan, release = entries[0] + assert plan.dataset == "populace_us_2024" + assert plan.dataset != manifest["regional_dataset_defaults"]["us"]["state"] + assert plan.source_uri == release["default_dataset_uri"] + + def test_install_bundle_package_only_uses_explicit_python(monkeypatch, tmp_path): calls = [] diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 485cc1d0..dffdd72e 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -12,6 +12,7 @@ from types import ModuleType, SimpleNamespace from unittest.mock import MagicMock, patch +import pytest from requests import Timeout from policyengine.core.tax_benefit_model import TaxBenefitModel @@ -32,6 +33,7 @@ get_data_release_manifest, get_release_manifest, https_release_manifest_uri, + normalise_bundle_dataset_metadata, resolve_dataset_reference, resolve_default_datasets, resolve_managed_dataset_reference, @@ -61,12 +63,15 @@ US_RELEASE_MANIFEST_DATASET_URI = ( f"hf://policyengine/populace-us/populace_us_2024.h5@{US_DATA_RELEASE_REVISION}" ) -# Non-default local-area overlay: a staged Populace US artifact published in its -# own immutable release (Build L), loadable by name but never the default. +# Local-area overlay: a Populace US artifact published in its own immutable +# release. It remains non-default nationally and is the regional default for +# states and congressional districts. US_LOCAL_AREA_DATASET = "populace_us_2024_acs_local" -US_LOCAL_AREA_RELEASE_ID = "populace-us-2024-buildo-acs-local-77e2061-20260724T110908Z" +US_LOCAL_AREA_RELEASE_ID = ( + "populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z" +) US_LOCAL_AREA_SHA256 = ( - "71763290ded993789af0ad818fd833a6aabb9ca7ff7d14f78315340d8617c6f4" + "769756c31f3ca646d12c272511744dec04c0e68870c6946dd945fbba65b6a7ec" ) US_LOCAL_AREA_DATASET_URI = ( "hf://policyengine/populace-us/populace_us_2024_acs_local.h5" @@ -316,8 +321,8 @@ def test__given_no_dataset__then_managed_resolution_uses_certified_default(self) == get_release_manifest("us").default_dataset_uri ) - def test__given_local_area_overlay__then_default_resolution_is_unchanged(self): - """Regression: the non-default local-area overlay never changes defaults. + def test__given_local_area_overlay__then_national_default_is_unchanged(self): + """Regression: the local-area overlay never changes the national default. The overlay adds a loadable name; it must never become the certified default, and default resolution must keep pointing at the certified @@ -332,6 +337,13 @@ def test__given_local_area_overlay__then_default_resolution_is_unchanged(self): assert manifest.region_datasets["national"].path_template == ( "populace_us_2024.h5" ) + assert manifest.region_datasets["state"].path_template == ( + "populace_us_2024_acs_local.h5" + ) + assert ( + manifest.region_datasets["congressional_district"].path_template + == "populace_us_2024_acs_local.h5" + ) def test__given_local_area_name__then_resolves_to_its_immutable_tag(self): """The local-area overlay resolves to its own Build L release tag.""" @@ -357,19 +369,18 @@ def test__given_local_area_overlay__then_registered_with_sha_but_not_default(sel assert US_LOCAL_AREA_DATASET in resolve_default_datasets("us") assert US_LOCAL_AREA_DATASET != manifest.default_dataset - def test__given_us_manifest__then_has_no_inherited_area_artifacts(self): - manifest = get_release_manifest("us") - - assert "state" not in manifest.region_datasets - assert "congressional_district" not in manifest.region_datasets - assert resolve_region_dataset_path("us", "state", state_code="CA") is None + def test__given_us_manifest__then_local_regions_resolve_to_acs_local(self): + assert ( + resolve_region_dataset_path("us", "state", state_code="CA") + == US_LOCAL_AREA_DATASET_URI + ) assert ( resolve_region_dataset_path( "us", "congressional_district", district_code="CA-01", ) - is None + == US_LOCAL_AREA_DATASET_URI ) assert not any( key.startswith(("states/", "districts/")) @@ -1130,12 +1141,12 @@ def test__given_uk_unmanaged_dataset_uri__then_source_is_not_rewritten(self): class TestDatasetOverlays: - """Unit tests for the ``dataset_overlays`` merge layer. + """Unit tests for durable dataset metadata outside certification output. ``dataset_overlays`` is the hand-maintained sibling of ``data_releases``. ``certify_data_release`` rewrites ``data_releases.{country}`` wholesale, so - these tests pin the invariant that overlays are additive, survive - re-certification, and can never hijack default resolution. + these tests pin the invariant that overlays and regional defaults survive + re-certification without changing the country-wide default. """ def teardown_method(self): @@ -1179,16 +1190,21 @@ def _local_area_overlay(self) -> dict: } } - def test__given_recertified_release__then_overlay_survives(self): - """The overlay lives outside ``data_releases``, so a re-certified - payload that never mentions it still resolves it by name.""" + def test__given_recertified_release__then_regional_default_survives(self): + """Sibling metadata survives replacement of ``data_releases.us``.""" payload = self._recertified_us_payload() bundle = { "data_releases": {"us": payload}, "dataset_overlays": {"us": self._local_area_overlay()}, + "regional_dataset_defaults": { + "us": { + "state": US_LOCAL_AREA_DATASET, + "congressional_district": US_LOCAL_AREA_DATASET, + } + }, } - merged = _apply_dataset_overlays("us", payload, bundle) + merged = normalise_bundle_dataset_metadata(bundle)["data_releases"]["us"] manifest = CountryReleaseManifest.model_validate(merged) assert manifest.default_dataset == "populace_us_2024" @@ -1196,12 +1212,46 @@ def test__given_recertified_release__then_overlay_survives(self): reference = manifest.datasets[US_LOCAL_AREA_DATASET] assert reference.revision == US_LOCAL_AREA_RELEASE_ID assert reference.sha256 == US_LOCAL_AREA_SHA256 + assert manifest.region_datasets["state"].path_template == reference.path + assert ( + manifest.region_datasets["congressional_district"].path_template + == reference.path + ) + + def test__given_bundle__then_current_bundle_and_release_manifest_match(self): + bundle = normalise_bundle_dataset_metadata( + { + "data_releases": {"us": self._recertified_us_payload()}, + "dataset_overlays": {"us": self._local_area_overlay()}, + "regional_dataset_defaults": {"us": {"state": US_LOCAL_AREA_DATASET}}, + } + ) + + release = CountryReleaseManifest.model_validate(bundle["data_releases"]["us"]) + + assert US_LOCAL_AREA_DATASET in release.datasets + assert release.region_datasets["state"].path_template == ( + release.datasets[US_LOCAL_AREA_DATASET].path + ) def test__given_no_overlays__then_payload_is_returned_unchanged(self): payload = self._recertified_us_payload() assert _apply_dataset_overlays("us", payload, {"data_releases": {}}) is payload + def test__given_normalised_bundle__then_normalising_again_is_unchanged(self): + payload = self._recertified_us_payload() + bundle = { + "data_releases": {"us": payload}, + "dataset_overlays": {"us": self._local_area_overlay()}, + "regional_dataset_defaults": {"us": {"state": US_LOCAL_AREA_DATASET}}, + } + + once = normalise_bundle_dataset_metadata(bundle) + twice = normalise_bundle_dataset_metadata(once) + + assert twice == once + def test__given_overlay_shadowing_default__then_raises(self): payload = self._recertified_us_payload() bundle = { @@ -1236,3 +1286,54 @@ def test__given_overlay_colliding_with_certified_dataset__then_raises(self): assert "collides with a certified dataset" in str(error) else: raise AssertionError("Expected overlay colliding with a dataset to fail") + + def test__given_unknown_regional_dataset__then_raises(self): + payload = self._recertified_us_payload() + bundle = { + "data_releases": {"us": payload}, + "regional_dataset_defaults": {"us": {"state": "missing_dataset"}}, + } + + with pytest.raises(ValueError, match="unknown dataset 'missing_dataset'"): + normalise_bundle_dataset_metadata(bundle) + + def test__given_national_regional_default__then_raises(self): + payload = self._recertified_us_payload() + bundle = { + "data_releases": {"us": payload}, + "regional_dataset_defaults": {"us": {"national": "populace_us_2024"}}, + } + + with pytest.raises(ValueError, match="must not override the national"): + normalise_bundle_dataset_metadata(bundle) + + def test__given_conflicting_certified_region_template__then_raises(self): + payload = self._recertified_us_payload() + payload["region_datasets"]["state"] = { + "path_template": "states/{state_code}.h5" + } + bundle = { + "data_releases": {"us": payload}, + "dataset_overlays": {"us": self._local_area_overlay()}, + "regional_dataset_defaults": {"us": {"state": US_LOCAL_AREA_DATASET}}, + } + + with pytest.raises(ValueError, match="conflicts with certified region dataset"): + normalise_bundle_dataset_metadata(bundle) + + def test__given_identical_certified_region_template__then_accepts_it(self): + payload = self._recertified_us_payload() + payload["region_datasets"]["state"] = { + "path_template": "populace_us_2024_acs_local.h5" + } + bundle = { + "data_releases": {"us": payload}, + "dataset_overlays": {"us": self._local_area_overlay()}, + "regional_dataset_defaults": {"us": {"state": US_LOCAL_AREA_DATASET}}, + } + + normalised = normalise_bundle_dataset_metadata(bundle) + + assert normalised["data_releases"]["us"]["region_datasets"]["state"] == { + "path_template": "populace_us_2024_acs_local.h5" + } From 88522cfcfbcc42f64356f26d49c9178332eabe0c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:42:50 +0400 Subject: [PATCH 2/7] Use ACS-local data for US regional simulations --- src/policyengine/countries/us/regions.py | 14 +++++- tests/test_us_regions.py | 58 ++++++++++++++++++++---- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/policyengine/countries/us/regions.py b/src/policyengine/countries/us/regions.py index 266fb5c4..7f6ceb87 100644 --- a/src/policyengine/countries/us/regions.py +++ b/src/policyengine/countries/us/regions.py @@ -43,7 +43,7 @@ def build_us_region_registry() -> RegionRegistry: ) ) - # 2. State regions (filtered from the certified national dataset) + # 2. State regions (filtered from the configured state dataset) for abbrev, name in US_STATES.items(): regions.append( Region( @@ -51,6 +51,11 @@ def build_us_region_registry() -> RegionRegistry: label=name, region_type="state", parent_code="us", + dataset_path=resolve_region_dataset_path( + "us", + "state", + state_code=abbrev, + ), scoping_strategy=RowFilterStrategy( variable_name="state_fips", variable_value=US_STATE_FIPS[abbrev], @@ -60,7 +65,7 @@ def build_us_region_registry() -> RegionRegistry: ) ) - # 3. Congressional district regions (filtered from the national dataset) + # 3. Congressional districts (filtered from the configured district dataset) for state_abbrev, count in DISTRICT_COUNTS.items(): state_name = US_STATES[state_abbrev] state_fips = US_STATE_FIPS[state_abbrev] @@ -81,6 +86,11 @@ def build_us_region_registry() -> RegionRegistry: label=label, region_type="congressional_district", parent_code=f"state/{state_abbrev.lower()}", + dataset_path=resolve_region_dataset_path( + "us", + "congressional_district", + district_code=district_code, + ), scoping_strategy=RowFilterStrategy( variable_name="congressional_district_geoid", variable_value=district_geoid, diff --git a/tests/test_us_regions.py b/tests/test_us_regions.py index 57414498..d1ccd09d 100644 --- a/tests/test_us_regions.py +++ b/tests/test_us_regions.py @@ -1,12 +1,21 @@ """Tests for US region definitions.""" -from policyengine.countries.us.data import DISTRICT_COUNTS, US_STATE_FIPS, US_STATES +from policyengine.countries.us.data import ( + DISTRICT_COUNTS, + US_STATE_FIPS, + US_STATES, +) from policyengine.countries.us.regions import ( build_us_region_registry, us_region_registry, ) from policyengine.provenance.manifest import CountryReleaseManifest +US_LOCAL_AREA_DATASET_URI = ( + "hf://policyengine/populace-us/populace_us_2024_acs_local.h5" + "@populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z" +) + class TestUSStates: """Tests for US state definitions.""" @@ -134,7 +143,7 @@ def test__given_california_region__then_has_correct_format(self): assert ca.label == "California" assert ca.region_type == "state" assert ca.parent_code == "us" - assert ca.dataset_path is None + assert ca.dataset_path == US_LOCAL_AREA_DATASET_URI assert ca.requires_filter assert ca.scoping_strategy is not None assert ca.scoping_strategy.variable_name == "state_fips" @@ -167,7 +176,7 @@ def test__given_ca_first_district__then_has_correct_format(self): assert "1st" in ca01.label.lower() or "1 " in ca01.label assert ca01.region_type == "congressional_district" assert ca01.parent_code == "state/ca" - assert ca01.dataset_path is None + assert ca01.dataset_path == US_LOCAL_AREA_DATASET_URI assert ca01.requires_filter assert ca01.scoping_strategy is not None assert ca01.scoping_strategy.variable_name == "congressional_district_geoid" @@ -250,24 +259,51 @@ def test__given_california__then_children_include_districts_and_places( assert len(district_children) == DISTRICT_COUNTS["CA"] assert len(place_children) >= 10 # CA has many large cities - def test__given_us_registry__then_dataset_regions_are_national_only(self): + def test__given_us_registry__then_all_filter_regions_have_datasets(self): """Given: US region registry When: Getting regions with datasets - Then: Only the national canonical Populace dataset is dedicated + Then: National, state, and district regions declare their input dataset """ # When dataset_regions = us_region_registry.get_dataset_regions() # Then - assert len(dataset_regions) == 1 - assert dataset_regions[0].region_type == "national" + assert len(dataset_regions) == 1 + 51 + 436 + assert {region.region_type for region in dataset_regions} == { + "national", + "state", + "congressional_district", + } + + def test__given_states_and_districts__then_all_use_local_area_dataset(self): + local_regions = [ + *us_region_registry.get_by_type("state"), + *us_region_registry.get_by_type("congressional_district"), + ] + + assert len(local_regions) == 51 + 436 + assert {region.dataset_path for region in local_regions} == { + US_LOCAL_AREA_DATASET_URI + } + assert all(region.requires_filter for region in local_regions) - def test__given_certified_state_template__then_state_filters_national_dataset( + def test__given_dc_state_and_district__then_both_use_local_area_dataset(self): + dc = us_region_registry.get("state/dc") + dc_al = us_region_registry.get("congressional_district/DC-01") + + assert dc is not None + assert dc_al is not None + assert dc.dataset_path == US_LOCAL_AREA_DATASET_URI + assert dc_al.dataset_path == US_LOCAL_AREA_DATASET_URI + assert dc.scoping_strategy is not None + assert dc_al.scoping_strategy is not None + + def test__given_certified_state_template__then_state_uses_it_and_filters_rows( self, monkeypatch ): """Given: US bundle manifest with a certified state template When: Building the region registry - Then: State regions still filter the national certified dataset + Then: State regions use the declared dataset and retain row filtering """ manifest = CountryReleaseManifest.model_validate( { @@ -312,7 +348,9 @@ def test__given_certified_state_template__then_state_filters_national_dataset( ca = registry.get("state/ca") assert ca is not None - assert ca.dataset_path is None + assert ca.dataset_path == ( + "hf://policyengine/policyengine-us-data/states/CA.h5@1.115.5" + ) assert ca.requires_filter assert ca.scoping_strategy is not None assert ca.scoping_strategy.variable_name == "state_fips" From 44faa269f3afca7e7774fa6b7803ded820aa591d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:42:54 +0400 Subject: [PATCH 3/7] Document regional dataset selection --- changelog.d/551.changed.md | 1 + docs/bundles.md | 16 ++++++--- .../build-m-us-populace-certification.md | 12 +++---- docs/engineering/skills/data-certification.md | 13 +++++-- docs/microsim.md | 11 +++--- docs/regions.md | 15 ++++---- docs/release-bundles.md | 36 +++++++++++++++---- 7 files changed, 72 insertions(+), 32 deletions(-) create mode 100644 changelog.d/551.changed.md diff --git a/changelog.d/551.changed.md b/changelog.d/551.changed.md new file mode 100644 index 00000000..80b935fa --- /dev/null +++ b/changelog.d/551.changed.md @@ -0,0 +1 @@ +Use the ACS-local Populace dataset by default for US state and congressional-district simulations while retaining the sparse Populace dataset as the national default. diff --git a/docs/bundles.md b/docs/bundles.md index 4c5b642b..00104854 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -146,11 +146,17 @@ python scripts/bundle.py certify-data \ --model-version ``` -US state and congressional-district regions scope the certified national -Populace dataset with row filters. If a Populace release also publishes derived -`states/*.h5` or `districts/*.h5` area slices, the bundle certification omits -those slices from `data_releases.us.datasets`; they are not runtime dataset -dependencies. +US state and congressional-district regions scope the ACS-local Populace +dataset with row filters. The durable `regional_dataset_defaults.us` mapping +selects that named dataset after `dataset_overlays.us` is merged into the +certified release. Certification may replace `data_releases.us` without +removing either sibling block. If a release publishes derived `states/*.h5` or +`districts/*.h5` slices, certification still omits them: the runtime loads one +ACS-local national file and filters its rows. + +`policyengine bundle install` continues to materialize only each country's +certified `default_dataset`. Regional runtimes materialize additional datasets +when the selected region declares one. Use `python scripts/bundle.py generate` to regenerate derived bundle metadata, and `python scripts/bundle.py generate --include-tros` when TRACE TRO sidecars diff --git a/docs/engineering/runbooks/build-m-us-populace-certification.md b/docs/engineering/runbooks/build-m-us-populace-certification.md index dd0d628f..61c3dab6 100644 --- a/docs/engineering/runbooks/build-m-us-populace-certification.md +++ b/docs/engineering/runbooks/build-m-us-populace-certification.md @@ -97,15 +97,15 @@ M was built against a newer `policyengine-us`: `PE_UPDATE_SNAPSHOTS=1 pytest tests/test_household_calculator_snapshot.py` and commit `tests/fixtures/household_calculator_snapshots/`. -## Step 5 — verify the local-area overlay survived +## Step 5 — verify the local-area metadata survived -Certification rewrites only `data_releases.us`, so the non-default -`dataset_overlays.us.populace_us_2024_acs_local` entry (see -[Non-default dataset overlays](../../release-bundles.md#non-default-dataset-overlays)) -must still be present and resolvable. No manual re-add is needed — confirm it: +Certification rewrites only `data_releases.us`, so both the +`dataset_overlays.us.populace_us_2024_acs_local` entry and the state/district +entries under `regional_dataset_defaults.us` must remain present and +resolvable. No manual re-add is needed—confirm them: ```bash -python -c "import json; b=json.load(open('src/policyengine/data/bundle/manifest.json')); assert 'populace_us_2024_acs_local' in b['dataset_overlays']['us'], 'overlay lost'; print('overlay preserved')" +python -c "import json; b=json.load(open('src/policyengine/data/bundle/manifest.json')); assert 'populace_us_2024_acs_local' in b['dataset_overlays']['us'], 'overlay lost'; assert set(b['regional_dataset_defaults']['us']) == {'state', 'congressional_district'}, 'regional defaults lost'; print('local-area metadata preserved')" pytest tests/test_release_manifests.py -k "local_area or DatasetOverlays" -q ``` diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index f5319517..740f8c38 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -34,8 +34,8 @@ python scripts/bundle.py certify-data --country us --data-producer populace \ --model-version "" ``` -US state and congressional-district regions are row filters over the certified -national Populace dataset. Certification writes: +US state and congressional-district regions are row filters over the ACS-local +Populace dataset. Certification still writes only its national template: ```json "region_datasets": { @@ -43,9 +43,16 @@ national Populace dataset. Certification writes: } ``` +The hand-maintained sibling blocks `dataset_overlays.us` and +`regional_dataset_defaults.us` register the ACS-local artifact and select it +for state and congressional-district regions. Bundle normalization combines +those blocks with the newly certified release. Certification must preserve both +sibling blocks. + If the Populace release publishes derived `states/*.h5` or `districts/*.h5` files for compatibility checks, certification omits them from the runtime -bundle. The national H5 is the canonical `.py` dataset. +bundle. The country-wide default and ACS-local dataset are each single national +files; region registries select one and then filter its rows. The script fetches and validates the manifest (every artifact must carry a revision pin; the certified dataset must be reachable), writes the canonical diff --git a/docs/microsim.md b/docs/microsim.md index 55820429..f9e5d8ec 100644 --- a/docs/microsim.md +++ b/docs/microsim.md @@ -134,15 +134,16 @@ pe.us.load_datasets() # or pe.uk.load_datasets() ### US local-area dataset -Alongside the certified national default, the bundle registers a **non-default** -US dataset for finer geographic work: `populace_us_2024_acs_local`. It is a +Alongside the certified national default, the bundle registers a US dataset for +finer geographic work: `populace_us_2024_acs_local`. It is a Populace US 2024 build of roughly **1.6 million households** on an **ACS 2024 multispine**, with each household **PUMA-assigned** to a 119th-Congress congressional district, county, and state, and calibrated to **state administrative totals and state and congressional-district population**. Its -release gate summary records **four reviewed limitations**, so read that gate -summary before relying on it. It ships in its own immutable release and is never -selected implicitly — you load it by name. +release validation summary records **four reviewed limitations**, so read that +summary before relying on it. It ships in its own immutable release. State and +congressional-district region simulations select it through the bundle's +regional-default metadata; direct microsimulations can still load it by name. Two-line load: diff --git a/docs/regions.md b/docs/regions.md index 3bf64d73..6d703c9f 100644 --- a/docs/regions.md +++ b/docs/regions.md @@ -23,9 +23,9 @@ ca_snap = Aggregate( ca_snap.run() ``` -Each state is a region in the US registry. State regions scope the certified -national Populace dataset by `state_fips`; they do not require separate state -H5 files: +Each state is a region in the US registry. State regions load the ACS-local +Populace dataset declared by `region.dataset_path`, then scope its rows by +`state_fips`. They do not require separate state H5 files: ```python states = pe.us.model.region_registry.get_by_type("state") @@ -33,8 +33,9 @@ for region in states: print(region.code, region.label, region.scoping_strategy) ``` -For state-specific simulations, pass `scoping_strategy=region.scoping_strategy` -with the certified national dataset. +For state-specific simulations, use both `region.dataset_path` and +`region.scoping_strategy`. The regional simulation API performs both steps from +the registry entry. ## US congressional districts @@ -49,7 +50,9 @@ for row in impacts.district_results: print(row["district_geoid"], row["avg_change"], row["winner_percentage"]) ``` -`district_geoid` is the SSDD integer (state FIPS × 100 + district number; at-large districts use `00`). Congressional district regions scope the certified national Populace dataset by `congressional_district_geoid`. +`district_geoid` is the SSDD integer (state FIPS × 100 + district number; +at-large districts use `00`). Congressional district regions load the same +ACS-local file and scope its rows by `congressional_district_geoid`. ## UK parliamentary constituencies diff --git a/docs/release-bundles.md b/docs/release-bundles.md index 8abd1c1b..c854db7c 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -205,11 +205,11 @@ python scripts/bundle.py certify-data --country us --data-producer populace \ ``` That produces one US bundle manifest entry containing the Populace national -default dataset. State and congressional-district regions are runtime row -filters over that national dataset, so derived `states/*.h5` or +default dataset. State and congressional-district simulations use one +ACS-local dataset and runtime row filters, so derived `states/*.h5` or `districts/*.h5` files are not vendored into `data_releases.us.datasets`. -### Non-default dataset overlays +### Dataset overlays and regional defaults `certify_data_release` rewrites `data_releases.{country}` wholesale from the certified release manifest, so that block only ever holds datasets from the @@ -235,10 +235,32 @@ sibling `dataset_overlays.{country}` map: `get_release_manifest` merges overlays into the resolvable dataset registry, so `resolve_dataset_reference` and `managed_microsimulation` load them by name at their own pinned, sha-verified revision. Overlays are strictly additive: an -overlay may not shadow the certified default or any certified dataset, so -default resolution is untouched. Because certification only rewrites -`data_releases`, overlays survive re-certification without any manual -re-add step. +overlay may not shadow the certified default or conflict with any certified +dataset, so country-wide default resolution is untouched. Re-normalizing an +already merged bundle accepts an identical entry. + +A second sibling mapping selects an overlaid or certified dataset for a region +type: + +```json +"regional_dataset_defaults": { + "us": { + "state": "populace_us_2024_acs_local", + "congressional_district": "populace_us_2024_acs_local" + } +} +``` + +Bundle normalization merges overlays first, validates every regional logical +name, and then emits the corresponding `region_datasets` path templates. The +mapping cannot replace the national default or conflict with a region template +from the certified release. `get_current_bundle` and `get_release_manifest` +share this normalization path. + +Because certification only rewrites `data_releases`, both sibling blocks +survive re-certification without a manual re-add step. Ordinary bundle +installation still downloads only `default_dataset`; a regional runtime is +responsible for materializing its selected regional dataset. Cross-package overlays must declare `data_package_name` and `repo_type` explicitly. Ordinary artifacts inherit these values from the country release's From 2346afd12ad4c2ec41f16f8a676294a504e8a11f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:03:08 +0400 Subject: [PATCH 4/7] fix: certify ACS-local regional dataset --- CHANGELOG.md | 2 +- docs/bundles.md | 18 +-- .../build-m-us-populace-certification.md | 20 +-- docs/engineering/skills/data-certification.md | 25 +-- docs/microsim.md | 2 +- docs/release-bundles.md | 62 +++----- scripts/bundle.py | 4 +- scripts/certify_data_release.py | 4 +- src/policyengine/data/bundle/manifest.json | 31 ++-- .../data/bundle/uk.trace.tro.jsonld | 4 +- .../data/bundle/us.trace.tro.jsonld | 4 +- src/policyengine/provenance/certification.py | 118 ++++++++++++++- src/policyengine/provenance/manifest.py | 122 ++------------- tests/test_bundle.py | 4 +- tests/test_certify_data_release.py | 143 +++++++++++++++++- tests/test_release_manifests.py | 123 ++------------- 16 files changed, 355 insertions(+), 331 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4b716db..8a980353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -167,7 +167,7 @@ ### Changed -- Repin the US local-area overlay (`populace_us_2024_acs_local`) to the buildo-acs-local release built on the certified Build O lineage: consumer-loadable bytes, 4,461-target local surface, immutable tag, default resolution unchanged. +- Certify the US ACS-local dataset (`populace_us_2024_acs_local`) for state and congressional-district simulations using the immutable buildo-acs-local release, while retaining the existing national default dataset. ## [4.22.2] - 2026-07-23 diff --git a/docs/bundles.md b/docs/bundles.md index 00104854..dc06c6ca 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -143,20 +143,20 @@ python scripts/bundle.py certify-data \ --country us \ --data-producer populace \ --manifest-uri hf://dataset/policyengine/populace-us@/releases//release_manifest.json \ + --regional-manifest-uri hf://dataset/policyengine/populace-us@/releases//release_manifest.json \ --model-version ``` -US state and congressional-district regions scope the ACS-local Populace -dataset with row filters. The durable `regional_dataset_defaults.us` mapping -selects that named dataset after `dataset_overlays.us` is merged into the -certified release. Certification may replace `data_releases.us` without -removing either sibling block. If a release publishes derived `states/*.h5` or -`districts/*.h5` slices, certification still omits them: the runtime loads one -ACS-local national file and filters its rows. +US state and congressional-district regions scope one certified ACS-local +Populace dataset with row filters. Pass its non-default local-area release +manifest through `--regional-manifest-uri`; certification adds the pinned H5 +artifact to `data_releases.us.datasets` and writes both regional path templates +under `data_releases.us.region_datasets`. If a legacy release publishes derived +`states/*.h5` or `districts/*.h5` slices, certification still omits them. `policyengine bundle install` continues to materialize only each country's -certified `default_dataset`. Regional runtimes materialize additional datasets -when the selected region declares one. +certified national `default_dataset`. Regional runtimes materialize the +additional certified dataset when the selected region declares it. Use `python scripts/bundle.py generate` to regenerate derived bundle metadata, and `python scripts/bundle.py generate --include-tros` when TRACE TRO sidecars diff --git a/docs/engineering/runbooks/build-m-us-populace-certification.md b/docs/engineering/runbooks/build-m-us-populace-certification.md index 61c3dab6..ec279505 100644 --- a/docs/engineering/runbooks/build-m-us-populace-certification.md +++ b/docs/engineering/runbooks/build-m-us-populace-certification.md @@ -25,10 +25,11 @@ it was built with is known. UK TRO in the `--include-tros` step and to run the UK data-release fetch in the test suite. The US populace repo is public. -## Fill in these three values +## Fill in these four values ``` BUILD_M_RELEASE_ID = populace-us-2024-buildm-sparse-rmloss100--Z +LOCAL_AREA_RELEASE_ID = populace-us-2024-buildo-acs-local--Z MODEL_VERSION = 1.764.6 # policyengine-us Build M was built with CURRENT_RELEASE_ID = populace-us-2024-buildj-sparse-rmloss100-75d5add-20260710T094201Z ``` @@ -44,7 +45,8 @@ python scripts/certify_data_release.py \ --country us \ --data-producer populace \ --model-version "$MODEL_VERSION" \ - --manifest-uri "hf://dataset/policyengine/populace-us@$BUILD_M_RELEASE_ID/releases/$BUILD_M_RELEASE_ID/release_manifest.json" + --manifest-uri "hf://dataset/policyengine/populace-us@$BUILD_M_RELEASE_ID/releases/$BUILD_M_RELEASE_ID/release_manifest.json" \ + --regional-manifest-uri "hf://dataset/policyengine/populace-us@$LOCAL_AREA_RELEASE_ID/releases/$LOCAL_AREA_RELEASE_ID/release_manifest.json" ``` This one command: @@ -97,16 +99,16 @@ M was built against a newer `policyengine-us`: `PE_UPDATE_SNAPSHOTS=1 pytest tests/test_household_calculator_snapshot.py` and commit `tests/fixtures/household_calculator_snapshots/`. -## Step 5 — verify the local-area metadata survived +## Step 5 — certify and verify the local-area release -Certification rewrites only `data_releases.us`, so both the -`dataset_overlays.us.populace_us_2024_acs_local` entry and the state/district -entries under `regional_dataset_defaults.us` must remain present and -resolvable. No manual re-add is needed—confirm them: +The step 1 command passes the immutable ACS-local release manifest with +`--regional-manifest-uri`. Certification merges its one non-default microdata artifact into +`data_releases.us.datasets` and records its path for both supported regional +types. Confirm that the resulting release is self-contained: ```bash -python -c "import json; b=json.load(open('src/policyengine/data/bundle/manifest.json')); assert 'populace_us_2024_acs_local' in b['dataset_overlays']['us'], 'overlay lost'; assert set(b['regional_dataset_defaults']['us']) == {'state', 'congressional_district'}, 'regional defaults lost'; print('local-area metadata preserved')" -pytest tests/test_release_manifests.py -k "local_area or DatasetOverlays" -q +python -c "import json; r=json.load(open('src/policyengine/data/bundle/manifest.json'))['data_releases']['us']; assert 'populace_us_2024_acs_local' in r['datasets']; assert set(r['region_datasets']) >= {'national', 'state', 'congressional_district'}; print('local-area release certified')" +pytest tests/test_certify_data_release.py tests/test_release_manifests.py -k "local_area" -q ``` ## Step 6 — check, format, lint, test diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 740f8c38..3f688af8 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -31,27 +31,29 @@ For US Populace certification, certify the Populace release manifest directly: ```bash python scripts/bundle.py certify-data --country us --data-producer populace \ --manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ + --regional-manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ --model-version "" ``` -US state and congressional-district regions are row filters over the ACS-local -Populace dataset. Certification still writes only its national template: +US state and congressional-district regions are row filters over one shared, +certified ACS-local Populace dataset. Its release manifest must declare +`dataset_role: non_default_local_area`, `is_default: false`, no default +datasets, and exactly one pinned H5 microdata artifact. Certification writes: ```json "region_datasets": { - "national": {"path_template": "populace_us_2024.h5"} + "national": {"path_template": "populace_us_2024.h5"}, + "state": {"path_template": "populace_us_2024_acs_local.h5"}, + "congressional_district": {"path_template": "populace_us_2024_acs_local.h5"} } ``` -The hand-maintained sibling blocks `dataset_overlays.us` and -`regional_dataset_defaults.us` register the ACS-local artifact and select it -for state and congressional-district regions. Bundle normalization combines -those blocks with the newly certified release. Certification must preserve both -sibling blocks. +The ACS-local artifact is a second entry in `data_releases.us.datasets`; it is +not a dataset overlay and it never replaces the national `default_dataset`. If the Populace release publishes derived `states/*.h5` or `districts/*.h5` files for compatibility checks, certification omits them from the runtime -bundle. The country-wide default and ACS-local dataset are each single national +bundle. The country-wide default and ACS-local dataset are each single shared files; region registries select one and then filter its rows. The script fetches and validates the manifest (every artifact must carry a @@ -81,8 +83,9 @@ A certification PR should normally change only: Hard failures (certification refuses): missing national default dataset, default dataset absent from artifacts, any artifact without a revision pin, unreachable certified dataset, missing required supplemental release files -(for example Populace-US `us_source_coverage.json`), missing or malformed US -state overlay artifacts when `--regional-manifest-uri` is used, unknown country. +(for example Populace-US `us_source_coverage.json`), a malformed shared +local-area release or malformed legacy per-state artifacts when +`--regional-manifest-uri` is used, unknown country. Certification gate: the model version must either exactly match the build-time model (`compatibility_basis: built_with_model_package`) or be diff --git a/docs/microsim.md b/docs/microsim.md index f9e5d8ec..98cdc7bc 100644 --- a/docs/microsim.md +++ b/docs/microsim.md @@ -143,7 +143,7 @@ administrative totals and state and congressional-district population**. Its release validation summary records **four reviewed limitations**, so read that summary before relying on it. It ships in its own immutable release. State and congressional-district region simulations select it through the bundle's -regional-default metadata; direct microsimulations can still load it by name. +`region_datasets` metadata; direct microsimulations can still load it by name. Two-line load: diff --git a/docs/release-bundles.md b/docs/release-bundles.md index c854db7c..80b6f850 100644 --- a/docs/release-bundles.md +++ b/docs/release-bundles.md @@ -201,31 +201,32 @@ US Populace certification uses the Populace release manifest directly: ```bash python scripts/bundle.py certify-data --country us --data-producer populace \ --manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ + --regional-manifest-uri "hf://dataset/policyengine/populace-us@/releases//release_manifest.json" \ --model-version "" ``` That produces one US bundle manifest entry containing the Populace national -default dataset. State and congressional-district simulations use one -ACS-local dataset and runtime row filters, so derived `states/*.h5` or -`districts/*.h5` files are not vendored into `data_releases.us.datasets`. +default dataset and the certified ACS-local regional dataset. State and +congressional-district simulations use the latter with runtime row filters, so +derived `states/*.h5` or `districts/*.h5` files are not vendored into +`data_releases.us.datasets`. -### Dataset overlays and regional defaults +### Non-default dataset overlays `certify_data_release` rewrites `data_releases.{country}` wholesale from the -certified release manifest, so that block only ever holds datasets from the -certified release. A staged artifact published in its own release — one that is -deliberately not the certified default — is registered instead under the +certified release manifest or manifests. A staged artifact that is not part of +that certification and is deliberately not a default is registered under the sibling `dataset_overlays.{country}` map: ```json "dataset_overlays": { - "us": { - "populace_us_2024_acs_local": { + "uk": { + "populace_uk_2023": { "data_package_name": "populace-data", - "path": "populace_us_2024_acs_local.h5", - "repo_id": "policyengine/populace-us", + "path": "populace_uk_2023.h5", + "repo_id": "policyengine/populace-uk-private", "repo_type": "dataset", - "revision": "populace-us-2024-buildo-acs-local-...", + "revision": "populace-uk-2023-...", "sha256": "..." } } @@ -235,32 +236,17 @@ sibling `dataset_overlays.{country}` map: `get_release_manifest` merges overlays into the resolvable dataset registry, so `resolve_dataset_reference` and `managed_microsimulation` load them by name at their own pinned, sha-verified revision. Overlays are strictly additive: an -overlay may not shadow the certified default or conflict with any certified -dataset, so country-wide default resolution is untouched. Re-normalizing an -already merged bundle accepts an identical entry. - -A second sibling mapping selects an overlaid or certified dataset for a region -type: - -```json -"regional_dataset_defaults": { - "us": { - "state": "populace_us_2024_acs_local", - "congressional_district": "populace_us_2024_acs_local" - } -} -``` - -Bundle normalization merges overlays first, validates every regional logical -name, and then emits the corresponding `region_datasets` path templates. The -mapping cannot replace the national default or conflict with a region template -from the certified release. `get_current_bundle` and `get_release_manifest` -share this normalization path. - -Because certification only rewrites `data_releases`, both sibling blocks -survive re-certification without a manual re-add step. Ordinary bundle -installation still downloads only `default_dataset`; a regional runtime is -responsible for materializing its selected regional dataset. +overlay may not shadow the certified default or any certified dataset, so +default resolution is untouched. Because certification only rewrites +`data_releases`, overlays survive re-certification without any manual re-add +step. + +Datasets used automatically for a region type are not overlays. They are +certified into `data_releases.{country}.datasets`, and +`data_releases.{country}.region_datasets` maps each region type to its artifact +path. The national `default_dataset` remains unchanged. Ordinary bundle +installation downloads only that national default; a regional runtime is +responsible for materializing the additional certified dataset. Cross-package overlays must declare `data_package_name` and `repo_type` explicitly. Ordinary artifacts inherit these values from the country release's diff --git a/scripts/bundle.py b/scripts/bundle.py index 4c535e21..121ab8e2 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -202,11 +202,11 @@ def _parser() -> argparse.ArgumentParser: ) certify.add_argument( "--regional-artifact-prefix", - help="Regional artifact prefix to import. Defaults to states/.", + help="Legacy per-state artifact prefix to import. Defaults to states/.", ) certify.add_argument( "--regional-path-template", - help="Region dataset path template to certify.", + help="Legacy per-state dataset path template to certify.", ) certify.add_argument( "--no-generate", diff --git a/scripts/certify_data_release.py b/scripts/certify_data_release.py index c4c1ec54..8842adbb 100644 --- a/scripts/certify_data_release.py +++ b/scripts/certify_data_release.py @@ -60,12 +60,12 @@ def main(argv=None) -> int: parser.add_argument( "--regional-artifact-prefix", default="states/", - help="Regional artifact path prefix to import. Defaults to states/.", + help="Legacy per-state artifact path prefix to import. Defaults to states/.", ) parser.add_argument( "--regional-path-template", default="states/{state_code}.h5", - help="Region dataset path template to certify into the bundle manifest.", + help="Legacy per-state dataset path template to certify.", ) parser.add_argument( "--model-version", diff --git a/src/policyengine/data/bundle/manifest.json b/src/policyengine/data/bundle/manifest.json index fbb6c3e5..723705b3 100644 --- a/src/policyengine/data/bundle/manifest.json +++ b/src/policyengine/data/bundle/manifest.json @@ -171,6 +171,13 @@ "revision": "populace-us-2024-spm-20260915", "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" }, + "populace_us_2024_acs_local": { + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "revision": "populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z", + "sha256": "769756c31f3ca646d12c272511744dec04c0e68870c6946dd945fbba65b6a7ec" + }, "source_enrichment": { "path": "releases/populace-us-2024-spm-20260915/source_enrichment.json", "repo_id": "policyengine/populace-us", @@ -212,10 +219,18 @@ }, "policyengine_version": "6.2.1", "region_datasets": { + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, "national": { "path_template": "populace_us_2024.h5" + }, + "state": { + "path_template": "populace_us_2024_acs_local.h5" } }, + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z/releases/populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z/releases/populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z/release_manifest.json", "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", "schema_version": 1, "source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", @@ -248,16 +263,6 @@ "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" } - }, - "us": { - "populace_us_2024_acs_local": { - "data_package_name": "populace-data", - "path": "populace_us_2024_acs_local.h5", - "repo_id": "policyengine/populace-us", - "repo_type": "dataset", - "revision": "populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z", - "sha256": "769756c31f3ca646d12c272511744dec04c0e68870c6946dd945fbba65b6a7ec" - } } }, "extras": { @@ -330,11 +335,5 @@ } }, "policyengine_version": "6.2.1", - "regional_dataset_defaults": { - "us": { - "congressional_district": "populace_us_2024_acs_local", - "state": "populace_us_2024_acs_local" - } - }, "schema_version": 2 } diff --git a/src/policyengine/data/bundle/uk.trace.tro.jsonld b/src/policyengine/data/bundle/uk.trace.tro.jsonld index 9c749246..a6036246 100644 --- a/src/policyengine/data/bundle/uk.trace.tro.jsonld +++ b/src/policyengine/data/bundle/uk.trace.tro.jsonld @@ -75,7 +75,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for uk", "trov:mimeType": "application/json", - "trov:sha256": "bf23eb08a005de9496ad7305db9797587e946c24d1d0473c7fde6b7005b50197" + "trov:sha256": "83f4217fb6c917bec0fb1a0e3f31581f68f730d7cb697065d2c49a39725f7606" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -102,7 +102,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "6843d5810a942503c6dd088edcf9ccdb1b10c232fd822199bfb2ac2861825a37" + "trov:sha256": "d07b591e5233a5b4c0d8029aac8562df60efcf04e4534d10e8fa6e49c6417f42" } }, "trov:hasPerformance": { diff --git a/src/policyengine/data/bundle/us.trace.tro.jsonld b/src/policyengine/data/bundle/us.trace.tro.jsonld index 3f64d7e9..eefa6c02 100644 --- a/src/policyengine/data/bundle/us.trace.tro.jsonld +++ b/src/policyengine/data/bundle/us.trace.tro.jsonld @@ -74,7 +74,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "bf23eb08a005de9496ad7305db9797587e946c24d1d0473c7fde6b7005b50197" + "trov:sha256": "83f4217fb6c917bec0fb1a0e3f31581f68f730d7cb697065d2c49a39725f7606" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -101,7 +101,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "3a2b8b8e6d4d899c093942144e82643b754673f2d65d52e26fad68f520a3b2c3" + "trov:sha256": "3534b9ac087d3a69265d9cce82b073df648fa6946cb99df427ec7bd72b045fd5" } }, "trov:hasPerformance": { diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 19d08ff0..ccdefb33 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -472,6 +472,108 @@ def merge_us_state_release_manifest( return DataReleaseManifest.model_validate(primary_payload) +def merge_us_local_area_release_manifest( + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, +) -> DataReleaseManifest: + """Add one certified local-area dataset to a primary US release. + + The supplemental release remains non-default nationally. State and + congressional-district simulations load the shared artifact and apply the + existing row-filtering strategy at runtime. + """ + + if regional_manifest.dataset_role != "non_default_local_area": + raise CertificationError( + "US local-area release must declare dataset_role='non_default_local_area'." + ) + if regional_manifest.is_default is not False: + raise CertificationError("US local-area release must declare is_default=false.") + if regional_manifest.default_datasets: + raise CertificationError( + "US local-area release must not declare default datasets." + ) + + local_area_artifacts = [ + (name, artifact) + for name, artifact in regional_manifest.artifacts.items() + if artifact.kind == "microdata" + ] + if len(local_area_artifacts) != 1: + raise CertificationError( + "US local-area release must contain exactly one microdata artifact." + ) + artifact_name, artifact = local_area_artifacts[0] + if not artifact.path.endswith(".h5"): + raise CertificationError("US local-area artifact must be an H5 file.") + if not artifact.repo_id or not artifact.revision or not artifact.sha256: + raise CertificationError( + "US local-area artifact must declare repo_id, revision, and sha256." + ) + + primary_payload = primary_manifest.model_dump(mode="json", exclude_none=True) + merged_artifacts = primary_payload.setdefault("artifacts", {}) + if artifact_name in merged_artifacts: + raise CertificationError( + f"Regional artifact {artifact_name!r} conflicts with the primary manifest." + ) + if any(item.get("path") == artifact.path for item in merged_artifacts.values()): + raise CertificationError( + f"Regional artifact path {artifact.path!r} conflicts with the primary manifest." + ) + merged_artifacts[artifact_name] = artifact.model_dump( + mode="json", + exclude_none=True, + ) + + metadata = primary_payload.setdefault("metadata", {}) + region_datasets = metadata.setdefault("region_datasets", {}) + default_dataset = primary_manifest.default_datasets["national"] + default_artifact = primary_manifest.artifacts[default_dataset] + region_datasets.setdefault( + "national", + {"path_template": default_artifact.path}, + ) + regional_template = {"path_template": artifact.path} + for region_type in ("state", "congressional_district"): + existing = region_datasets.get(region_type) + if existing is not None and existing != regional_template: + raise CertificationError( + f"US local-area release conflicts with the {region_type!r} " + "dataset template in the primary manifest." + ) + region_datasets[region_type] = regional_template + + return DataReleaseManifest.model_validate(primary_payload) + + +def merge_us_regional_release_manifest( + primary_manifest: DataReleaseManifest, + regional_manifest: DataReleaseManifest, + *, + artifact_prefix: str = "states/", + path_template: str = "states/{state_code}.h5", +) -> DataReleaseManifest: + """Merge a typed shared local-area release or a legacy per-state release.""" + + if regional_manifest.dataset_role == "non_default_local_area": + return merge_us_local_area_release_manifest( + primary_manifest, + regional_manifest, + ) + if regional_manifest.dataset_role is not None: + raise CertificationError( + "Unsupported US regional release dataset_role " + f"{regional_manifest.dataset_role!r}." + ) + return merge_us_state_release_manifest( + primary_manifest, + regional_manifest, + artifact_prefix=artifact_prefix, + path_template=path_template, + ) + + def build_country_manifest_payload( *, country: str, @@ -518,14 +620,18 @@ def build_country_manifest_payload( raw_regions = manifest.metadata.get("region_datasets") if isinstance(raw_regions, dict): for region, template in sorted(raw_regions.items()): + if not isinstance(template, dict) or "path_template" not in template: + continue + path_template = template.get("path_template") if ( country == "us" and manifest.data_package.name in POPULACE_DATA_PACKAGES and region in {"state", "congressional_district"} + and isinstance(path_template, str) + and "{" in path_template ): continue - if isinstance(template, dict) and "path_template" in template: - region_datasets[region] = {"path_template": template["path_template"]} + region_datasets[region] = {"path_template": template["path_template"]} region_datasets.setdefault( "national", {"path_template": default_artifact.path}, @@ -681,15 +787,15 @@ def certify( if regional_manifest_uri is not None: if country != "us": raise CertificationError( - "Regional data release overlays are only supported for US " + "Regional data release manifests are only supported for US " "Populace certification." ) - state_manifest, _, regional_uri_parts = fetch_release_manifest( + regional_manifest, _, regional_uri_parts = fetch_release_manifest( regional_manifest_uri, token=token ) - manifest = merge_us_state_release_manifest( + manifest = merge_us_regional_release_manifest( manifest, - state_manifest, + regional_manifest, artifact_prefix=regional_artifact_prefix, path_template=regional_path_template, ) diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index a155186d..fdb8ff0a 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -4,7 +4,7 @@ from functools import lru_cache from importlib.resources import files from pathlib import Path -from typing import Any, Literal, Mapping, Optional +from typing import Literal, Optional from urllib.parse import quote import requests @@ -125,6 +125,10 @@ def uri(self) -> str: class DataReleaseManifest(BaseModel): schema_version: int data_package: PackageVersion + dataset_role: Optional[str] = None + """Producer-declared role for a release that supplements a default release.""" + is_default: Optional[bool] = None + """Whether the producer intends this release to provide default datasets.""" compatible_model_packages: list[CompatiblePackage] = Field(default_factory=list) compatible_core_packages: list[CompatiblePackage] = Field(default_factory=list) default_datasets: dict[str, str] = Field(default_factory=dict) @@ -265,7 +269,6 @@ def fetch_pypi_wheel_metadata(name: str, version: str) -> dict[str, Optional[str DATASET_OVERLAYS_KEY = "dataset_overlays" -REGIONAL_DATASET_DEFAULTS_KEY = "regional_dataset_defaults" def _apply_dataset_overlays( @@ -286,9 +289,8 @@ def _apply_dataset_overlays( ``data_releases``, overlays survive re-certification untouched. Overlays are strictly additive: an overlay may not shadow the certified - default dataset or conflict with a certified dataset entry, so it can never - alter default resolution. An identical existing entry is accepted to make - bundle normalization idempotent. + default dataset or any certified dataset entry, so it can never alter + default resolution. """ overlays = (bundle.get(DATASET_OVERLAYS_KEY) or {}).get(country_id) or {} if not overlays: @@ -304,8 +306,6 @@ def _apply_dataset_overlays( "change default resolution." ) if overlay_name in certified_datasets: - if certified_datasets[overlay_name] == overlay_reference: - continue raise ValueError( f"Dataset overlay '{overlay_name}' for country '{country_id}' " "collides with a certified dataset entry. Overlays must be " @@ -316,109 +316,6 @@ def _apply_dataset_overlays( return {**release_payload, "datasets": certified_datasets} -def _apply_regional_dataset_defaults( - country_id: str, - release_payload: dict, - bundle: Mapping[str, Any], -) -> dict: - """Apply durable region-to-dataset selections to one country release. - - Certification replaces ``data_releases.{country}`` as a unit. Regional - defaults therefore live beside ``data_releases`` and refer to logical - dataset names after overlays have been merged. The country-wide default - remains owned by the certified release. - """ - - defaults_by_country = bundle.get(REGIONAL_DATASET_DEFAULTS_KEY) or {} - if not isinstance(defaults_by_country, Mapping): - raise ValueError(f"{REGIONAL_DATASET_DEFAULTS_KEY} must be a mapping.") - regional_defaults = defaults_by_country.get(country_id) or {} - if not isinstance(regional_defaults, Mapping): - raise ValueError( - f"{REGIONAL_DATASET_DEFAULTS_KEY}.{country_id} must be a mapping." - ) - if not regional_defaults: - return release_payload - - datasets = release_payload.get("datasets") or {} - if not isinstance(datasets, Mapping): - raise ValueError(f"Datasets for country '{country_id}' must be a mapping.") - region_datasets = dict(release_payload.get("region_datasets") or {}) - - for region_type, dataset_name in regional_defaults.items(): - if region_type == "national": - raise ValueError( - f"Regional dataset defaults for country '{country_id}' must not " - "override the national certified default." - ) - if not isinstance(dataset_name, str) or dataset_name not in datasets: - raise ValueError( - f"Regional dataset default for country '{country_id}' and region " - f"type '{region_type}' references unknown dataset '{dataset_name}'." - ) - reference = datasets[dataset_name] - if not isinstance(reference, Mapping) or not isinstance( - reference.get("path"), str - ): - raise ValueError( - f"Regional dataset default '{dataset_name}' for country " - f"'{country_id}' has no artifact path." - ) - regional_template = {"path_template": reference["path"]} - certified_template = region_datasets.get(region_type) - if certified_template is not None and certified_template != regional_template: - raise ValueError( - f"Regional dataset default for country '{country_id}' and region " - f"type '{region_type}' conflicts with certified region dataset " - f"template {certified_template!r}." - ) - region_datasets[region_type] = regional_template - - return {**release_payload, "region_datasets": region_datasets} - - -def normalise_bundle_dataset_metadata(bundle: Mapping[str, Any]) -> dict[str, Any]: - """Return a bundle with overlays and regional defaults applied. - - Both the public bundle API and country release loading use this function so - callers observe the same dataset registry and region mapping. - """ - - payload = dict(bundle) - releases = payload.get("data_releases") - if releases is None: - return payload - if not isinstance(releases, Mapping): - raise ValueError("data_releases must be a mapping.") - - defaults_by_country = payload.get(REGIONAL_DATASET_DEFAULTS_KEY) or {} - if not isinstance(defaults_by_country, Mapping): - raise ValueError(f"{REGIONAL_DATASET_DEFAULTS_KEY} must be a mapping.") - unknown_countries = set(defaults_by_country) - set(releases) - if unknown_countries: - raise ValueError( - "Regional dataset defaults reference countries without data releases: " - f"{sorted(unknown_countries)}." - ) - - normalised_releases: dict[str, Any] = {} - for country_id, release_payload in releases.items(): - if not isinstance(country_id, str) or not isinstance(release_payload, dict): - raise ValueError("Each data release must be a country-keyed mapping.") - release_payload = _apply_dataset_overlays( - country_id, - release_payload, - payload, - ) - normalised_releases[country_id] = _apply_regional_dataset_defaults( - country_id, - release_payload, - payload, - ) - payload["data_releases"] = normalised_releases - return payload - - @lru_cache def get_release_manifest(country_id: str) -> CountryReleaseManifest: manifest_path = files("policyengine").joinpath("data", "bundle", "manifest.json") @@ -428,11 +325,10 @@ def get_release_manifest(country_id: str) -> CountryReleaseManifest: source_bytes = manifest_path.read_text().encode() bundle = json.loads(source_bytes) try: - release_payload = normalise_bundle_dataset_metadata(bundle)["data_releases"][ - country_id - ] + release_payload = bundle["data_releases"][country_id] except KeyError as exc: raise ValueError(f"No bundled data release for country '{country_id}'") from exc + release_payload = _apply_dataset_overlays(country_id, release_payload, bundle) manifest = CountryReleaseManifest.model_validate(release_payload) manifest.source_sha256 = hashlib.sha256(source_bytes).hexdigest() return manifest diff --git a/tests/test_bundle.py b/tests/test_bundle.py index 75cb4ecc..b30db02e 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -49,7 +49,7 @@ def test_bundle_manifest_exposes_data_releases(): ) -def test_bundle_and_country_manifest_share_normalised_dataset_metadata(): +def test_bundle_and_country_manifest_share_certified_dataset_metadata(): bundle_release = bundle.get_current_bundle()["data_releases"]["us"] country_release = get_release_manifest("us") @@ -105,7 +105,7 @@ def test_selected_us_dataset_plan_installs_only_national_default(tmp_path): assert len(entries) == 1 plan, release = entries[0] assert plan.dataset == "populace_us_2024" - assert plan.dataset != manifest["regional_dataset_defaults"]["us"]["state"] + assert "populace_us_2024_acs_local" in release["datasets"] assert plan.source_uri == release["default_dataset_uri"] diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 9b25a1db..672d95c1 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -14,6 +14,8 @@ CertificationError, build_country_manifest_payload, certify_data_release, + merge_us_local_area_release_manifest, + merge_us_regional_release_manifest, merge_us_state_release_manifest, parse_manifest_uri, required_supplemental_release_files, @@ -36,6 +38,11 @@ "hf://model/policyengine/policyengine-us-data" f"@{US_DATA_VERSION}/releases/{US_DATA_VERSION}/release_manifest.json" ) +US_LOCAL_AREA_TAG = "populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z" +US_LOCAL_AREA_MANIFEST_URI = ( + "hf://dataset/policyengine/populace-us" + f"@{US_LOCAL_AREA_TAG}/releases/{US_LOCAL_AREA_TAG}/release_manifest.json" +) def _release_manifest_payload() -> dict: @@ -157,6 +164,26 @@ def _state_release_manifest_payload( } +def _local_area_release_manifest_payload() -> dict: + return { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "dataset_role": "non_default_local_area", + "is_default": False, + "default_datasets": {}, + "artifacts": { + "populace_us_2024_acs_local": { + "kind": "microdata", + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "revision": US_LOCAL_AREA_TAG, + "sha256": "7" * 64, + "size_bytes": 1, + } + }, + } + + def _manifest() -> DataReleaseManifest: return DataReleaseManifest.model_validate(_release_manifest_payload()) @@ -444,6 +471,104 @@ def test__given_duplicate_state_artifact__then_warns_and_ignores_duplicate(self) ) +class TestMergeUSLocalAreaReleaseManifest: + def test__given_typed_local_area_release__then_certifies_shared_dataset(self): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + regional = DataReleaseManifest.model_validate( + _local_area_release_manifest_payload() + ) + + merged = merge_us_local_area_release_manifest(primary, regional) + payload = build_country_manifest_payload( + country="us", + manifest=merged, + uri_parts=parse_manifest_uri(MANIFEST_URI), + policyengine_version="9.9.9", + model_package="policyengine-us", + model_version="1.723.0", + model_wheel={}, + ) + + local_dataset = payload["datasets"]["populace_us_2024_acs_local"] + assert local_dataset == { + "path": "populace_us_2024_acs_local.h5", + "revision": US_LOCAL_AREA_TAG, + "sha256": "7" * 64, + "repo_id": "policyengine/populace-us", + } + assert payload["region_datasets"] == { + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, + "national": {"path_template": "populace_us_2024.h5"}, + "state": {"path_template": "populace_us_2024_acs_local.h5"}, + } + + @pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("dataset_role", "other", "dataset_role='non_default_local_area'"), + ("is_default", True, "is_default=false"), + ("default_datasets", {"national": "local"}, "default datasets"), + ], + ) + def test__given_invalid_release_role__then_rejects_it( + self, + field, + value, + message, + ): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + regional_payload = _local_area_release_manifest_payload() + regional_payload[field] = value + regional = DataReleaseManifest.model_validate(regional_payload) + + with pytest.raises(CertificationError, match=message): + merge_us_local_area_release_manifest(primary, regional) + + def test__given_more_than_one_microdata_artifact__then_rejects_it(self): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + regional_payload = _local_area_release_manifest_payload() + regional_payload["artifacts"]["second"] = { + **regional_payload["artifacts"]["populace_us_2024_acs_local"], + "path": "second.h5", + } + regional = DataReleaseManifest.model_validate(regional_payload) + + with pytest.raises(CertificationError, match="exactly one microdata"): + merge_us_local_area_release_manifest(primary, regional) + + def test__given_legacy_regional_release__then_uses_state_merge(self): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + states = DataReleaseManifest.model_validate(_state_release_manifest_payload()) + + merged = merge_us_regional_release_manifest(primary, states) + + assert "states/CA" in merged.artifacts + assert merged.metadata["region_datasets"]["state"] == { + "path_template": "states/{state_code}.h5" + } + + def test__given_unknown_regional_role__then_rejects_it(self): + primary = DataReleaseManifest.model_validate( + _populace_manifest_payload_without_regions() + ) + regional_payload = _local_area_release_manifest_payload() + regional_payload["dataset_role"] = "unknown" + regional = DataReleaseManifest.model_validate(regional_payload) + + with pytest.raises(CertificationError, match="Unsupported.*dataset_role"): + merge_us_regional_release_manifest(primary, regional) + + class TestCertifyDataRelease: def test__given_fetched_populace_manifest__then_updates_bundle_manifest( self, tmp_path @@ -501,7 +626,7 @@ def test__given_fetched_populace_manifest__then_updates_bundle_manifest( assert result.build_id == UK_TAG assert result.bundle_path == bundle_path - def test__given_us_regional_manifest__then_validates_but_does_not_vendor_state_artifacts( + def test__given_us_local_area_manifest__then_certifies_shared_regional_dataset( self, tmp_path ): bundle_path = tmp_path / "manifest.json" @@ -514,7 +639,7 @@ def test__given_us_regional_manifest__then_validates_but_does_not_vendor_state_a regional_response = MagicMock() regional_response.status_code = 200 regional_response.content = json.dumps( - _state_release_manifest_payload() + _local_area_release_manifest_payload() ).encode() with ( @@ -547,7 +672,7 @@ def test__given_us_regional_manifest__then_validates_but_does_not_vendor_state_a country="us", data_producer="populace", manifest_uri=MANIFEST_URI, - regional_manifest_uri=US_DATA_MANIFEST_URI, + regional_manifest_uri=US_LOCAL_AREA_MANIFEST_URI, model_version="1.723.0", bundle_path=bundle_path, ) @@ -555,12 +680,16 @@ def test__given_us_regional_manifest__then_validates_but_does_not_vendor_state_a written = json.loads(bundle_path.read_text()) release = written["data_releases"]["us"] assert release["source_manifest_uri"] == MANIFEST_URI - assert release["regional_source_manifest_uri"] == US_DATA_MANIFEST_URI + assert release["regional_source_manifest_uri"] == US_LOCAL_AREA_MANIFEST_URI assert release["region_datasets"] == { - "national": {"path_template": "populace_us_2024.h5"} + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, + "national": {"path_template": "populace_us_2024.h5"}, + "state": {"path_template": "populace_us_2024_acs_local.h5"}, } - assert "states/CA" not in release["datasets"] - assert result.dataset_count == 4 + assert "populace_us_2024_acs_local" in release["datasets"] + assert result.dataset_count == 5 def test__given_us_without_data_producer__then_legacy_update_is_explicitly_unsupported( self, tmp_path diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index dffdd72e..e1c73d83 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -12,7 +12,6 @@ from types import ModuleType, SimpleNamespace from unittest.mock import MagicMock, patch -import pytest from requests import Timeout from policyengine.core.tax_benefit_model import TaxBenefitModel @@ -33,7 +32,6 @@ get_data_release_manifest, get_release_manifest, https_release_manifest_uri, - normalise_bundle_dataset_metadata, resolve_dataset_reference, resolve_default_datasets, resolve_managed_dataset_reference, @@ -63,8 +61,8 @@ US_RELEASE_MANIFEST_DATASET_URI = ( f"hf://policyengine/populace-us/populace_us_2024.h5@{US_DATA_RELEASE_REVISION}" ) -# Local-area overlay: a Populace US artifact published in its own immutable -# release. It remains non-default nationally and is the regional default for +# Certified local-area dataset: a Populace US artifact published in its own +# immutable release. It remains non-default nationally and is selected for # states and congressional districts. US_LOCAL_AREA_DATASET = "populace_us_2024_acs_local" US_LOCAL_AREA_RELEASE_ID = ( @@ -321,13 +319,8 @@ def test__given_no_dataset__then_managed_resolution_uses_certified_default(self) == get_release_manifest("us").default_dataset_uri ) - def test__given_local_area_overlay__then_national_default_is_unchanged(self): - """Regression: the local-area overlay never changes the national default. - - The overlay adds a loadable name; it must never become the certified - default, and default resolution must keep pointing at the certified - national Populace artifact. - """ + def test__given_local_area_dataset__then_national_default_is_unchanged(self): + """The certified regional dataset never changes the national default.""" manifest = get_release_manifest("us") assert manifest.default_dataset == "populace_us_2024" @@ -346,7 +339,7 @@ def test__given_local_area_overlay__then_national_default_is_unchanged(self): ) def test__given_local_area_name__then_resolves_to_its_immutable_tag(self): - """The local-area overlay resolves to its own Build L release tag.""" + """The local-area dataset resolves to its own immutable release tag.""" assert ( resolve_dataset_reference("us", US_LOCAL_AREA_DATASET) == US_LOCAL_AREA_DATASET_URI @@ -358,7 +351,7 @@ def test__given_local_area_name__then_resolves_to_its_immutable_tag(self): == US_LOCAL_AREA_DATASET_URI ) - def test__given_local_area_overlay__then_registered_with_sha_but_not_default(self): + def test__given_local_area_dataset__then_registered_with_sha_but_not_default(self): manifest = get_release_manifest("us") reference = manifest.datasets[US_LOCAL_AREA_DATASET] @@ -1141,12 +1134,12 @@ def test__given_uk_unmanaged_dataset_uri__then_source_is_not_rewritten(self): class TestDatasetOverlays: - """Unit tests for durable dataset metadata outside certification output. + """Unit tests for the ``dataset_overlays`` merge layer. ``dataset_overlays`` is the hand-maintained sibling of ``data_releases``. ``certify_data_release`` rewrites ``data_releases.{country}`` wholesale, so - these tests pin the invariant that overlays and regional defaults survive - re-certification without changing the country-wide default. + these tests pin the invariant that overlays are additive, survive + re-certification, and can never hijack default resolution. """ def teardown_method(self): @@ -1190,21 +1183,16 @@ def _local_area_overlay(self) -> dict: } } - def test__given_recertified_release__then_regional_default_survives(self): - """Sibling metadata survives replacement of ``data_releases.us``.""" + def test__given_recertified_release__then_overlay_survives(self): + """The overlay lives outside ``data_releases``, so a re-certified + payload that never mentions it still resolves it by name.""" payload = self._recertified_us_payload() bundle = { "data_releases": {"us": payload}, "dataset_overlays": {"us": self._local_area_overlay()}, - "regional_dataset_defaults": { - "us": { - "state": US_LOCAL_AREA_DATASET, - "congressional_district": US_LOCAL_AREA_DATASET, - } - }, } - merged = normalise_bundle_dataset_metadata(bundle)["data_releases"]["us"] + merged = _apply_dataset_overlays("us", payload, bundle) manifest = CountryReleaseManifest.model_validate(merged) assert manifest.default_dataset == "populace_us_2024" @@ -1212,46 +1200,12 @@ def test__given_recertified_release__then_regional_default_survives(self): reference = manifest.datasets[US_LOCAL_AREA_DATASET] assert reference.revision == US_LOCAL_AREA_RELEASE_ID assert reference.sha256 == US_LOCAL_AREA_SHA256 - assert manifest.region_datasets["state"].path_template == reference.path - assert ( - manifest.region_datasets["congressional_district"].path_template - == reference.path - ) - - def test__given_bundle__then_current_bundle_and_release_manifest_match(self): - bundle = normalise_bundle_dataset_metadata( - { - "data_releases": {"us": self._recertified_us_payload()}, - "dataset_overlays": {"us": self._local_area_overlay()}, - "regional_dataset_defaults": {"us": {"state": US_LOCAL_AREA_DATASET}}, - } - ) - - release = CountryReleaseManifest.model_validate(bundle["data_releases"]["us"]) - - assert US_LOCAL_AREA_DATASET in release.datasets - assert release.region_datasets["state"].path_template == ( - release.datasets[US_LOCAL_AREA_DATASET].path - ) def test__given_no_overlays__then_payload_is_returned_unchanged(self): payload = self._recertified_us_payload() assert _apply_dataset_overlays("us", payload, {"data_releases": {}}) is payload - def test__given_normalised_bundle__then_normalising_again_is_unchanged(self): - payload = self._recertified_us_payload() - bundle = { - "data_releases": {"us": payload}, - "dataset_overlays": {"us": self._local_area_overlay()}, - "regional_dataset_defaults": {"us": {"state": US_LOCAL_AREA_DATASET}}, - } - - once = normalise_bundle_dataset_metadata(bundle) - twice = normalise_bundle_dataset_metadata(once) - - assert twice == once - def test__given_overlay_shadowing_default__then_raises(self): payload = self._recertified_us_payload() bundle = { @@ -1286,54 +1240,3 @@ def test__given_overlay_colliding_with_certified_dataset__then_raises(self): assert "collides with a certified dataset" in str(error) else: raise AssertionError("Expected overlay colliding with a dataset to fail") - - def test__given_unknown_regional_dataset__then_raises(self): - payload = self._recertified_us_payload() - bundle = { - "data_releases": {"us": payload}, - "regional_dataset_defaults": {"us": {"state": "missing_dataset"}}, - } - - with pytest.raises(ValueError, match="unknown dataset 'missing_dataset'"): - normalise_bundle_dataset_metadata(bundle) - - def test__given_national_regional_default__then_raises(self): - payload = self._recertified_us_payload() - bundle = { - "data_releases": {"us": payload}, - "regional_dataset_defaults": {"us": {"national": "populace_us_2024"}}, - } - - with pytest.raises(ValueError, match="must not override the national"): - normalise_bundle_dataset_metadata(bundle) - - def test__given_conflicting_certified_region_template__then_raises(self): - payload = self._recertified_us_payload() - payload["region_datasets"]["state"] = { - "path_template": "states/{state_code}.h5" - } - bundle = { - "data_releases": {"us": payload}, - "dataset_overlays": {"us": self._local_area_overlay()}, - "regional_dataset_defaults": {"us": {"state": US_LOCAL_AREA_DATASET}}, - } - - with pytest.raises(ValueError, match="conflicts with certified region dataset"): - normalise_bundle_dataset_metadata(bundle) - - def test__given_identical_certified_region_template__then_accepts_it(self): - payload = self._recertified_us_payload() - payload["region_datasets"]["state"] = { - "path_template": "populace_us_2024_acs_local.h5" - } - bundle = { - "data_releases": {"us": payload}, - "dataset_overlays": {"us": self._local_area_overlay()}, - "regional_dataset_defaults": {"us": {"state": US_LOCAL_AREA_DATASET}}, - } - - normalised = normalise_bundle_dataset_metadata(bundle) - - assert normalised["data_releases"]["us"]["region_datasets"]["state"] == { - "path_template": "populace_us_2024_acs_local.h5" - } From 426e79472d3f4cd34eda34eda6261be8b6cc8735 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:26:59 +0400 Subject: [PATCH 5/7] fix: keep bundle exports on certified schema --- scripts/export_bundle_release_assets.py | 4 +--- src/policyengine/bundle.py | 9 +-------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/scripts/export_bundle_release_assets.py b/scripts/export_bundle_release_assets.py index 0c10721b..c0ff3a08 100644 --- a/scripts/export_bundle_release_assets.py +++ b/scripts/export_bundle_release_assets.py @@ -8,8 +8,6 @@ from generate_bundle_artifacts import BUNDLE_MANIFEST, REPO_ROOT -from policyengine.provenance.manifest import normalise_bundle_dataset_metadata - def _write_json(dist_dir: Path, name: str, payload: object) -> Path: path = dist_dir / name @@ -28,7 +26,7 @@ def main() -> int: parser.add_argument("--dist-dir", type=Path, default=REPO_ROOT / "dist") args = parser.parse_args() - bundle = normalise_bundle_dataset_metadata(json.loads(BUNDLE_MANIFEST.read_text())) + bundle = json.loads(BUNDLE_MANIFEST.read_text()) version = bundle["bundle_version"] args.dist_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/policyengine/bundle.py b/src/policyengine/bundle.py index 69c5610a..cb27430a 100644 --- a/src/policyengine/bundle.py +++ b/src/policyengine/bundle.py @@ -29,10 +29,7 @@ _resolve_bundle_dataset, _reuse_or_download_bundle_files, ) -from policyengine.provenance.manifest import ( - CountryReleaseManifest, - normalise_bundle_dataset_metadata, -) +from policyengine.provenance.manifest import CountryReleaseManifest from policyengine.utils.hashing import sha256_file BUNDLE_MANIFEST_RESOURCE = ("data", "bundle", "manifest.json") @@ -75,10 +72,6 @@ def _normalise_manifest(manifest: Mapping[str, Any]) -> dict[str, Any]: payload.setdefault("packages", {}) payload.setdefault("extras", {}) payload.setdefault("data_releases", _data_releases_from_countries(payload)) - try: - payload = normalise_bundle_dataset_metadata(payload) - except ValueError as exc: - raise BundleError(str(exc)) from exc try: validate_bundle_measurements(payload) except ValueError as exc: From 8737dae05e67fb3a50924c1cb8daa7b2c34e3ac9 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:46:24 +0400 Subject: [PATCH 6/7] fix: preserve regional certification metadata --- CHANGELOG.md | 2 +- .../build-m-us-populace-certification.md | 147 ------------------ .../runbooks/us-populace-certification.md | 126 +++++++++++++++ docs/engineering/skills/data-certification.md | 4 +- src/policyengine/core/region.py | 20 ++- src/policyengine/provenance/certification.py | 14 +- src/policyengine/provenance/manifest.py | 1 + tests/test_certify_data_release.py | 10 +- 8 files changed, 160 insertions(+), 164 deletions(-) delete mode 100644 docs/engineering/runbooks/build-m-us-populace-certification.md create mode 100644 docs/engineering/runbooks/us-populace-certification.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a980353..b4b716db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -167,7 +167,7 @@ ### Changed -- Certify the US ACS-local dataset (`populace_us_2024_acs_local`) for state and congressional-district simulations using the immutable buildo-acs-local release, while retaining the existing national default dataset. +- Repin the US local-area overlay (`populace_us_2024_acs_local`) to the buildo-acs-local release built on the certified Build O lineage: consumer-loadable bytes, 4,461-target local surface, immutable tag, default resolution unchanged. ## [4.22.2] - 2026-07-23 diff --git a/docs/engineering/runbooks/build-m-us-populace-certification.md b/docs/engineering/runbooks/build-m-us-populace-certification.md deleted file mode 100644 index ec279505..00000000 --- a/docs/engineering/runbooks/build-m-us-populace-certification.md +++ /dev/null @@ -1,147 +0,0 @@ -# Build M US Populace certification runbook - -Fill-in-the-id runbook for certifying the next US Populace default -(`sparse-rmloss100` lineage: Build I → J → **M**) into the -`policyengine.py` bundle. It replays the exact steps that landed -[#470](https://github.com/PolicyEngine/policyengine.py/pull/470) (the Build J -certification), parameterized for the Build M release id. Read the -[data certification](../skills/data-certification.md) skill first for the -validation semantics; this file is the concrete checklist. - -## When to use - -A new US Populace `sparse-rmloss100` release has been published to -`policyengine/populace-us` and is ready to become the certified default. You -have the release id, the release manifest is reachable, and the model version -it was built with is known. - -## Prerequisites - -- A clean worktree branched from current `origin/main` - (`git fetch origin && git checkout -b certify-us-buildm origin/main`). -- Network access: certification fetches the release manifest and PyPI wheel - metadata and runs reachability `HEAD` checks against Hugging Face. -- `HUGGING_FACE_TOKEN` (or `HF_TOKEN`) exported — required to regenerate the - UK TRO in the `--include-tros` step and to run the UK data-release fetch in - the test suite. The US populace repo is public. - -## Fill in these four values - -``` -BUILD_M_RELEASE_ID = populace-us-2024-buildm-sparse-rmloss100--Z -LOCAL_AREA_RELEASE_ID = populace-us-2024-buildo-acs-local--Z -MODEL_VERSION = 1.764.6 # policyengine-us Build M was built with -CURRENT_RELEASE_ID = populace-us-2024-buildj-sparse-rmloss100-75d5add-20260710T094201Z -``` - -`CURRENT_RELEASE_ID` is the outgoing default (Build J) — the string you are -replacing in the pinned test constants. `MODEL_VERSION` stays `1.764.6` unless -Build M was built against a newer `policyengine-us`; if it was, see step 4. - -## Step 1 — certify the release - -```bash -python scripts/certify_data_release.py \ - --country us \ - --data-producer populace \ - --model-version "$MODEL_VERSION" \ - --manifest-uri "hf://dataset/policyengine/populace-us@$BUILD_M_RELEASE_ID/releases/$BUILD_M_RELEASE_ID/release_manifest.json" \ - --regional-manifest-uri "hf://dataset/policyengine/populace-us@$LOCAL_AREA_RELEASE_ID/releases/$LOCAL_AREA_RELEASE_ID/release_manifest.json" -``` - -This one command: - -- rewrites `data_releases.us` in `src/policyengine/data/bundle/manifest.json` - from the Build M release manifest (default dataset, per-artifact - repo/revision/sha256 pins, certified artifact, certification block); -- runs `generate(check=False)`, which re-normalizes `manifest.json` and - updates `pyproject.toml` **only if the model pins moved**; -- writes the changelog fragment - `changelog.d/certify-us-$BUILD_M_RELEASE_ID.changed.md`. - -## Step 2 — regenerate the US TRO sidecar - -The certify step does not touch TRO sidecars. Rebind them so -`src/policyengine/data/bundle/us.trace.tro.jsonld` records the new -`manifest.json` sha256 (the `bundle_manifest` artifact hash is asserted by -`tests/test_certify_data_release.py::TestVendoredSidecarBinding`): - -```bash -HUGGING_FACE_TOKEN="$HUGGING_FACE_TOKEN" python scripts/bundle.py generate --include-tros -``` - -Only `us.trace.tro.jsonld` should change (UK regenerates identically because -only US was certified). If UK cannot be reached, the run writes a *limited* UK -TRO — do not commit a degraded `uk.trace.tro.jsonld`; `git checkout` it and -rerun with a valid token. - -## Step 3 — update the pinned test constants - -Three files hard-code the certified release id. Replace `CURRENT_RELEASE_ID` -with `BUILD_M_RELEASE_ID` in each (this is the whole of #470's test diff): - -- `tests/test_release_manifests.py` — `US_DATA_RELEASE_ID`. -- `tests/test_models.py` — the `us_latest.default_dataset_uri` `@` - assertion. -- `tests/test_us_regions.py` — the national `dataset_path` `@` assertion. - -## Step 4 — only if the model version changed - -Build J → M on the same `policyengine-us` needs nothing here (this is the -common case; #470 left `pyproject.toml` and the snapshots untouched). If Build -M was built against a newer `policyengine-us`: - -- `pyproject.toml` model pins are already rewritten by step 1's `generate`; - commit them. -- Update `US_MODEL_VERSION` and `US_BUILT_WITH_MODEL_VERSION` in - `tests/test_release_manifests.py`. -- Refresh the household snapshots: - `PE_UPDATE_SNAPSHOTS=1 pytest tests/test_household_calculator_snapshot.py` - and commit `tests/fixtures/household_calculator_snapshots/`. - -## Step 5 — certify and verify the local-area release - -The step 1 command passes the immutable ACS-local release manifest with -`--regional-manifest-uri`. Certification merges its one non-default microdata artifact into -`data_releases.us.datasets` and records its path for both supported regional -types. Confirm that the resulting release is self-contained: - -```bash -python -c "import json; r=json.load(open('src/policyengine/data/bundle/manifest.json'))['data_releases']['us']; assert 'populace_us_2024_acs_local' in r['datasets']; assert set(r['region_datasets']) >= {'national', 'state', 'congressional_district'}; print('local-area release certified')" -pytest tests/test_certify_data_release.py tests/test_release_manifests.py -k "local_area" -q -``` - -## Step 6 — check, format, lint, test - -```bash -python scripts/bundle.py check # must exit 0 -make format -make lint -make test # needs HUGGING_FACE_TOKEN for UK -``` - -## Step 7 — commit exactly these files - -Matches #470's file set (add `pyproject.toml` and the snapshot dir only when -step 4 applied): - -- `src/policyengine/data/bundle/manifest.json` -- `src/policyengine/data/bundle/us.trace.tro.jsonld` -- `changelog.d/certify-us-$BUILD_M_RELEASE_ID.changed.md` -- `tests/test_release_manifests.py` -- `tests/test_models.py` -- `tests/test_us_regions.py` - -## Step 8 — open the PR - -Follow [github-prs](../skills/github-prs.md): open/find the issue, put -`Fixes #ISSUE` first, push to the canonical repo, and open a **draft** PR. The -certify step already wrote the changelog fragment, so the changelog check -passes. - -## Verified against #470 - -Every step above is checked against the merged Build J certification (#470), -whose diff was exactly: `manifest.json`, `us.trace.tro.jsonld`, the three test -constants, and the auto-written changelog fragment — with `pyproject.toml` and -the snapshots untouched because `policyengine-us` stayed `1.764.6`. diff --git a/docs/engineering/runbooks/us-populace-certification.md b/docs/engineering/runbooks/us-populace-certification.md new file mode 100644 index 00000000..9f590438 --- /dev/null +++ b/docs/engineering/runbooks/us-populace-certification.md @@ -0,0 +1,126 @@ +# US Populace certification runbook + +Use this runbook to certify a new national US Populace release together with +the shared ACS-local release used for state and congressional-district +simulations. Read the [data certification](../skills/data-certification.md) +skill first for validation semantics. + +## When to use + +Both release manifests have been published to `policyengine/populace-us`, and +the model version used by the national release is known. The national release +will remain the default dataset; the local-area release will remain +non-default and be selected only for supported regional runs. + +## Prerequisites + +- A clean worktree branched from current `origin/main`. +- Network access: certification fetches the release manifest and PyPI wheel + metadata and runs reachability `HEAD` checks against Hugging Face. +- `HUGGING_FACE_TOKEN` (or `HF_TOKEN`) exported — required to regenerate the + UK TRO in the `--include-tros` step and to run the UK data-release fetch in + the test suite. The US populace repo is public. + +## Step 1 — identify the releases + +Set values from the two published release manifests rather than copying an +older certification: + +``` +NATIONAL_RELEASE_ID = populace-us-2024---Z +LOCAL_AREA_RELEASE_ID = populace-us-2024---Z +MODEL_VERSION = +``` + +The local-area manifest must declare `dataset_role: non_default_local_area`, +`is_default: false`, no default datasets, and exactly one pinned H5 microdata +artifact. + +## Step 2 — certify both releases + +```bash +python scripts/bundle.py certify-data \ + --country us \ + --data-producer populace \ + --model-version "$MODEL_VERSION" \ + --manifest-uri "hf://dataset/policyengine/populace-us@$NATIONAL_RELEASE_ID/releases/$NATIONAL_RELEASE_ID/release_manifest.json" \ + --regional-manifest-uri "hf://dataset/policyengine/populace-us@$LOCAL_AREA_RELEASE_ID/releases/$LOCAL_AREA_RELEASE_ID/release_manifest.json" +``` + +This one command: + +- rewrites `data_releases.us` in `src/policyengine/data/bundle/manifest.json` + from the national release manifest (default dataset, per-artifact + repo/revision/sha256 pins, certified artifact, certification block); +- runs `generate(check=False)`, which re-normalizes `manifest.json` and + updates `pyproject.toml` **only if the model pins moved**; +- writes the changelog fragment + `changelog.d/certify-us-$NATIONAL_RELEASE_ID.changed.md`. + +## Step 3 — regenerate TRACE sidecars + +The certify step does not touch TRACE sidecars. Regenerate them so they record +the new bundle-manifest SHA-256: + +```bash +HUGGING_FACE_TOKEN="$HUGGING_FACE_TOKEN" python scripts/bundle.py generate --include-tros +``` + +Both country sidecars contain the bundle-manifest hash, so both may change when +the bundle changes. If UK cannot be reached, the run writes a *limited* UK +sidecar; do not commit that degraded output. + +## Step 4 — update pinned expectations + +Update tests that intentionally pin the outgoing national release, local-area +release, model version, or artifact hashes. Derive every replacement from the +new manifests; do not copy identifiers from this runbook. + +If the model version changed, refresh the household snapshots with: + +```bash +PE_UPDATE_SNAPSHOTS=1 pytest tests/test_household_calculator_snapshot.py +``` + +## Step 5 — certify and verify the local-area release + +The certification command passes the immutable ACS-local release manifest with +`--regional-manifest-uri`. Confirm all of the following: + +- `default_dataset` remains the national Populace dataset; +- the local-area artifact appears in `data_releases.us.datasets` with its + repository type, immutable revision, and SHA-256; +- `region_datasets.national` selects the national artifact; +- `region_datasets.state` and `region_datasets.congressional_district` select + the shared local-area artifact; +- no derived `states/*.h5` or `districts/*.h5` artifacts are runtime inputs. + +```bash +pytest tests/test_certify_data_release.py \ + tests/test_release_manifests.py \ + tests/test_us_regions.py \ + -q +``` + +## Step 6 — check, format, lint, test + +```bash +python scripts/bundle.py check # must exit 0 +make format +make lint +make test # needs HUGGING_FACE_TOKEN for UK +``` + +## Step 7 — inspect and commit the generated changes + +The expected diff normally includes the bundle manifest, TRACE sidecars, one +Towncrier fragment, and pinned test expectations. Include package pins and +snapshots only when the certified model version changed. Investigate any other +generated difference before committing it. + +## Step 8 — open the PR + +Follow [github-prs](../skills/github-prs.md): open/find the issue, put +`Fixes #ISSUE` first, push to the canonical repo, and open a **draft** PR. The +certify step already wrote the changelog fragment, so the changelog check +passes. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 3f688af8..cf9bb601 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -102,8 +102,8 @@ publisher-claim basis above. Concrete, fill-in-the-id runbooks that replay a specific certification live under `docs/engineering/runbooks/`. See -`runbooks/build-m-us-populace-certification.md` for the next US Populace -`sparse-rmloss100` default. +`runbooks/us-populace-certification.md` for the reusable US national and +local-area certification procedure. ## Legacy paths diff --git a/src/policyengine/core/region.py b/src/policyengine/core/region.py index d5d177a6..20be0f2e 100644 --- a/src/policyengine/core/region.py +++ b/src/policyengine/core/region.py @@ -1,10 +1,10 @@ """Region definitions for geographic simulations. This module provides the Region and RegionRegistry classes for defining -geographic regions that a tax-benefit model supports. Regions can have: -1. A dedicated dataset, usually for the national default. -2. A scoping strategy that derives the region from a parent dataset - (row filter or weight replacement). +geographic regions that a tax-benefit model supports. Regions can select an +input dataset, apply a scoping strategy to a selected dataset, or do both. For +example, US states and congressional districts select the shared ACS-local +dataset and then filter its rows. """ from typing import Literal, Optional, Union @@ -22,10 +22,8 @@ class Region(BaseModel): """Geographic region for tax-benefit simulations. - Regions can either have: - 1. A dedicated dataset (``dataset_path`` is set). - 2. A scoping strategy that derives the region from a parent dataset - (``scoping_strategy`` is set). + ``dataset_path`` selects the input dataset. ``scoping_strategy`` optionally + scopes that dataset to the requested geography. A region may declare both. The unique identifier is the code field, which uses a prefixed format: - National: "us", "uk" @@ -56,7 +54,7 @@ class Region(BaseModel): # Dataset configuration dataset_path: Optional[str] = Field( default=None, - description="URI to a dedicated dataset when the region has one.", + description="URI to the input dataset selected before regional scoping.", ) # Scoping strategy for regions that derive from a parent dataset @@ -173,11 +171,11 @@ def get_children(self, parent_code: str) -> list[Region]: return [r for r in self.regions if r.parent_code == parent_code] def get_dataset_regions(self) -> list[Region]: - """Get all regions that have a dedicated dataset on disk.""" + """Get all regions that select an input dataset.""" return [r for r in self.regions if r.dataset_path is not None] def get_filter_regions(self) -> list[Region]: - """Get all regions that derive from a parent dataset via a scoping strategy.""" + """Get all regions that apply a scoping strategy to their input dataset.""" return [r for r in self.regions if r.scoping_strategy is not None] def __len__(self) -> int: diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index ccdefb33..3b4b27f1 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -42,7 +42,7 @@ import warnings from dataclasses import dataclass, field from pathlib import Path -from typing import Optional +from typing import Literal, Optional import requests @@ -475,6 +475,8 @@ def merge_us_state_release_manifest( def merge_us_local_area_release_manifest( primary_manifest: DataReleaseManifest, regional_manifest: DataReleaseManifest, + *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, ) -> DataReleaseManifest: """Add one certified local-area dataset to a primary US release. @@ -521,10 +523,13 @@ def merge_us_local_area_release_manifest( raise CertificationError( f"Regional artifact path {artifact.path!r} conflicts with the primary manifest." ) - merged_artifacts[artifact_name] = artifact.model_dump( + artifact_payload = artifact.model_dump( mode="json", exclude_none=True, ) + if regional_repo_type is not None: + artifact_payload["repo_type"] = regional_repo_type + merged_artifacts[artifact_name] = artifact_payload metadata = primary_payload.setdefault("metadata", {}) region_datasets = metadata.setdefault("region_datasets", {}) @@ -551,6 +556,7 @@ def merge_us_regional_release_manifest( primary_manifest: DataReleaseManifest, regional_manifest: DataReleaseManifest, *, + regional_repo_type: Optional[Literal["model", "dataset"]] = None, artifact_prefix: str = "states/", path_template: str = "states/{state_code}.h5", ) -> DataReleaseManifest: @@ -560,6 +566,7 @@ def merge_us_regional_release_manifest( return merge_us_local_area_release_manifest( primary_manifest, regional_manifest, + regional_repo_type=regional_repo_type, ) if regional_manifest.dataset_role is not None: raise CertificationError( @@ -614,6 +621,8 @@ def build_country_manifest_payload( payload["sha256"] = artifact.sha256 if artifact.repo_id: payload["repo_id"] = artifact.repo_id + if artifact.repo_type: + payload["repo_type"] = artifact.repo_type datasets[name] = payload region_datasets = {} @@ -796,6 +805,7 @@ def certify( manifest = merge_us_regional_release_manifest( manifest, regional_manifest, + regional_repo_type=regional_uri_parts["repo_type"], artifact_prefix=regional_artifact_prefix, path_template=regional_path_template, ) diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index fdb8ff0a..0995c0d5 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -106,6 +106,7 @@ class DataReleaseArtifact(BaseModel): path: str repo_id: str revision: str + repo_type: Optional[Literal["model", "dataset"]] = None sha256: Optional[str] = None size_bytes: Optional[int] = None preservation_mirrors: list[PreservationMirror] = Field(default_factory=list) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 672d95c1..b18a77f0 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -480,7 +480,11 @@ def test__given_typed_local_area_release__then_certifies_shared_dataset(self): _local_area_release_manifest_payload() ) - merged = merge_us_local_area_release_manifest(primary, regional) + merged = merge_us_local_area_release_manifest( + primary, + regional, + regional_repo_type="dataset", + ) payload = build_country_manifest_payload( country="us", manifest=merged, @@ -497,6 +501,7 @@ def test__given_typed_local_area_release__then_certifies_shared_dataset(self): "revision": US_LOCAL_AREA_TAG, "sha256": "7" * 64, "repo_id": "policyengine/populace-us", + "repo_type": "dataset", } assert payload["region_datasets"] == { "congressional_district": { @@ -689,6 +694,9 @@ def test__given_us_local_area_manifest__then_certifies_shared_regional_dataset( "state": {"path_template": "populace_us_2024_acs_local.h5"}, } assert "populace_us_2024_acs_local" in release["datasets"] + assert ( + release["datasets"]["populace_us_2024_acs_local"]["repo_type"] == "dataset" + ) assert result.dataset_count == 5 def test__given_us_without_data_producer__then_legacy_update_is_explicitly_unsupported( From 5815bde77c1c26e0eee90198335cc4c03fae40dc Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:35:28 +0400 Subject: [PATCH 7/7] docs: preserve certification runbook path --- ...certification.md => build-m-us-populace-certification.md} | 3 +++ docs/engineering/skills/data-certification.md | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) rename docs/engineering/runbooks/{us-populace-certification.md => build-m-us-populace-certification.md} (97%) diff --git a/docs/engineering/runbooks/us-populace-certification.md b/docs/engineering/runbooks/build-m-us-populace-certification.md similarity index 97% rename from docs/engineering/runbooks/us-populace-certification.md rename to docs/engineering/runbooks/build-m-us-populace-certification.md index 9f590438..b63f919b 100644 --- a/docs/engineering/runbooks/us-populace-certification.md +++ b/docs/engineering/runbooks/build-m-us-populace-certification.md @@ -5,6 +5,9 @@ the shared ACS-local release used for state and congressional-district simulations. Read the [data certification](../skills/data-certification.md) skill first for validation semantics. +This historical filename is retained so existing links remain valid; the +procedure itself is release-independent. + ## When to use Both release manifests have been published to `policyengine/populace-us`, and diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index cf9bb601..5f1d9abe 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -102,8 +102,9 @@ publisher-claim basis above. Concrete, fill-in-the-id runbooks that replay a specific certification live under `docs/engineering/runbooks/`. See -`runbooks/us-populace-certification.md` for the reusable US national and -local-area certification procedure. +`runbooks/build-m-us-populace-certification.md` for the reusable US national +and local-area certification procedure. The historical filename is retained +so existing links remain valid. ## Legacy paths