diff --git a/.github/workflows/certify-uk-release.yaml b/.github/workflows/certify-uk-release.yaml index a1dd71962..3f27350ee 100644 --- a/.github/workflows/certify-uk-release.yaml +++ b/.github/workflows/certify-uk-release.yaml @@ -7,7 +7,8 @@ # mandatory gates on that exact commit, replayed by policyengine-scorecard, # and recorded in the uk-qualification-ledger branch. Promotion is # serialized: it readies the newest promotable candidate PR that advances -# main, closes superseded ones, and (cos d1344) fast-forwards main to its +# main, closes superseded ones and those of pairs from another data line +# (which are never promoted), and (cos d1344) fast-forwards main to its # qualified commit once every check on it has passed, which succeeds only # while main is still the commit it was built on; otherwise the catch-up # requalifies it on current main. diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index fe77ff440..3d09625a1 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -316,6 +316,11 @@ jobs: VERSION: ${{ needs.Publish.outputs.version }} HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} run: python scripts/release_consumers.py scorecard-request --version "$VERSION" > "$RUNNER_TEMP/scorecard-request.json" + - name: Warn that no released-bundle replay was requested + if: steps.released.outcome == 'failure' || steps.scorecard-request.outcome == 'failure' + env: + VERSION: ${{ needs.Publish.outputs.version }} + run: echo "::warning title=No released-bundle replay::The scorecard's replay request for policyengine $VERSION could not be built; see the steps above." - name: Generate GitHub App token for the scorecard if: steps.scorecard-request.outcome == 'success' id: scorecard-token diff --git a/README.md b/README.md index f92796d6d..b7da84050 100644 --- a/README.md +++ b/README.md @@ -70,9 +70,10 @@ print(f"Total UC spending: £{agg.result / 1e9:.1f}bn") For baseline-vs-reform comparisons, see `pe.uk.economic_impact_analysis` and its US counterpart. -UK population data is stored in a private Hugging Face model repository. Set -`HUGGING_FACE_TOKEN` to a token from an account with access before running UK -population examples. To download the raw `.h5` file directly, see +UK population data is stored in private Hugging Face repositories: the +Microcosm default in a dataset repository, and the enhanced FRS in a model +repository. Set `HUGGING_FACE_TOKEN` to a token from an account with access +before running UK population examples. To download the raw `.h5` file directly, see [Microsimulation](docs/microsim.md#uk-private-data-and-raw-h5-access). ## Documentation diff --git a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md new file mode 100644 index 000000000..1a6ef427f --- /dev/null +++ b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.changed.md @@ -0,0 +1 @@ +Switch the UK default dataset from the Enhanced FRS 2024-25 (policyengine-uk-data 1.58.1) to Microcosm UK 2024-25 (`microcosm_uk_2024_25`, cut `microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71`), certified with policyengine-uk 2.132.6 and policyengine-core 3.33.3, the versions it was built with. Constituency and local-authority regions now load the Microcosm local-area release (`microcosm_uk_2024_25_dense`, cut `microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe`) and are in the default UK region registry. UK results change with the data. The national file has no constituency or local-authority codes, so constituency and local-authority outputs (`compute_uk_geography_disclosure` at those levels, `compute_uk_constituency_impacts`, `compute_uk_local_authority_impacts`) need simulations on `microcosm_uk_2024_25_dense`, and receipts for those runs are uncertified: the certificate covers the national dataset only. The Enhanced FRS 2024-25 datasets stay loadable by name. Data certificates now say how they cover policyengine-core: by default one follows the bundle's core pin (`core_binding: bundle_pin`, as the UK certificate does; the Microcosm releases claim core 3.33.3), and certification and `bundle.py check --include-tros --strict-tros` warn when the pin is outside a release's core claims, as it is for the US release today; `--core-version` binds the core instead (`core_binding: certified`). With a Hugging Face token set, importing policyengine with any policyengine-uk other than 2.132.6 raises, as it did before for versions the Enhanced FRS release did not claim. The certified default is below the Universal Credit caseload floor (0.897 of the floor's 6.7m benchmark, policyengine-uk-data's derivation; Microcosm calibrates to DWP Stat-Xplore's 6.2m), an exception accepted and recorded in the data certification skill. UK candidate promotion now closes the pull requests of pairs from another data line than main's certificate, at the commit their receipt qualified, and recovery no longer re-dispatches attempts for such pairs. diff --git a/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.removed.md b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.removed.md new file mode 100644 index 000000000..17a271aa4 --- /dev/null +++ b/changelog.d/certify-uk-microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71.removed.md @@ -0,0 +1 @@ +The Enhanced FRS weight matrices (`parliamentary_constituency_weights`, `local_authority_weights`, `local_geography_weights`) no longer resolve by name, now that the UK default is Microcosm UK and its area regions use the Microcosm local-area release. diff --git a/docs/bundles.md b/docs/bundles.md index dc06c6cad..21bcd37a9 100644 --- a/docs/bundles.md +++ b/docs/bundles.md @@ -127,6 +127,14 @@ python scripts/bundle.py update-packages \ --uk 2.91.0 ``` +By default a data certificate follows the core pin (`core_binding: +bundle_pin`), so `--core` can move it; `bundle.py check --include-tros +--strict-tros` then warns when a data release does not claim the new core. A +certificate that binds its core (`certify-data --core-version`, +`core_binding: certified`) instead fails that check until the country is +re-certified for the new core. See "The core" in the data certification +skill. + To certify a new data release from a data-producer manifest, run: ```bash diff --git a/docs/countries.md b/docs/countries.md index 4e2227a58..256ac9077 100644 --- a/docs/countries.md +++ b/docs/countries.md @@ -33,7 +33,7 @@ Override in any output with `income_variable=`. | | Dataset | |---|---| | US | Populace US 2024 (`populace_us_2024.h5`) | -| UK | Enhanced FRS 2024-25 (`enhanced_frs_2024_25.h5`, policyengine-uk-data) | +| UK | Microcosm UK 2024-25 (`microcosm_uk_2024_25.h5`); constituency and local-authority regions use its local-area release (`microcosm_uk_2024_25_dense.h5`) | ## State / regional breakdown diff --git a/docs/disclosure.md b/docs/disclosure.md index b6af7dca0..b66c76969 100644 --- a/docs/disclosure.md +++ b/docs/disclosure.md @@ -22,6 +22,10 @@ impact.findings # what blocks publication, if anything impact.public # the rows that may be published, or None while blocked ``` +For constituency and local-authority levels, `baseline` and `reform` must run +on a dataset whose households carry those codes: the local-area dataset +`microcosm_uk_2024_25_dense`, not the Microcosm national default. + The UK datasets derive from surveys under the UK Data Service End User Licence. The defaults below follow its disclosure rules. You can raise them for a study with stricter conditions; you cannot lower them. diff --git a/docs/engineering/runbooks/uk-qualification.md b/docs/engineering/runbooks/uk-qualification.md index 41e245ce4..9392e340c 100644 --- a/docs/engineering/runbooks/uk-qualification.md +++ b/docs/engineering/runbooks/uk-qualification.md @@ -118,3 +118,23 @@ write here, such as renaming another branch into main to make a run against another base look like the candidate's: main is not protected, so they could push it directly. Treat any such change as an incident, not as something the gate will catch. + +## A pair from another data line + +Promotion compares only pairs on main's data line (its UK certificate's data +package and repository). While Microcosm UK is the default, the +policyengine-uk-data candidates the workflow resolves are `other_line`. An +unforced run of one stops there and writes nothing to the ledger. Recovery and +check refreshes skip such pairs, and promotion closes their open pull requests; +the ledger keeps their evidence. + +`force` still qualifies such a pair, for evidence only. Its candidate commit +drops main's area routing (`--drop-regional-release`), because the routing +belongs to the line the pair would replace, and leaves the test constants that +pin main's certificate as they are, so the candidate's own tests fail. Its pull +request says it is never promoted, and promotion closes it once its receipt is +published, at the commit that receipt qualified. The catch-up does not recover +such a run: re-run it by hand with `force`. A run whose attempt dies before it +publishes a receipt leaves its pull request open; close that one by hand. Changing the data line +is a reviewed certification pull request (see +`docs/engineering/skills/data-certification.md`), not a promotion. diff --git a/docs/engineering/skills/data-certification.md b/docs/engineering/skills/data-certification.md index 42eec89be..23c5d8133 100644 --- a/docs/engineering/skills/data-certification.md +++ b/docs/engineering/skills/data-certification.md @@ -80,9 +80,30 @@ Microcosm national release with one of: `dataset_overlays.uk` entry. Either way, certification checks that the regional release serves the pinned -model, and the pinned core when `--core-version` is given, by its own build or -compatibility claims (a `--qualified-pair-id` names the primary release only), -and HEAD-checks every artifact it carries. +model by its own build or compatibility claims (a `--qualified-pair-id` names +the primary release only), checks its core claims as for the primary release +(below), and HEAD-checks every artifact it carries. + +### The core + +The bundle has one core pin, `packages.policyengine-core`, shared by every +country. A certificate says how it covers it: + +- **`core_binding: bundle_pin` (the default, no `--core-version`).** The + certificate names no core and certifies whatever core the bundle pins, so + the pin can move with another country without re-certifying this one. + Certification checks the pin against each release's build-time core and + core claims, and warns, without refusing, when the pin is outside them. + `bundle.py check --include-tros --strict-tros` warns the same way when the + pin later moves outside the claims. Receipts then certify the pin without + the release's claim; the warning, the certificate and its TRO say so. +- **`core_binding: certified` (`--core-version`).** The certificate names the + core (`certified_for_core_version`). Certification refuses a core that a + release neither was built with nor claims, and the strict check refuses a + pin that differs from the named core, so moving the core needs a + re-certification first. A qualified exact pair always binds its core. + +Use the default unless a release must not run on any other core. ### Re-certifying a routed bundle @@ -92,6 +113,19 @@ a run without `--regional-manifest-uri` is refused; pass the current regional release again (or its replacement), or `--drop-regional-release` to remove the routing on purpose. The two flags cannot be combined. +An overlay may never shadow a certified dataset name, so certification removes +any `dataset_overlays.` entry whose name the new release certifies. A +rollback to policyengine-uk-data with `--drop-regional-release`, for example, +removes the enhanced FRS overlays. `bundle.py check` refuses a bundle with an +overlay that shadows a certified name. + +Removed overlays do not come back by themselves. Certification warns about +every dataset name that stops resolving. After a certification that stops +certifying a name that must stay loadable (rolling forward after a rollback, +or a ship-together run after a national-first one), re-add its overlay from +the previous bundle, as #685 did for `enhanced_frs_2024_25`, `frs_2024_25` and +their `_tiny` files. + ### US For US Populace certification, certify the Populace release manifest directly: @@ -182,9 +216,34 @@ python scripts/check_uk_uc_floor.py --dataset =3.0", ] models = [ - "policyengine-core==3.33.0", + "policyengine-core==3.33.3", "policyengine-us==2.2.1", - "policyengine-uk==2.130.1", + "policyengine-uk==2.132.6", "spm-calculator==1.0.0", ] uk = [ - "policyengine-core==3.33.0", - "policyengine-uk==2.130.1", + "policyengine-core==3.33.3", + "policyengine-uk==2.132.6", ] us = [ - "policyengine-core==3.33.0", + "policyengine-core==3.33.3", "policyengine-us==2.2.1", "spm-calculator==1.0.0", ] @@ -70,9 +70,9 @@ dev = [ "pytest-cov>=5.0.0", "hypothesis>=6.100.0", "rfc8785>=0.1.4", - "policyengine-core==3.33.0", + "policyengine-core==3.33.3", "policyengine-us==2.2.1", - "policyengine-uk==2.130.1", + "policyengine-uk==2.132.6", "spm-calculator==1.0.0", ] diff --git a/scripts/bundle.py b/scripts/bundle.py index 42179f4fe..53b74a26a 100644 --- a/scripts/bundle.py +++ b/scripts/bundle.py @@ -90,11 +90,12 @@ def _check(args: argparse.Namespace) -> int: generate, load_bundle_manifest, validate_bundle_measurements, + validate_dataset_overlays, ) - validate_bundle_measurements( - load_bundle_manifest(), require_published_spm=args.published_spm - ) + bundle = load_bundle_manifest() + validate_bundle_measurements(bundle, require_published_spm=args.published_spm) + validate_dataset_overlays(bundle) result = generate(check=True) if not args.include_tros: return result diff --git a/scripts/check_uk_candidate.py b/scripts/check_uk_candidate.py index b82c432c4..7920b8c33 100644 --- a/scripts/check_uk_candidate.py +++ b/scripts/check_uk_candidate.py @@ -34,6 +34,7 @@ import subprocess import sys import time +from collections.abc import Sequence from functools import lru_cache from pathlib import Path from typing import Any, Optional @@ -75,6 +76,8 @@ candidate_action, candidate_branch, candidate_pr_body, + certified_data_line, + data_line, expected_release_bundle, finalize_attempt, load_attempt_budget_inputs, @@ -464,10 +467,14 @@ def cmd_finalize(args: argparse.Namespace) -> int: attempt, Path(args.ledger), issued_by=issued_by(), completed_at=utc_now() ) payload = read_json(attempt / "candidate.json") - body = candidate_pr_body( - receipt, receipt.pair, ledger_url=ledger_url(receipt.pair_id) - ) if args.pr_body: + line = certified_data_line(json.loads(BUNDLE_PATH.read_text())) + body = candidate_pr_body( + receipt, + receipt.pair, + ledger_url=ledger_url(receipt.pair_id), + on_main_line=data_line(receipt.pair) == line, + ) Path(args.pr_body).write_text(body) print(f"verdict {receipt.verdict}; receipt {receipt.receipt_sha256}") set_output( @@ -528,6 +535,7 @@ def owner_run_failed(run_id: str) -> bool: def cmd_recoverable(args: argparse.Namespace) -> int: now = utc_now() + line = certified_data_line(json.loads(BUNDLE_PATH.read_text())) for pair_dir in sorted((Path(args.ledger) / "uk").glob("*")): receipt = pair_dir / "qualification_receipt.json" stages = list((pair_dir / "attempts").glob("*/staging.json")) @@ -544,8 +552,22 @@ def cmd_recoverable(args: argparse.Namespace) -> int: attempt, terminal ): continue - if attempt_is_recoverable(attempt, now=now, owner_is_failed=owner_run_failed): - print(json.dumps(read_json(attempt / "candidate.json"))) + if not attempt_is_recoverable( + attempt, now=now, owner_is_failed=owner_run_failed + ): + continue + candidate = read_json(attempt / "candidate.json") + # The catch-up re-dispatches without force, and a pair from another + # data line then resolves as other_line before any recovery, so the + # dispatch would clear nothing. A forced run is re-run by hand. + if data_line(CandidatePair.model_validate(candidate["pair"])) != line: + print( + f"::notice::not recovering {pair_dir.name}: its pair is not on " + "main's data line; re-run it by hand with force", + file=sys.stderr, + ) + continue + print(json.dumps(candidate)) return 0 @@ -856,7 +878,12 @@ def cmd_refresh_pr_checks(args: argparse.Namespace) -> int: pull_requests = {pr["headRefName"]: pr for pr in _candidate_pull_requests()} now = _now() + line = certified_data_line(json.loads(BUNDLE_PATH.read_text())) for receipt in latest_receipts(Path(args.ledger)): + if data_line(receipt.pair) != line: + # Promotion never takes a pair from another line, so its runs + # are not worth re-running. + continue branch = candidate_branch(receipt.pair) head = receipt.harness.git_commit pull_request = pull_requests.get(branch) @@ -909,6 +936,35 @@ def cmd_refresh_pr_checks(args: argparse.Namespace) -> int: return 0 +def _close_pull_requests(closes: Sequence[tuple[str, str]]) -> set[str]: + """Close each ``(number, comment)``; return the numbers GitHub closed. + + A failed close is reported, and the pull request stays open for the next + run to close. + """ + + closed = set() + for number, comment in closes: + result = subprocess.run( + [ + "gh", + "pr", + "close", + number, + "--repo", + LEDGER_REPOSITORY, + "--comment", + comment, + ], + check=False, + ) + if result.returncode == 0: + closed.add(number) + else: + print(f"::warning::could not close pull request {number}") + return closed + + def cmd_promote(args: argparse.Namespace) -> int: bundle = json.loads(BUNDLE_PATH.read_text()) problems: list[str] = [] @@ -930,29 +986,37 @@ def cmd_promote(args: argparse.Namespace) -> int: ) for branch in plan.close_branches: print(f"superseded {branch}") + for branch in plan.close_off_line: + print(f"off main's data line {branch}") if not args.apply: return 0 - for branch in plan.close_branches: - subprocess.run( - [ - "gh", - "pr", - "close", - numbers.get(branch, branch), - "--repo", - LEDGER_REPOSITORY, - "--comment", - f"Superseded: {plan.reason}. The ledger keeps this candidate's evidence.", - ], - check=False, - ) + closed = _close_pull_requests( + [ + ( + numbers[branch], + f"Superseded: {plan.reason}. The ledger keeps this candidate's " + "evidence.", + ) + for branch in plan.close_branches + ] + + [ + ( + numbers[branch], + "This pair is not on main's data line, so it is never promoted. " + "The ledger keeps this candidate's evidence.", + ) + for branch in plan.close_off_line + ] + ) # Only the plan's target may be ready, and only while _promote finds it # on current main: any other ready candidate (left by a run that could # not undo it, or by older code) goes back to draft. returned = [ _set_ready(str(pr["number"]), False) for pr in sorted(pull_requests, key=lambda pr: pr["number"]) - if not pr["isDraft"] and pr["headRefName"] != plan.ready_branch + if not pr["isDraft"] + and pr["headRefName"] != plan.ready_branch + and str(pr["number"]) not in closed ] safe = True if plan.ready_branch is not None and plan.ready_commit is not None: diff --git a/scripts/generate_bundle_artifacts.py b/scripts/generate_bundle_artifacts.py index cdbfaea79..e941bb856 100644 --- a/scripts/generate_bundle_artifacts.py +++ b/scripts/generate_bundle_artifacts.py @@ -37,6 +37,7 @@ _VALIDATION_MODULE = importlib.util.module_from_spec(_VALIDATION_SPEC) _VALIDATION_SPEC.loader.exec_module(_VALIDATION_MODULE) validate_bundle_measurements = _VALIDATION_MODULE.validate_bundle_measurements +validate_dataset_overlays = _VALIDATION_MODULE.validate_dataset_overlays OPTIONAL_DEPENDENCIES_HEADER = "[project.optional-dependencies]" NEXT_SECTION_PATTERN = re.compile(r"\n\[tool\.setuptools\]", re.MULTILINE) diff --git a/scripts/generate_trace_tros.py b/scripts/generate_trace_tros.py index b34562f82..656d1868b 100644 --- a/scripts/generate_trace_tros.py +++ b/scripts/generate_trace_tros.py @@ -22,7 +22,10 @@ from jsonschema import Draft202012Validator from release_lock import PYPI, load_toml, validate_registry_artifacts -from policyengine.provenance.certification import validate_release_manifest +from policyengine.provenance.certification import ( + core_claim_warning, + validate_release_manifest, +) from policyengine.provenance.manifest import ( CountryReleaseManifest, DataReleaseManifest, @@ -138,16 +141,44 @@ def _validate_runtime_model(bundle: dict, country: CountryReleaseManifest) -> No model = country.model_package prefix = f"{country.country_id}: strict TRACE release" certification = country.certification - if certification and certification.compatibility_basis == "qualified_exact_pair": - core = bundle.get("packages", {}).get("policyengine-core", {}) - if ( - not certification.qualification_pair_id - or not certification.certified_for_core_version - or core.get("version") != certification.certified_for_core_version - ): - raise ValueError( - f"{prefix}: runtime core differs from the qualified exact pair" - ) + core = bundle.get("packages", {}).get("policyengine-core", {}) + exact_pair = ( + certification is not None + and certification.compatibility_basis == "qualified_exact_pair" + ) + if exact_pair and not ( + certification.qualification_pair_id and certification.certified_for_core_version + ): + raise ValueError( + f"{prefix}: a qualified exact pair must name its pair and core" + ) + if ( + certification is not None + and certification.core_binding is not None + and (certification.core_binding == "certified") + != (certification.certified_for_core_version is not None) + ): + raise ValueError( + f"{prefix}: core_binding {certification.core_binding!r} disagrees with " + "certified_for_core_version" + ) + if ( + certification is not None + and certification.certified_for_core_version + and core.get("version") != certification.certified_for_core_version + ): + # Receipts certify the core the certificate names, not the pin, so a + # core-only pin change would leave every receipt uncertified. + remedy = ( + "requalify the pair for the new core" + if exact_pair + else "re-certify the data release for the new core, or without " + "--core-version so that the certificate follows the core pin" + ) + raise ValueError( + f"{prefix}: runtime core {core.get('version')} differs from the " + f"certified core {certification.certified_for_core_version}; {remedy}" + ) if ( model.name != key or component.get("name") != model.name @@ -222,6 +253,20 @@ def generated_tros(*, strict: bool = False) -> list[tuple[Path, bytes]]: ) if strict: _validate_reviewed_inputs(country_manifest, data_release_manifest, tro_path) + certification = country_manifest.certification + core = ( + bundle.get("packages", {}).get("policyengine-core", {}).get("version") + ) + if ( + certification is not None + and certification.certified_for_core_version is None + and core is not None + ): + # The certificate follows the core pin: an unclaimed pin is + # reported, not refused. + warning = core_claim_warning(data_release_manifest, core) + if warning is not None: + print(f"warning: {country_id}: {warning}", file=sys.stderr) tro = build_trace_tro_from_release_bundle( country_manifest, data_release_manifest, diff --git a/scripts/release_consumers.py b/scripts/release_consumers.py index 57ab70cf8..9c96a06a5 100644 --- a/scripts/release_consumers.py +++ b/scripts/release_consumers.py @@ -29,6 +29,10 @@ import requests # noqa: E402 +from policyengine.provenance.manifest import ( # noqa: E402 + https_dataset_uri, + hugging_face_auth_headers, +) from policyengine.provenance.qualification_ledger import ( # noqa: E402 fetch_ledger_receipt, ) @@ -52,17 +56,48 @@ def _pypi(path: str) -> dict: return response.json() -def _hf_file(repo_id: str, revision: str, path: str) -> bytes: - token = os.environ.get("HUGGING_FACE_TOKEN") or os.environ.get("HF_TOKEN") +def _hf_file(repo_id: str, revision: str, path: str, repo_type: str = "model") -> bytes: response = requests.get( - f"https://huggingface.co/{repo_id}/resolve/{revision}/{path}", - headers={"Authorization": f"Bearer {token}"} if token else {}, + https_dataset_uri(repo_id, path, revision, repo_type=repo_type), + headers=hugging_face_auth_headers(), timeout=TIMEOUT, ) response.raise_for_status() return response.content +def _hf_size(repo_id: str, revision: str, path: str, repo_type: str = "model") -> int: + """A file's size from Hugging Face's response headers, without downloading it. + + A large-file pointer answers HEAD with a redirect that carries + ``X-Linked-Size``. A small file answers with its own ``Content-Length``, + or with a redirect that leads to it, which is then followed. + """ + + url = https_dataset_uri(repo_id, path, revision, repo_type=repo_type) + # A compressed answer has no Content-Length, so ask for the bytes as they + # are, as huggingface_hub does. + headers = {**hugging_face_auth_headers(), "Accept-Encoding": "identity"} + response = requests.head( + url, headers=headers, timeout=TIMEOUT, allow_redirects=False + ) + response.raise_for_status() + size = response.headers.get("X-Linked-Size") + if size is None and response.status_code == 200: + size = response.headers.get("Content-Length") + elif size is None: + followed = requests.head( + url, headers=headers, timeout=TIMEOUT, allow_redirects=True + ) + followed.raise_for_status() + size = followed.headers.get("Content-Length") + if not size: + raise SystemExit( + f"Hugging Face reports no size for {repo_id}/{path}@{revision}." + ) + return int(size) + + def cmd_payload(args: argparse.Namespace) -> int: payload = release_dispatch_payload( _bundle(), version=args.version, fetch_receipt=fetch_ledger_receipt @@ -126,6 +161,7 @@ def cmd_scorecard_request(args: argparse.Namespace) -> int: ).hexdigest(), release_bundle=release_bundle, hf_file=_hf_file, + hf_size=_hf_size, pypi_project=_pypi, requested_by={ "repository": repository, diff --git a/src/policyengine/data/bundle/manifest.json b/src/policyengine/data/bundle/manifest.json index cc56a3ebc..c6a8eff3d 100644 --- a/src/policyengine/data/bundle/manifest.json +++ b/src/policyengine/data/bundle/manifest.json @@ -16,99 +16,120 @@ }, "data_releases": { "uk": { - "build_id": "policyengine-uk-data-1.58.1", + "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", "bundle_id": "uk-6.9.0", "certification": { - "built_with_model_git_sha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", - "built_with_model_version": "2.122.2", + "built_with_model_version": "2.132.6", "certified_by": "policyengine.py bundle certification", - "certified_for_model_version": "2.130.1", - "compatibility_basis": "legacy_compatible_model_package", - "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", - "data_build_id": "policyengine-uk-data-1.58.1" + "certified_for_model_version": "2.132.6", + "compatibility_basis": "built_with_model_package", + "core_binding": "bundle_pin", + "data_build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" }, "certified_data_artifact": { - "build_id": "policyengine-uk-data-1.58.1", + "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", "data_package": { - "name": "policyengine-uk-data", - "version": "1.58.1" + "name": "microcosm-data", + "version": "0.1.0" }, - "dataset": "enhanced_frs_2024_25", - "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", - "uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1" + "dataset": "microcosm_uk_2024_25", + "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece", + "uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" }, "country_id": "uk", "data_package": { - "name": "policyengine-uk-data", - "release_manifest_path": "releases/1.58.1/release_manifest.json", - "release_manifest_revision": "89be6b73fc1e6eeba6dae2f9400676d8510bcd51", - "repo_id": "policyengine/policyengine-uk-data-private", - "repo_type": "model", - "version": "1.58.1" + "name": "microcosm-data", + "release_manifest_path": "releases/microcosm-uk-2024-25-national/release_manifest.json", + "release_manifest_revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "version": "0.1.0" }, "data_producer": "populace", "datasets": { - "enhanced_frs_2024_25": { - "path": "enhanced_frs_2024_25.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + "calibration_diagnostics": { + "path": "releases/microcosm-uk-2024-25-national/calibration_diagnostics.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "34978e2bfde3f2641769c9c3a8f60455c59ecae5a50524b359d74388d7edc888" + }, + "microcosm_uk_2024_25": { + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece" + }, + "microcosm_uk_2024_25_calibration": { + "path": "microcosm_uk_2024_25_calibration.npz", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "104a8253bf1aebc91147c9bbb40e2603d1025317ba8c6b991383d5e4d91852c2" }, - "enhanced_frs_2024_25_tiny": { - "path": "enhanced_frs_2024_25_tiny.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "d5d2bde80f1f028fec98391ea31bb5a0cf8449e3867fc3c1bfe23693b9faecf8" + "microcosm_uk_2024_25_dense": { + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe", + "sha256": "8e10e130d0260e0b52e2e29902f0502ebdff76a14a3e57d2c876bf4cd8db5872" }, - "frs_2024_25": { - "path": "frs_2024_25.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "97af00b2cd2739957f40dcc4de8e0383a6a91d47952c90d4b4921a6007bd2021" + "release_certification": { + "path": "releases/microcosm-uk-2024-25-national/release_certification.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "6a68cc93595fc5a7ad451d75a902523c25659de27014f50431dd5e19af5bd959" }, - "frs_2024_25_tiny": { - "path": "frs_2024_25_tiny.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "e4d7662cd3f053620224b7ceff17266783f09407002a72633b043b844cec3f5a" + "release_cut_gates": { + "path": "releases/microcosm-uk-2024-25-national/release_cut_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "c91c1364d9c3be98493039d4e1dd8c2256881a434c1f124dd2ed0701bfaf7751" }, - "local_authority_weights": { - "path": "local_authority_weights.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "2b7a2f7f9e7081a394b37cb84e909fd3c01d22e421808141684e7c3be59f2089" + "score_vs_enhanced_frs": { + "path": "releases/microcosm-uk-2024-25-national/score_vs_enhanced_frs.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "ba8a675f36e7c836bf5521e5933e21bb689442734b7340e05cabd3e0cc55323d" }, - "local_geography_weights": { - "path": "local_geography_weights.csv.gz", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "97c6e3542543baf737a713bb918ebc3d3dbc3f3765be9bf3414375c322d0e3d1" + "spine_gates": { + "path": "releases/microcosm-uk-2024-25-national/spine_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "6dbba976dd1304c07da531382cecb11c9cb69512de0ee65126b71f032a0f37d6" }, - "parliamentary_constituency_weights": { - "path": "parliamentary_constituency_weights.h5", - "repo_id": "policyengine/policyengine-uk-data-private", - "revision": "1.58.1", - "sha256": "8c696e4d65a22033deb4c4f69a767fdb46e0a4f5580f36b7fb47ee4722629b11" + "terminal_gates": { + "path": "releases/microcosm-uk-2024-25-national/terminal_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "e29439d5ce0ea098d32579284ac06f26f086feba2fb8d12d1e88ac9a54840808" } }, - "default_dataset": "enhanced_frs_2024_25", - "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1", + "default_dataset": "microcosm_uk_2024_25", + "default_dataset_uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", "model_package": { "name": "policyengine-uk", - "sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873", - "version": "2.130.1", - "wheel_url": "https://files.pythonhosted.org/packages/1a/21/88e8addba0eabd9a7d3a29fa5293e141feb405baf0432209387f9a1f467a/policyengine_uk-2.130.1-py3-none-any.whl" + "sha256": "cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354", + "version": "2.132.6", + "wheel_url": "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl" }, "policyengine_version": "6.9.0", "region_datasets": { + "constituency": { + "path_template": "microcosm_uk_2024_25_dense.h5" + }, + "local_authority": { + "path_template": "microcosm_uk_2024_25_dense.h5" + }, "national": { - "path_template": "enhanced_frs_2024_25.h5" + "path_template": "microcosm_uk_2024_25.h5" } }, - "release_manifest_uri": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json", + "regional_release_kind": "local_area", + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", + "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", "schema_version": 1, - "source_manifest_uri": "hf://model/policyengine/policyengine-uk-data-private@89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json", - "version": "policyengine-uk-data-1.58.1" + "source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", + "version": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" }, "us": { "build_id": "populace-us-2024-spm-20260915", @@ -247,6 +268,22 @@ "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" }, + "enhanced_frs_2024_25": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + }, + "enhanced_frs_2024_25_tiny": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "d5d2bde80f1f028fec98391ea31bb5a0cf8449e3867fc3c1bfe23693b9faecf8" + }, "frs_2023_24": { "data_package_name": "policyengine-uk-data", "path": "frs_2023_24.h5", @@ -255,6 +292,22 @@ "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", "sha256": "df26d4d7af9d164aa2d064181b39290292d2f62bb26fee6126fc095fc06da292" }, + "frs_2024_25": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "97af00b2cd2739957f40dcc4de8e0383a6a91d47952c90d4b4921a6007bd2021" + }, + "frs_2024_25_tiny": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "e4d7662cd3f053620224b7ceff17266783f09407002a72633b043b844cec3f5a" + }, "populace_uk_2023": { "data_package_name": "populace-data", "path": "populace_uk_2023.h5", @@ -302,18 +355,18 @@ }, "policyengine-core": { "import_name": "policyengine_core", - "install_requirement": "policyengine-core==3.33.0", + "install_requirement": "policyengine-core==3.33.3", "name": "policyengine-core", "role": "runtime_dependency", - "version": "3.33.0" + "version": "3.33.3" }, "policyengine-uk": { "country": "uk", "import_name": "policyengine_uk", - "install_requirement": "policyengine-uk==2.130.1", + "install_requirement": "policyengine-uk==2.132.6", "name": "policyengine-uk", "role": "country_model", - "version": "2.130.1" + "version": "2.132.6" }, "policyengine-us": { "country": "us", diff --git a/src/policyengine/data/bundle/uk.trace.tro.jsonld b/src/policyengine/data/bundle/uk.trace.tro.jsonld index 66d99811a..4fdaa92b7 100644 --- a/src/policyengine/data/bundle/uk.trace.tro.jsonld +++ b/src/policyengine/data/bundle/uk.trace.tro.jsonld @@ -17,7 +17,7 @@ "schema:name": "PolicyEngine", "schema:url": "https://policyengine.org" }, - "schema:dateCreated": "2026-10-14T07:43:49.920588Z", + "schema:dateCreated": "2026-10-19T10:17:15.545356+00:00", "schema:description": "TRACE TRO for certified runtime bundle uk-6.9.0 covering the bundle manifest, the certified dataset artifact, the country model wheel, and the country data release manifest when it is available.", "schema:name": "policyengine uk certified bundle TRO", "trov:createdWith": { @@ -45,7 +45,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/data_release_manifest" }, - "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json" }, { "@id": "arrangement/1/location/dataset", @@ -53,7 +53,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/dataset" }, - "trov:hasLocation": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/1.58.1/enhanced_frs_2024_25.h5" + "trov:hasLocation": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/microcosm_uk_2024_25.h5" }, { "@id": "arrangement/1/location/model_wheel", @@ -61,7 +61,7 @@ "trov:hasArtifact": { "@id": "composition/1/artifact/model_wheel" }, - "trov:hasLocation": "https://files.pythonhosted.org/packages/1a/21/88e8addba0eabd9a7d3a29fa5293e141feb405baf0432209387f9a1f467a/policyengine_uk-2.130.1-py3-none-any.whl" + "trov:hasLocation": "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl" } ] } @@ -75,52 +75,51 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for uk", "trov:mimeType": "application/json", - "trov:sha256": "c32e4566725879cda855ec3852e951ee9fda7146e288d5baf05ae628ae49acdd" + "trov:sha256": "6bbbfdfb433a5d8de4b1110238311338fd0425d92d4b34f06e7f8d3db3001fad" }, { "@id": "composition/1/artifact/data_release_manifest", "@type": "trov:ResearchArtifact", - "schema:name": "policyengine-uk-data release manifest 1.58.1", + "schema:name": "microcosm-data release manifest 0.1.0", "trov:mimeType": "application/json", - "trov:sha256": "545797bf231cd3681e64b677065d921304e9a32f1e46e757a940a412b4136c17" + "trov:sha256": "49f9fa820db2e6544ff5cdceeb2d868a504843a6a1c1c06b37188621e700066f" }, { "@id": "composition/1/artifact/dataset", "@type": "trov:ResearchArtifact", - "schema:name": "enhanced_frs_2024_25", + "schema:name": "microcosm_uk_2024_25", "trov:mimeType": "application/x-hdf5", - "trov:sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + "trov:sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece" }, { "@id": "composition/1/artifact/model_wheel", "@type": "trov:ResearchArtifact", - "schema:name": "policyengine-uk==2.130.1 wheel", + "schema:name": "policyengine-uk==2.132.6 wheel", "trov:mimeType": "application/zip", - "trov:sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873" + "trov:sha256": "cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354" } ], "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "e9b6a021cef492d7e045394a0d814345793fd141e40d9c715723c8cb49963b69" + "trov:sha256": "1cce237175e76a9e5f9dffeb6ec3573dbc78c5facc5a7c55d06886d453641e3a" } }, "trov:hasPerformance": { "@id": "trp/1", "@type": "trov:TransparentResearchPerformance", - "pe:builtWithModelGitSha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", - "pe:builtWithModelVersion": "2.122.2", + "pe:builtWithModelVersion": "2.132.6", "pe:certifiedBy": "policyengine.py bundle certification", - "pe:certifiedForModelVersion": "2.130.1", - "pe:compatibilityBasis": "legacy_compatible_model_package", - "pe:dataBuildFingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", - "pe:dataBuildId": "policyengine-uk-data-1.58.1", + "pe:certifiedForModelVersion": "2.132.6", + "pe:compatibilityBasis": "built_with_model_package", + "pe:coreBinding": "bundle_pin", + "pe:dataBuildId": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", "pe:emittedIn": "repository-bundle", - "rdfs:comment": "Certification of build policyengine-uk-data-1.58.1 for policyengine-uk 2.130.1.", + "rdfs:comment": "Certification of build microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71 for policyengine-uk 2.132.6.", "trov:accessedArrangement": { "@id": "arrangement/1" }, - "trov:startedAtTime": "2026-10-14T07:43:49.920588Z", + "trov:startedAtTime": "2026-10-19T10:17:15.545356+00:00", "trov:wasConductedBy": { "@id": "trs" } diff --git a/src/policyengine/data/bundle/us.trace.tro.jsonld b/src/policyengine/data/bundle/us.trace.tro.jsonld index 0b5e8cf35..dc9975833 100644 --- a/src/policyengine/data/bundle/us.trace.tro.jsonld +++ b/src/policyengine/data/bundle/us.trace.tro.jsonld @@ -74,7 +74,7 @@ "@type": "trov:ResearchArtifact", "schema:name": "policyengine.py bundle manifest for us", "trov:mimeType": "application/json", - "trov:sha256": "c32e4566725879cda855ec3852e951ee9fda7146e288d5baf05ae628ae49acdd" + "trov:sha256": "6bbbfdfb433a5d8de4b1110238311338fd0425d92d4b34f06e7f8d3db3001fad" }, { "@id": "composition/1/artifact/data_release_manifest", @@ -101,7 +101,7 @@ "trov:hasFingerprint": { "@id": "composition/1/fingerprint", "@type": "trov:CompositionFingerprint", - "trov:sha256": "86eb651ac706bcb1858e4b6b995f56f5e8fb868c9ae00ce25cee0207528012c5" + "trov:sha256": "35c18fb640a23dfd78be0933a636c3088973934d7eff61ed2223c020da065d79" } }, "trov:hasPerformance": { diff --git a/src/policyengine/outputs/uk_geography_disclosure.py b/src/policyengine/outputs/uk_geography_disclosure.py index 3e589f4a6..470557e4a 100644 --- a/src/policyengine/outputs/uk_geography_disclosure.py +++ b/src/policyengine/outputs/uk_geography_disclosure.py @@ -26,6 +26,7 @@ Release, estimate_change, ) +from policyengine.provenance.manifest import uk_area_dataset_hint if TYPE_CHECKING: from policyengine.core.simulation import Simulation @@ -252,9 +253,11 @@ def areas_for(level: str) -> Optional[AreaWeights]: if areas is None: if level in self.levels: columns = UK_DISCLOSURE_LEVEL_COLUMNS[level] + hint = uk_area_dataset_hint(level) raise ValueError( f"The household output gives no household an area in " f"any of {list(columns)} for level {level!r}" + + (f". {hint}" if hint else "") ) continue estimated[level] = areas diff --git a/src/policyengine/outputs/uk_geography_impact.py b/src/policyengine/outputs/uk_geography_impact.py index f6158d722..ded47ed28 100644 --- a/src/policyengine/outputs/uk_geography_impact.py +++ b/src/policyengine/outputs/uk_geography_impact.py @@ -14,8 +14,14 @@ default_local_search_dirs, packaged_uk_area_list_path, ) +from policyengine.provenance.manifest import uk_area_dataset_hint logger = logging.getLogger(__name__) +#: The region type whose dataset carries each longwise area column. +AREA_REGION_TYPES = { + "constituency_code_oa": "constituency", + "la_code_oa": "local_authority", +} def warn_unprotected_uk_geography_output(name: str, level: str) -> None: @@ -142,10 +148,12 @@ def compute_longwise_uk_geography_impacts( "household_weight", ]: if column not in baseline_household.columns: + hint = uk_area_dataset_hint(AREA_REGION_TYPES.get(column, "")) raise ValueError( f"UK geography impacts require baseline household column " f"'{column}'. Re-run the simulation with an output dataset " f"that preserves UK geography passthrough columns." + + (f" {hint}" if hint else "") ) if "household_net_income" not in reform_household.columns: raise ValueError( diff --git a/src/policyengine/provenance/bundle_validation.py b/src/policyengine/provenance/bundle_validation.py index 51324de80..51c474481 100644 --- a/src/policyengine/provenance/bundle_validation.py +++ b/src/policyengine/provenance/bundle_validation.py @@ -24,6 +24,46 @@ def _is_normalized_version(value: Any) -> bool: ) +def shadowed_dataset_overlays( + release: Mapping[str, Any], overlays: Mapping[str, Any] +) -> list[str]: + """The overlay names a country's certified release itself certifies. + + Overlays are strictly additive: one may never shadow the certified default + or any certified dataset, so it can never alter resolution. The runtime + refuses a shadowing overlay, maintenance checks refuse it first, and + certification removes the overlays a new release supersedes. + """ + certified = set(release.get("datasets") or {}) + if release.get("default_dataset"): + certified.add(release["default_dataset"]) + return sorted(set(overlays or {}) & certified) + + +def shadowing_overlays_message(country: str, shadowed: list[str]) -> str: + """The one refusal for overlays that shadow certified datasets.""" + + return ( + f"dataset_overlays.{country} shadows certified datasets: " + f"{', '.join(shadowed)}. Overlays must be additive and never change " + "resolution: remove those overlays." + ) + + +def validate_dataset_overlays(bundle: Mapping[str, Any]) -> None: + """Refuse an overlay that shadows a certified dataset name. + + The runtime refuses such a bundle on every release-manifest read, so + ``import policyengine`` would fail; maintenance checks refuse it first. + """ + overlays = bundle.get("dataset_overlays") or {} + for country, entries in overlays.items(): + release = (bundle.get("data_releases") or {}).get(country) or {} + shadowed = shadowed_dataset_overlays(release, entries) + if shadowed: + raise ValueError(shadowing_overlays_message(country, shadowed)) + + def validate_spm_configuration(configuration: Any) -> None: """Validate JSON defaults using only the standard library. diff --git a/src/policyengine/provenance/certification.py b/src/policyengine/provenance/certification.py index 31327f576..dc773e215 100644 --- a/src/policyengine/provenance/certification.py +++ b/src/policyengine/provenance/certification.py @@ -52,6 +52,7 @@ CONSTITUENCY_ASSET_SPEC, LOCAL_AUTHORITY_ASSET_SPEC, ) +from policyengine.provenance.bundle_validation import shadowed_dataset_overlays from policyengine.provenance.manifest import ( HF_REQUEST_TIMEOUT_SECONDS, DataReleaseArtifact, @@ -421,20 +422,8 @@ def _check_core_compatibility( case the certificate's basis must be the qualified exact pair. """ - build = manifest.build - built_with_core = ( - build.built_with_core_package.version - if build is not None and build.built_with_core_package is not None - else None - ) - core_claims = [ - package.specifier - for package in manifest.compatible_core_packages - if package.name == "policyengine-core" - ] - if built_with_core == core_version or any( - _specifier_matches(core_version, specifier) for specifier in core_claims - ): + built_with_core, core_claims = _core_claims(manifest) + if _core_is_claimed(manifest, core_version): return False if qualified_pair_id is not None: warnings.append( @@ -450,6 +439,53 @@ def _check_core_compatibility( ) +def _core_claims(manifest: DataReleaseManifest) -> tuple[Optional[str], list[str]]: + """A release's build-time core and its publisher core claims.""" + + build = manifest.build + built_with_core = ( + build.built_with_core_package.version + if build is not None and build.built_with_core_package is not None + else None + ) + core_claims = [ + package.specifier + for package in manifest.compatible_core_packages + if package.name == "policyengine-core" + ] + return built_with_core, core_claims + + +def _core_is_claimed(manifest: DataReleaseManifest, core_version: str) -> bool: + """Whether a release was built with, or claims, ``core_version``.""" + + built_with_core, core_claims = _core_claims(manifest) + return built_with_core == core_version or any( + _specifier_matches(core_version, specifier) for specifier in core_claims + ) + + +def core_claim_warning( + manifest: DataReleaseManifest, core_version: str +) -> Optional[str]: + """Why ``core_version`` is outside a release's core claims, or None. + + For a certificate that follows the bundle's core pin: the pin is not held + to the release's claims, so a pin outside them is reported, not refused. + """ + + if _core_is_claimed(manifest, core_version): + return None + built_with_core, core_claims = _core_claims(manifest) + return ( + f"the bundle's policyengine-core {core_version} is neither the " + f"build-time core ({built_with_core or 'unrecorded'}) nor in the " + f"release's core claims {core_claims or '(none declared)'}; the " + "certificate names no core, so it follows the core pin, and receipts " + "certify that core without the release's claim" + ) + + def head_artifact(artifact, token: Optional[str] = None) -> bool: # Repo type is not recorded per artifact in the schema; try dataset # first (the publishing convention), then model-style. @@ -1001,6 +1037,9 @@ def build_country_manifest_payload( } if core_version is not None: certification["certified_for_core_version"] = core_version + certification["core_binding"] = "certified" + else: + certification["core_binding"] = "bundle_pin" if qualified_pair_id is not None: certification["qualification_pair_id"] = qualified_pair_id certification["qualification_ledger_url"] = ( @@ -1109,6 +1148,7 @@ def certify( regional_release_kind: RegionalReleaseKind = "local_area", core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, + bundle_core_version: Optional[str] = None, ) -> tuple[dict, str, list[str]]: raise NotImplementedError @@ -1131,6 +1171,7 @@ def certify( regional_release_kind: RegionalReleaseKind = "local_area", core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, + bundle_core_version: Optional[str] = None, ) -> tuple[dict, str, list[str]]: raise CertificationError( "Legacy data-producer certification updates are not implemented in " @@ -1156,10 +1197,13 @@ def certify( regional_release_kind: RegionalReleaseKind = "local_area", core_version: Optional[str] = None, qualified_pair_id: Optional[str] = None, + bundle_core_version: Optional[str] = None, ) -> tuple[dict, str, list[str]]: manifest, manifest_sha256, uri_parts = fetch_release_manifest( manifest_uri, token=token ) + primary_manifest = manifest + regional_manifest = None regional_uri_parts = None recorded_release_kind = None regional_artifacts: dict[str, DataReleaseArtifact] = {} @@ -1209,6 +1253,18 @@ def certify( qualified_pair_id=qualified_pair_id, ) warnings = warnings + regional_warnings + if core_version is None and bundle_core_version is not None: + # The certificate follows the bundle's core pin: report, not + # refuse, a pin that a release does not claim. + for label, release in ( + ("data release", primary_manifest), + ("regional release", regional_manifest), + ): + if release is None: + continue + warning = core_claim_warning(release, bundle_core_version) + if warning is not None: + warnings.append(f"{label}: {warning}") for filename in required_supplemental_release_files( country, manifest, uri_parts @@ -1370,8 +1426,39 @@ def certify_data_release( regional_release_kind=regional_release_kind, core_version=core_version, qualified_pair_id=qualified_pair_id, + bundle_core_version=( + (bundle.get("packages") or {}).get("policyengine-core") or {} + ).get("version"), ) + previous_release = (bundle.get("data_releases") or {}).get(country) or {} bundle.setdefault("data_releases", {})[country] = data_release + # Overlays may never shadow a certified name, so a release that certifies + # a name the bundle keeps as an overlay (a rollback to the release it came + # from, or the dataset a national-first route carries) supersedes it. + overlays = (bundle.get("dataset_overlays") or {}).get(country) or {} + superseded = shadowed_dataset_overlays(data_release, overlays) + for name in superseded: + del overlays[name] + if superseded: + warnings = [ + *warnings, + f"Removed the dataset_overlays.{country} entries this release now " + f"certifies: {', '.join(superseded)}.", + ] + # A name the previous release certified, and neither the new release nor + # an overlay keeps, stops resolving. Say so, so that it can be re-added. + dropped = sorted( + set(previous_release.get("datasets") or {}) + - set(data_release["datasets"]) + - set(overlays) + ) + if dropped: + warnings = [ + *warnings, + f"These {country.upper()} datasets no longer resolve by name: " + f"{', '.join(dropped)}. Re-add any that must stay loadable under " + f"dataset_overlays.{country}.", + ] bundle.setdefault("countries", {}).setdefault(country, {})["model_package"] = ( model_package ) diff --git a/src/policyengine/provenance/dataset_materialization.py b/src/policyengine/provenance/dataset_materialization.py index 35824357f..28e8f60b9 100644 --- a/src/policyengine/provenance/dataset_materialization.py +++ b/src/policyengine/provenance/dataset_materialization.py @@ -5,7 +5,7 @@ import os import tempfile from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Literal, Optional import requests @@ -84,6 +84,37 @@ def metadata_destination(self) -> Path: return Path(f"{self.destination}.metadata.json") +def _bundle_destination(data_dir: Path, path: str) -> Path: + """Where a bundle artifact is cached. + + A top-level file is cached by its name. A nested one keeps its release + path, so two releases' files of one name (each country's + ``calibration_diagnostics.json``) never share a cache entry. + """ + + relative = PurePosixPath(path) + # Repository paths use forward slashes. A backslash, or a drive or root in + # any segment, would make the platform path leave data_dir on Windows. + if ( + relative.is_absolute() + or ".." in relative.parts + or "\\" in path + or any( + PureWindowsPath(part).drive or PureWindowsPath(part).root + for part in relative.parts + ) + ): + raise DatasetMaterializationError( + f"Bundle artifact path {path!r} must be relative to its repository." + ) + destination = data_dir.joinpath(*relative.parts) + if destination == data_dir or not destination.is_relative_to(data_dir): + raise DatasetMaterializationError( + f"Bundle artifact path {path!r} must be relative to its repository." + ) + return destination + + def _resolve_bundle_dataset( country_id: str, dataset: Optional[str] = None, @@ -116,7 +147,7 @@ def _resolve_bundle_dataset( revision=reference.revision or _artifact_revision(country_manifest.data_package), sha256=reference.sha256, - destination=data_dir / Path(reference.path).name, + destination=_bundle_destination(data_dir, reference.path), metadata_sha256=reference.metadata_sha256, ) diff --git a/src/policyengine/provenance/manifest.py b/src/policyengine/provenance/manifest.py index d43e3f74b..f3b15fb5d 100644 --- a/src/policyengine/provenance/manifest.py +++ b/src/policyengine/provenance/manifest.py @@ -10,6 +10,11 @@ import requests from pydantic import BaseModel, Field +from policyengine.provenance.bundle_validation import ( + shadowed_dataset_overlays, + shadowing_overlays_message, +) + HF_REQUEST_TIMEOUT_SECONDS = 30 PYPI_REQUEST_TIMEOUT_SECONDS = 30 @@ -167,6 +172,14 @@ class DataCertification(BaseModel): data_build_fingerprint: Optional[str] = None certified_by: Optional[str] = None certified_for_core_version: Optional[str] = None + #: How the certificate covers policyengine-core. ``certified``: it names + #: the core it was certified for (``certified_for_core_version``), and the + #: bundle's core pin must equal it. ``bundle_pin``: it certifies whatever + #: core the bundle pins, without holding the pin to the data release's + #: core claims; certification and the strict check warn when the pin is + #: outside them. Absent in certificates from before October 2026, which + #: follow the pin the same way. + core_binding: Optional[Literal["certified", "bundle_pin"]] = None qualification_pair_id: Optional[str] = None qualification_ledger_url: Optional[str] = None @@ -315,33 +328,23 @@ def _apply_dataset_overlays( registry of *additional* named datasets that resolution should expose without ever certifying them as the default — for example a staged artifact published in its own immutable release (see - ``docs/release-bundles.md``). Because certification only rewrites - ``data_releases``, overlays survive re-certification untouched. + ``docs/release-bundles.md``). Certification keeps every overlay except + one whose name the new release certifies itself, which it removes and + reports; re-add such an overlay if a later release stops certifying it. - Overlays are strictly additive: an overlay may not shadow the certified - default dataset or any certified dataset entry, so it can never alter - default resolution. + Overlays are strictly additive (``shadowed_dataset_overlays``): an overlay + may not shadow the certified default dataset or any certified dataset + entry, so it can never alter default resolution. """ overlays = (bundle.get(DATASET_OVERLAYS_KEY) or {}).get(country_id) or {} if not overlays: return release_payload + shadowed = shadowed_dataset_overlays(release_payload, overlays) + if shadowed: + raise ValueError(shadowing_overlays_message(country_id, shadowed)) certified_datasets = dict(release_payload.get("datasets") or {}) - default_dataset = release_payload.get("default_dataset") - for overlay_name, overlay_reference in overlays.items(): - if overlay_name == default_dataset: - raise ValueError( - f"Dataset overlay '{overlay_name}' for country '{country_id}' " - "shadows the certified default dataset. Overlays must never " - "change default resolution." - ) - if overlay_name in certified_datasets: - raise ValueError( - f"Dataset overlay '{overlay_name}' for country '{country_id}' " - "collides with a certified dataset entry. Overlays must be " - "additive, not overrides." - ) - certified_datasets[overlay_name] = overlay_reference + certified_datasets.update(overlays) return {**release_payload, "datasets": certified_datasets} @@ -706,6 +709,43 @@ def resolve_default_datasets(country_id: str) -> list[str]: return list(manifest.datasets.keys()) +def region_dataset_name(country_id: str, region_type: str) -> Optional[str]: + """The certified dataset that ``region_type`` regions are routed to, if any.""" + + manifest = get_release_manifest(country_id) + template = manifest.region_datasets.get(region_type) + if template is None: + return None + return next( + ( + name + for name, reference in manifest.datasets.items() + if reference.path == template.path_template + ), + None, + ) + + +def uk_area_dataset_hint(region_type: str) -> str: + """A sentence naming the dataset whose households carry an area's codes. + + The Microcosm national default has no constituency or local-authority + codes; the local-area release the bundle routes those regions to does. + """ + + try: + name = region_dataset_name("uk", region_type) + except ValueError: + return "" + if name is None: + return "" + return ( + f"The certified bundle routes {region_type} regions to the dataset " + f"{name!r}, whose households carry these codes: run the simulations " + "on that dataset." + ) + + def resolve_region_dataset_path( country_id: str, region_type: str, diff --git a/src/policyengine/provenance/receipt.py b/src/policyengine/provenance/receipt.py index c1a6c525a..12bcbcdbd 100644 --- a/src/policyengine/provenance/receipt.py +++ b/src/policyengine/provenance/receipt.py @@ -3785,7 +3785,16 @@ def _data_identity( f"The file is not the bundle's {dataset_name!r}: it hashes to " f"{dataset_sha256}, but {spec.source_uri} is {spec.sha256}." ) - release = dataset_name in (record.get("datasets") or {}) + datasets = record.get("datasets") or {} + entry = datasets.get(dataset_name) + primary = datasets.get(record.get("default_dataset")) or {} + # A regional release's artifacts are certified beside the primary release + # but are not part of it, so only entries at the primary release's own + # revision take its build and manifest identity. + release = entry is not None and entry.get("revision") in ( + None, + primary.get("revision"), + ) data_package = record["data_package"] certified_artifact = record.get("certified_data_artifact") or {} certification = record.get("certification") or {} diff --git a/src/policyengine/provenance/trace.py b/src/policyengine/provenance/trace.py index 350c676f5..bcffda78a 100644 --- a/src/policyengine/provenance/trace.py +++ b/src/policyengine/provenance/trace.py @@ -534,6 +534,12 @@ def _build_bundle_performance( certification.certified_for_model_version ) performance["pe:compatibilityBasis"] = certification.compatibility_basis + if certification.core_binding is not None: + performance["pe:coreBinding"] = certification.core_binding + if certification.certified_for_core_version is not None: + performance["pe:certifiedForCoreVersion"] = ( + certification.certified_for_core_version + ) if certification.built_with_model_version is not None: performance["pe:builtWithModelVersion"] = ( certification.built_with_model_version diff --git a/src/policyengine/provenance/uk_qualification.py b/src/policyengine/provenance/uk_qualification.py index cb1c9988c..0ae373212 100644 --- a/src/policyengine/provenance/uk_qualification.py +++ b/src/policyengine/provenance/uk_qualification.py @@ -14,9 +14,12 @@ ``prepare`` On a checkout of ``main``, certify the pair into the bundle manifest (exact ``==`` claim backed by the pair id when no publisher claim covers - it), regenerate pins, ``uv.lock``, TROs, the certified-version test - constants and household snapshots together, and commit that as the - candidate commit on ``candidate/uk/-``. + it), regenerate pins, ``uv.lock``, TROs and household snapshots + together, and the certified-version test constants for a pair on main's + data line, and commit that as the candidate commit on + ``candidate/uk/-``. On a main routed to a local-area release, + a pair from another line drops the routing, and a pair on main's own line + is refused: it needs a local-area cut certified by hand. ``gates`` Run the mandatory local gates on the candidate commit (:mod:`policyengine.provenance.uk_candidate`). @@ -27,7 +30,8 @@ Record the qualification receipt in the ledger. ``promote`` Pick the newest promotable pair that advances ``main`` and mark its - candidate PR ready, closing superseded candidates. + candidate PR ready, closing superseded candidates, and closing the PRs + of pairs from another data line at the commit their receipt qualified. Network, git and GitHub access are injected so every decision can be tested offline. @@ -352,11 +356,22 @@ def data_line(pair: CandidatePair) -> tuple[str, str]: return (pair.data.data_package, pair.data.repo_id) +def split_by_data_line( + receipts: Iterable[QualificationReceipt], bundle: Mapping[str, Any] +) -> tuple[list[QualificationReceipt], list[QualificationReceipt]]: + """``(on main's data line, from another line)``.""" + + line = certified_data_line(bundle) + same, other = [], [] + for receipt in receipts: + (same if data_line(receipt.pair) == line else other).append(receipt) + return same, other + + def _same_line_receipts( receipts: Iterable[QualificationReceipt], bundle: Mapping[str, Any] ) -> list[QualificationReceipt]: - line = certified_data_line(bundle) - return [receipt for receipt in receipts if data_line(receipt.pair) == line] + return split_by_data_line(receipts, bundle)[0] def certified_order_key(bundle: Mapping[str, Any]) -> tuple[str, str, str]: @@ -752,6 +767,34 @@ def _run( ) +def regional_certification_arguments( + bundle: Mapping[str, Any], pair: CandidatePair +) -> list[str]: + """``certify-data``'s regional flags for certifying ``pair`` onto ``bundle``. + + Certification refuses to drop a bundle's regional routing silently. A pair + from another data line (a forced qualification, never promoted) replaces + the line the routing belongs to, so the routing is dropped and the pair's + national dataset serves every region type. A pair on a routed main's own + line is refused: the routing would need a local-area cut for the pair's + versions, which a candidate pair does not carry, and automated + qualification checks no local-area file. + """ + + record = bundle["data_releases"]["uk"] + regional_uri = record.get("regional_source_manifest_uri") + if regional_uri is None: + return [] + if data_line(pair) != certified_data_line(bundle): + return ["--drop-regional-release"] + raise ValueError( + f"Main routes UK area regions to {regional_uri}. Automated " + "qualification cannot carry a local-area cut for a candidate on main's " + "own data line; certify such a pair by hand, with its local-area cut " + "(docs/engineering/skills/data-certification.md)." + ) + + def prepare_candidate_commit( repo_root: Path, resolved: ResolvedCandidate, @@ -770,7 +813,9 @@ def prepare_candidate_commit( bundle_path = ( repo_root / "src" / "policyengine" / "data" / "bundle" / "manifest.json" ) - old_record = json.loads(bundle_path.read_text())["data_releases"]["uk"] + old_bundle = json.loads(bundle_path.read_text()) + old_record = old_bundle["data_releases"]["uk"] + same_line = data_line(pair) == certified_data_line(old_bundle) run( [ python, @@ -789,6 +834,7 @@ def prepare_candidate_commit( "--qualified-pair-id", pair.pair_id, "--no-changelog", + *regional_certification_arguments(old_bundle, pair), ], cwd=repo_root, ) @@ -818,8 +864,16 @@ def prepare_candidate_commit( [python, "scripts/bundle.py", "generate", "--include-tros"], cwd=repo_root, ) - constants = rewrite_certified_constants( - repo_root, certified_constant_replacements(old_record, new_record) + # The constant files pin main's line. Rewriting them by value from + # another line's certificate would write that line's values into + # unrelated constants, so a cross-line candidate leaves them, and its + # tests show the change of line. + constants = ( + rewrite_certified_constants( + repo_root, certified_constant_replacements(old_record, new_record) + ) + if same_line + else [] ) basis = new_record["certification"]["compatibility_basis"] basis_note = { @@ -1547,6 +1601,9 @@ class PromotionPlan: #: The target, when its qualification compared against an earlier #: certificate than main's: it needs requalifying before promotion. requalify: Optional[CandidatePair] = None + #: Open candidate branches whose pair is not on main's data line. They are + #: never promoted, so their pull requests are closed. + close_off_line: list[str] = field(default_factory=list) def plan_promotion( @@ -1566,9 +1623,29 @@ def plan_promotion( A branch that moved since is left alone and reported. """ - # Only pairs on main's data line are ordered against it; others are left - # alone (neither promoted nor closed). - receipts = _same_line_receipts(receipts, bundle) + # Only pairs on main's data line are ordered against it. A pair from + # another line is never promoted, so its pull request is closed, but only + # at the commit its receipt qualified: a newer head is a forced re-run + # still in flight, which its own receipt decides. The ledger keeps the + # evidence. + same_line, off_line = split_by_data_line(receipts, bundle) + plan = _plan_same_line_promotion(same_line, bundle, open_candidate_branches) + plan.close_off_line = sorted( + { + candidate_branch(receipt.pair) + for receipt in off_line + if open_candidate_branches.get(candidate_branch(receipt.pair)) + == receipt.harness.git_commit + } + ) + return plan + + +def _plan_same_line_promotion( + receipts: Sequence[QualificationReceipt], + bundle: Mapping[str, Any], + open_candidate_branches: Mapping[str, str], +) -> PromotionPlan: current = certified_order_key(bundle) target = select_promotion_target(receipts, current) baseline_matches = target is not None and ( @@ -1643,9 +1720,14 @@ def candidate_pr_body( pair: CandidatePair, *, ledger_url: str, + on_main_line: bool, prepared: Optional[Mapping[str, Any]] = None, ) -> str: - """Markdown body for a candidate PR: the pair, the gates and the verdict.""" + """Markdown body for a candidate PR: the pair, the gates and the verdict. + + ``on_main_line`` is false for a pair from another data line than main's + certificate (a forced qualification), which is never promoted. + """ lines = [ f"Automated certification candidate for **policyengine-uk {pair.model.version}**, " @@ -1667,11 +1749,20 @@ def candidate_pr_body( summary = gate.summary.replace("|", "\\|").replace("\n", " ") lines.append(f"| `{gate_id}` | {gate.status} | {summary} |") verdict = receipt.verdict if receipt else "pending" + if on_main_line: + outcome = ( + "Merging this PR promotes the pair; the certification evidence test " + "refuses an exact-pair certificate without a promotable ledger receipt." + ) + else: + outcome = ( + "This pair is not on main's data line, so it was qualified for " + "evidence only and is never promoted. Its tests fail by design, and " + "promotion closes this PR; the ledger keeps the evidence." + ) lines += [ "", - f"**Verdict: {verdict}.** Merging this PR promotes the pair. Until cos d1344 " - "is ruled, promotion is a reviewed merge; the certification evidence test " - "refuses an exact-pair certificate without a promotable ledger receipt.", + f"**Verdict: {verdict}.** {outcome}", "", "🤖 Generated with [Claude Code](https://claude.com/claude-code)", ] @@ -1757,18 +1848,26 @@ def released_bundle_request( wheel_sha256: str, packaged_manifest_sha256: str, release_bundle: Mapping[str, Optional[str]], - hf_file: Callable[[str, str, str], bytes], + hf_file: Callable[[str, str, str, str], bytes], pypi_project: Callable[[str], Mapping[str, Any]], requested_by: Mapping[str, Any], + hf_size: Optional[Callable[[str, str, str, str], int]] = None, ) -> ReplayQualificationRequest: - """``released_bundle`` replay request for a published policyengine release.""" + """``released_bundle`` replay request for a published policyengine release. + + ``hf_file`` and ``hf_size`` take a repository id, revision, path and + repository type. The dataset's size comes from the release manifest, or + from ``hf_size`` when the manifest records none, as Microcosm's do not. + """ record = bundle["data_releases"]["uk"] data_package = record["data_package"] + repo_type = data_package.get("repo_type", "model") manifest_bytes = hf_file( data_package["repo_id"], data_package["release_manifest_revision"], data_package["release_manifest_path"], + repo_type, ) manifest = DataReleaseManifest.model_validate_json(manifest_bytes) dataset = record["default_dataset"] @@ -1778,6 +1877,18 @@ def released_bundle_request( raise ValueError( "The release manifest's dataset hash differs from the bundle's." ) + data_repo_id = reference.get("repo_id") or data_package["repo_id"] + data_repo_type = reference.get("repo_type") or repo_type + size_bytes = artifact.size_bytes + if size_bytes is None: + if hf_size is None: + raise ValueError( + f"The release manifest records no size for {dataset!r}; pass " + "hf_size to read it from Hugging Face." + ) + size_bytes = hf_size( + data_repo_id, reference["revision"], reference["path"], data_repo_type + ) core_version = bundle["packages"][CORE_PACKAGE]["version"] core_wheel = _wheel( pypi_project(CORE_PACKAGE)["releases"][core_version], CORE_PACKAGE, core_version @@ -1805,13 +1916,12 @@ def released_bundle_request( "build_id": record["certified_data_artifact"].get("build_id") or record["build_id"], "dataset": dataset, - "repo_id": reference.get("repo_id") or data_package["repo_id"], - "repo_type": reference.get("repo_type") - or data_package.get("repo_type", "model"), + "repo_id": data_repo_id, + "repo_type": data_repo_type, "revision": reference["revision"], "path": reference["path"], "sha256": reference["sha256"], - "size_bytes": artifact.size_bytes, + "size_bytes": size_bytes, "release_manifest_sha256": hashlib.sha256( manifest_bytes ).hexdigest(), diff --git a/tests/fixtures/household_calculator_snapshots/uk_model_surface.json b/tests/fixtures/household_calculator_snapshots/uk_model_surface.json index e3d367c18..25270d576 100644 --- a/tests/fixtures/household_calculator_snapshots/uk_model_surface.json +++ b/tests/fixtures/household_calculator_snapshots/uk_model_surface.json @@ -1,6 +1,6 @@ { "country_id": "uk", - "data_package_name": "policyengine-uk-data", + "data_package_name": "microcosm-data", "has_employment_income": true, "has_income_tax": true, "has_region_registry": true, diff --git a/tests/fixtures/uk_qualification/microcosm_routed_bundle.json b/tests/fixtures/uk_qualification/microcosm_routed_bundle.json new file mode 100644 index 000000000..c6a8eff3d --- /dev/null +++ b/tests/fixtures/uk_qualification/microcosm_routed_bundle.json @@ -0,0 +1,392 @@ +{ + "bundle_version": "6.9.0", + "citation": { + "publisher": "PolicyEngine", + "title": "PolicyEngine bundle 6.9.0", + "type": "software-bundle", + "version": "6.9.0" + }, + "countries": { + "uk": { + "model_package": "policyengine-uk" + }, + "us": { + "model_package": "policyengine-us" + } + }, + "data_releases": { + "uk": { + "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "bundle_id": "uk-6.9.0", + "certification": { + "built_with_model_version": "2.132.6", + "certified_by": "policyengine.py bundle certification", + "certified_for_model_version": "2.132.6", + "compatibility_basis": "built_with_model_package", + "core_binding": "bundle_pin", + "data_build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + }, + "certified_data_artifact": { + "build_id": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "data_package": { + "name": "microcosm-data", + "version": "0.1.0" + }, + "dataset": "microcosm_uk_2024_25", + "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece", + "uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + }, + "country_id": "uk", + "data_package": { + "name": "microcosm-data", + "release_manifest_path": "releases/microcosm-uk-2024-25-national/release_manifest.json", + "release_manifest_revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "version": "0.1.0" + }, + "data_producer": "populace", + "datasets": { + "calibration_diagnostics": { + "path": "releases/microcosm-uk-2024-25-national/calibration_diagnostics.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "34978e2bfde3f2641769c9c3a8f60455c59ecae5a50524b359d74388d7edc888" + }, + "microcosm_uk_2024_25": { + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "810ba980005a8b907cb51464279fa5e0125464b7af9a4871b7f6744c30951ece" + }, + "microcosm_uk_2024_25_calibration": { + "path": "microcosm_uk_2024_25_calibration.npz", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "104a8253bf1aebc91147c9bbb40e2603d1025317ba8c6b991383d5e4d91852c2" + }, + "microcosm_uk_2024_25_dense": { + "path": "microcosm_uk_2024_25_dense.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe", + "sha256": "8e10e130d0260e0b52e2e29902f0502ebdff76a14a3e57d2c876bf4cd8db5872" + }, + "release_certification": { + "path": "releases/microcosm-uk-2024-25-national/release_certification.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "6a68cc93595fc5a7ad451d75a902523c25659de27014f50431dd5e19af5bd959" + }, + "release_cut_gates": { + "path": "releases/microcosm-uk-2024-25-national/release_cut_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "c91c1364d9c3be98493039d4e1dd8c2256881a434c1f124dd2ed0701bfaf7751" + }, + "score_vs_enhanced_frs": { + "path": "releases/microcosm-uk-2024-25-national/score_vs_enhanced_frs.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "ba8a675f36e7c836bf5521e5933e21bb689442734b7340e05cabd3e0cc55323d" + }, + "spine_gates": { + "path": "releases/microcosm-uk-2024-25-national/spine_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "6dbba976dd1304c07da531382cecb11c9cb69512de0ee65126b71f032a0f37d6" + }, + "terminal_gates": { + "path": "releases/microcosm-uk-2024-25-national/terminal_gates.json", + "repo_id": "policyengine/populace-uk-private", + "revision": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "sha256": "e29439d5ce0ea098d32579284ac06f26f086feba2fb8d12d1e88ac9a54840808" + } + }, + "default_dataset": "microcosm_uk_2024_25", + "default_dataset_uri": "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71", + "model_package": { + "name": "policyengine-uk", + "sha256": "cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354", + "version": "2.132.6", + "wheel_url": "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl" + }, + "policyengine_version": "6.9.0", + "region_datasets": { + "constituency": { + "path_template": "microcosm_uk_2024_25_dense.h5" + }, + "local_authority": { + "path_template": "microcosm_uk_2024_25_dense.h5" + }, + "national": { + "path_template": "microcosm_uk_2024_25.h5" + } + }, + "regional_release_kind": "local_area", + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe/releases/microcosm-uk-2024-25-dense/release_manifest.json", + "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-uk-private/resolve/microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", + "schema_version": 1, + "source_manifest_uri": "hf://dataset/policyengine/populace-uk-private@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71/releases/microcosm-uk-2024-25-national/release_manifest.json", + "version": "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + }, + "us": { + "build_id": "populace-us-2024-spm-20260915", + "bundle_id": "us-6.9.0", + "certification": { + "built_with_model_version": "2.2.1", + "certified_by": "policyengine.py bundle certification", + "certified_for_model_version": "2.2.1", + "compatibility_basis": "built_with_model_package", + "data_build_id": "populace-us-2024-spm-20260915" + }, + "certified_data_artifact": { + "build_id": "populace-us-2024-spm-20260915", + "data_package": { + "name": "microcosm-data", + "version": "0.1.0" + }, + "dataset": "populace_us_2024", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915" + }, + "country_id": "us", + "data_package": { + "name": "microcosm-data", + "release_manifest_path": "releases/populace-us-2024-spm-20260915/release_manifest.json", + "release_manifest_revision": "populace-us-2024-spm-20260915", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "version": "0.1.0" + }, + "data_producer": "populace", + "datasets": { + "build_manifest": { + "path": "releases/populace-us-2024-spm-20260915/build_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "f3fd734867324bca7619f5f083a8115e32104bea3948d48723b345cfd39fa03b" + }, + "calibration_diagnostics": { + "path": "releases/populace-us-2024-spm-20260915/calibration_diagnostics.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "870449b44e86b13b25bcea1a57f0e7af37f4d4db18be815eea3acdf9fe6eb40e" + }, + "parent_build_manifest": { + "path": "releases/populace-us-2024-spm-20260915/parent_build_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "1990e8fd37ce66f499b8e6600c5896701bb07015be132a12de64e321a2edc67e" + }, + "parent_release_manifest": { + "path": "releases/populace-us-2024-spm-20260915/parent_release_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "dd949ba3c4c7a56aff8442c6db5a031d2c84c3da59f5d14832f468c358604506" + }, + "populace_us_2024": { + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" + }, + "populace_us_2024_acs_local": { + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "revision": "populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z", + "sha256": "3ce781c440a7f933b1d952ff4e9d9f65f4698c64b94d2fad165af0c662838d6c" + }, + "source_enrichment": { + "path": "releases/populace-us-2024-spm-20260915/source_enrichment.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "85329907b2b9c4299855b0970b1f24ede5819d3b16c312eb7400b4203bd71203" + }, + "source_enrichment_compatibility": { + "path": "releases/populace-us-2024-spm-20260915/source_enrichment_compatibility.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "63109acffd486453c99c476dece6b0154cc5388e5a430830a30744a2249444d1" + }, + "source_spm_independence_provenance": { + "path": "releases/populace-us-2024-spm-20260915/source_spm_independence_provenance.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "265e844178ed3856d68c1638820f3089e0e1e1363aa4677e33bebb37cbd5bff3" + }, + "source_spm_person_independence": { + "path": "releases/populace-us-2024-spm-20260915/source_spm_person_independence.csv", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "22b5968d90fecfeef7614583e493fe10cc16bda8b5be82e6f49a5bc2102d3ce5" + }, + "us_source_coverage": { + "path": "releases/populace-us-2024-spm-20260915/us_source_coverage.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "6406c8686c292015a5bc7265a42402f89935f9395a8b63510efdd4d9562e7ff5" + } + }, + "default_dataset": "populace_us_2024", + "default_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915", + "model_package": { + "name": "policyengine-us", + "sha256": "0993a6c73fcdfbe171a796aca9d302741bd8e00ab83388ad81c15a08740318c6", + "version": "2.2.1", + "wheel_url": "https://files.pythonhosted.org/packages/5e/09/86ac3968d2a18e28fce3ba1186fc21728e9f299b96a806d6f204f1e0aaed/policyengine_us-2.2.1-py3-none-any.whl" + }, + "policyengine_version": "6.9.0", + "region_datasets": { + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, + "national": { + "path_template": "populace_us_2024.h5" + }, + "state": { + "path_template": "populace_us_2024_acs_local.h5" + } + }, + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/releases/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/releases/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/release_manifest.json", + "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", + "schema_version": 1, + "source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", + "version": "populace-us-2024-spm-20260915" + } + }, + "dataset_overlays": { + "uk": { + "enhanced_frs_2023_24": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2023_24.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" + }, + "enhanced_frs_2024_25": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + }, + "enhanced_frs_2024_25_tiny": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "d5d2bde80f1f028fec98391ea31bb5a0cf8449e3867fc3c1bfe23693b9faecf8" + }, + "frs_2023_24": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2023_24.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "sha256": "df26d4d7af9d164aa2d064181b39290292d2f62bb26fee6126fc095fc06da292" + }, + "frs_2024_25": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "97af00b2cd2739957f40dcc4de8e0383a6a91d47952c90d4b4921a6007bd2021" + }, + "frs_2024_25_tiny": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "1.58.1", + "sha256": "e4d7662cd3f053620224b7ceff17266783f09407002a72633b043b844cec3f5a" + }, + "populace_uk_2023": { + "data_package_name": "populace-data", + "path": "populace_uk_2023.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" + } + } + }, + "extras": { + "models": [ + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "spm-calculator" + ], + "uk": [ + "policyengine-core", + "policyengine-uk" + ], + "us": [ + "policyengine-core", + "policyengine-us", + "spm-calculator" + ] + }, + "measurements": { + "spm": { + "as_of": null, + "county_vintage": "2020", + "forecast_content_sha256": "3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173", + "geography_id": null, + "geography_kind": "county", + "scenario": "ce_trend" + } + }, + "packages": { + "policyengine": { + "import_name": "policyengine", + "install_requirement": "policyengine==6.9.0", + "name": "policyengine", + "role": "bundle_carrier", + "version": "6.9.0" + }, + "policyengine-core": { + "import_name": "policyengine_core", + "install_requirement": "policyengine-core==3.33.3", + "name": "policyengine-core", + "role": "runtime_dependency", + "version": "3.33.3" + }, + "policyengine-uk": { + "country": "uk", + "import_name": "policyengine_uk", + "install_requirement": "policyengine-uk==2.132.6", + "name": "policyengine-uk", + "role": "country_model", + "version": "2.132.6" + }, + "policyengine-us": { + "country": "us", + "import_name": "policyengine_us", + "install_requirement": "policyengine-us==2.2.1", + "name": "policyengine-us", + "role": "country_model", + "version": "2.2.1" + }, + "spm-calculator": { + "country": "us", + "import_name": "spm_calculator", + "install_requirement": "spm-calculator==1.0.0", + "name": "spm-calculator", + "role": "runtime_dependency", + "sha256": "e354937a5e1a4045d4966ed594a528d8b02866fabaac9bb5672017004b627305", + "version": "1.0.0", + "wheel_url": "https://files.pythonhosted.org/packages/19/a0/c484f69a0ebf88a9b9fd0ac28176f8df66550f46714f1b0c5ec0b360824e/spm_calculator-1.0.0-py3-none-any.whl" + } + }, + "policyengine_version": "6.9.0", + "schema_version": 2 +} diff --git a/tests/fixtures/uk_qualification/uk_data_line_bundle.json b/tests/fixtures/uk_qualification/uk_data_line_bundle.json new file mode 100644 index 000000000..cc56a3ebc --- /dev/null +++ b/tests/fixtures/uk_qualification/uk_data_line_bundle.json @@ -0,0 +1,339 @@ +{ + "bundle_version": "6.9.0", + "citation": { + "publisher": "PolicyEngine", + "title": "PolicyEngine bundle 6.9.0", + "type": "software-bundle", + "version": "6.9.0" + }, + "countries": { + "uk": { + "model_package": "policyengine-uk" + }, + "us": { + "model_package": "policyengine-us" + } + }, + "data_releases": { + "uk": { + "build_id": "policyengine-uk-data-1.58.1", + "bundle_id": "uk-6.9.0", + "certification": { + "built_with_model_git_sha": "daad71cd26c107f1b784ff600d1464ba59c70e4a", + "built_with_model_version": "2.122.2", + "certified_by": "policyengine.py bundle certification", + "certified_for_model_version": "2.130.1", + "compatibility_basis": "legacy_compatible_model_package", + "data_build_fingerprint": "sha256:c094d7006fdd81df220090284212f16bbffb15624643bc442c4bb3e1b1e5dac8", + "data_build_id": "policyengine-uk-data-1.58.1" + }, + "certified_data_artifact": { + "build_id": "policyengine-uk-data-1.58.1", + "data_package": { + "name": "policyengine-uk-data", + "version": "1.58.1" + }, + "dataset": "enhanced_frs_2024_25", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784", + "uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1" + }, + "country_id": "uk", + "data_package": { + "name": "policyengine-uk-data", + "release_manifest_path": "releases/1.58.1/release_manifest.json", + "release_manifest_revision": "89be6b73fc1e6eeba6dae2f9400676d8510bcd51", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "version": "1.58.1" + }, + "data_producer": "populace", + "datasets": { + "enhanced_frs_2024_25": { + "path": "enhanced_frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "056e80fd8e9151d78f432a25cde863340baee81304c6216a37fb575ce8563784" + }, + "enhanced_frs_2024_25_tiny": { + "path": "enhanced_frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "d5d2bde80f1f028fec98391ea31bb5a0cf8449e3867fc3c1bfe23693b9faecf8" + }, + "frs_2024_25": { + "path": "frs_2024_25.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "97af00b2cd2739957f40dcc4de8e0383a6a91d47952c90d4b4921a6007bd2021" + }, + "frs_2024_25_tiny": { + "path": "frs_2024_25_tiny.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "e4d7662cd3f053620224b7ceff17266783f09407002a72633b043b844cec3f5a" + }, + "local_authority_weights": { + "path": "local_authority_weights.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "2b7a2f7f9e7081a394b37cb84e909fd3c01d22e421808141684e7c3be59f2089" + }, + "local_geography_weights": { + "path": "local_geography_weights.csv.gz", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "97c6e3542543baf737a713bb918ebc3d3dbc3f3765be9bf3414375c322d0e3d1" + }, + "parliamentary_constituency_weights": { + "path": "parliamentary_constituency_weights.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "revision": "1.58.1", + "sha256": "8c696e4d65a22033deb4c4f69a767fdb46e0a4f5580f36b7fb47ee4722629b11" + } + }, + "default_dataset": "enhanced_frs_2024_25", + "default_dataset_uri": "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1", + "model_package": { + "name": "policyengine-uk", + "sha256": "f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873", + "version": "2.130.1", + "wheel_url": "https://files.pythonhosted.org/packages/1a/21/88e8addba0eabd9a7d3a29fa5293e141feb405baf0432209387f9a1f467a/policyengine_uk-2.130.1-py3-none-any.whl" + }, + "policyengine_version": "6.9.0", + "region_datasets": { + "national": { + "path_template": "enhanced_frs_2024_25.h5" + } + }, + "release_manifest_uri": "https://huggingface.co/policyengine/policyengine-uk-data-private/resolve/89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json", + "schema_version": 1, + "source_manifest_uri": "hf://model/policyengine/policyengine-uk-data-private@89be6b73fc1e6eeba6dae2f9400676d8510bcd51/releases/1.58.1/release_manifest.json", + "version": "policyengine-uk-data-1.58.1" + }, + "us": { + "build_id": "populace-us-2024-spm-20260915", + "bundle_id": "us-6.9.0", + "certification": { + "built_with_model_version": "2.2.1", + "certified_by": "policyengine.py bundle certification", + "certified_for_model_version": "2.2.1", + "compatibility_basis": "built_with_model_package", + "data_build_id": "populace-us-2024-spm-20260915" + }, + "certified_data_artifact": { + "build_id": "populace-us-2024-spm-20260915", + "data_package": { + "name": "microcosm-data", + "version": "0.1.0" + }, + "dataset": "populace_us_2024", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915" + }, + "country_id": "us", + "data_package": { + "name": "microcosm-data", + "release_manifest_path": "releases/populace-us-2024-spm-20260915/release_manifest.json", + "release_manifest_revision": "populace-us-2024-spm-20260915", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "version": "0.1.0" + }, + "data_producer": "populace", + "datasets": { + "build_manifest": { + "path": "releases/populace-us-2024-spm-20260915/build_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "f3fd734867324bca7619f5f083a8115e32104bea3948d48723b345cfd39fa03b" + }, + "calibration_diagnostics": { + "path": "releases/populace-us-2024-spm-20260915/calibration_diagnostics.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "870449b44e86b13b25bcea1a57f0e7af37f4d4db18be815eea3acdf9fe6eb40e" + }, + "parent_build_manifest": { + "path": "releases/populace-us-2024-spm-20260915/parent_build_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "1990e8fd37ce66f499b8e6600c5896701bb07015be132a12de64e321a2edc67e" + }, + "parent_release_manifest": { + "path": "releases/populace-us-2024-spm-20260915/parent_release_manifest.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "dd949ba3c4c7a56aff8442c6db5a031d2c84c3da59f5d14832f468c358604506" + }, + "populace_us_2024": { + "path": "populace_us_2024.h5", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" + }, + "populace_us_2024_acs_local": { + "path": "populace_us_2024_acs_local.h5", + "repo_id": "policyengine/populace-us", + "repo_type": "dataset", + "revision": "populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z", + "sha256": "3ce781c440a7f933b1d952ff4e9d9f65f4698c64b94d2fad165af0c662838d6c" + }, + "source_enrichment": { + "path": "releases/populace-us-2024-spm-20260915/source_enrichment.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "85329907b2b9c4299855b0970b1f24ede5819d3b16c312eb7400b4203bd71203" + }, + "source_enrichment_compatibility": { + "path": "releases/populace-us-2024-spm-20260915/source_enrichment_compatibility.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "63109acffd486453c99c476dece6b0154cc5388e5a430830a30744a2249444d1" + }, + "source_spm_independence_provenance": { + "path": "releases/populace-us-2024-spm-20260915/source_spm_independence_provenance.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "265e844178ed3856d68c1638820f3089e0e1e1363aa4677e33bebb37cbd5bff3" + }, + "source_spm_person_independence": { + "path": "releases/populace-us-2024-spm-20260915/source_spm_person_independence.csv", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "22b5968d90fecfeef7614583e493fe10cc16bda8b5be82e6f49a5bc2102d3ce5" + }, + "us_source_coverage": { + "path": "releases/populace-us-2024-spm-20260915/us_source_coverage.json", + "repo_id": "policyengine/populace-us", + "revision": "populace-us-2024-spm-20260915", + "sha256": "6406c8686c292015a5bc7265a42402f89935f9395a8b63510efdd4d9562e7ff5" + } + }, + "default_dataset": "populace_us_2024", + "default_dataset_uri": "hf://policyengine/populace-us/populace_us_2024.h5@populace-us-2024-spm-20260915", + "model_package": { + "name": "policyengine-us", + "sha256": "0993a6c73fcdfbe171a796aca9d302741bd8e00ab83388ad81c15a08740318c6", + "version": "2.2.1", + "wheel_url": "https://files.pythonhosted.org/packages/5e/09/86ac3968d2a18e28fce3ba1186fc21728e9f299b96a806d6f204f1e0aaed/policyengine_us-2.2.1-py3-none-any.whl" + }, + "policyengine_version": "6.9.0", + "region_datasets": { + "congressional_district": { + "path_template": "populace_us_2024_acs_local.h5" + }, + "national": { + "path_template": "populace_us_2024.h5" + }, + "state": { + "path_template": "populace_us_2024_acs_local.h5" + } + }, + "regional_release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/releases/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/release_manifest.json", + "regional_source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/releases/populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/release_manifest.json", + "release_manifest_uri": "https://huggingface.co/datasets/policyengine/populace-us/resolve/populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", + "schema_version": 1, + "source_manifest_uri": "hf://dataset/policyengine/populace-us@populace-us-2024-spm-20260915/releases/populace-us-2024-spm-20260915/release_manifest.json", + "version": "populace-us-2024-spm-20260915" + } + }, + "dataset_overlays": { + "uk": { + "enhanced_frs_2023_24": { + "data_package_name": "policyengine-uk-data", + "path": "enhanced_frs_2023_24.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "sha256": "584ae33d80ca0431254610a3f8254d132da73477d31966d6446282861ecae50d" + }, + "frs_2023_24": { + "data_package_name": "policyengine-uk-data", + "path": "frs_2023_24.h5", + "repo_id": "policyengine/policyengine-uk-data-private", + "repo_type": "model", + "revision": "655dd07e4bb9c777b00dac044949611f1feb824f", + "sha256": "df26d4d7af9d164aa2d064181b39290292d2f62bb26fee6126fc095fc06da292" + }, + "populace_uk_2023": { + "data_package_name": "populace-data", + "path": "populace_uk_2023.h5", + "repo_id": "policyengine/populace-uk-private", + "repo_type": "dataset", + "revision": "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z", + "sha256": "f17306ccb2aad7ff0130be3589b560afb2e2a12a943570911cd0c77f07934833" + } + } + }, + "extras": { + "models": [ + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "spm-calculator" + ], + "uk": [ + "policyengine-core", + "policyengine-uk" + ], + "us": [ + "policyengine-core", + "policyengine-us", + "spm-calculator" + ] + }, + "measurements": { + "spm": { + "as_of": null, + "county_vintage": "2020", + "forecast_content_sha256": "3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173", + "geography_id": null, + "geography_kind": "county", + "scenario": "ce_trend" + } + }, + "packages": { + "policyengine": { + "import_name": "policyengine", + "install_requirement": "policyengine==6.9.0", + "name": "policyengine", + "role": "bundle_carrier", + "version": "6.9.0" + }, + "policyengine-core": { + "import_name": "policyengine_core", + "install_requirement": "policyengine-core==3.33.0", + "name": "policyengine-core", + "role": "runtime_dependency", + "version": "3.33.0" + }, + "policyengine-uk": { + "country": "uk", + "import_name": "policyengine_uk", + "install_requirement": "policyengine-uk==2.130.1", + "name": "policyengine-uk", + "role": "country_model", + "version": "2.130.1" + }, + "policyengine-us": { + "country": "us", + "import_name": "policyengine_us", + "install_requirement": "policyengine-us==2.2.1", + "name": "policyengine-us", + "role": "country_model", + "version": "2.2.1" + }, + "spm-calculator": { + "country": "us", + "import_name": "spm_calculator", + "install_requirement": "spm-calculator==1.0.0", + "name": "spm-calculator", + "role": "runtime_dependency", + "sha256": "e354937a5e1a4045d4966ed594a528d8b02866fabaac9bb5672017004b627305", + "version": "1.0.0", + "wheel_url": "https://files.pythonhosted.org/packages/19/a0/c484f69a0ebf88a9b9fd0ac28176f8df66550f46714f1b0c5ec0b360824e/spm_calculator-1.0.0-py3-none-any.whl" + } + }, + "policyengine_version": "6.9.0", + "schema_version": 2 +} diff --git a/tests/qualification_fixtures.py b/tests/qualification_fixtures.py index bf74fc960..724ab2b0f 100644 --- a/tests/qualification_fixtures.py +++ b/tests/qualification_fixtures.py @@ -20,6 +20,44 @@ ) FIXTURES = Path(__file__).parent / "fixtures" / "uk_replay" +REPO_ROOT = Path(__file__).resolve().parents[1] +#: The last packaged bundle certified on the policyengine-uk-data line (main at +#: 5f97905e, the 6.9.0 release, before Microcosm UK became the default). +#: ``make_pair`` builds candidates on that line, so qualification tests run +#: against this bundle. +UK_DATA_LINE_BUNDLE_PATH = ( + Path(__file__).parent / "fixtures" / "uk_qualification" / "uk_data_line_bundle.json" +) +UK_DATA_LINE_BUNDLE = json.loads(UK_DATA_LINE_BUNDLE_PATH.read_text()) +#: A main certified on the Microcosm line with its area regions routed to a +#: local-area release: the bundle #685 certified, frozen, so that tests of +#: other-line pairs do not depend on what main certifies now. +MICROCOSM_ROUTED_BUNDLE_PATH = ( + Path(__file__).parent + / "fixtures" + / "uk_qualification" + / "microcosm_routed_bundle.json" +) +MICROCOSM_ROUTED_BUNDLE = json.loads(MICROCOSM_ROUTED_BUNDLE_PATH.read_text()) + + +def load_candidate_cli( + name: str = "check_uk_candidate_for_test", + bundle_path: Path = UK_DATA_LINE_BUNDLE_PATH, +): + """``scripts/check_uk_candidate.py``, reading ``bundle_path`` as main's bundle.""" + + import importlib.util + + spec = importlib.util.spec_from_file_location( + name, REPO_ROOT / "scripts" / "check_uk_candidate.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.BUNDLE_PATH = bundle_path + return module + + VENDORED_SCHEMAS = ( Path(__file__).resolve().parents[1] / "src" diff --git a/tests/test_bundle.py b/tests/test_bundle.py index b30db02ea..e04fb0eb2 100644 --- a/tests/test_bundle.py +++ b/tests/test_bundle.py @@ -36,16 +36,15 @@ def test_bundle_manifest_exposes_data_releases(): assert manifest["bundle_version"] == manifest["policyengine_version"] assert manifest["data_releases"]["us"]["data_producer"] == "populace" assert manifest["data_releases"]["us"]["version"].startswith("populace-us-2024-") - # UK certified default reverted to the policyengine-uk-data enhanced FRS per - # the UC caseload postmortem (policyengine-uk-data discussion #464); the - # "populace" producer label names the strict verification contract, not the - # producing repo. + # The UK certified default is a Microcosm national release; the "populace" + # producer label names the strict verification contract, not the producing + # repo. assert manifest["data_releases"]["uk"]["data_producer"] == "populace" assert manifest["data_releases"]["uk"]["version"].startswith( - "policyengine-uk-data-" + "microcosm-uk-2024-25-national-" ) assert manifest["data_releases"]["uk"]["default_dataset_uri"].startswith( - "hf://policyengine/policyengine-uk-data-private/enhanced_frs_" + "hf://policyengine/populace-uk-private/microcosm_uk_" ) @@ -89,11 +88,14 @@ def test_selected_dataset_plan_uses_certified_release_metadata(tmp_path): bundle.get_current_bundle(), ["uk"], data_dir=tmp_path ) + # Only the national default: area regions' local-area dataset is not + # installed with the bundle. + assert [entry_plan.dataset for entry_plan, _ in entries] == ["microcosm_uk_2024_25"] plan, release = entries[0] assert plan.country_id == "uk" - assert plan.data_package_name == "policyengine-uk-data" - assert plan.repo_type == "model" - assert plan.destination == tmp_path / "enhanced_frs_2024_25.h5" + assert plan.data_package_name == "microcosm-data" + assert plan.repo_type == "dataset" + assert plan.destination == tmp_path / "microcosm_uk_2024_25.h5" assert plan.sha256 == release["datasets"][plan.dataset]["sha256"] @@ -202,7 +204,7 @@ def fake_materialize(plan): ) assert [plan.country_id for plan in calls] == ["uk"] - assert result["datasets"][0]["data_package_name"] == "policyengine-uk-data" + assert result["datasets"][0]["data_package_name"] == "microcosm-data" assert result["datasets"][0]["installed_sha256"] == calls[0].sha256 receipt = bundle.read_receipt(tmp_path) assert receipt is not None diff --git a/tests/test_bundle_metadata.py b/tests/test_bundle_metadata.py index 85dd2d016..937e4f88f 100644 --- a/tests/test_bundle_metadata.py +++ b/tests/test_bundle_metadata.py @@ -66,12 +66,12 @@ def test_bundle_manifest_carries_uk_data_release(): release = bundle.get_current_bundle()["data_releases"]["uk"] assert release["data_producer"] == "populace" - assert release["data_package"]["name"] == "policyengine-uk-data" - assert release["data_package"]["repo_type"] == "model" - assert release["default_dataset"] == "enhanced_frs_2024_25" - assert release["version"].startswith("policyengine-uk-data-") + assert release["data_package"]["name"] == "microcosm-data" + assert release["data_package"]["repo_type"] == "dataset" + assert release["default_dataset"] == "microcosm_uk_2024_25" + assert release["version"].startswith("microcosm-uk-2024-25-national-") assert release["default_dataset_uri"].startswith( - "hf://policyengine/policyengine-uk-data-private/" + "hf://policyengine/populace-uk-private/" ) @@ -143,3 +143,21 @@ def test_export_release_assets_writes_bundle_assets(monkeypatch, tmp_path): "PolicyEngine bundle" in (tmp_path / f"policyengine-bundle-{version}.citation.txt").read_text() ) + + +def test_dataset_overlays_may_not_shadow_certified_names(): + import copy + + import pytest + + from policyengine.provenance.bundle_validation import validate_dataset_overlays + + current = bundle.get_current_bundle() + validate_dataset_overlays(current) + + release = current["data_releases"]["uk"] + for name in (release["default_dataset"], next(iter(release["datasets"]))): + shadowing = copy.deepcopy(current) + shadowing["dataset_overlays"]["uk"][name] = {"path": f"{name}.h5"} + with pytest.raises(ValueError, match=f"shadows certified datasets: {name}"): + validate_dataset_overlays(shadowing) diff --git a/tests/test_certify_data_release.py b/tests/test_certify_data_release.py index 74964a50f..092420f61 100644 --- a/tests/test_certify_data_release.py +++ b/tests/test_certify_data_release.py @@ -29,6 +29,7 @@ from policyengine.provenance.manifest import ( CountryReleaseManifest, DataReleaseManifest, + _apply_dataset_overlays, ) TAG = "populace-us-2024-aaaaaaa-20260101" @@ -1171,7 +1172,9 @@ def _response(payload: dict) -> MagicMock: return response -def _certify_uk(bundle_path: Path, responses: list, **kwargs): +def _certify_uk( + bundle_path: Path, responses: list, manifest_uri: str = UK_NATIONAL_URI, **kwargs +): with ( patch( "policyengine.provenance.certification.requests.get", @@ -1197,7 +1200,7 @@ def _certify_uk(bundle_path: Path, responses: list, **kwargs): return certify_data_release( country="uk", data_producer="populace", - manifest_uri=UK_NATIONAL_URI, + manifest_uri=manifest_uri, model_version=UK_MODEL_VERSION, bundle_path=bundle_path, **kwargs, @@ -1444,8 +1447,45 @@ def test__given_pinned_core__then_the_regional_release_must_cover_it( ) release = json.loads(bundle_path.read_text())["data_releases"]["uk"] assert release["certification"]["certified_for_core_version"] == "3.33.0" + assert release["certification"]["core_binding"] == "certified" assert release["regional_release_kind"] == "area_coded_national" + def test__given_no_core_version__then_the_certificate_follows_the_pin_openly( + self, tmp_path + ): + """Without --core-version the certificate follows the bundle's core pin. + + It says so (core_binding bundle_pin), and certification reports a + release that does not claim the pin instead of refusing it. + """ + national = _microcosm_uk_national_payload() + national["compatible_core_packages"] = [ + {"name": "policyengine-core", "specifier": "==3.33.3"} + ] + bundle_path = _tmp_bundle(tmp_path) + bundle = json.loads(bundle_path.read_text()) + bundle["packages"]["policyengine-core"] = { + "name": "policyengine-core", + "version": "3.33.3", + } + bundle_path.write_text(json.dumps(bundle)) + + result = _certify_uk( + bundle_path, + [_response(national), _response(_microcosm_uk_dense_payload())], + regional_manifest_uri=UK_DENSE_URI, + ) + + written = json.loads(bundle_path.read_text()) + certification = written["data_releases"]["uk"]["certification"] + assert certification["core_binding"] == "bundle_pin" + assert "certified_for_core_version" not in certification + assert written["packages"]["policyengine-core"]["version"] == "3.33.3" + # The national release claims the pin; the local-area one claims no core. + unclaimed = [w for w in result.warnings if "follows the core pin" in w] + assert len(unclaimed) == 1 + assert unclaimed[0].startswith("regional release: ") + def test__given_unreachable_regional_artifact__then_refuses(self, tmp_path): with ( patch( @@ -1498,6 +1538,55 @@ def test__given_routed_bundle__then_recertifying_without_regional_refuses( assert "regional_release_kind" not in release assert set(release["region_datasets"]) == {"national"} + def test__given_a_rollback_to_overlaid_names__then_their_overlays_go( + self, tmp_path + ): + """A rollback certifies names the bundle keeps as overlays. + + Overlays may not shadow certified names, so a bundle that kept them + would refuse every release-manifest read, and import with it. + """ + bundle_path = _tmp_bundle(tmp_path) + _certify_uk( + bundle_path, + [ + _response(_microcosm_uk_national_payload()), + _response(_microcosm_uk_dense_payload()), + ], + regional_manifest_uri=UK_DENSE_URI, + ) + bundle = json.loads(bundle_path.read_text()) + kept = {"path": "populace_uk_2023.h5", "repo_id": "x", "revision": "r"} + bundle["dataset_overlays"] = { + "uk": { + "enhanced_frs_2024_25": {"path": "enhanced_frs_2024_25.h5"}, + "frs_2024_25": {"path": "frs_2024_25.h5"}, + "populace_uk_2023": kept, + } + } + bundle_path.write_text(json.dumps(bundle)) + + result = _certify_uk( + bundle_path, + [_response(_efrs_release_payload())], + manifest_uri=EFRS_URI, + drop_regional_release=True, + ) + + bundle = json.loads(bundle_path.read_text()) + assert bundle["data_releases"]["uk"]["default_dataset"] == ( + "enhanced_frs_2024_25" + ) + assert bundle["dataset_overlays"]["uk"] == {"populace_uk_2023": kept} + assert any("enhanced_frs_2024_25, frs_2024_25" in w for w in result.warnings) + # The Microcosm names stop resolving, and certification says so. + dropped = next(w for w in result.warnings if "no longer resolve" in w) + assert "microcosm_uk_2024_25_dense" in dropped + manifest = CountryReleaseManifest.model_validate( + _apply_dataset_overlays("uk", bundle["data_releases"]["uk"], bundle) + ) + assert "populace_uk_2023" in manifest.datasets + def test__given_drop_and_a_regional_release__then_refuses_both(self, tmp_path): # Dropping the routing and routing to a release are opposite requests; # neither silently wins. diff --git a/tests/test_constituency_impact.py b/tests/test_constituency_impact.py index 59820857b..64500c9da 100644 --- a/tests/test_constituency_impact.py +++ b/tests/test_constituency_impact.py @@ -252,12 +252,16 @@ def test_compute_constituency_impacts_requires_longwise_geography_column(): } ) - with pytest.raises(ValueError, match="constituency_code_oa"): + with pytest.raises(ValueError, match="constituency_code_oa") as error: compute_uk_constituency_impacts( baseline, reform, download_missing_assets=False, ) + # The error names the dataset whose households carry the codes. + from policyengine.provenance.manifest import uk_area_dataset_hint + + assert uk_area_dataset_hint("constituency") in str(error.value) def test_constituency_impacts_warn_that_they_are_not_disclosure_controlled(caplog): diff --git a/tests/test_dataset_materialization.py b/tests/test_dataset_materialization.py index 256e004da..5180ebd88 100644 --- a/tests/test_dataset_materialization.py +++ b/tests/test_dataset_materialization.py @@ -252,3 +252,34 @@ def test_authentication_failure_does_not_retry_repository_type(tmp_path, status_ ) assert len(session.calls) == 1 + + +def test_same_named_artifacts_of_two_releases_never_share_a_cache_path(tmp_path): + from policyengine.provenance.dataset_materialization import ( + DatasetMaterializationError, + _bundle_destination, + _resolve_bundle_dataset, + ) + + uk = _resolve_bundle_dataset("uk", "calibration_diagnostics", data_dir=tmp_path) + us = _resolve_bundle_dataset("us", "calibration_diagnostics", data_dir=tmp_path) + assert uk.destination.name == us.destination.name == "calibration_diagnostics.json" + assert uk.destination != us.destination + # A top-level file keeps its cache path. + default = _resolve_bundle_dataset("uk", data_dir=tmp_path) + assert default.destination == tmp_path / default.path + for path in ( + "/etc/passwd", + "../outside.h5", + "releases/../../outside.h5", + "..\\..\\outside.h5", + "C:\\outside.h5", + "C:outside.h5", + "\\outside.h5", + "\\\\server\\share\\outside.h5", + "releases/D:outside.h5", + "a/b/E:/outside.h5", + "", + ): + with pytest.raises(DatasetMaterializationError, match="must be relative"): + _bundle_destination(tmp_path, path) diff --git a/tests/test_dataset_runtime.py b/tests/test_dataset_runtime.py index 76cab79b6..44083c06d 100644 --- a/tests/test_dataset_runtime.py +++ b/tests/test_dataset_runtime.py @@ -143,15 +143,15 @@ def test_uk_load_datasets_defaults_to_certified_bundle_dataset(monkeypatch): ) resolve = Mock( return_value=( - "hf://policyengine/policyengine-uk-data-private/" - "enhanced_frs_2024_25.h5@1.56.16" + "hf://policyengine/populace-uk-private/" + "microcosm_uk_2024_25.h5@microcosm-uk-2024-25-national-cut" ) ) monkeypatch.setattr(uk_datasets, "resolve_dataset_reference", resolve) assert uk_datasets.load_datasets(years=[]) == {} - resolve.assert_called_once_with("uk", "enhanced_frs_2024_25") + resolve.assert_called_once_with("uk", "microcosm_uk_2024_25") def test_uk_ensure_datasets_defaults_to_certified_bundle_dataset(monkeypatch): @@ -165,7 +165,7 @@ def test_uk_ensure_datasets_defaults_to_certified_bundle_dataset(monkeypatch): assert uk_datasets.ensure_datasets(years=[]) == {} load.assert_called_once_with( - datasets=["enhanced_frs_2024_25"], + datasets=["microcosm_uk_2024_25"], years=[], data_folder="./data", ) diff --git a/tests/test_execution_receipt.py b/tests/test_execution_receipt.py index 33e9b5e58..66fe4d220 100644 --- a/tests/test_execution_receipt.py +++ b/tests/test_execution_receipt.py @@ -132,6 +132,8 @@ def _bundle_for( record["certified_data_artifact"]["sha256"] = digest record["datasets"][record["default_dataset"]]["sha256"] = digest record["certification"]["certified_for_model_version"] = model_version + if "certified_for_core_version" in record["certification"]: + record["certification"]["certified_for_core_version"] = core_version bundle["packages"]["policyengine-core"]["version"] = core_version bundle["policyengine_version"] = INSTALLED_WRAPPER record["policyengine_version"] = INSTALLED_WRAPPER @@ -303,16 +305,50 @@ def test_any_uncovered_component_makes_the_receipt_uncertified( elif change == "dataset name": # Same bytes registered under another certified dataset name. record = bundle["data_releases"]["uk"] - record["datasets"]["frs_2024_25"]["sha256"] = record["certified_data_artifact"][ - "sha256" - ] - kwargs["dataset"] = "frs_2024_25" + record["datasets"]["same_bytes_other_name"] = dict( + record["datasets"][record["default_dataset"]] + ) + kwargs["dataset"] = "same_bytes_other_name" receipt = _build(path, bundle, **kwargs) assert receipt.certification_status == "uncertified" assert receipt.certificate is None assert receipt.model.version == INSTALLED_UK # installed, never the manifest's +def test_a_core_the_certificate_names_decides_over_the_pin(dataset_file): + # bundle.py check refuses a pin that differs from a named core + # (scripts/generate_trace_tros.py), because receipts follow the certificate. + bundle = _bundle_for(dataset_file.read_bytes()) + bundle["data_releases"]["uk"]["certification"]["certified_for_core_version"] = ( + INSTALLED_CORE + ) + bundle["packages"]["policyengine-core"]["version"] = "0.0.1" + + receipt = _build(dataset_file, bundle) + + assert receipt.certification_status == "certified" + assert receipt.certificate.certified_for_core_version == INSTALLED_CORE + + +@pytest.mark.parametrize("pin", [INSTALLED_CORE, "0.0.1"]) +def test_a_certificate_without_a_core_certifies_the_pinned_core(dataset_file, pin): + # Certificates from before --core-version, and those certified without + # it, name no core: the pin is the certified core. + bundle = _bundle_for(dataset_file.read_bytes()) + bundle["data_releases"]["uk"]["certification"].pop( + "certified_for_core_version", None + ) + bundle["packages"]["policyengine-core"]["version"] = pin + + receipt = _build(dataset_file, bundle) + + if pin == INSTALLED_CORE: + assert receipt.certification_status == "certified" + assert receipt.certificate.certified_for_core_version == INSTALLED_CORE + else: + assert receipt.certification_status == "uncertified" + + @pytest.mark.parametrize( "years", [[2027.9], [True], "2027", [], [2027, "2028"], 2027, [np.float64(2027)]], @@ -376,6 +412,27 @@ def test_overlay_datasets_receive_uncertified_receipts(dataset_file): assert receipt.data.build_id is None +def test_a_regional_release_dataset_keeps_its_own_identity(dataset_file): + """A local-area file is certified beside the national release, not in it.""" + + bundle = _bundle_for(dataset_file.read_bytes()) + record = bundle["data_releases"]["uk"] + record["datasets"]["area_file"] = { + "path": "area_file.h5", + "repo_id": record["data_package"]["repo_id"], + "revision": "regional-release-cut", + "sha256": hashlib.sha256(dataset_file.read_bytes()).hexdigest(), + } + + receipt = _build(dataset_file, bundle, dataset="area_file") + + assert receipt.certification_status == "uncertified" + assert receipt.data.dataset_revision == "regional-release-cut" + assert receipt.data.release_version == "regional-release-cut" + assert receipt.data.build_id is None + assert receipt.data.release_manifest_revision is None + + def test_an_omitted_artifact_revision_is_the_release_manifest_revision(dataset_file): """Round-4 review M5: the receipt names what the materializer downloads.""" @@ -548,6 +605,16 @@ def test_a_forged_receipt_with_the_genuine_record_digest_does_not_verify( assert any("data.dataset_sha256" in p for p in problems) +#: A valid compatibility basis other than the committed certificate's. +_OTHER_BASIS = next( + basis + for basis in ("built_with_model_package", "legacy_compatible_model_package") + if basis + != json.loads(BUNDLE_PATH.read_text())["data_releases"]["uk"]["certification"][ + "compatibility_basis" + ] +) + #: Consistent edits a forger could make, each of which the model accepts. FORGERIES = { "bundle version": { @@ -567,7 +634,7 @@ def test_a_forged_receipt_with_the_genuine_record_digest_does_not_verify( "certificate__certified_dataset_sha256": "e" * 64, }, "record digest": {"certificate__certificate_sha256": "e" * 64}, - "basis": {"certificate__compatibility_basis": "built_with_model_package"}, + "basis": {"certificate__compatibility_basis": _OTHER_BASIS}, "certifier": {"certificate__certified_by": "someone else"}, "dataset name": {"data__dataset": "frs_2024_25"}, "data release": {"data__release_version": "9.9.9"}, @@ -621,6 +688,14 @@ def test_every_consistent_forgery_is_reported(tmp_path_factory, names): assert verify_certificate(_forge(genuine, **changes), bundle) +def _certify_core(bundle: dict, version: str) -> None: + """Change the core version a bundle certifies, wherever it states it.""" + certification = bundle["data_releases"]["uk"]["certification"] + if "certified_for_core_version" in certification: + certification["certified_for_core_version"] = version + bundle["packages"]["policyengine-core"].update(version=version) + + @pytest.mark.parametrize( "edit", [ @@ -633,7 +708,7 @@ def test_every_consistent_forgery_is_reported(tmp_path_factory, names): ), lambda b: b["data_releases"]["uk"].update(policyengine_version="0.0.0"), lambda b: b.update(policyengine_version="0.0.0"), - lambda b: b["packages"]["policyengine-core"].update(version="0.0.1"), + lambda b: _certify_core(b, "0.0.1"), ], ids=[ "certifier", diff --git a/tests/test_models.py b/tests/test_models.py index c2ad28752..682925a09 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -43,13 +43,13 @@ def test_has_release_manifest_metadata(self): assert uk_latest.release_manifest is not None assert uk_latest.release_manifest.country_id == "uk" assert uk_latest.model_package.name == "policyengine-uk" - assert uk_latest.model_package.version == "2.130.1" - assert uk_latest.data_package.name == "policyengine-uk-data" - assert uk_latest.data_package.version == "1.58.1" + assert uk_latest.model_package.version == "2.132.6" + assert uk_latest.data_package.name == "microcosm-data" + assert uk_latest.data_package.version == "0.1.0" assert ( uk_latest.default_dataset_uri - == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5" - "@1.58.1" + == "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" + "@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" ) def test_has_hundreds_of_parameters(self): diff --git a/tests/test_qualification_ledger_concurrency.py b/tests/test_qualification_ledger_concurrency.py index 3ca61a03d..95b374168 100644 --- a/tests/test_qualification_ledger_concurrency.py +++ b/tests/test_qualification_ledger_concurrency.py @@ -56,14 +56,20 @@ stage_attempt, write_gate_results, ) -from tests.qualification_fixtures import HARNESS, ISSUED, gates, golden, make_pair +from tests.qualification_fixtures import ( + HARNESS, + ISSUED, + UK_DATA_LINE_BUNDLE, + gates, + golden, + make_pair, +) ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / ".github" / "qualification-ledger.sh" CLI = ROOT / "scripts" / "check_uk_candidate.py" -BASE = json.loads( - (ROOT / "src" / "policyengine" / "data" / "bundle" / "manifest.json").read_text() -) +# PAIR is a policyengine-uk-data pair, so main is a bundle on that line. +BASE = UK_DATA_LINE_BUNDLE PAIR = make_pair( model="2.200.0", data=BASE["data_releases"]["uk"]["data_package"]["version"] ) diff --git a/tests/test_release_manifests.py b/tests/test_release_manifests.py index 2586c593d..aff21577a 100644 --- a/tests/test_release_manifests.py +++ b/tests/test_release_manifests.py @@ -73,12 +73,21 @@ "hf://policyengine/populace-us/populace_us_2024_acs_local.h5" f"@{US_LOCAL_AREA_RELEASE_ID}" ) -UK_MODEL_VERSION = "2.130.1" -UK_BUILT_WITH_MODEL_VERSION = "2.122.2" -UK_DATA_RELEASE_VERSION = "1.58.1" -UK_DATA_RELEASE_ID = "policyengine-uk-data-1.58.1" -UK_DATA_RELEASE_REVISION = "89be6b73fc1e6eeba6dae2f9400676d8510bcd51" -UK_DATA_RELEASE_PATH = "releases/1.58.1/release_manifest.json" +UK_MODEL_VERSION = "2.132.6" +UK_BUILT_WITH_MODEL_VERSION = "2.132.6" +UK_DATA_RELEASE_VERSION = "0.1.0" +UK_DATA_RELEASE_ID = "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" +UK_DATA_RELEASE_REVISION = "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" +UK_DATA_RELEASE_PATH = "releases/microcosm-uk-2024-25-national/release_manifest.json" +UK_LOCAL_AREA_RELEASE_ID = "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe" +# The prior certified default (policyengine-uk-data 1.58.1), kept resolvable +# via dataset_overlays. +UK_EFRS_DATASET_URI = ( + "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1" +) +UK_EFRS_FRS_DATASET_URI = ( + "hf://policyengine/policyengine-uk-data-private/frs_2024_25.h5@1.58.1" +) # The prior certified default, kept resolvable via dataset_overlays. UK_POPULACE_RELEASE_ID = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z" UK_POPULACE_DATASET_URI = ( @@ -87,7 +96,12 @@ ) UK_CERTIFICATION_SOURCE = "policyengine.py bundle certification" UK_CERTIFIED_DATASET_URI = ( - "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5@1.58.1" + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" + f"@{UK_DATA_RELEASE_ID}" +) +UK_LOCAL_AREA_DATASET_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25_dense.h5" + f"@{UK_LOCAL_AREA_RELEASE_ID}" ) @@ -226,25 +240,20 @@ def test__given_uk_manifest__then_has_pinned_model_and_data_packages(self): assert manifest.policyengine_version == POLICYENGINE_VERSION assert manifest.model_package.name == "policyengine-uk" assert manifest.model_package.version == UK_MODEL_VERSION - assert manifest.data_package.name == "policyengine-uk-data" + assert manifest.data_package.name == "microcosm-data" assert manifest.data_package.version == UK_DATA_RELEASE_VERSION - assert manifest.data_package.repo_id == ( - "policyengine/policyengine-uk-data-private" - ) + assert manifest.data_package.repo_id == "policyengine/populace-uk-private" assert manifest.data_package.release_manifest_path == UK_DATA_RELEASE_PATH assert ( manifest.data_package.release_manifest_revision == UK_DATA_RELEASE_REVISION ) assert manifest.certified_data_artifact is not None assert manifest.certified_data_artifact.build_id == UK_DATA_RELEASE_ID - assert manifest.certified_data_artifact.dataset == "enhanced_frs_2024_25" + assert manifest.certified_data_artifact.dataset == "microcosm_uk_2024_25" assert manifest.certified_data_artifact.uri == UK_CERTIFIED_DATASET_URI assert manifest.certification is not None assert manifest.certification.data_build_id == UK_DATA_RELEASE_ID - assert ( - manifest.certification.compatibility_basis - == "legacy_compatible_model_package" - ) + assert manifest.certification.compatibility_basis == "built_with_model_package" assert manifest.certification.certified_by == UK_CERTIFICATION_SOURCE assert ( manifest.certification.built_with_model_version @@ -277,10 +286,27 @@ def test__given_dataset_explicit_revision__then_resolves_to_that_revision(self): ) def test__given_uk_dataset_name__then_resolves_to_versioned_hf_url(self): - resolved = resolve_dataset_reference("uk", "enhanced_frs_2024_25") + resolved = resolve_dataset_reference("uk", "microcosm_uk_2024_25") assert resolved == UK_CERTIFIED_DATASET_URI + def test__given_uk_manifest__then_area_regions_route_to_local_release(self): + manifest = get_release_manifest("uk") + + assert manifest.regional_release_kind == "local_area" + assert { + region_type: template.path_template + for region_type, template in manifest.region_datasets.items() + } == { + "national": "microcosm_uk_2024_25.h5", + "constituency": "microcosm_uk_2024_25_dense.h5", + "local_authority": "microcosm_uk_2024_25_dense.h5", + } + assert ( + resolve_dataset_reference("uk", "microcosm_uk_2024_25_dense") + == UK_LOCAL_AREA_DATASET_URI + ) + def test__given_uk_legacy_dataset_names__then_resolves_bundled_aliases(self): assert ( resolve_dataset_reference("uk", "frs_2023_24") == UK_LEGACY_FRS_DATASET_URI @@ -293,6 +319,11 @@ def test__given_uk_legacy_dataset_names__then_resolves_bundled_aliases(self): resolve_dataset_reference("uk", "populace_uk_2023") == UK_POPULACE_DATASET_URI ) + assert ( + resolve_dataset_reference("uk", "enhanced_frs_2024_25") + == UK_EFRS_DATASET_URI + ) + assert resolve_dataset_reference("uk", "frs_2024_25") == UK_EFRS_FRS_DATASET_URI def test__given_explicit_url__then_resolution_is_noop(self): url = "hf://policyengine/policyengine-us-data/cps_2023.h5@1.73.0" @@ -915,11 +946,11 @@ def test__given_manifest_certification__then_release_bundle_exposes_it(self): bundle = model_version.release_bundle assert bundle["bundle_id"] == f"uk-{POLICYENGINE_VERSION}" - assert bundle["default_dataset"] == "enhanced_frs_2024_25" + assert bundle["default_dataset"] == "microcosm_uk_2024_25" assert bundle["default_dataset_uri"] == manifest.default_dataset_uri assert bundle["certified_data_build_id"] == UK_DATA_RELEASE_ID assert bundle["data_build_model_version"] == UK_BUILT_WITH_MODEL_VERSION - assert bundle["compatibility_basis"] == "legacy_compatible_model_package" + assert bundle["compatibility_basis"] == "built_with_model_package" assert bundle["certified_by"] == UK_CERTIFICATION_SOURCE def test__given_runtime_certification__then_release_bundle_prefers_runtime_value( @@ -1099,30 +1130,30 @@ def test__given_uk_managed_dataset_name__then_resolves_within_bundle(self): "materialize_dataset", return_value=_dataset_source( "uk", - "enhanced_frs_2024_25", - "/tmp/enhanced_frs_2024_25.h5", + "microcosm_uk_2024_25", + "/tmp/microcosm_uk_2024_25.h5", ), ): microsim = uk_model.managed_microsimulation( - dataset="enhanced_frs_2024_25" + dataset="microcosm_uk_2024_25" ) dataset = mock_microsimulation.call_args.kwargs["dataset"] - assert dataset == "/tmp/enhanced_frs_2024_25.h5" + assert dataset == "/tmp/microcosm_uk_2024_25.h5" assert ( microsim.policyengine_bundle["policyengine_version"] == POLICYENGINE_VERSION ) - assert microsim.policyengine_bundle["runtime_dataset"] == "enhanced_frs_2024_25" + assert microsim.policyengine_bundle["runtime_dataset"] == "microcosm_uk_2024_25" assert microsim.policyengine_bundle["runtime_dataset_uri"] == ( UK_CERTIFIED_DATASET_URI ) dataset_source = microsim.policyengine_bundle["runtime_dataset_source"] - assert dataset_source == "/tmp/enhanced_frs_2024_25.h5" + assert dataset_source == "/tmp/microcosm_uk_2024_25.h5" assert ( microsim.policyengine_bundle["runtime_dataset_sha256"] - == get_release_manifest("uk").datasets["enhanced_frs_2024_25"].sha256 + == get_release_manifest("uk").datasets["microcosm_uk_2024_25"].sha256 ) - assert microsim.policyengine_bundle["runtime_dataset_repo_type"] == "model" + assert microsim.policyengine_bundle["runtime_dataset_repo_type"] == "dataset" def test__given_uk_unmanaged_dataset_uri__then_source_is_not_rewritten(self): dataset = "hf://policyengine/policyengine-uk-data-private/frs_2022_23.h5@1.40.4" @@ -1171,8 +1202,9 @@ class TestDatasetOverlays: ``dataset_overlays`` is the hand-maintained sibling of ``data_releases``. ``certify_data_release`` rewrites ``data_releases.{country}`` wholesale, so - these tests pin the invariant that overlays are additive, survive - re-certification, and can never hijack default resolution. + these tests pin the invariant that overlays are additive, survive a + re-certification that does not certify their names, and can never hijack + default resolution. """ def teardown_method(self): @@ -1251,7 +1283,7 @@ def test__given_overlay_shadowing_default__then_raises(self): try: _apply_dataset_overlays("us", payload, bundle) except ValueError as error: - assert "shadows the certified default" in str(error) + assert "shadows certified datasets: populace_us_2024" in str(error) else: raise AssertionError("Expected overlay shadowing the default to fail") @@ -1270,6 +1302,6 @@ def test__given_overlay_colliding_with_certified_dataset__then_raises(self): try: _apply_dataset_overlays("us", payload, bundle) except ValueError as error: - assert "collides with a certified dataset" in str(error) + assert "shadows certified datasets: demographics" in str(error) else: raise AssertionError("Expected overlay colliding with a dataset to fail") diff --git a/tests/test_release_tro_generation.py b/tests/test_release_tro_generation.py index 69ffd6a82..d0aa9410d 100644 --- a/tests/test_release_tro_generation.py +++ b/tests/test_release_tro_generation.py @@ -114,6 +114,57 @@ def test_runtime_pin_update_requires_matching_certified_model(release): assert maintenance._check_tros(strict=True) == 0 +def test_runtime_core_must_match_a_core_the_certificate_names(release): + """A core-only pin change must not leave receipts on another core.""" + + path = release[0] / "manifest.json" + payload = json.loads(path.read_bytes()) + # The fixture release was built with, and claims, this core. + certified_core = "3.26.11" + payload["data_releases"]["us"]["certification"]["certified_for_core_version"] = ( + certified_core + ) + payload["packages"]["policyengine-core"]["version"] = certified_core + path.write_text(json.dumps(payload)) + manifest.get_release_manifest.cache_clear() + generator.generated_tros(strict=True) + + payload["packages"]["policyengine-core"]["version"] = "99.0.0" + path.write_text(json.dumps(payload)) + manifest.get_release_manifest.cache_clear() + with pytest.raises(ValueError, match="runtime core 99.0.0 differs"): + generator.generated_tros(strict=True) + + +@pytest.mark.parametrize(("pin", "warned"), [("3.26.11", False), ("99.0.0", True)]) +def test_a_certificate_that_follows_the_pin_reports_an_unclaimed_pin( + release, capsys, pin, warned +): + """A certificate that names no core follows the core pin: an unclaimed + pin is reported, not refused.""" + + path = release[0] / "manifest.json" + payload = json.loads(path.read_bytes()) + payload["data_releases"]["us"]["certification"]["core_binding"] = "bundle_pin" + payload["packages"]["policyengine-core"]["version"] = pin + path.write_text(json.dumps(payload)) + manifest.get_release_manifest.cache_clear() + + generator.generated_tros(strict=True) + + assert ("follows the core pin" in capsys.readouterr().err) == warned + + +def test_a_core_binding_must_agree_with_the_named_core(release): + path = release[0] / "manifest.json" + payload = json.loads(path.read_bytes()) + payload["data_releases"]["us"]["certification"]["core_binding"] = "certified" + path.write_text(json.dumps(payload)) + manifest.get_release_manifest.cache_clear() + with pytest.raises(ValueError, match="core_binding 'certified' disagrees"): + generator.generated_tros(strict=True) + + @pytest.mark.parametrize("field", ["name", "version", "sha256", "wheel_url"]) def test_runtime_model_identity_must_match_certification(release, field): path = release[0] / "manifest.json" diff --git a/tests/test_uk_geography_disclosure.py b/tests/test_uk_geography_disclosure.py index d4245ab10..073346926 100644 --- a/tests/test_uk_geography_disclosure.py +++ b/tests/test_uk_geography_disclosure.py @@ -464,3 +464,27 @@ def test_the_release_is_available_only_after_a_run(): def test_the_adapter_is_exported_from_outputs(): assert outputs.UKGeographyDisclosure is UKGeographyDisclosure assert outputs.compute_uk_geography_disclosure is compute_uk_geography_disclosure + + +def test_a_missing_area_column_names_the_dataset_that_carries_it(): + from policyengine.provenance.manifest import uk_area_dataset_hint + + baseline, reform = _households() + for frame in (baseline, reform): + del frame["constituency_code_oa"] + with pytest.raises(ValueError, match="no household an area") as error: + _compute(baseline, reform, levels=("constituency",)) + hint = uk_area_dataset_hint("constituency") + assert "microcosm_uk_2024_25_dense" in hint + assert hint in str(error.value) + + +def test_no_hint_when_nothing_routes_the_area(monkeypatch): + from policyengine.provenance import manifest + + record = manifest.get_release_manifest("uk") + unrouted = record.model_copy( + update={"region_datasets": {"national": record.region_datasets["national"]}} + ) + monkeypatch.setattr(manifest, "get_release_manifest", lambda country: unrouted) + assert manifest.uk_area_dataset_hint("constituency") == "" diff --git a/tests/test_uk_qualification_pipeline.py b/tests/test_uk_qualification_pipeline.py index 5d94bf5b4..ea07e2d0f 100644 --- a/tests/test_uk_qualification_pipeline.py +++ b/tests/test_uk_qualification_pipeline.py @@ -17,6 +17,7 @@ from __future__ import annotations import hashlib +import importlib.util import json import os import shutil @@ -24,6 +25,7 @@ import sys from pathlib import Path from types import SimpleNamespace +from unittest.mock import MagicMock import jsonschema import pytest @@ -36,6 +38,7 @@ build_replay_request, candidate_action, candidate_branch, + candidate_pr_body, certified_constant_replacements, choose_core_version, core_requirement, @@ -49,11 +52,15 @@ from tests.qualification_fixtures import ( HARNESS, ISSUED, + MICROCOSM_ROUTED_BUNDLE, + MICROCOSM_ROUTED_BUNDLE_PATH, + UK_DATA_LINE_BUNDLE, VENDORED_SCHEMAS, gates, golden, graphql_base_changes_output, jq, + load_candidate_cli, make_pair, make_receipt, rest_pulls_output, @@ -61,9 +68,13 @@ ) REPO_ROOT = Path(__file__).resolve().parents[1] -BUNDLE = json.loads( +#: The bundle this checkout packages. +PACKAGED_BUNDLE = json.loads( (REPO_ROOT / "src/policyengine/data/bundle/manifest.json").read_text() ) +#: The candidates these tests build are on the policyengine-uk-data line, so +#: they are qualified against the last bundle certified on that line. +BUNDLE = UK_DATA_LINE_BUNDLE def _wheel(name, version, digest): @@ -150,14 +161,7 @@ def test_resolve_refuses_a_pair_other_than_the_one_requested( resolve to another (a moved data tag, another core), the run fails and nothing else is qualified.""" - import importlib.util - - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_pair_pin", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + cli = load_candidate_cli("check_uk_candidate_pair_pin") pair = make_pair(model="2.200.0") resolved = cli.ResolvedCandidate( pair, @@ -471,14 +475,8 @@ def test_requalify_names_the_target_only_while_its_baseline_is_stale( monkeypatch, tmp_path, capsys ): import argparse - import importlib.util - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_requalify", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + cli = load_candidate_cli("check_uk_candidate_requalify") current = _promotion_receipt(pair=_promotable_pair_and_branch()) stale = current.model_copy( update={ @@ -522,13 +520,16 @@ def test_incomplete_pairs_requalify_on_a_new_base_only(tmp_path): ) -def _new_record(old): +SUCCESSOR_CUT = "microcosm-uk-2024-25-national-20261101T000000Z-aaaaaaaa" + + +def _successor_record(old): + """The next certificate on the packaged bundle's own data line.""" new = json.loads(json.dumps(old)) - new["model_package"]["version"] = "2.126.8" - new["certification"]["built_with_model_version"] = "2.126.0" - new["certification"]["data_build_id"] = "policyengine-uk-data-1.59.0" - new["data_package"]["version"] = "1.59.0" - new["data_package"]["release_manifest_revision"] = "1" * 40 + new["model_package"]["version"] = "2.140.0" + new["certification"]["built_with_model_version"] = "2.140.0" + new["certification"]["data_build_id"] = SUCCESSOR_CUT + new["data_package"]["release_manifest_revision"] = SUCCESSOR_CUT return new @@ -540,8 +541,10 @@ def test_constant_rewrite_leaves_no_old_values(tmp_path): ): (tmp_path / relative).parent.mkdir(parents=True, exist_ok=True) shutil.copy(REPO_ROOT / relative, tmp_path / relative) - old = BUNDLE["data_releases"]["uk"] - new = _new_record(old) + # The constant files pin the packaged certificate, so promotion rewrites + # them from it. + old = PACKAGED_BUNDLE["data_releases"]["uk"] + new = _successor_record(old) replacements = certified_constant_replacements(old, new) changed = rewrite_certified_constants(tmp_path, replacements) assert set(changed) == { @@ -554,17 +557,23 @@ def test_constant_rewrite_leaves_no_old_values(tmp_path): for before, after in replacements: assert f'"{before}"' not in text and f'@{before}"' not in text manifests = (tmp_path / "tests/test_release_manifests.py").read_text() - assert 'UK_MODEL_VERSION = "2.126.8"' in manifests - assert 'UK_DATA_RELEASE_PATH = "releases/1.59.0/release_manifest.json"' in manifests - assert 'UK_DATA_RELEASE_REVISION = "' + "1" * 40 + '"' in manifests + assert 'UK_MODEL_VERSION = "2.140.0"' in manifests + assert f'UK_DATA_RELEASE_REVISION = "{SUCCESSOR_CUT}"' in manifests assert rewrite_certified_constants(tmp_path, replacements) == [] +@pytest.mark.parametrize( + "routed", [False, True], ids=["uk-data main", "main routed to Microcosm"] +) @pytest.mark.parametrize("qualification_only", [False, True]) def test_preparation_generates_new_sidecars_and_verifies_the_exact_pair( - tmp_path, monkeypatch, qualification_only + tmp_path, monkeypatch, qualification_only, routed ): - """Run preparation, certification and real TRO generation without a network.""" + """Run preparation, certification and real TRO generation without a network. + + On a main routed to another line's local-area release, the uk-data + candidate (a forced qualification) drops the routing it cannot keep. + """ from policyengine.provenance import certification, manifest from policyengine.provenance.uk_qualification import prepare_candidate_commit @@ -591,7 +600,7 @@ def test_preparation_generates_new_sidecars_and_verifies_the_exact_pair( pair = resolved.pair bundle_dir = tmp_path / "src/policyengine/data/bundle" bundle_dir.mkdir(parents=True) - source = json.loads(json.dumps(BUNDLE)) + source = json.loads(json.dumps(MICROCOSM_ROUTED_BUNDLE if routed else BUNDLE)) source["data_releases"].pop("us") source["packages"].pop("policyengine-us") for packages in source["extras"].values(): @@ -688,14 +697,60 @@ def run(command, *, cwd, env=None): assert (tmp_path / prepared["changelog_fragment"]).is_file() assert maintenance.main(["check", "--include-tros", "--strict-tros"]) == 0 assert calls[-1][2:] == ["check", "--include-tros", "--strict-tros"] + certify = next(command for command in calls if "certify-data" in command) + record = json.loads(bundle_path.read_bytes())["data_releases"]["uk"] + assert ("--drop-regional-release" in certify) == routed + assert "regional_source_manifest_uri" not in record + if routed: + # The constant files pin main's line, not the candidate's. + assert prepared["constant_files"] == [] if qualification_only: changed = json.loads(bundle_path.read_bytes()) changed["packages"]["policyengine-core"]["version"] = "99.0.0" write_json(bundle_path, changed) - with pytest.raises(ValueError, match="runtime core.*qualified exact pair"): + with pytest.raises(ValueError, match="runtime core 99.0.0 differs.*requalify"): tros.generated_tros(strict=True) +def test_regional_flags_keep_routing_on_its_line_and_drop_it_across_lines(): + from policyengine.provenance.uk_qualification import ( + regional_certification_arguments, + ) + + uk_data_pair = make_pair() + record = MICROCOSM_ROUTED_BUNDLE["data_releases"]["uk"] + + def microcosm_pair(model, core): + pair = make_pair(model=model, core=core) + return pair.model_copy( + update={ + "data": pair.data.model_copy( + update={ + "data_package": record["data_package"]["name"], + "repo_id": record["data_package"]["repo_id"], + } + ) + } + ) + + certified = ( + record["model_package"]["version"], + MICROCOSM_ROUTED_BUNDLE["packages"]["policyengine-core"]["version"], + ) + + assert regional_certification_arguments(BUNDLE, uk_data_pair) == [] + assert regional_certification_arguments(MICROCOSM_ROUTED_BUNDLE, uk_data_pair) == [ + "--drop-regional-release" + ] + # A same-line pair needs a local-area cut for its versions, which no + # candidate carries, whatever those versions are. + for model, core in (certified, ("2.140.0", certified[1]), (certified[0], "3.34.0")): + with pytest.raises(ValueError, match="certify such a pair by hand"): + regional_certification_arguments( + MICROCOSM_ROUTED_BUNDLE, microcosm_pair(model, core) + ) + + def test_replay_request_is_valid_for_the_scorecard(): pair = make_pair() release_bundle = dict( @@ -762,7 +817,7 @@ def test_the_real_cli_generates_replay_requests_despite_country_import_suppressi candidate = request["candidate"] assert candidate["policyengine"]["source"]["git"]["commit"] == HARNESS.git_commit bundle = candidate["expected_release_bundle"] - record = BUNDLE["data_releases"]["uk"] + record = PACKAGED_BUNDLE["data_releases"]["uk"] assert bundle["model_version"] == record["model_package"]["version"] assert bundle["data_version"] == record["data_package"]["version"] assert bundle["default_dataset"] == record["default_dataset"] @@ -930,8 +985,8 @@ def test_a_pair_from_another_data_line_is_never_ordered_against_main(tmp_path): """A uk-data release number says nothing about a Microcosm build. With a Microcosm certificate on main, every uk-data pair compared as newer (1.58.x > 0.1.0) and would have been qualified and promoted back. Now - automated qualification skips it unless forced, and promotion neither - readies nor closes it.""" + automated qualification skips it unless forced, and promotion never + readies it and closes its pull request.""" from policyengine.provenance.uk_qualification import requalification_target @@ -951,6 +1006,9 @@ def test_a_pair_from_another_data_line_is_never_ordered_against_main(tmp_path): plan = plan_promotion([promotable], bundle, {branch: promotable.harness.git_commit}) assert plan.target is None and plan.ready_branch is None assert branch not in plan.close_branches + assert plan.close_off_line == [branch] + body = candidate_pr_body(promotable, uk_data, ledger_url="u", on_main_line=False) + assert "never promoted" in body and "promotes the pair" not in body assert requalification_target([promotable], bundle, tmp_path, "b" * 40) is None # On main's own data line the same pair still advances. @@ -968,6 +1026,127 @@ def _promotable_pair_and_branch(core=None): return make_pair(model="2.200.0", data=current, core=core) +def _other_line_pair(model="2.900.0"): + """A pair on the Microcosm line, which BUNDLE (a uk-data main) is not.""" + + pair = make_pair(model=model) + return pair.model_copy( + update={ + "data": pair.data.model_copy( + update={ + "data_package": "microcosm-data", + "repo_id": "policyengine/populace-uk-private", + } + ) + } + ) + + +def test_promotion_closes_other_line_prs_only_at_their_qualified_commit(): + target = _promotion_receipt(pair=_promotable_pair_and_branch()) + other = _promotion_receipt(pair=_other_line_pair()) + target_branch = candidate_branch(target.pair) + other_branch = candidate_branch(other.pair) + + plan = plan_promotion( + [target, other], + BUNDLE, + { + target_branch: target.harness.git_commit, + other_branch: other.harness.git_commit, + }, + ) + assert plan.ready_branch == target_branch + assert plan.close_off_line == [other_branch] + assert other_branch not in plan.close_branches + + # A newer head is a forced re-run still in flight: its own receipt decides. + in_flight = plan_promotion( + [target, other], + BUNDLE, + {target_branch: target.harness.git_commit, other_branch: "f" * 40}, + ) + assert in_flight.ready_branch == target_branch + assert in_flight.close_off_line == [] + + +def test_promote_closes_other_line_prs_and_never_drafts_a_closed_one(monkeypatch): + import argparse + + cli = load_candidate_cli("check_uk_promote_line", MICROCOSM_ROUTED_BUNDLE_PATH) + receipt = _promotion_receipt(pair=make_pair(model="2.200.0")) + branch = candidate_branch(receipt.pair) + monkeypatch.setattr(cli, "latest_receipts", lambda ledger, problems=None: [receipt]) + monkeypatch.setattr( + cli, + "_candidate_pull_requests", + lambda: [ + { + "number": 7, + "headRefName": branch, + "headRefOid": receipt.harness.git_commit, + "isDraft": False, + "isCrossRepository": False, + } + ], + ) + commands = [] + monkeypatch.setattr( + cli.subprocess, + "run", + lambda command, **kwargs: ( + commands.append(command) or subprocess.CompletedProcess(command, 0) + ), + ) + drafted = [] + monkeypatch.setattr( + cli, "_set_ready", lambda number, ready: drafted.append(number) or True + ) + + assert cli.cmd_promote(argparse.Namespace(ledger="ledger", apply=True)) == 0 + assert commands == [ + [ + "gh", + "pr", + "close", + "7", + "--repo", + cli.LEDGER_REPOSITORY, + "--comment", + "This pair is not on main's data line, so it is never promoted. " + "The ledger keeps this candidate's evidence.", + ] + ] + # gh refuses to return a closed pull request to draft. + assert drafted == [] + + +def test_finalize_says_an_other_line_pair_is_never_promoted(monkeypatch, tmp_path): + import argparse + + cli = load_candidate_cli("check_uk_finalize_line", MICROCOSM_ROUTED_BUNDLE_PATH) + receipt = _promotion_receipt(pair=make_pair(model="2.200.0")) + attempt = tmp_path / "attempt" + write_json( + attempt / "candidate.json", + { + "pair": receipt.pair.canonical_payload(), + "branch": candidate_branch(receipt.pair), + }, + ) + monkeypatch.setattr(cli, "finalize_attempt", lambda *args, **kwargs: receipt) + monkeypatch.setattr(cli, "issued_by", lambda: ISSUED) + monkeypatch.setattr(cli, "set_output", lambda **kwargs: None) + body = tmp_path / "body.md" + + namespace = argparse.Namespace( + attempt=str(attempt), ledger=str(tmp_path / "ledger"), pr_body=str(body) + ) + assert cli.cmd_finalize(namespace) == 0 + text = body.read_text() + assert "never promoted" in text and "promotes the pair" not in text + + def test_pairs_that_share_versions_never_share_a_branch(): """Interrupted review of #574: a qualified pair readied the branch of an incomplete pair with the same model and data but another core.""" @@ -1368,7 +1547,6 @@ def test_promotion_makes_main_exactly_the_qualified_commit( moves main to it with a push leased on that parent.""" import argparse - import importlib.util checks, runs, at_commit, main, push_code, automatic, promotes, readiness = ( PROMOTION_CASES[case] @@ -1377,12 +1555,7 @@ def test_promotion_makes_main_exactly_the_qualified_commit( branch = candidate_branch(qualified.pair) commit = qualified.harness.git_commit parent = "p" * 40 - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_for_test", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + cli = load_candidate_cli("check_uk_candidate_for_test") calls = [] target = { @@ -1499,8 +1672,6 @@ def _sweep_cli( ``shared_fails`` fails the REST listing of a branch's pull requests ("listing") or the base-change query ("history").""" - import importlib.util - qualified = _promotion_receipt(pair=_promotable_pair_and_branch()) branch = candidate_branch(qualified.pair) commit = qualified.harness.git_commit @@ -1508,12 +1679,7 @@ def _sweep_cli( if superseded: receipts.append(_promotion_receipt(pair=SUPERSEDED_PAIR)) stale = "candidate/uk/stale-pair" - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_for_test", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + cli = load_candidate_cli("check_uk_candidate_for_test") calls = [] listing = [ { @@ -1657,14 +1823,7 @@ def test_a_candidate_listing_at_its_limit_is_refused(monkeypatch, listed, refuse listing that reaches it may have missed the target; promotion refuses to go on rather than act on part of the list.""" - import importlib.util - - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_for_test", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + cli = load_candidate_cli("check_uk_candidate_for_test") listing = [ { "number": number, @@ -1882,14 +2041,7 @@ def test_only_pull_requests_to_main_are_candidates(monkeypatch): branch cannot collapse into one entry that sends readiness changes to the wrong one.""" - import importlib.util - - spec = importlib.util.spec_from_file_location( - "check_uk_candidate_for_test", - Path(__file__).resolve().parents[1] / "scripts" / "check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + cli = load_candidate_cli("check_uk_candidate_for_test") calls = [] listing = [ { @@ -2691,8 +2843,6 @@ def _publish_as_attempt(ledger, pair, receipt, key): def test_forced_stage_recovery_binds_the_terminal_receipt_it_supersedes( tmp_path, verdict, main_advanced, staged_base, monkeypatch, capsys ): - import importlib.util - from policyengine.provenance.qualification_ledger import attempt_directory from policyengine.provenance.uk_qualification import stage_attempt @@ -2716,11 +2866,7 @@ def test_forced_stage_recovery_binds_the_terminal_receipt_it_supersedes( candidate_action(pair, BUNDLE, tmp_path, base_commit=base, now=now)[0] == "qualify" ) - spec = importlib.util.spec_from_file_location( - "check_uk_bound_recovery_for_test", REPO_ROOT / "scripts/check_uk_candidate.py" - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + cli = load_candidate_cli("check_uk_bound_recovery_for_test") monkeypatch.setattr(cli, "utc_now", lambda: now) assert cli.main(["recoverable", "--ledger", str(tmp_path)]) == 0 assert json.loads(capsys.readouterr().out) == payload @@ -2790,17 +2936,10 @@ def test_forced_recovery_preserves_newer_work_on_another_base(tmp_path, protecte def test_resolution_preserves_authorization_to_recover_forced_work( tmp_path, monkeypatch, verdict, main_advanced, abandonment ): - import importlib.util - from policyengine.provenance.qualification_ledger import attempt_directory from policyengine.provenance.uk_qualification import stage_attempt - spec = importlib.util.spec_from_file_location( - "check_uk_resolve_recovery_for_test", - REPO_ROOT / "scripts/check_uk_candidate.py", - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + cli = load_candidate_cli("check_uk_resolve_recovery_for_test") pair = make_pair(model="2.200.0") terminal = make_receipt( pair=pair, @@ -2919,13 +3058,7 @@ def test_cross_base_recovery_allocates_new_keys_without_overwriting_evidence(tmp def test_recovery_requires_a_confirmed_failed_owner( monkeypatch, status, conclusion, expected ): - import importlib.util - - spec = importlib.util.spec_from_file_location( - "check_uk_owner_for_test", REPO_ROOT / "scripts/check_uk_candidate.py" - ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + cli = load_candidate_cli("check_uk_owner_for_test") calls = [] def run(argv, **kwargs): @@ -2946,19 +3079,76 @@ def run(argv, **kwargs): assert not cli.owner_run_failed("42") -def test_catchup_finds_abandoned_stages_without_replay_requests( - tmp_path, monkeypatch, capsys +@pytest.mark.parametrize("force", [False, True]) +def test_recovery_skips_pairs_from_another_data_line( + tmp_path, monkeypatch, capsys, force ): - import importlib.util + """The catch-up re-dispatches without force, so even a forced run of a + pair from another line would resolve as other_line and clear nothing.""" from policyengine.provenance.qualification_ledger import attempt_directory from policyengine.provenance.uk_qualification import stage_attempt - spec = importlib.util.spec_from_file_location( - "check_uk_recovery_for_test", REPO_ROOT / "scripts/check_uk_candidate.py" + # Main is a routed Microcosm bundle; make_pair builds a uk-data pair. + cli = load_candidate_cli("check_uk_recovery_line", MICROCOSM_ROUTED_BUNDLE_PATH) + pair = make_pair() + attempt = attempt_directory(tmp_path, "uk", pair.pair_id, "a" * 12) + payload = {"pair": pair.canonical_payload(), "force": force} + stage_attempt(attempt, payload, HARNESS) + write_json( + attempt / "staging.json", + {"sequence": 1, "owner_run_id": "42", "expires_at": "2026-10-14T14:00:00Z"}, ) - cli = importlib.util.module_from_spec(spec) - spec.loader.exec_module(cli) + monkeypatch.setattr(cli, "utc_now", lambda: "2026-10-14T11:00:00Z") + + # A live attempt is not recoverable, and logs nothing. + monkeypatch.setattr(cli, "owner_run_failed", lambda run_id: False) + assert cli.main(["recoverable", "--ledger", str(tmp_path)]) == 0 + assert capsys.readouterr() == ("", "") + + monkeypatch.setattr(cli, "owner_run_failed", lambda run_id: run_id == "42") + assert cli.main(["recoverable", "--ledger", str(tmp_path)]) == 0 + out = capsys.readouterr() + assert out.out == "" + assert "not on main's data line" in out.err + + +def test_check_refresh_skips_pairs_from_another_data_line(tmp_path, monkeypatch): + import argparse + + cli = load_candidate_cli("check_uk_refresh_line", MICROCOSM_ROUTED_BUNDLE_PATH) + receipt = _promotion_receipt(pair=make_pair(model="2.200.0")) + assert receipt.promotable + branch = candidate_branch(receipt.pair) + monkeypatch.setattr(cli, "latest_receipts", lambda ledger: [receipt]) + monkeypatch.setattr( + cli, + "_candidate_pull_requests", + lambda: [ + { + "number": 1, + "headRefName": branch, + "headRefOid": receipt.harness.git_commit, + } + ], + ) + + def unexpected(*args, **kwargs): + raise AssertionError("a pair from another data line was refreshed") + + monkeypatch.setattr(cli, "_shared_branch_pull_requests", unexpected) + monkeypatch.setattr(cli, "_candidate_runs", unexpected) + + assert cli.cmd_refresh_pr_checks(argparse.Namespace(ledger=str(tmp_path))) == 0 + + +def test_catchup_finds_abandoned_stages_without_replay_requests( + tmp_path, monkeypatch, capsys +): + from policyengine.provenance.qualification_ledger import attempt_directory + from policyengine.provenance.uk_qualification import stage_attempt + + cli = load_candidate_cli("check_uk_recovery_for_test") pair = make_pair() attempt = attempt_directory(tmp_path, "uk", pair.pair_id, "a" * 12) payload = {"pair": pair.canonical_payload()} @@ -3291,39 +3481,58 @@ def test_release_payload_names_the_qualification_receipt(): } -def test_released_bundle_request_is_valid_for_the_scorecard(): +@pytest.mark.parametrize( + ("bundle", "manifest_size"), + [(BUNDLE, 128369110), (MICROCOSM_ROUTED_BUNDLE, None)], + ids=["uk-data model repo", "Microcosm dataset repo without sizes"], +) +def test_released_bundle_request_is_valid_for_the_scorecard(bundle, manifest_size): from policyengine.provenance.uk_qualification import released_bundle_request - record = BUNDLE["data_releases"]["uk"] + record = bundle["data_releases"]["uk"] dataset = record["default_dataset"] + reference = record["datasets"][dataset] + artifact = { + "kind": "microdata", + "path": reference["path"], + "repo_id": reference.get("repo_id") or record["data_package"]["repo_id"], + "revision": reference["revision"], + "sha256": reference["sha256"], + } + if manifest_size is not None: + artifact["size_bytes"] = manifest_size manifest = { "schema_version": 1, "data_package": { - "name": "policyengine-uk-data", + "name": record["data_package"]["name"], "version": record["data_package"]["version"], }, "default_datasets": {"national": dataset}, - "artifacts": { - dataset: { - "kind": "microdata", - "path": record["datasets"][dataset]["path"], - "repo_id": record["datasets"][dataset]["repo_id"], - "revision": record["datasets"][dataset]["revision"], - "sha256": record["datasets"][dataset]["sha256"], - "size_bytes": 128369110, - } - }, + "artifacts": {dataset: artifact}, } - core = BUNDLE["packages"]["policyengine-core"]["version"] + data_package = record["data_package"] + repo_type = data_package["repo_type"] + fetched = [] + + def hf_file(*request): + fetched.append(("file", *request)) + return json.dumps(manifest).encode() + + def hf_size(*request): + fetched.append(("size", *request)) + return 321798995 + + core = bundle["packages"]["policyengine-core"]["version"] request = released_bundle_request( - BUNDLE, - version=BUNDLE["policyengine_version"], + bundle, + version=bundle["policyengine_version"], wheel_sha256=sha("7"), packaged_manifest_sha256=sha("8"), release_bundle=golden("qualification_request_golden")["candidate"][ "expected_release_bundle" ], - hf_file=lambda repo, rev, path: json.dumps(manifest).encode(), + hf_file=hf_file, + hf_size=hf_size, pypi_project=lambda name: { "releases": {core: [_wheel("policyengine-core", core, sha("6"))]} }, @@ -3341,6 +3550,215 @@ def test_released_bundle_request_is_valid_for_the_scorecard(): assert request.document()["candidate"]["policyengine"]["source"] == { "pypi": {"wheel_sha256": sha("7")} } + data = request.document()["candidate"]["data"] + assert data["repo_type"] == repo_type + # The manifest's size when it records one, else the file's own size. + assert data["size_bytes"] == manifest_size or ( + manifest_size is None and data["size_bytes"] == 321798995 + ) + manifest_request = ( + "file", + data_package["repo_id"], + data_package["release_manifest_revision"], + data_package["release_manifest_path"], + repo_type, + ) + size_request = ( + "size", + reference.get("repo_id") or data_package["repo_id"], + reference["revision"], + reference["path"], + reference.get("repo_type") or repo_type, + ) + assert fetched == ( + [manifest_request] + if manifest_size is not None + else [manifest_request, size_request] + ) + + +def test_released_bundle_request_uses_the_datasets_own_location(): + from policyengine.provenance.uk_qualification import released_bundle_request + + bundle = json.loads(json.dumps(MICROCOSM_ROUTED_BUNDLE)) + record = bundle["data_releases"]["uk"] + reference = record["datasets"]["microcosm_uk_2024_25"] + reference.update( + repo_id="policyengine/other-repo", revision="other-revision", repo_type="model" + ) + manifest = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "default_datasets": {"national": "microcosm_uk_2024_25"}, + "artifacts": { + "microcosm_uk_2024_25": { + "kind": "microdata", + "path": "microcosm_uk_2024_25.h5", + "repo_id": "policyengine/other-repo", + "revision": "other-revision", + "sha256": reference["sha256"], + } + }, + } + manifest_bytes = json.dumps(manifest).encode() + fetched = [] + + def hf_file(*request): + fetched.append(("file", *request)) + return manifest_bytes + + def hf_size(*request): + fetched.append(("size", *request)) + return 1234 + + core = bundle["packages"]["policyengine-core"]["version"] + request = released_bundle_request( + bundle, + version=bundle["policyengine_version"], + wheel_sha256=sha("7"), + packaged_manifest_sha256=sha("8"), + release_bundle=golden("qualification_request_golden")["candidate"][ + "expected_release_bundle" + ], + hf_file=hf_file, + hf_size=hf_size, + pypi_project=lambda name: { + "releases": {core: [_wheel("policyengine-core", core, sha("6"))]} + }, + requested_by={ + "repository": "PolicyEngine/policyengine.py", + "workflow_run_id": 5, + "commit": "e" * 40, + }, + ) + + cut = "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" + assert fetched == [ + ( + "file", + "policyengine/populace-uk-private", + cut, + "releases/microcosm-uk-2024-25-national/release_manifest.json", + "dataset", + ), + ( + "size", + "policyengine/other-repo", + "other-revision", + "microcosm_uk_2024_25.h5", + "model", + ), + ] + assert request.document()["candidate"]["data"] == { + "build_id": cut, + "dataset": "microcosm_uk_2024_25", + "repo_id": "policyengine/other-repo", + "repo_type": "model", + "revision": "other-revision", + "path": "microcosm_uk_2024_25.h5", + "sha256": reference["sha256"], + "size_bytes": 1234, + "release_manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + } + + +def test_released_bundle_request_needs_a_size_source(): + from policyengine.provenance.uk_qualification import released_bundle_request + + record = MICROCOSM_ROUTED_BUNDLE["data_releases"]["uk"] + dataset = record["default_dataset"] + reference = record["datasets"][dataset] + manifest = { + "schema_version": 1, + "data_package": {"name": "microcosm-data", "version": "0.1.0"}, + "default_datasets": {"national": dataset}, + "artifacts": { + dataset: { + "kind": "microdata", + "path": reference["path"], + "repo_id": reference["repo_id"], + "revision": reference["revision"], + "sha256": reference["sha256"], + } + }, + } + with pytest.raises(ValueError, match="records no size"): + released_bundle_request( + MICROCOSM_ROUTED_BUNDLE, + version=MICROCOSM_ROUTED_BUNDLE["policyengine_version"], + wheel_sha256=sha("7"), + packaged_manifest_sha256=sha("8"), + release_bundle={}, + hf_file=lambda *args: json.dumps(manifest).encode(), + pypi_project=lambda name: {}, + requested_by={}, + ) + + +def test_release_consumers_address_dataset_repositories(monkeypatch): + monkeypatch.setenv("POLICYENGINE_SKIP_COUNTRY_IMPORTS", "1") + for name in ("HUGGING_FACE_TOKEN", "HF_TOKEN"): + monkeypatch.delenv(name, raising=False) + # The shared helper reads every token variable the script lists. + monkeypatch.setenv("HUGGINGFACE_HUB_TOKEN", "hub-token") + spec = importlib.util.spec_from_file_location( + "release_consumers_for_test", REPO_ROOT / "scripts" / "release_consumers.py" + ) + consumers = importlib.util.module_from_spec(spec) + spec.loader.exec_module(consumers) + url = "https://huggingface.co/datasets/org/data/resolve/v1/releases/a%20b.h5" + authorized = {"Authorization": "Bearer hub-token"} + # A compressed answer carries no Content-Length. + sized = {**authorized, "Accept-Encoding": "identity"} + calls = [] + + def response(status, headers, content=b""): + answer = MagicMock(status_code=status, headers=headers, content=content) + answer.raise_for_status = lambda: None + return answer + + def get(target, **kwargs): + calls.append(("get", target, kwargs["headers"])) + return response(200, {}, b"manifest") + + answers = [] + + def head(target, **kwargs): + calls.append(("head", target, kwargs["headers"], kwargs["allow_redirects"])) + return answers.pop(0) + + monkeypatch.setattr(consumers.requests, "get", get) + monkeypatch.setattr(consumers.requests, "head", head) + + assert consumers._hf_file("org/data", "v1", "releases/a b.h5", "dataset") == ( + b"manifest" + ) + assert calls.pop() == ("get", url, authorized) + + # A large file's redirect carries its size. + answers.append(response(302, {"X-Linked-Size": "321", "Content-Length": "1152"})) + assert consumers._hf_size("org/data", "v1", "releases/a b.h5", "dataset") == 321 + assert calls == [("head", url, sized, False)] + calls.clear() + + # A small file answers with its own length, or with a redirect to it. + answers.append(response(200, {"Content-Length": "4288"})) + assert consumers._hf_size("org/data", "v1", "releases/a b.h5", "dataset") == 4288 + answers.extend( + [ + response(307, {"Content-Length": "0"}), + response(200, {"Content-Length": "77"}), + ] + ) + assert consumers._hf_size("org/data", "v1", "releases/a b.h5", "dataset") == 77 + assert calls[-2:] == [ + ("head", url, sized, False), + ("head", url, sized, True), + ] + + model_url = "https://huggingface.co/org/model/resolve/v1/a.h5" + consumers._hf_file("org/model", "v1", "a.h5", "model") + assert calls.pop() == ("get", model_url, authorized) def test_expected_release_bundle_uses_the_scorecard_certification_environment( diff --git a/tests/test_uk_qualification_workflow.py b/tests/test_uk_qualification_workflow.py index 10f1c2a8b..1e07a780e 100644 --- a/tests/test_uk_qualification_workflow.py +++ b/tests/test_uk_qualification_workflow.py @@ -3,7 +3,6 @@ from __future__ import annotations import argparse -import importlib.util import json import os import subprocess @@ -23,6 +22,7 @@ HARNESS, gates, graphql_base_changes_output, + load_candidate_cli, make_pair, make_receipt, rest_pulls_output, @@ -35,12 +35,7 @@ def _cli(): - spec = importlib.util.spec_from_file_location( - "qualification_workflow_cli", REPO_ROOT / "scripts/check_uk_candidate.py" - ) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + return load_candidate_cli("qualification_workflow_cli") def _workflow(): diff --git a/tests/test_uk_regions.py b/tests/test_uk_regions.py index 77d250bcb..29c6b27eb 100644 --- a/tests/test_uk_regions.py +++ b/tests/test_uk_regions.py @@ -1,5 +1,6 @@ """Tests for UK region definitions.""" +from collections import Counter from unittest.mock import patch import pandas as pd @@ -19,6 +20,18 @@ ) from policyengine.provenance.manifest import CountryReleaseManifest +# The certified bundle's national dataset, and the Microcosm local-area +# release its constituency and local-authority regions are routed to. +UK_CERTIFIED_NATIONAL_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25.h5" + "@microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71" +) +UK_CERTIFIED_LOCAL_AREA_URI = ( + "hf://policyengine/populace-uk-private/microcosm_uk_2024_25_dense.h5" + "@microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe" +) +UK_AREA_REGION_COUNT = 650 + 361 + class TestUKCountries: """Tests for UK country definitions.""" @@ -78,11 +91,7 @@ def test__given_uk_registry__then_has_national_region(self): assert national.code == "uk" assert national.label == "United Kingdom" assert national.region_type == "national" - assert ( - national.dataset_path - == "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2024_25.h5" - "@1.58.1" - ) + assert national.dataset_path == UK_CERTIFIED_NATIONAL_URI assert not national.requires_filter def test__given_uk_registry__then_has_four_country_regions(self): @@ -171,49 +180,65 @@ def test__given_northern_ireland_region__then_filters_from_national(self): assert ni.requires_filter assert ni.scoping_strategy.variable_value == "NORTHERN_IRELAND" - def test__given_uk_national__then_children_are_countries(self): + def test__given_uk_national__then_children_are_countries_and_areas(self): """Given: UK national region When: Getting its children - Then: All children are country regions + Then: The 4 countries, 650 constituencies and 361 local authorities """ # When uk_children = uk_region_registry.get_children("uk") # Then - assert len(uk_children) == 4 - assert all(c.region_type == "country" for c in uk_children) + assert Counter(c.region_type for c in uk_children) == { + "country": 4, + "constituency": 650, + "local_authority": 361, + } - def test__given_uk_registry__then_only_national_has_dataset(self): - """Given: UK region registry + def test__given_uk_registry__then_national_and_area_regions_have_datasets( + self, + ): + """Given: UK region registry routed to a Microcosm local-area release When: Getting dataset regions - Then: Only national has dedicated dataset + Then: National loads the national dataset, area regions the local one """ # When dataset_regions = uk_region_registry.get_dataset_regions() # Then - assert len(dataset_regions) == 1 - assert dataset_regions[0].code == "uk" + assert len(dataset_regions) == 1 + UK_AREA_REGION_COUNT + paths = {r.region_type: set() for r in dataset_regions} + for region in dataset_regions: + paths[region.region_type].add(region.dataset_path) + assert paths == { + "national": {UK_CERTIFIED_NATIONAL_URI}, + "constituency": {UK_CERTIFIED_LOCAL_AREA_URI}, + "local_authority": {UK_CERTIFIED_LOCAL_AREA_URI}, + } - def test__given_uk_registry__then_filter_regions_are_countries(self): + def test__given_uk_registry__then_countries_and_areas_filter(self): """Given: UK region registry When: Getting filter regions - Then: All 4 countries require filter + Then: The 4 countries and every area region require a filter """ # When filter_regions = uk_region_registry.get_filter_regions() # Then - assert len(filter_regions) == 4 - assert all(r.region_type == "country" for r in filter_regions) + assert len(filter_regions) == 4 + UK_AREA_REGION_COUNT + assert {r.region_type for r in filter_regions} == { + "country", + "constituency", + "local_authority", + } - def test__given_default_registry__then_has_5_regions(self): - """Given: Default UK registry + def test__given_default_registry__then_has_1016_regions(self): + """Given: Default UK registry routed to a Microcosm local-area release When: Counting regions - Then: Contains 1 national + 4 countries = 5 + Then: 1 national + 4 countries + 650 constituencies + 361 LAs = 1,016 """ # Then - assert len(uk_region_registry) == 5 + assert len(uk_region_registry) == 5 + UK_AREA_REGION_COUNT class TestUKRegionRegistryBuilder: @@ -373,19 +398,36 @@ def _uk_routed_manifest(kind: str) -> CountryReleaseManifest: ) +def _uk_unrouted_manifest() -> CountryReleaseManifest: + """A certified national bundle that routes no area regions.""" + manifest = _uk_routed_manifest("local_area") + return manifest.model_copy( + update={ + "region_datasets": {"national": manifest.region_datasets["national"]}, + "regional_release_kind": None, + } + ) + + class TestUKAreaRegionRouting: """Area regions load the dataset certification routes them to.""" def test__given_unrouted_bundle__then_area_regions_filter_the_national_dataset( - self, + self, monkeypatch ): + monkeypatch.setattr( + "policyengine.provenance.manifest.get_release_manifest", + lambda country_id: _uk_unrouted_manifest(), + ) + + default = build_uk_region_registry() registry = build_uk_region_registry( include_constituencies=True, include_local_authorities=True ) assert registry.get("constituency/E14001063").dataset_path is None assert registry.get("local_authority/N09000011").dataset_path is None - assert len(uk_region_registry.get_by_type("constituency")) == 0 + assert len(default) == 5 def test__given_national_first_bundle__then_area_regions_load_the_efrs( self, monkeypatch diff --git a/uv.lock b/uv.lock index a5807ea08..38ce23003 100644 --- a/uv.lock +++ b/uv.lock @@ -1881,13 +1881,13 @@ requires-dist = [ { name = "pandas", specifier = ">=2.0.0" }, { name = "plotly", marker = "extra == 'dev'", specifier = ">=5.0.0" }, { name = "plotly", marker = "extra == 'plotting'", specifier = ">=5.0.0" }, - { name = "policyengine-core", marker = "extra == 'dev'", specifier = "==3.33.0" }, - { name = "policyengine-core", marker = "extra == 'models'", specifier = "==3.33.0" }, - { name = "policyengine-core", marker = "extra == 'uk'", specifier = "==3.33.0" }, - { name = "policyengine-core", marker = "extra == 'us'", specifier = "==3.33.0" }, - { name = "policyengine-uk", marker = "extra == 'dev'", specifier = "==2.130.1" }, - { name = "policyengine-uk", marker = "extra == 'models'", specifier = "==2.130.1" }, - { name = "policyengine-uk", marker = "extra == 'uk'", specifier = "==2.130.1" }, + { name = "policyengine-core", marker = "extra == 'dev'", specifier = "==3.33.3" }, + { name = "policyengine-core", marker = "extra == 'models'", specifier = "==3.33.3" }, + { name = "policyengine-core", marker = "extra == 'uk'", specifier = "==3.33.3" }, + { name = "policyengine-core", marker = "extra == 'us'", specifier = "==3.33.3" }, + { name = "policyengine-uk", marker = "extra == 'dev'", specifier = "==2.132.6" }, + { name = "policyengine-uk", marker = "extra == 'models'", specifier = "==2.132.6" }, + { name = "policyengine-uk", marker = "extra == 'uk'", specifier = "==2.132.6" }, { name = "policyengine-us", marker = "extra == 'dev'", specifier = "==2.2.1" }, { name = "policyengine-us", marker = "extra == 'models'", specifier = "==2.2.1" }, { name = "policyengine-us", marker = "extra == 'us'", specifier = "==2.2.1" }, @@ -1909,7 +1909,7 @@ provides-extras = ["plotting", "graph", "models", "uk", "us", "dev"] [[package]] name = "policyengine-core" -version = "3.33.0" +version = "3.33.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -1930,14 +1930,14 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/90/d7/70936abfab724399ea3d8db04bbd5808339ee1534819bf25ab46d00c8d32/policyengine_core-3.33.0.tar.gz", hash = "sha256:5c923828dc84a7a1ff8f6f407f7a704fe067246dd277ede04c15f409804b9c19", size = 674785, upload-time = "2026-10-10T09:43:12.12Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/af/e8449e8a488468e5ce28b6b9066a2372e05bc7971ad9a441eb4e6efbb740/policyengine_core-3.33.3.tar.gz", hash = "sha256:7da78ae9cc7b1d6833dc30651a0a4775edd2d5e205a73bbafcc0b15c988c880e", size = 693203, upload-time = "2026-10-12T20:53:17.299Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/02/190e06717785ff974120a9136c6c5408a00beaa8cddd792f557a6bf974c4/policyengine_core-3.33.0-py3-none-any.whl", hash = "sha256:2e32d02a8910ab683a2b7600987904b351d058fc2bd5910a8ea6641438bcd789", size = 299495, upload-time = "2026-10-10T09:43:10.318Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/253cc99d853e36996f264138f4f8a2c662047da3eda8e37ddf39959397fc/policyengine_core-3.33.3-py3-none-any.whl", hash = "sha256:5847d9938412095fe9848c2ce535838a7dc8fac9238d8fd9e6c9d0a0648e0888", size = 301905, upload-time = "2026-10-12T20:53:15.706Z" }, ] [[package]] name = "policyengine-uk" -version = "2.130.1" +version = "2.132.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1945,9 +1945,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/27/82a97f5ec344d4be34db17bd26fe35077f1fe22f15c99818514d704262e9/policyengine_uk-2.130.1.tar.gz", hash = "sha256:1ce0a924a847b4e3e8ea8ba9702a84b267a0490c84a55ef94da6ed2efd965b34", size = 2496922, upload-time = "2026-10-14T01:58:47.232Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/88/bc06328a739b335d4ef5f2d07fd36cc6a3a5e0f6ec1b2c774a14e77e43a3/policyengine_uk-2.132.6.tar.gz", hash = "sha256:f9ce261d9881eee32fab96341c7080cd0c86717b40785b5cdaff44b07262a771", size = 2560464, upload-time = "2026-10-15T10:08:14.263Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/21/88e8addba0eabd9a7d3a29fa5293e141feb405baf0432209387f9a1f467a/policyengine_uk-2.130.1-py3-none-any.whl", hash = "sha256:f8b1b8b917c9297ff66937811dbebd9a62d3b1796ce894c326e98051cac4c873", size = 3804651, upload-time = "2026-10-14T01:58:44.937Z" }, + { url = "https://files.pythonhosted.org/packages/11/f2/c003cd7a95cb499737a5b8af0d619ba4e759c8bc7e55a2bb5c7efa46abd1/policyengine_uk-2.132.6-py3-none-any.whl", hash = "sha256:cc54025f2a3bc54be4f8d024d1e6f11ed29fed75ca513b214f2d2963c5bf4354", size = 3882707, upload-time = "2026-10-15T10:08:11.572Z" }, ] [[package]]