diff --git a/.gitignore b/.gitignore index 2bd2cfa..93d3f28 100644 --- a/.gitignore +++ b/.gitignore @@ -29,5 +29,7 @@ policyengine-uk-data # Metadata and registry snapshots generated at build time public/metadata-*.json +public/variable-tree-*.json +public/variable-modules-*.json public/param-tree-*.json public/programs-*.json diff --git a/README.md b/README.md index aaeb940..18ad185 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,22 @@ version is included when its metadata endpoint responds. The coverage page uses this provenance to show which model version the coverage reflects and, when available, the version served by the production API. +The same run snapshots each repository's `parameters/` and `variables/` file trees +(`public/param-tree-.json`, `public/variable-tree-.json`) at that +commit. Both trees record `{ source, paths }`. The UK variable tree also records +`modules`, an exact class-to-module map read from that commit's source archive, +because the UK metadata API serves no module names. The UK coverage tracker +verifies parameter, source and computation-tree links only against trees whose +repository and commit match its programme registry snapshot. + +If the archive is unavailable, module enrichment falls back to unique file stems. +UK variables whose modules remain unknown appear under “Unclassified variables”. +The registry and both trees are cleared before each refresh so a failed fetch +cannot reuse a stale snapshot. + The program loader tries the static snapshot first, then production API metadata, -then the bundled hardcoded programs. Snapshot URLs use the public base prefix so +then the bundled hardcoded programs (US only; the UK page says the registry could +not be loaded). Snapshot URLs use the public base prefix so they work both locally and under `/us/model`. ## Daily registry refresh @@ -55,3 +69,47 @@ The workflow sends a POST request to the hook. If the secret is absent, it logs notice and exits successfully. The workflow must be on the default branch for scheduled runs to apply. The Vercel build must run `bun run build` so each deployment fetches a fresh snapshot. + +## UK data release pages + +The UK pipeline, calibration targets and validation pages quote +`src/data/ukDataRelease.json` and `src/data/ukTargetChecks.json`, not typed-in +numbers. Both describe the UK data release that policyengine.py pins +(`src/policyengine/data/bundle/manifest.json`, `data_releases.uk`). When that pin +moves, regenerate them: + +1. Put the pinned release's files in one folder: the bundle manifest, the + release manifest at the revision the bundle names, `frs_.h5`, + `enhanced_frs_.h5`, both weight files (from the gated + `policyengine/policyengine-uk-data-private` Hugging Face repo), and the two + calibration logs (GitHub Actions artifacts of the uk-data publish run: + `gh run download -R PolicyEngine/policyengine-uk-data`). +2. Dump the release's target registry with `scripts/dump_uk_target_registry.py`, + and score the saved file with the pinned model with + `scripts/run_uk_target_check.py` (commands in each script's docstring). +3. Run `scripts/extract_uk_release.py --artifacts --out + src/data/ukDataRelease.json --rows-out src/data/ukTargetChecks.json`. It checks + every file's sha256 against the release manifest and refuses a release other + than the pin. + The target check records the dataset's sha256, data package version and + year, and the extractor refuses a check of any other file, package or year. +4. Run `UK_RELEASE_ARTIFACTS= UK_DATA_SOURCE= uv run --no-project --with pytest --with hypothesis --with h5py + --with tables --with pandas --with numpy pytest + scripts/test_extract_uk_release.py`, and + `UK_MODEL_SOURCE= bun run test`. The first + checks the extractor's labels, areas and notes against the data package's + code; the second runs policyengine-uk's own behavioural functions against + the behavioural page. Then reread the pipeline stage copy in + `src/data/ukPipelineStages.ts` against the new release's + `datasets/create_datasets.py`: the counts update themselves, the mechanisms + do not. `UK_DATA_VERSION_DESCRIBED` (pipeline copy) and `UK_MODEL_VERSION` + (behavioural page) must name the new pin before the tests pass. + +The household files hold UK Data Service microdata. Only the aggregates the +extractor writes may leave that folder. + +The UK behavioural responses page mirrors policyengine-uk's parameters and +`dynamics/progression.py` (`src/data/ukBehaviouralResponses.ts`); run `bun run +test` with `UK_MODEL_SOURCE` set to a policyengine-uk checkout at the pinned +version to compare them. diff --git a/app/data/calibration/page.tsx b/app/data/calibration/page.tsx index da0c921..52d460e 100644 --- a/app/data/calibration/page.tsx +++ b/app/data/calibration/page.tsx @@ -1,4 +1,5 @@ import CalibrationPage from '../../../src/views/data/CalibrationPageClient'; +import UkCalibrationPage from '../../../src/views/data/UkCalibrationPage'; import CalibrationComparison from '../../../src/components/comparison/CalibrationComparison'; import { loadComparisonData } from '../../../src/data/comparisons'; import { detectComparePeers } from '../../../src/components/comparison/detectComparePeers'; @@ -18,7 +19,7 @@ export default async function CalibrationRoute({ ); if (!parsed.compareMode) { - return ; + return country === 'uk' ? : ; } const data = loadComparisonData(); diff --git a/app/data/page.tsx b/app/data/page.tsx index 6f5812d..387486c 100644 --- a/app/data/page.tsx +++ b/app/data/page.tsx @@ -1,31 +1,20 @@ import SectionOverview from '../../src/components/layout/SectionOverview'; +import { detectCountry } from '../../src/components/comparison/detectCountry'; +import { dataSection } from '../../src/data/sectionOverviews'; -export default function DataOverviewPage() { +export default async function DataOverviewPage({ + searchParams, +}: { + searchParams: Promise<{ country?: string | string[] }>; +}) { + const country = await detectCountry((await searchParams).country); + const copy = dataSection[country]; return ( ); } diff --git a/app/data/validation/page.tsx b/app/data/validation/page.tsx index ac4f570..a9ab14b 100644 --- a/app/data/validation/page.tsx +++ b/app/data/validation/page.tsx @@ -2,6 +2,7 @@ import ValidationPage from '../../../src/views/data/ValidationPage'; import ValidationComparison from '../../../src/components/comparison/ValidationComparison'; import { loadComparisonData } from '../../../src/data/comparisons'; import { detectComparePeers } from '../../../src/components/comparison/detectComparePeers'; +import UkValidationPage from '../../../src/views/data/UkValidationPage'; export default async function ValidationRoute({ searchParams, @@ -17,6 +18,11 @@ export default async function ValidationRoute({ sp.country, ); + // The UK page reports the pinned UK release; TAXSIM covers US taxes only. + if (!parsed.compareMode && country === 'uk') { + return ; + } + if (!parsed.compareMode) { return ; } diff --git a/app/rules/page.tsx b/app/rules/page.tsx index e1134e0..a468f4b 100644 --- a/app/rules/page.tsx +++ b/app/rules/page.tsx @@ -1,31 +1,20 @@ import SectionOverview from '../../src/components/layout/SectionOverview'; +import { detectCountry } from '../../src/components/comparison/detectCountry'; +import { rulesSection } from '../../src/data/sectionOverviews'; -export default function RulesOverviewPage() { +export default async function RulesOverviewPage({ + searchParams, +}: { + searchParams: Promise<{ country?: string | string[] }>; +}) { + const country = await detectCountry((await searchParams).country); + const copy = rulesSection[country]; return ( ); } diff --git a/bun.lock b/bun.lock index 335d7de..e3446ca 100644 --- a/bun.lock +++ b/bun.lock @@ -29,6 +29,7 @@ "eslint": "^9.39.1", "eslint-plugin-jsx-a11y": "^6.10.2", "eslint-plugin-react-hooks": "^7.0.1", + "fast-check": "4.3.0", "globals": "^16.5.0", "jsdom": "^28.1.0", "typescript": "~5.9.3", @@ -706,6 +707,8 @@ "expect-type": ["expect-type@1.3.0", "", {}, "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA=="], + "fast-check": ["fast-check@4.3.0", "", { "dependencies": { "pure-rand": "^7.0.0" } }, "sha512-JVw/DJSxVKl8uhCb7GrwanT9VWsCIdBkK3WpP37B/Au4pyaspriSjtrY2ApbSFwTg3ViPfniT13n75PhzE7VEQ=="], + "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], "fast-json-stable-stringify": ["fast-json-stable-stringify@2.1.0", "", {}, "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="], @@ -976,6 +979,8 @@ "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], + "pure-rand": ["pure-rand@7.0.1", "", {}, "sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ=="], + "radix-ui": ["radix-ui@1.4.3", "", { "dependencies": { "@radix-ui/primitive": "1.1.3", "@radix-ui/react-accessible-icon": "1.1.7", "@radix-ui/react-accordion": "1.2.12", "@radix-ui/react-alert-dialog": "1.1.15", "@radix-ui/react-arrow": "1.1.7", "@radix-ui/react-aspect-ratio": "1.1.7", "@radix-ui/react-avatar": "1.1.10", "@radix-ui/react-checkbox": "1.3.3", "@radix-ui/react-collapsible": "1.1.12", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-context-menu": "2.2.16", "@radix-ui/react-dialog": "1.1.15", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.11", "@radix-ui/react-dropdown-menu": "2.1.16", "@radix-ui/react-focus-guards": "1.1.3", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-form": "0.1.8", "@radix-ui/react-hover-card": "1.1.15", "@radix-ui/react-label": "2.1.7", "@radix-ui/react-menu": "2.1.16", "@radix-ui/react-menubar": "1.1.16", "@radix-ui/react-navigation-menu": "1.2.14", "@radix-ui/react-one-time-password-field": "0.1.8", "@radix-ui/react-password-toggle-field": "0.1.3", "@radix-ui/react-popover": "1.1.15", "@radix-ui/react-popper": "1.2.8", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.5", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-progress": "1.1.7", "@radix-ui/react-radio-group": "1.3.8", "@radix-ui/react-roving-focus": "1.1.11", "@radix-ui/react-scroll-area": "1.2.10", "@radix-ui/react-select": "2.2.6", "@radix-ui/react-separator": "1.1.7", "@radix-ui/react-slider": "1.3.6", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-switch": "1.2.6", "@radix-ui/react-tabs": "1.1.13", "@radix-ui/react-toast": "1.2.15", "@radix-ui/react-toggle": "1.1.10", "@radix-ui/react-toggle-group": "1.1.11", "@radix-ui/react-toolbar": "1.1.11", "@radix-ui/react-tooltip": "1.2.8", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-escape-keydown": "1.1.1", "@radix-ui/react-use-is-hydrated": "0.1.0", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-size": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-aWizCQiyeAenIdUbqEpXgRA1ya65P13NKn/W8rWkcN0OPkRDxdBVLWnIEDsS2RpwCK2nobI7oMUSmexzTDyAmA=="], "react": ["react@19.2.4", "", {}, "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ=="], diff --git a/package.json b/package.json index ec6b659..6eb5f8b 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "eslint": "^9.39.1", "eslint-plugin-jsx-a11y": "^6.10.2", "eslint-plugin-react-hooks": "^7.0.1", + "fast-check": "4.3.0", "globals": "^16.5.0", "jsdom": "^28.1.0", "typescript": "~5.9.3", diff --git a/proxy.ts b/proxy.ts index 32afc70..95b98c6 100644 --- a/proxy.ts +++ b/proxy.ts @@ -100,5 +100,9 @@ export const config = { '/((?!_next|.*\\..*).*)', '/:country(us|uk)/programs-:registry(us|uk).json', '/:country(us|uk)/model/programs-:registry(us|uk).json', + '/:country(us|uk)/param-tree-:registry(us|uk).json', + '/:country(us|uk)/model/param-tree-:registry(us|uk).json', + '/:country(us|uk)/variable-tree-:registry(us|uk).json', + '/:country(us|uk)/model/variable-tree-:registry(us|uk).json', ], }; diff --git a/scripts/dump_uk_target_registry.py b/scripts/dump_uk_target_registry.py new file mode 100644 index 0000000..93270e6 --- /dev/null +++ b/scripts/dump_uk_target_registry.py @@ -0,0 +1,32 @@ +"""Dump policyengine-uk-data's calibration target registry to JSON. + +Run it against the data package at the release tag, with the model version +that built the release, e.g. for policyengine-uk-data 1.58.1: + + git -C policyengine-uk-data archive 1.58.1 | tar -x -C /tmp/uk-data-1.58.1 + uv run --no-project --python 3.13 --with policyengine-uk==2.122.2 \\ + --with policyengine-core==3.32.13 --with pydantic --with pyyaml \\ + --with /tmp/uk-data-1.58.1 python scripts/dump_uk_target_registry.py registry_targets.json + +The registry reads only target statistics bundled with the package, no microdata. +""" + +import json +import sys + +from policyengine_uk_data.targets.registry import get_all_targets + + +def main(out_path: str) -> None: + targets = [] + for target in get_all_targets(): + row = target.model_dump(exclude={"custom_compute"}) + row["values"] = {str(year): value for year, value in row["values"].items()} + targets.append(row) + with open(out_path, "w") as f: + json.dump(targets, f, indent=1, default=str) + print(f"wrote {len(targets)} targets to {out_path}") + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/scripts/extract_uk_release.py b/scripts/extract_uk_release.py new file mode 100644 index 0000000..d135526 --- /dev/null +++ b/scripts/extract_uk_release.py @@ -0,0 +1,1175 @@ +"""Summarise the UK data release that policyengine.py pins, for the model pages. + +The UK pipeline, calibration and validation pages quote this file, not +typed-in numbers. Regenerate it whenever policyengine.py moves its UK pin: + + uv run --no-project --with h5py --with tables --with pandas --with numpy \\ + python scripts/extract_uk_release.py --artifacts \\ + --out src/data/ukDataRelease.json --rows-out src/data/ukTargetChecks.json + +```` holds the pinned release's files, unmodified: + +- ``bundle_manifest.json``: policyengine.py ``src/policyengine/data/bundle/manifest.json`` + at the pinned tag. +- ``release_manifest.json``: the data release's manifest, from the revision the + bundle names (``data_package.release_manifest_revision``). +- ``frs_2024_25.h5``, ``enhanced_frs_2024_25.h5``, + ``parliamentary_constituency_weights.h5``, ``local_authority_weights.h5``: + release files; their sha256 must match the release manifest. +- ``constituency_calibration_log.csv``, ``la_calibration_log.csv``: the + calibration logs the publish run uploads as GitHub Actions artifacts. +- ``registry_targets.json``: the release's national target registry, dumped + with ``scripts/dump_uk_target_registry.py`` from the data package at the + release tag. +- ``pinned_target_check.csv``: every national target scored on the released + file by the model version the bundle pins, from + ``scripts/run_uk_target_check.py``. + +Only aggregates are written. No household record leaves the input files. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path + +import numpy as np +import pandas as pd + +SCHEMA_VERSION = 1 +WITHIN = 0.10 # "within 10%" is |estimate / target - 1| < 0.10 + +# Who publishes each registry `source`. Labels follow GOV.UK's organisation titles. +PUBLISHERS = { + "dwp": "DWP", + "ehs": "English Housing Survey and ONS", + "hmrc": "HMRC", + "hmrc_spi": "HMRC Survey of Personal Incomes", + "nts": "DfT National Travel Survey", + "obr": "OBR", + "ons": "ONS", + "scotland_census": "Scotland's Census", + "scottish_government": "Scottish Government", + "slc": "Student Loans Company", + "voa": "Valuation Office Agency", +} + +# Registry targets whose `source` code names the wrong publisher; the +# reference URL in the registry is the authority. +PUBLISHER_BY_TARGET = { + "obr/private_school_students": "Independent Schools Council census", + "obr/salary_sacrifice_users_total": "HM Treasury and HMRC", + "obr/salary_sacrifice_users_below_cap": "HM Treasury and HMRC", + "obr/salary_sacrifice_users_above_cap": "HM Treasury and HMRC", + # targets/sources/housing.py: monthly payment x owners, both set in code. + "housing/total_mortgage": "Assumption set in the build", +} + +# Publishers by target-name prefix, where one family mixes publishers. +PUBLISHER_BY_PREFIX = { + # voa_council_tax.py takes Scotland's band counts from gov.scot. + "voa/council_tax/SCOTLAND/": "Scottish Government", + # slc.py: borrower counts come from Student loan forecasts for England, + # on Explore Education Statistics. + "slc/plan_": "Explore Education Statistics", +} + + +def publisher_of(metric: str, source: str) -> str: + if metric in PUBLISHER_BY_TARGET: + return PUBLISHER_BY_TARGET[metric] + for prefix, publisher in PUBLISHER_BY_PREFIX.items(): + if metric.startswith(prefix): + return publisher + return PUBLISHERS.get(source, source) + + +# The area a target covers, where the registry leaves it unset (meaning UK) +# but the source or the model says otherwise. +AREA_BY_TARGET = { + # OBR table 4.9 lists these under "DWP social security", which excludes + # Northern Ireland's separate "NI social security" rows (obr.py says so + # for Universal Credit, which it restricts to Great Britain). These are + # scored on every UK household. + **{ + f"obr/{name}": "Great Britain" + for name in [ + "pip", + "esa", + "attendance_allowance", + "carers_allowance", + "statutory_maternity_pay", + "winter_fuel_allowance", + "state_pension", + "jobseekers_allowance", + ] + }, + "obr/domestic_rates": "Northern Ireland", + # policyengine-uk's sdlt_liable: England and Northern Ireland. + "obr/stamp_duty_land_tax": "England and Northern Ireland", +} +# slc.py: every SLC target counts England's borrowers and students. +AREA_BY_PREFIX = { + "slc/plan_": "England", + "slc/maintenance_loan": "England", + "slc/parents_learning_allowance": "England", + "slc/adult_dependants_grant": "England", +} + + +def area_override(metric: str) -> str | None: + if metric in AREA_BY_TARGET: + return AREA_BY_TARGET[metric] + for prefix, area in AREA_BY_PREFIX.items(): + if metric.startswith(prefix): + return area + return None + + +# How a target family is built or scored, where its label alone would +# mislead. Every note restates code in the release. +GB_ON_UK_NOTE = "Scored on UK households (policyengine-uk-data#588)" +# hmrc_spi.py's 2025 projections reuse the base-year targets' names, and +# registry.get_all_targets keeps the first target of each name, so the +# calibration fits the 2023-24 figures. +SPI_NOTE = ( + "HMRC's 2023-24 figures, fitted unchanged for 2025: the build's 2025 " + "projections share their names and are dropped (fixed after this release, " + "policyengine-uk-data#608)" +) +SPI_GROUPS = [ + "employment_income", + "self_employment_income", + "private_pension_income", + "state_pension", + "property_income", + "dividend_income", +] +GROUP_NOTES = { + ("obr", "ni_employee", "national", None): ( + "OBR's total includes employer and self-employed contributions " + "(policyengine-uk-data#537)" + ), + ("obr", "esa_income", "national", None): ( + "OBR's line also covers incapacity benefit, severe disablement " + "allowance and the incapacity part of Income Support; scored on UK " + "households (policyengine-uk-data#582, #588)" + ), + ("obr", "council_tax", "national", None): ( + "OBR's UK total also includes Northern Ireland domestic rates and an " + "accruals adjustment; the model sums council tax net of reductions " + "(policyengine-uk-data#584)" + ), + ("obr", "pip", "national", None): ( + "Scored on UK households (policyengine-uk-data#583, #588)" + ), + ("obr", "attendance_allowance", "national", None): GB_ON_UK_NOTE, + ("obr", "carers_allowance", "national", None): GB_ON_UK_NOTE, + ("obr", "statutory_maternity_pay", "national", None): GB_ON_UK_NOTE, + ("obr", "winter_fuel_allowance", "national", None): GB_ON_UK_NOTE, + ("obr", "state_pension", "national", None): GB_ON_UK_NOTE, + ("obr", "jsa", "national", None): GB_ON_UK_NOTE, + ( + "ons", + "mortgage_capital_repayment", + "national", + None, + ): "£1,100 a month for 7.5 million mortgaged owners, figures set in the build", + ("ehs", "rent", "national", None): ( + "Average rents times renter numbers, set in the build" + ), + ( + "hmrc_spi", + "property_income", + "national", + "total_income", + ): ( + "Amounts scaled up 1.9 times, the build's ratio of HMRC's property " + "rental income statistics to the SPI. " + SPI_NOTE + ), + # targets/sources/_land.py and mhclg_regional_land.py + ("ons", "land_value", "national", None): "ONS's 2024 total, held flat for 2025", + ("ons", "household_land_value", "national", None): ( + "ONS's 2024 total land value times the 2020 household share, held flat for 2025" + ), + ("ons", "corporate_land_value", "national", None): ( + "ONS's 2024 total land value times the 2020 corporate share, held flat for 2025" + ), + ("ons", "household_land_value", "region", None): ( + "The household total shared out by each region's dwellings times its " + "average house price" + ), + # targets/sources/obr.py and targets/compute/income.py + ("obr", "attends_private_school", "national", None): ( + "557,000 in every year from 2018 to 2031, a figure set in the build" + ), + ( + "obr", + "pension_contributions_via_salary_sacrifice", + "national", + None, + ): ( + "2024 users set in the build (7.7 million; 4.3 million at or below the " + "cap, 3.3 million above), grown 2.4% a year; the £2,000 test uses " + "contributions as the model runs them, which it does not uprate" + ), + ("hmrc", "capital_gains", "national", None): ( + "2023-24 outturn: gains uprated to 2025, taxpayer counts held flat" + ), +} + +# One row per (source, variable, geographic level, breakdown). Every group in +# the release must have a label here, or extraction fails. +GROUP_LABELS = { + ( + "dwp", + "benefit_cap_reduction", + "national", + None, + ): "Benefit cap: households capped and total reduction", + ("dwp", "esa", "national", None): "Employment and Support Allowance claimants", + ("dwp", "esa_contrib", "national", None): "Contribution-based ESA claimants", + ("dwp", "esa_income", "national", None): "Income-related ESA claimants", + ("dwp", "jsa_contrib", "national", None): "Jobseeker's Allowance claimants", + ( + "dwp", + "housing_benefit", + "national", + None, + ): "Housing Benefit claims and spending, by age group", + ("dwp", "pension_credit", "national", None): "Pension Credit claims and spending", + ( + "dwp", + "pip_dl_category", + "national", + None, + ): "PIP daily living claimants, standard and enhanced rate", + ( + "dwp", + "universal_credit", + "national", + None, + ): "Universal Credit households: total, by family type and number of children, and Scottish households with a child under 1", + ( + "dwp", + "universal_credit", + "national", + "universal_credit", + ): "Universal Credit households by family type and annual award band", + ("ehs", "rent", "national", None): "Private and social rent paid", + ( + "hmrc", + "capital_gains", + "national", + None, + ): "Capital gains: taxpayers and gains, in total and by size of gain", + ( + "hmrc", + "income_tax", + "national", + None, + ): "Salary sacrifice: income tax relief by rate band", + ( + "hmrc", + "ni_employee", + "national", + None, + ): "Salary sacrifice: employee National Insurance relief", + ( + "hmrc", + "ni_employer", + "national", + None, + ): "Salary sacrifice: employer National Insurance relief", + ( + "hmrc", + "pension_contributions_via_salary_sacrifice", + "national", + None, + ): "Pension contributions made through salary sacrifice", + ( + "hmrc_spi", + "employment_income", + "national", + "total_income", + ): "Employment income: taxpayers and amounts, by total income band", + ( + "hmrc_spi", + "self_employment_income", + "national", + "total_income", + ): "Self-employment income: taxpayers and amounts, by total income band", + ( + "hmrc_spi", + "private_pension_income", + "national", + "total_income", + ): "Private pension income: taxpayers and amounts, by total income band", + ( + "hmrc_spi", + "state_pension", + "national", + "total_income", + ): "State Pension: taxpayers and amounts, by total income band", + ( + "hmrc_spi", + "property_income", + "national", + "total_income", + ): "Property income: taxpayers and amounts, by total income band", + ( + "hmrc_spi", + "dividend_income", + "national", + "total_income", + ): "Dividend income: taxpayers and amounts, by total income band", + ( + "nts", + "num_vehicles", + "national", + None, + ): "Households with no, one, and two or more vehicles", + ("obr", "income_tax", "national", None): "Income tax receipts", + ( + "obr", + "ni_employee", + "national", + None, + ): "National Insurance receipts, total and employee, both scored against employee contributions", + ("obr", "ni_employer", "national", None): "Employer National Insurance receipts", + ( + "obr", + "ni_self_employed", + "national", + None, + ): "Self-employed National Insurance receipts", + ("obr", "vat", "national", None): "VAT receipts", + ( + "obr", + "fuel_duty", + "national", + None, + ): "Fuel duty receipts, scored against duty on households' car fuel", + ("obr", "capital_gains_tax", "national", None): "Capital gains tax receipts", + ("obr", "stamp_duty_land_tax", "national", None): "Stamp Duty Land Tax receipts", + ("obr", "council_tax", "national", None): "Council tax receipts, UK and by nation", + ( + "obr", + "domestic_rates", + "national", + None, + ): "Northern Ireland domestic rates receipts", + ("obr", "tv_licence", "national", None): "TV licence fee receipts", + ("obr", "universal_credit", "national", None): "Universal Credit spending", + ("obr", "state_pension", "national", None): "State Pension spending", + ( + "obr", + "pip", + "national", + None, + ): "Disability Living Allowance and PIP spending, scored against PIP alone", + ("obr", "child_benefit", "national", None): "Child Benefit spending", + ("obr", "attendance_allowance", "national", None): "Attendance Allowance spending", + ("obr", "carers_allowance", "national", None): "Carer's Allowance spending", + ( + "obr", + "esa_income", + "national", + None, + ): "Incapacity benefits spending, scored against ESA (income-related and contributory)", + ("obr", "jsa", "national", None): "Jobseeker's Allowance spending", + ("obr", "winter_fuel_allowance", "national", None): "Winter Fuel Payment spending", + ( + "obr", + "statutory_maternity_pay", + "national", + None, + ): "Statutory Maternity Pay spending", + ( + "obr", + "attends_private_school", + "national", + None, + ): "Pupils at independent schools", + ( + "obr", + "pension_contributions_via_salary_sacrifice", + "national", + None, + ): "Salary sacrifice users, in total and either side of the £2,000 cap", + ( + "ons", + "age", + "national", + None, + ): "UK population, in total and by sex and five-year age band", + ( + "ons", + "age", + "region", + None, + ): "Population by region and ten-year age band, and Scottish children under 1 and under 16", + ("ons", "family_type", "national", None): "Households by family type", + ( + "ons", + "tenure_type", + "country", + None, + ): "Dwellings in England, in total and by tenure", + ( + "ons", + "employment_status", + "national", + None, + ): "Employees and self-employed people", + ("ons", "employment_sector", "national", None): "Public sector employment", + ("ons", "savings_interest_income", "national", None): "Household savings interest", + ( + "ons", + "mortgage_capital_repayment", + "national", + None, + ): "Mortgage payments, capital and interest", + ("ons", "land_value", "national", None): "Total land value", + ("ons", "household_land_value", "national", None): "Household land value", + ("ons", "household_land_value", "region", None): "Household land value by region", + ("ons", "corporate_land_value", "national", None): "Corporate land value", + ( + "scotland_census", + "is_child", + "country", + None, + ): "Scottish households with three or more children", + ( + "scottish_government", + "scottish_child_payment", + "country", + None, + ): "Scottish Child Payment spending", + ( + "slc", + "maintenance_loan", + "national", + None, + ): "Maintenance loans: recipients and spending", + ( + "slc", + "adult_dependants_grant", + "national", + None, + ): "Adult Dependants' Grant: recipients and spending", + ( + "slc", + "parents_learning_allowance", + "national", + None, + ): "Parents' Learning Allowance: recipients and spending", + ( + "slc", + "student_loan_plan", + "national", + None, + ): "Plan 2 and Plan 5 borrowers, liable and above the repayment threshold", + ( + "slc", + "student_loan_repayment", + "country", + None, + ): "Student loan repayments, by nation and plan", + ( + "voa", + "council_tax_band", + "region", + None, + ): "Dwellings by region, in total and by council tax band", +} + +for _variable in SPI_GROUPS: + GROUP_NOTES.setdefault( + ("hmrc_spi", _variable, "national", "total_income"), SPI_NOTE + ) + +# Constituency and local authority loss builders (datasets/local_areas/*/loss.py). +INCOME_NOTE = "Rescaled so the areas add to a projected UK total" +INCOME_COUNT_NOTE = "Scaled by the same factor as the amounts" +# local_uc.py returns counts in the workbook's row order, and loss.py assigns +# them by position to areas in another order. +UC_ORDER_NOTE = ( + "The build assigns these counts to areas by row order, which does not " + "match the areas, so this fit is not a check on Universal Credit by area " + "(policyengine-uk-data#468)" +) + +# Local (per-area) metric families. Sources are the ones each loss builder's +# docstring and source module name. +LOCAL_FAMILIES = [ + ( + "age/", + "Population by ten-year age band, 0 to 79", + "ONS mid-year population estimates", + "Rescaled so the bands across all areas together add to 90% of the UK " + "population", + ), + ( + "hmrc/employment_income/amount", + "Employment income of income taxpayers", + "HMRC Survey of Personal Incomes, table 3.15", + INCOME_NOTE, + ), + ( + "hmrc/employment_income/count", + "Income taxpayers with employment income", + "HMRC Survey of Personal Incomes, table 3.15", + INCOME_COUNT_NOTE, + ), + ( + "hmrc/self_employment_income/", + "Self-employment income: taxpayers and amount", + "HMRC Survey of Personal Incomes, table 3.15", + INCOME_NOTE, + ), + ( + "uc_households", + "Universal Credit households", + "DWP Stat-Xplore and DfC Northern Ireland statistics", + UC_ORDER_NOTE, + ), + ( + "uc_hh_", + "Universal Credit households by number of children", + "DWP Stat-Xplore and DfC Northern Ireland statistics", + "Each area's total split by its nation's shares (Northern Ireland " + "takes Great Britain's). " + UC_ORDER_NOTE, + ), + ( + "housing/scotland_", + "Private renters and rent in Scottish constituencies", + "Country totals set in the build, shared by population", + None, + ), + ( + "housing/wales_", + "Private renters and rent in Welsh constituencies", + "Country totals set in the build, shared by population", + None, + ), + ( + "ons/equiv_", + "Equivalised income before and after housing costs, and housing costs", + "ONS small area income estimates", + "The average of each authority's small-area estimates, times its " + "households, uprated", + ), + ( + "tenure/", + "Households by tenure", + "English Housing Survey", + "Tenure shares times households", + ), + ( + "rent/private_rent", + "Private rent paid", + "VOA private rental market statistics", + "Median rent times private renting households, England only", + ), + ( + "voa/council_tax/", + "Dwellings by council tax band, A to H", + "VOA council tax stock of properties", + "VOA counts dwellings, including empty ones; the model counts households", + ), + ( + "housing/council_tax_net", + "Council tax paid, net of Council Tax Reduction", + "MHCLG and Welsh Government council tax statistics", + "Taxbase times Band D (England) or council tax income (Wales), against " + "households' reported bills net of reductions", + ), +] + +# The build logs these by area but does not train on them (create_datasets.py, +# VALIDATION_ONLY_LOCAL_TARGETS). +HELD_OUT_LOCAL = "hmrc/employment_income/count" + + +class ExtractionError(ValueError): + """The inputs do not describe one consistent, pinned release.""" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def fit_share( + estimate: pd.Series, target: pd.Series, tolerance: float = WITHIN +) -> float | None: + """Share of targets whose estimate is within `tolerance` of the target. + + Relative error is undefined for a zero target, so zero targets are left + out (and counted separately by the callers that can have them). Returns + None when no target is scorable. + """ + scorable = target != 0 + if not scorable.any(): + return None + error = (estimate[scorable] / target[scorable] - 1).abs() + return round(float((error < tolerance).mean()), 6) + + +def pinned_release(bundle: dict) -> dict: + try: + release = bundle["data_releases"]["uk"] + return { + "policyengine": bundle["bundle_version"], + "modelVersion": release["model_package"]["version"], + "dataPackage": release["data_package"]["name"], + "dataVersion": release["data_package"]["version"], + "releaseManifestRevision": release["data_package"][ + "release_manifest_revision" + ], + "defaultDataset": release["default_dataset"], + "datasetSha256": release["datasets"][release["default_dataset"]]["sha256"], + "regionDatasets": sorted(release.get("region_datasets", {})), + } + except KeyError as missing: + raise ExtractionError( + f"bundle manifest has no UK release field {missing}" + ) from None + + +def check_release(pin: dict, manifest: dict, artifacts: Path) -> dict: + if manifest["data_package"]["version"] != pin["dataVersion"]: + raise ExtractionError( + f"release manifest is {manifest['data_package']['version']}, bundle pins {pin['dataVersion']}" + ) + if manifest["default_datasets"].get("national") != pin["defaultDataset"]: + raise ExtractionError( + "release manifest's national default is not the bundle's default dataset" + ) + files = {} + for name in ( + pin["defaultDataset"], + "frs_2024_25", + "parliamentary_constituency_weights", + "local_authority_weights", + ): + entry = manifest["artifacts"].get(name) + if entry is None: + raise ExtractionError(f"release manifest lists no artifact {name}") + path = artifacts / entry["path"] + actual = sha256(path) + if actual != entry["sha256"]: + raise ExtractionError( + f"{entry['path']} sha256 {actual} does not match the release manifest" + ) + files[name] = path + if manifest["artifacts"][pin["defaultDataset"]]["sha256"] != pin["datasetSha256"]: + raise ExtractionError( + "bundle and release manifest disagree on the default dataset's sha256" + ) + return files + + +def read_tables(path: Path) -> dict[str, pd.DataFrame]: + return { + table: pd.read_hdf(path, table) + for table in ("household", "benunit", "person", "time_period") + } + + +def household_summary(tables: dict[str, pd.DataFrame]) -> dict: + household = tables["household"] + return { + "households": len(household), + "benefitUnits": len(tables["benunit"]), + "people": len(tables["person"]), + "weightedHouseholds": int(round(float(household["household_weight"].sum()))), + "timePeriod": str(np.asarray(tables["time_period"]).ravel()[0]), + } + + +def composition(household: pd.DataFrame, base_households: int) -> dict: + spi = household["household_is_spi_synthetic"].astype(bool) + gains = household["household_is_capital_gains_clone"].astype(bool) + donor = household["household_is_cgt_band_donor"].astype(bool) + frs = ~spi & ~gains & ~donor + spi_copies = spi & ~gains & ~donor + if int(frs.sum()) != base_households: + raise ExtractionError( + f"{int(frs.sum())} unflagged households, base FRS has {base_households}" + ) + if int(gains.sum() - (gains & donor).sum()) != int(frs.sum() + spi_copies.sum()): + raise ExtractionError( + "capital gains copies do not pair one-to-one with FRS and SPI households" + ) + weight = household["household_weight"] + return { + "frs": int(frs.sum()), + "spiCopies": int(spi_copies.sum()), + "capitalGainsCopies": int((gains & ~donor).sum()), + "cgtBandDonors": int(donor.sum()), + "spiWeightShare": round(float(weight[spi].sum() / weight.sum()), 4), + } + + +def geography(household: pd.DataFrame) -> dict: + constituency = household["constituency_code_oa"].astype(str).str.strip() + authority = household["la_code_oa"].astype(str).str.strip() + has_area = constituency.str.len() > 0 + if not (has_area == (authority.str.len() > 0)).all(): + raise ExtractionError( + "constituency and local authority codes are not assigned together" + ) + region = household["region"].astype(str) + without = region[~has_area].unique().tolist() + clones = household["clone_index"].nunique() + if household["source_household_id"].nunique() != len(household): + raise ExtractionError( + "source households repeat: the file has more than one Output Area draw per household" + ) + per_area = constituency[has_area].value_counts() + return { + "outputAreaDrawsPerHousehold": int(clones), + "householdsWithArea": int(has_area.sum()), + "householdsWithoutArea": int((~has_area).sum()), + "regionsWithoutArea": sorted(without), + "constituenciesWithHouseholds": int(per_area.size), + "householdsPerConstituency": { + "min": int(per_area.min()), + "median": float(per_area.median()), + "max": int(per_area.max()), + }, + } + + +def weight_matrix(path: Path, households: int) -> dict: + import h5py + + with h5py.File(path, "r") as f: + keys = list(f.keys()) + if len(keys) != 1: + raise ExtractionError( + f"{path.name} holds {keys}, expected one calibration year" + ) + matrix = f[keys[0]][:] + if matrix.ndim != 2 or matrix.shape[1] != households: + raise ExtractionError( + f"{path.name} has shape {matrix.shape}, expected (areas, {households})" + ) + return { + "areas": int(matrix.shape[0]), + "households": int(matrix.shape[1]), + "year": keys[0], + "matrix": matrix, + } + + +def last_epoch(log: pd.DataFrame) -> pd.DataFrame: + epoch = log["epoch"].max() + return log[log["epoch"] == epoch].copy() + + +GREAT_BRITAIN = {"ENGLAND", "SCOTLAND", "WALES"} +# The areas national targets describe, broadest first. +AREA_ORDER = [ + "UK", + "Great Britain", + "England and Northern Ireland", + "England", + "Scotland", + "Northern Ireland", + "Nation", + "Region", +] + + +def area_label(level: str, group: pd.DataFrame) -> str: + """The area a target family covers: an override for the source's own + coverage, else the registry's country scope or nation, else its level.""" + overrides = {area_override(m) for m in group["metric"]} + if len(overrides) != 1: + raise ExtractionError(f"a target family mixes area overrides: {overrides}") + override = overrides.pop() + if override is not None: + return override + scopes = { + tuple(sorted(c)) if isinstance(c, (list, tuple)) else None + for c in group["countries"] + } + if len(scopes) != 1: + raise ExtractionError(f"a target family mixes country scopes: {scopes}") + scope = scopes.pop() + if scope is None: + if level == "country": + names = set(group["geo_name"].dropna()) + if len(names) == 1 and group["geo_name"].notna().all(): + return names.pop() + return {"national": "UK", "region": "Region", "country": "Nation"}[level] + if set(scope) == GREAT_BRITAIN: + return "Great Britain" + return ", ".join(c.replace("_", " ").title() for c in scope) + + +def national_groups( + national: pd.DataFrame, registry: pd.DataFrame, pinned: pd.DataFrame +) -> tuple[list[dict], list[dict]]: + missing = sorted(set(national["metric"]) - set(registry.index)) + if missing: + raise ExtractionError( + f"logged national targets missing from the registry: {missing[:5]}" + ) + rows = national.join(registry, on="metric") + rows = rows.join( + pinned.set_index("metric")[["target", "estimate"]].add_prefix("pinned_"), + on="metric", + ) + rows["breakdown"] = rows["breakdown_variable"].where( + rows["breakdown_variable"].notna(), None + ) + groups = [] + target_rows = [] + for key, group in rows.groupby( + ["source", "variable", "geographic_level", "breakdown"], + dropna=False, + sort=False, + ): + source, variable, level, breakdown = key + breakdown = ( + None + if breakdown is None + or (isinstance(breakdown, float) and math.isnan(breakdown)) + else breakdown + ) + label = GROUP_LABELS.get((source, variable, level, breakdown)) + if label is None: + raise ExtractionError( + f"no label for target group {(source, variable, level, breakdown)}" + ) + publishers = sorted({publisher_of(m, source) for m in group["metric"]}) + units = sorted(set(group["unit"])) + entry = { + "source": source, + "label": label, + "publisher": " and ".join(publishers), + "geography": area_label(level, group), + "note": GROUP_NOTES.get((source, variable, level, breakdown)), + "units": units, + "targets": len(group), + "zeroTargets": int((group["target"] == 0).sum()), + "within10": fit_share(group["estimate"], group["target"]), + "pinnedWithin10": fit_share(group["pinned_estimate"], group["target"]), + "urls": sorted(set(group["reference_url"].dropna())), + "vintages": sorted(set(group["forecast_vintage"].dropna())), + } + if len(group) == 1: + entry["value"] = float(group["target"].iloc[0]) + groups.append(entry) + for _, row in group.iterrows(): + target_rows.append( + { + "id": row["metric"], + "group": label, + "unit": row["unit"], + "target": float(row["target"]), + "estimate": float(row["pinned_estimate"]), + "calibrationEstimate": float(row["estimate"]), + } + ) + order = [ + "obr", + "hmrc", + "hmrc_spi", + "dwp", + "ons", + "voa", + "slc", + "nts", + "ehs", + "scottish_government", + "scotland_census", + ] + groups.sort( + key=lambda g: ( + order.index(g["source"]) if g["source"] in order else len(order), + -g["targets"], + g["label"], + ) + ) + target_rows.sort(key=lambda r: r["id"]) + return groups, target_rows + + +def read_pinned_check( + path: Path, model_version: str, pin_checks: dict | None = None +) -> tuple[dict, pd.DataFrame]: + meta = {} + with open(path) as f: + for line in f: + if not line.startswith("# "): + break + key, value = line[2:].strip().split("=", 1) + meta[key] = value + if meta.get("policyengine_uk") != model_version: + raise ExtractionError( + f"pinned check ran policyengine-uk {meta.get('policyengine_uk')}, the bundle pins {model_version}" + ) + for key, expected in (pin_checks or {}).items(): + if meta.get(key) != expected: + raise ExtractionError( + f"pinned check has {key}={meta.get(key)}, expected {expected}" + ) + checked = pd.read_csv(path, comment="#") + if ( + checked["metric"].duplicated().any() + or not np.isfinite(checked[["target", "estimate"]].to_numpy()).all() + ): + raise ExtractionError("pinned check has repeated or non-finite rows") + return meta, checked + + +def local_summary(local: pd.DataFrame, areas: int) -> dict: + if local["index"].nunique() != areas: + raise ExtractionError( + f"log covers {local['index'].nunique()} areas, the weight matrix {areas}" + ) + finite = local[np.isfinite(local["target"])] + trained = finite[finite["metric"] != HELD_OUT_LOCAL] + nonzero = trained[trained["target"] != 0] + zero = trained[trained["target"] == 0] + if (zero["estimate"].abs() > 1e-6).any(): + raise ExtractionError( + "a zero local target has a non-zero estimate; it is not a structural zero" + ) + held = finite[finite["metric"] == HELD_OUT_LOCAL] + families = [] + claimed: set[str] = set() + for prefix, label, source, note in LOCAL_FAMILIES: + metrics = sorted( + m + for m in local["metric"].unique() + if m.startswith(prefix) and m not in claimed + ) + if prefix == "uc_households": + metrics = [m for m in metrics if m == "uc_households"] + if not metrics: + continue + claimed.update(metrics) + cells = nonzero[nonzero["metric"].isin(metrics)] + held_family = held[held["metric"].isin(metrics)] + family = { + "label": label, + "source": source, + "note": note, + "metrics": len(metrics), + "heldOut": bool(len(held_family)) and not len(cells), + } + scored = held_family if family["heldOut"] else cells + family["cells"] = len(scored) + family["within10"] = fit_share(scored["estimate"], scored["target"]) + families.append(family) + unclaimed = sorted(set(local["metric"]) - claimed) + if unclaimed: + raise ExtractionError(f"local metrics with no family: {unclaimed}") + return { + "areas": int(areas), + "metrics": int(local["metric"].nunique()), + "cellsWithTarget": len(finite), + "cellsWithoutSource": int(len(local) - len(finite)), + "structuralZeroCells": len(zero), + "trainingCells": len(nonzero), + "trainingWithin10": fit_share(nonzero["estimate"], nonzero["target"]), + "heldOut": { + "metric": HELD_OUT_LOCAL, + "cells": len(held), + "within10": fit_share(held["estimate"], held["target"]), + }, + "families": families, + } + + +def calibration( + log_path: Path, registry: pd.DataFrame, areas: int +) -> tuple[dict, pd.DataFrame]: + columns = ["index", "name", "metric", "estimate", "target", "validation", "epoch"] + log = last_epoch(pd.read_csv(log_path, usecols=columns)) + national = log[log["name"] == "UK"] + local = log[log["name"] != "UK"] + if national["metric"].duplicated().any(): + raise ExtractionError(f"{log_path.name} repeats a national target") + if national["validation"].astype(bool).any(): + raise ExtractionError( + "a national target is marked validation-only; the pages assume all train" + ) + flagged = set(local.loc[local["validation"].astype(bool), "metric"]) + if flagged != {HELD_OUT_LOCAL}: + raise ExtractionError( + f"held-out local metrics are {sorted(flagged)}, expected [{HELD_OUT_LOCAL}]" + ) + summary = { + "epoch": int(log["epoch"].iloc[0]), + "nationalTargets": len(national), + "nationalWithin10": fit_share(national["estimate"], national["target"]), + "nationalWithin5": fit_share(national["estimate"], national["target"], 0.05), + "nationalZeroTargets": int((national["target"] == 0).sum()), + "local": local_summary(local, areas), + } + return summary, national + + +def extract(artifacts: Path, registry_path: Path) -> dict: + bundle = json.loads((artifacts / "bundle_manifest.json").read_text()) + manifest = json.loads((artifacts / "release_manifest.json").read_text()) + pin = pinned_release(bundle) + files = check_release(pin, manifest, artifacts) + + base = read_tables(files["frs_2024_25"]) + enhanced = read_tables(files[pin["defaultDataset"]]) + base_summary = household_summary(base) + enhanced_summary = household_summary(enhanced) + household = enhanced["household"] + + constituency = weight_matrix( + files["parliamentary_constituency_weights"], len(household) + ) + authority = weight_matrix(files["local_authority_weights"], len(household)) + ratio = ( + constituency["matrix"].sum(axis=0) / household["household_weight"].to_numpy() + ) + if ratio.max() - ratio.min() > 1e-4: + raise ExtractionError( + "constituency matrix column sums are not one uniform multiple of the saved weights" + ) + + registry = pd.DataFrame(json.loads(registry_path.read_text())).set_index("name") + # The check must have run the pinned file, from the pinned data package, + # for the year the weight matrices were fitted to. + run, pinned = read_pinned_check( + artifacts / "pinned_target_check.csv", + pin["modelVersion"], + { + "dataset_sha256": pin["datasetSha256"], + "policyengine_uk_data": pin["dataVersion"], + "year": constituency["year"], + }, + ) + if authority["year"] != constituency["year"]: + raise ExtractionError("the two weight matrices were fitted to different years") + constituency_cal, national = calibration( + artifacts / "constituency_calibration_log.csv", registry, constituency["areas"] + ) + authority_cal, authority_national = calibration( + artifacts / "la_calibration_log.csv", registry, authority["areas"] + ) + if set(national["metric"]) != set(authority_national["metric"]): + raise ExtractionError("the two calibrations fit different national targets") + if set(pinned["metric"]) != set(national["metric"]): + raise ExtractionError( + "the pinned check scored a different set of national targets from calibration" + ) + groups, target_rows = national_groups(national, registry, pinned) + # build_loss_matrix._resolve_value: a target with no value for the year + # takes its nearest earlier listed year's, within three years. + year = str(int(run["year"])) + carried = [m for m in national["metric"] if year not in registry.loc[m, "values"]] + unknown = {g["geography"] for g in groups} - set(AREA_ORDER) + if unknown: + raise ExtractionError( + f"national targets cover areas the pages don't name: {unknown}" + ) + # The pinned run rebuilds the targets with today's code, and a few values + # can move (a source download can fall back, say). Validation scores the + # pinned estimates against the values calibration fitted, and lists the + # targets whose rebuilt value differs. + scored = national.set_index("metric")[["target"]].join( + pinned.set_index("metric")[["target", "estimate"]].add_prefix("pinned_") + ) + revised = scored[ + ~np.isclose(scored["pinned_target"], scored["target"], rtol=1e-9, atol=0) + ] + + build = manifest["build"] + return { + "schemaVersion": SCHEMA_VERSION, + "release": { + **{k: v for k, v in pin.items() if k != "datasetSha256"}, + "datasetSha256": pin["datasetSha256"], + "builtAt": build["built_at"], + "builtWithModelVersion": build["built_with_model_package"]["version"], + "builtWithCoreVersion": build["built_with_core_package"]["version"], + "dataGitSha": build["metadata"]["data_package_git_sha"], + }, + "base": {**base_summary, "dataset": "frs_2024_25"}, + "enhanced": { + **enhanced_summary, + "dataset": pin["defaultDataset"], + "composition": composition(household, base_summary["households"]), + "geography": geography(household), + }, + "weightMatrices": { + "constituency": {k: v for k, v in constituency.items() if k != "matrix"}, + "localAuthority": {k: v for k, v in authority.items() if k != "matrix"}, + "columnSumPerSavedWeight": round(float(ratio.mean()), 6), + }, + "calibration": { + "epoch": constituency_cal["epoch"], + "nationalTargets": constituency_cal["nationalTargets"], + "nationalZeroTargets": constituency_cal["nationalZeroTargets"], + "nationalWithin10": constituency_cal["nationalWithin10"], + "nationalWithin5": constituency_cal["nationalWithin5"], + "constituency": constituency_cal["local"], + "localAuthority": authority_cal["local"], + "targetsWithoutYearValue": len(carried), + "nationalTargetsByArea": { + area: sum(g["targets"] for g in groups if g["geography"] == area) + for area in AREA_ORDER + if any(g["geography"] == area for g in groups) + }, + "nationalGroups": groups, + }, + "validation": { + "modelVersion": run["policyengine_uk"], + "coreVersion": run["policyengine_core"], + "year": int(run["year"]), + "people": int(round(float(run["people"]))), + "households": int(round(float(run["households"]))), + "targets": len(scored), + "zeroTargets": int((scored["target"] == 0).sum()), + "within10": fit_share(scored["pinned_estimate"], scored["target"]), + "within5": fit_share(scored["pinned_estimate"], scored["target"], 0.05), + "revisedTargets": sorted(revised.index), + "rows": target_rows, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--artifacts", type=Path, required=True) + parser.add_argument( + "--registry", type=Path, help="defaults to /registry_targets.json" + ) + parser.add_argument("--out", type=Path, required=True, help="release summary JSON") + parser.add_argument( + "--rows-out", type=Path, required=True, help="per-target check rows JSON" + ) + args = parser.parse_args() + summary = extract( + args.artifacts, args.registry or args.artifacts / "registry_targets.json" + ) + rows = summary["validation"].pop("rows") + args.out.write_text(json.dumps(summary, indent=2, allow_nan=False) + "\n") + args.rows_out.write_text( + json.dumps(rows, separators=(",", ":"), allow_nan=False) + "\n" + ) + print(f"wrote {args.out} and {args.rows_out} ({len(rows)} targets)") + + +if __name__ == "__main__": + main() diff --git a/scripts/fetch-metadata.js b/scripts/fetch-metadata.js index 8a57424..fb9fa67 100644 --- a/scripts/fetch-metadata.js +++ b/scripts/fetch-metadata.js @@ -1,5 +1,5 @@ /** - * Snapshot the default-branch program registries and parameter YAML file trees. + * Snapshot the default-branch program registries and source file trees. * Run: node scripts/fetch-metadata.js (also called by `bun run build`). * Fetch failures are warnings: the app can fall back to API metadata. */ @@ -7,7 +7,9 @@ import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { gunzipSync } from 'node:zlib'; import yaml from 'js-yaml'; +import { extractVariableModules } from './variable-modules.js'; const PUBLIC_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', 'public'); const REPO_MAP = { @@ -112,25 +114,91 @@ export async function fetchRegistrySnapshot(country, source, { return snapshot; } -/** Fetch only the parameter subtree, using the same resolved commit as coverage. */ -export async function fetchParameterYamlPaths(country, source, fetchImpl = globalThis.fetch) { +/** List every file in one directory of the country package at the resolved commit. */ +async function fetchPackageSubtree(country, source, directory, fetchImpl) { const treeRoot = `https://api.github.com/repos/${source.repo}/git/trees`; const root = await (await request(`${treeRoot}/${source.commit}`, fetchImpl)).json(); const packageTree = root.tree?.find(entry => entry.path === `policyengine_${country}`); if (!packageTree?.sha) throw new Error('Package directory missing from repository tree'); const packageFiles = await (await request(`${treeRoot}/${packageTree.sha}`, fetchImpl)).json(); - const parameters = packageFiles.tree?.find(entry => entry.path === 'parameters'); - if (!parameters?.sha) throw new Error('Parameter directory missing from repository tree'); - - const files = await (await request(`${treeRoot}/${parameters.sha}?recursive=1`, fetchImpl)).json(); - if (files.truncated) throw new Error('GitHub returned a truncated parameter tree'); - if (!Array.isArray(files.tree)) throw new Error('GitHub did not return a parameter tree'); - return files.tree - .filter(entry => entry.type === 'blob' - && entry.path.endsWith('.yaml') - && !entry.path.startsWith('calibration/')) - .map(entry => entry.path); + const subtree = packageFiles.tree?.find(entry => entry.path === directory); + if (!subtree?.sha) throw new Error(`${directory} directory missing from repository tree`); + + const files = await (await request(`${treeRoot}/${subtree.sha}?recursive=1`, fetchImpl)).json(); + if (files.truncated) throw new Error(`GitHub returned a truncated ${directory} tree`); + if (!Array.isArray(files.tree)) throw new Error(`GitHub did not return a ${directory} tree`); + return files.tree.filter(entry => entry.type === 'blob').map(entry => entry.path); +} + +/** Fetch only the parameter subtree, using the same resolved commit as coverage. */ +export async function fetchParameterYamlPaths(country, source, fetchImpl = globalThis.fetch) { + const paths = await fetchPackageSubtree(country, source, 'parameters', fetchImpl); + return paths.filter(path => path.endsWith('.yaml') && !path.startsWith('calibration/')); +} + +/** + * Variable module files at the same commit, e.g. `gov/hmrc/income_tax/income_tax.py`. + * The UK metadata API serves no `moduleName`, so the explorer maps variable names to + * these files to group variables by department and link their source. + */ +export async function fetchVariableModulePaths(country, source, fetchImpl = globalThis.fetch) { + const paths = await fetchPackageSubtree(country, source, 'variables', fetchImpl); + return paths.filter(path => path.endsWith('.py') && !path.endsWith('__init__.py')); +} + +/** Map UK Variable classes from the very same commit as the file tree. */ +export async function fetchVariableTreeSnapshot(country, source, { + fetchImpl = globalThis.fetch, + logger = console, +} = {}) { + if (country === 'uk') { + try { + const response = await request(`https://codeload.github.com/${source.repo}/tar.gz/${source.commit}`, fetchImpl); + const tar = gunzipSync(Buffer.from(await response.arrayBuffer())); + return { source, ...extractVariableModules(country, tar) }; + } catch (error) { + logger.warn(` ${country}: class-to-module map unavailable, using file tree — ${error.message}`); + } + } + return { source, paths: await fetchVariableModulePaths(country, source, fetchImpl) }; +} + +/** The country package version the production API serves. */ +export async function fetchApiModelVersion(country, fetchImpl = globalThis.fetch) { + const metadata = await (await request(`https://api.policyengine.org/${country}/metadata`, fetchImpl)).json(); + const version = (metadata.result ?? metadata).version; + if (typeof version !== 'string' || !/^\d+\.\d+\.\d+$/.test(version)) { + throw new Error('production API returned no package version'); + } + return version; +} + +/** The commit a release tag names, following an annotated tag to its commit. */ +export async function resolveTagCommit(repo, tag, fetchImpl = globalThis.fetch) { + const ref = await (await request(`https://api.github.com/repos/${repo}/git/ref/tags/${tag}`, fetchImpl)).json(); + let target = ref.object; + if (target?.type === 'tag') { + target = (await (await request(`https://api.github.com/repos/${repo}/git/tags/${target.sha}`, fetchImpl)).json()).object; + } + if (target?.type !== 'commit' || !/^[0-9a-f]{40}$/.test(target.sha ?? '')) { + throw new Error(`tag ${tag} does not resolve to a commit`); + } + return target.sha; +} + +/** + * The class-to-module map for the package release the production API serves. + * The explorer's variables come from that API, so the map is taken from the + * release with the same version, not from the default branch, and the browser + * applies it only when the versions still match. + */ +export async function fetchVariableModuleSnapshot(country, repo, fetchImpl = globalThis.fetch) { + const version = await fetchApiModelVersion(country, fetchImpl); + const commit = await resolveTagCommit(repo, version, fetchImpl); + const response = await request(`https://codeload.github.com/${repo}/tar.gz/${commit}`, fetchImpl); + const { modules } = extractVariableModules(country, gunzipSync(Buffer.from(await response.arrayBuffer()))); + return { source: { repo, version, commit }, modules }; } export async function fetchMetadataFiles({ @@ -149,10 +217,15 @@ export async function fetchMetadataFiles({ for (const country of countries) { const snapshotPath = join(outputDir, `programs-${country}.json`); + const parameterTreePath = join(outputDir, `param-tree-${country}.json`); + const variableTreePath = join(outputDir, `variable-tree-${country}.json`); + const variableModulesPath = join(outputDir, `variable-modules-${country}.json`); let source; try { - // A failed refresh must not leave a snapshot from an earlier build behind. - rmSync(snapshotPath, { force: true }); + // Clear every snapshot before resolving HEAD, including when resolution fails. + for (const path of [snapshotPath, parameterTreePath, variableTreePath, variableModulesPath]) { + rmSync(path, { force: true }); + } source = await resolveRepository(country, fetchImpl); } catch (error) { logger.warn(` ${country}: could not resolve registry source — ${error.message}`); @@ -172,13 +245,32 @@ export async function fetchMetadataFiles({ (async () => { try { const paths = await fetchParameterYamlPaths(country, source, fetchImpl); - const treePath = join(outputDir, `param-tree-${country}.json`); - writeFileSync(treePath, JSON.stringify(paths)); - logger.log(` ${country}: wrote ${treePath} (${paths.length} files)`); + writeFileSync(parameterTreePath, JSON.stringify({ source, paths })); + logger.log(` ${country}: wrote ${parameterTreePath} (${paths.length} files)`); } catch (error) { logger.warn(` ${country}: parameter tree skipped — ${error.message}`); } })(), + (async () => { + try { + const tree = await fetchVariableTreeSnapshot(country, source, { fetchImpl, logger }); + writeFileSync(variableTreePath, JSON.stringify(tree)); + logger.log(` ${country}: wrote ${variableTreePath} (${tree.paths.length} files)`); + } catch (error) { + logger.warn(` ${country}: variable tree skipped — ${error.message}`); + } + })(), + (async () => { + // The UK API serves no module names; the US API does. + if (country !== 'uk') return; + try { + const snapshot = await fetchVariableModuleSnapshot(country, source.repo, fetchImpl); + writeFileSync(variableModulesPath, JSON.stringify(snapshot)); + logger.log(` ${country}: wrote ${variableModulesPath} (${Object.keys(snapshot.modules).length} classes, ${snapshot.source.version})`); + } catch (error) { + logger.warn(` ${country}: variable module map skipped — ${error.message}`); + } + })(), ]); } logger.log('Done.'); diff --git a/scripts/run_uk_target_check.py b/scripts/run_uk_target_check.py new file mode 100644 index 0000000..3ad4450 --- /dev/null +++ b/scripts/run_uk_target_check.py @@ -0,0 +1,60 @@ +"""Run the pinned UK model on the pinned dataset and score every national target. + +This is the check behind the UK validation page: the released household file, +run by the model version policyengine.py pins, for the calibration year, +against the same national targets the release was calibrated to. It writes +one row per target (aggregates only). + + git -C policyengine-uk-data archive 1.58.1 | tar -x -C /tmp/uk-data-1.58.1 + uv run --no-project --python 3.13 --with policyengine-uk==2.130.1 \\ + --with policyengine-core==3.33.0 --with pydantic --with pyyaml \\ + --with /tmp/uk-data-1.58.1 python scripts/run_uk_target_check.py \\ + /enhanced_frs_2024_25.h5 2025 /pinned_target_check.csv +""" + +import hashlib +import sys +from importlib.metadata import version + +import pandas as pd +from policyengine_uk import Microsimulation +from policyengine_uk.data import UKSingleYearDataset +from policyengine_uk_data.targets.build_loss_matrix import create_target_matrix + + +def sha256(path: str) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(dataset_path: str, year: str, out_path: str) -> None: + dataset = UKSingleYearDataset(file_path=dataset_path) + matrix, targets = create_target_matrix(dataset, time_period=year) + sim = Microsimulation(dataset=dataset) + weights = sim.calculate("household_weight", int(year)).values + estimates = weights @ matrix.values + out = pd.DataFrame( + {"metric": targets.index, "target": targets.values, "estimate": estimates} + ) + out.attrs = {} + meta = { + "policyengine_uk": version("policyengine-uk"), + "policyengine_core": version("policyengine-core"), + "policyengine_uk_data": version("policyengine-uk-data"), + "dataset_sha256": sha256(dataset_path), + "year": year, + "people": float(sim.calculate("people", int(year)).sum()), + "households": float(weights.sum()), + } + with open(out_path, "w") as f: + for key, value in meta.items(): + f.write(f"# {key}={value}\n") + out.to_csv(f, index=False) + print(meta, len(out), "targets") + + +if __name__ == "__main__": + main(*sys.argv[1:4]) diff --git a/scripts/test_extract_uk_release.py b/scripts/test_extract_uk_release.py new file mode 100644 index 0000000..9170d2b --- /dev/null +++ b/scripts/test_extract_uk_release.py @@ -0,0 +1,885 @@ +"""Tests for extract_uk_release.py on a small synthetic release. + + uv run --no-project --with pytest --with hypothesis --with h5py --with tables \\ + --with pandas --with numpy pytest scripts/test_extract_uk_release.py +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +import extract_uk_release as ex +import h5py +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +N_FRS = 4 +N_SPI = 2 + + +def _households() -> pd.DataFrame: + """4 FRS households, 2 SPI copies, a capital gains copy of each, 1 donor.""" + rows = [] + for i in range(N_FRS): + rows.append( + dict( + spi=False, + gains=False, + donor=False, + region="WALES" if i == 0 else "LONDON", + ) + ) + for i in range(N_SPI): + rows.append(dict(spi=True, gains=False, donor=False, region="LONDON")) + for r in list(rows): + rows.append({**r, "gains": True}) + rows.append(dict(spi=False, gains=True, donor=True, region="NORTHERN_IRELAND")) + hh = pd.DataFrame(rows) + n = len(hh) + ni = hh["region"] == "NORTHERN_IRELAND" + return pd.DataFrame( + { + "household_id": np.arange(n), + "household_weight": np.linspace(100, 200, n), + "region": hh["region"], + "household_is_spi_synthetic": hh["spi"], + "household_is_capital_gains_clone": hh["gains"], + "household_is_cgt_band_donor": hh["donor"], + "source_household_id": np.arange(n) + 1000, + "clone_index": 0, + "constituency_code_oa": np.where( + ni, "", np.where(hh["region"] == "WALES", "W07000001", "E14000001") + ), + "la_code_oa": np.where(ni, "", "E06000001"), + } + ) + + +def _write_dataset( + path: Path, household: pd.DataFrame, people_per_household: int = 2 +) -> None: + n = len(household) + person = pd.DataFrame( + { + "person_household_id": np.repeat( + household["household_id"], people_per_household + ) + } + ) + benunit = pd.DataFrame({"benunit_id": np.arange(n)}) + household.to_hdf(path, key="household", format="table") + person.to_hdf(path, key="person", format="table") + benunit.to_hdf(path, key="benunit", format="table") + pd.Series(["2024"]).to_hdf(path, key="time_period", format="table") + + +def _write_weights(path: Path, areas: int, weights: np.ndarray, ratio: float) -> None: + matrix = np.tile(weights * ratio / areas, (areas, 1)) + with h5py.File(path, "w") as f: + f.create_dataset("2025", data=matrix.astype(np.float64)) + + +def _log( + areas: int, + national: list[tuple[str, float, float]], + local_metrics: dict[str, tuple[float, float]], +) -> pd.DataFrame: + rows = [] + for epoch in (0, 10): + for a in range(areas): + for metric, (target, estimate) in local_metrics.items(): + rows.append( + dict( + index=f"A{a}", + name=f"Area {a}", + metric=metric, + estimate=estimate if epoch else 0.0, + target=target, + validation=metric == ex.HELD_OUT_LOCAL, + epoch=epoch, + ) + ) + for metric, target, estimate in national: + rows.append( + dict( + index=0, + name="UK", + metric=metric, + estimate=estimate if epoch else 0.0, + target=target, + validation=False, + epoch=epoch, + ) + ) + return pd.DataFrame(rows) + + +NATIONAL = [ + ("obr/income_tax", 100.0, 105.0), + ("obr/vat", 50.0, 80.0), + ("slc/plan_5_borrowers_above_threshold", 0.0, 3.0), +] +LOCAL = { + "age/0_10": (10.0, 10.5), + "hmrc/employment_income/amount": (20.0, 30.0), + ex.HELD_OUT_LOCAL: (5.0, 9.0), + "housing/wales_private_rent_amount": (0.0, 0.0), +} +REGISTRY = [ + dict( + name="obr/income_tax", + variable="income_tax", + source="obr", + unit="gbp", + geographic_level="national", + breakdown_variable=None, + reference_url="https://obr.uk/x", + forecast_vintage="march_2026", + countries=None, + geo_name=None, + values={"2025": 100.0}, + ), + dict( + name="obr/vat", + variable="vat", + source="obr", + unit="gbp", + geographic_level="national", + breakdown_variable=None, + reference_url="https://obr.uk/x", + forecast_vintage="march_2026", + countries=None, + geo_name=None, + values={"2025": 50.0}, + ), + dict( + name="slc/plan_5_borrowers_above_threshold", + variable="student_loan_plan", + source="slc", + unit="count", + geographic_level="national", + breakdown_variable=None, + reference_url="https://slc.example", + forecast_vintage=None, + countries=["ENGLAND", "SCOTLAND", "WALES"], + geo_name=None, + values={"2024": 0.0}, + ), +] + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.fixture +def artifacts(tmp_path: Path) -> Path: + d = tmp_path / "release" + d.mkdir() + base = ( + _households() + .query( + "~household_is_spi_synthetic and ~household_is_capital_gains_clone and ~household_is_cgt_band_donor" + ) + .copy() + ) + _write_dataset(d / "frs_2024_25.h5", base) + enhanced = _households() + _write_dataset(d / "enhanced_frs_2024_25.h5", enhanced) + _write_weights( + d / "parliamentary_constituency_weights.h5", + 3, + enhanced["household_weight"].to_numpy(), + 1.01, + ) + _write_weights( + d / "local_authority_weights.h5", + 2, + enhanced["household_weight"].to_numpy(), + 1.0, + ) + _log(3, NATIONAL, LOCAL).to_csv(d / "constituency_calibration_log.csv", index=False) + _log(2, NATIONAL, LOCAL).to_csv(d / "la_calibration_log.csv", index=False) + (d / "registry_targets.json").write_text(json.dumps(REGISTRY)) + with open(d / "pinned_target_check.csv", "w") as f: + f.write( + "# policyengine_uk=2.126.12\n# policyengine_core=3.33.0\n" + "# policyengine_uk_data=1.58.0\n" + f"# dataset_sha256={_sha(d / 'enhanced_frs_2024_25.h5')}\n" + "# year=2025\n# people=40\n# households=1500\n" + ) + pd.DataFrame( + [(m, t, e * 1.02) for m, t, e in NATIONAL], + columns=["metric", "target", "estimate"], + ).to_csv(f, index=False) + names = [ + "enhanced_frs_2024_25", + "frs_2024_25", + "parliamentary_constituency_weights", + "local_authority_weights", + ] + files = { + "enhanced_frs_2024_25": "enhanced_frs_2024_25.h5", + "frs_2024_25": "frs_2024_25.h5", + "parliamentary_constituency_weights": "parliamentary_constituency_weights.h5", + "local_authority_weights": "local_authority_weights.h5", + } + manifest = { + "data_package": {"name": "policyengine-uk-data", "version": "1.58.0"}, + "default_datasets": {"national": "enhanced_frs_2024_25"}, + "artifacts": { + n: {"path": files[n], "sha256": _sha(d / files[n])} for n in names + }, + "build": { + "built_at": "2026-10-07T17:31:53Z", + "built_with_model_package": {"version": "2.122.2"}, + "built_with_core_package": {"version": "3.32.13"}, + "metadata": {"data_package_git_sha": "abc"}, + }, + } + (d / "release_manifest.json").write_text(json.dumps(manifest)) + bundle = { + "bundle_version": "6.3.4", + "data_releases": { + "uk": { + "model_package": {"version": "2.126.12"}, + "data_package": { + "name": "policyengine-uk-data", + "version": "1.58.0", + "release_manifest_revision": "rev", + }, + "default_dataset": "enhanced_frs_2024_25", + "datasets": { + "enhanced_frs_2024_25": { + "sha256": manifest["artifacts"]["enhanced_frs_2024_25"][ + "sha256" + ] + } + }, + "region_datasets": {"national": {}}, + } + }, + } + (d / "bundle_manifest.json").write_text(json.dumps(bundle)) + return d + + +def run(d: Path) -> dict: + return ex.extract(d, d / "registry_targets.json") + + +def test_extracts_counts_composition_and_geography(artifacts): + out = run(artifacts) + assert ( + out["release"]["dataVersion"] == "1.58.0" + and out["release"]["modelVersion"] == "2.126.12" + ) + assert out["base"]["households"] == N_FRS and out["base"]["people"] == 2 * N_FRS + c = out["enhanced"]["composition"] + assert (c["frs"], c["spiCopies"], c["capitalGainsCopies"], c["cgtBandDonors"]) == ( + N_FRS, + N_SPI, + N_FRS + N_SPI, + 1, + ) + assert ( + c["frs"] + c["spiCopies"] + c["capitalGainsCopies"] + c["cgtBandDonors"] + == out["enhanced"]["households"] + ) + g = out["enhanced"]["geography"] + assert g["householdsWithoutArea"] == 1 and g["regionsWithoutArea"] == [ + "NORTHERN_IRELAND" + ] + assert g["outputAreaDrawsPerHousehold"] == 1 + assert out["weightMatrices"]["columnSumPerSavedWeight"] == pytest.approx(1.01) + + +def test_scores_fit_on_nonzero_targets_and_holds_out_the_validation_metric(artifacts): + out = run(artifacts) + cal = out["calibration"] + assert cal["nationalTargets"] == 3 and cal["nationalZeroTargets"] == 1 + assert ( + cal["nationalWithin10"] == 0.5 + ) # income tax within 10%, VAT not, zero target unscored + local = cal["constituency"] + assert ( + local["structuralZeroCells"] == 3 + ) # the Welsh rent anchor, zero and met in every area + assert local["trainingCells"] == 6 and local["trainingWithin10"] == 0.5 + assert local["heldOut"] == { + "metric": ex.HELD_OUT_LOCAL, + "cells": 3, + "within10": 0.0, + } + held = [f for f in local["families"] if f["heldOut"]] + assert ( + len(held) == 1 and held[0]["label"] == "Income taxpayers with employment income" + ) + v = out["validation"] + assert ( + v["targets"] == 3 and v["zeroTargets"] == 1 and v["modelVersion"] == "2.126.12" + ) + assert len(v["rows"]) == 3 and {r["id"] for r in v["rows"]} == { + m for m, *_ in NATIONAL + } + + +def test_scores_the_pinned_run_against_the_targets_calibration_fitted(artifacts): + """A target the pinned run rebuilds at another value is still scored at the + value calibration fitted, and listed as revised.""" + path = artifacts / "pinned_target_check.csv" + text = path.read_text().replace("obr/vat,50.0,", "obr/vat,81.0,") + path.write_text(text) + v = run(artifacts)["validation"] + assert v["revisedTargets"] == ["obr/vat"] + assert v["within10"] == 0.5 # VAT's 81.6 still misses the fitted 50 + vat = next(r for r in v["rows"] if r["id"] == "obr/vat") + assert vat["target"] == 50.0 + + +@pytest.mark.parametrize( + "line, replacement", + [ + ("# dataset_sha256=", "# dataset_sha256=0"), # another file + ("# policyengine_uk_data=1.58.0", "# policyengine_uk_data=1.58.1"), + ("# year=2025", "# year=2026"), + ], +) +def test_rejects_a_pinned_check_of_another_file_package_or_year( + artifacts, line, replacement +): + path = artifacts / "pinned_target_check.csv" + path.write_text(path.read_text().replace(line, replacement, 1)) + _expect_error(artifacts, "pinned check has") + + +def test_is_deterministic_and_independent_of_log_row_order(artifacts): + first = run(artifacts) + log = pd.read_csv(artifacts / "constituency_calibration_log.csv") + log.sample(frac=1, random_state=3).to_csv( + artifacts / "constituency_calibration_log.csv", index=False + ) + assert run(artifacts) == first + + +def _expect_error(d: Path, match: str): + with pytest.raises(ex.ExtractionError, match=match): + run(d) + + +def test_rejects_a_file_whose_hash_differs_from_the_release(artifacts): + with h5py.File(artifacts / "local_authority_weights.h5", "a") as f: + f["2025"][0, 0] = 1e9 + _expect_error(artifacts, "sha256") + + +def test_rejects_a_release_other_than_the_pin(artifacts): + manifest = json.loads((artifacts / "release_manifest.json").read_text()) + manifest["data_package"]["version"] = "1.58.1" + (artifacts / "release_manifest.json").write_text(json.dumps(manifest)) + _expect_error(artifacts, "bundle pins") + + +def test_rejects_an_unlabelled_target_group(artifacts, monkeypatch): + monkeypatch.delitem(ex.GROUP_LABELS, ("obr", "vat", "national", None)) + _expect_error(artifacts, "no label") + + +def test_rejects_a_local_metric_with_no_family(artifacts): + log = pd.read_csv(artifacts / "la_calibration_log.csv") + extra = log[log["metric"] == "age/0_10"].assign(metric="mystery/metric") + pd.concat([log, extra]).to_csv(artifacts / "la_calibration_log.csv", index=False) + _expect_error(artifacts, "no family") + + +def test_rejects_a_pinned_check_from_another_model_version(artifacts): + text = ( + (artifacts / "pinned_target_check.csv") + .read_text() + .replace("2.126.12", "2.130.1") + ) + (artifacts / "pinned_target_check.csv").write_text(text) + _expect_error(artifacts, "bundle pins") + + +def test_rejects_a_zero_local_target_with_a_nonzero_estimate(artifacts): + log = pd.read_csv(artifacts / "constituency_calibration_log.csv") + log.loc[ + (log["metric"] == "housing/wales_private_rent_amount") & (log["epoch"] == 10), + "estimate", + ] = 4.0 + log.to_csv(artifacts / "constituency_calibration_log.csv", index=False) + _expect_error(artifacts, "structural zero") + + +def test_rejects_repeated_source_households(artifacts): + tables = { + k: pd.read_hdf(artifacts / "enhanced_frs_2024_25.h5", k) + for k in ("household", "person", "benunit", "time_period") + } + hh = tables["household"] + hh.loc[1, "source_household_id"] = hh.loc[0, "source_household_id"] + path = artifacts / "enhanced_frs_2024_25.h5" + path.unlink() + for k, frame in tables.items(): + frame.to_hdf(path, key=k, format="table") + manifest = json.loads((artifacts / "release_manifest.json").read_text()) + manifest["artifacts"]["enhanced_frs_2024_25"]["sha256"] = _sha(path) + (artifacts / "release_manifest.json").write_text(json.dumps(manifest)) + bundle = json.loads((artifacts / "bundle_manifest.json").read_text()) + bundle["data_releases"]["uk"]["datasets"]["enhanced_frs_2024_25"]["sha256"] = _sha( + path + ) + (artifacts / "bundle_manifest.json").write_text(json.dumps(bundle)) + check = artifacts / "pinned_target_check.csv" + check.write_text( + re.sub( + r"# dataset_sha256=\w+", f"# dataset_sha256={_sha(path)}", check.read_text() + ) + ) + _expect_error(artifacts, "more than one Output Area") + + +finite = st.floats( + min_value=-1e12, max_value=1e12, allow_nan=False, allow_infinity=False +) + + +@settings(max_examples=200, deadline=None) +@given( + st.lists(st.tuples(finite, finite), min_size=1, max_size=40), st.floats(0.001, 2) +) +def test_fit_share_is_a_share_of_the_nonzero_targets(pairs, tolerance): + targets = pd.Series([t for t, _ in pairs]) + estimates = pd.Series([e for _, e in pairs]) + share = ex.fit_share(estimates, targets, tolerance) + scorable = [(t, e) for t, e in pairs if t != 0] + if not scorable: + assert share is None + return + expected = sum(abs(e / t - 1) < tolerance for t, e in scorable) / len(scorable) + assert 0 <= share <= 1 + assert share == round(expected, 6) + assert ( + ex.fit_share(estimates, targets, tolerance * 2) >= share + ) # wider tolerance never scores fewer + + +REPO = Path(__file__).resolve().parent.parent + + +@pytest.mark.skipif( + not __import__("os").environ.get("UK_RELEASE_ARTIFACTS"), + reason="set UK_RELEASE_ARTIFACTS to the pinned release's files to run the differential check", +) +def test_committed_summary_equals_a_fresh_extraction(): + artifacts = Path(__import__("os").environ["UK_RELEASE_ARTIFACTS"]) + fresh = ex.extract(artifacts, artifacts / "registry_targets.json") + rows = fresh["validation"].pop("rows") + assert json.loads((REPO / "src/data/ukDataRelease.json").read_text()) == json.loads( + json.dumps(fresh) + ) + assert json.loads( + (REPO / "src/data/ukTargetChecks.json").read_text() + ) == json.loads(json.dumps(rows)) + + +def test_labels_a_family_by_its_country_scope(artifacts): + groups = {g["label"]: g for g in run(artifacts)["calibration"]["nationalGroups"]} + assert groups["Income tax receipts"]["geography"] == "UK" + assert ( + groups["Plan 2 and Plan 5 borrowers, liable and above the repayment threshold"][ + "geography" + ] + == "England" # slc.py counts England's borrowers, whatever the registry scope + ) + + +def test_names_publishers_and_counts_targets_carried_from_an_earlier_year(artifacts): + cal = run(artifacts)["calibration"] + plan = next(g for g in cal["nationalGroups"] if g["source"] == "slc") + assert plan["publisher"] == "Explore Education Statistics" + assert cal["targetsWithoutYearValue"] == 1 # the Plan 5 target lists only 2024 + assert cal["nationalTargetsByArea"] == {"UK": 2, "England": 1} + + +# The extractor's labels, areas, publishers and notes restate the release's +# code. Re-extracting the release can't catch a restatement that has gone +# wrong, so this reads the data package's source directly: +# UK_DATA_SOURCE=/path/to/policyengine-uk-data (at the pinned release). +DATA_SOURCE = __import__("os").environ.get("UK_DATA_SOURCE") + + +def _read(path: str) -> str: + return (Path(DATA_SOURCE) / "policyengine_uk_data" / path).read_text() + + +def _figure(text: str) -> float: + """1_100 -> 1100.0; 7.5e6 -> 7500000.0.""" + return float(text.replace("_", "")) + + +def _millions(x: float) -> str: + return f"{x / 1e6:g} million" + + +def restatement_mismatches(ex, source: Path) -> list[str]: + """Each claim the extractor makes about how a target is built, checked + against the figure or rule the data package's code holds. Returns one + message per mismatch, so a missing, stale or altered note fails.""" + import re + + def read(path: str) -> str: + return (source / "policyengine_uk_data" / path).read_text() + + problems: list[str] = [] + + def need(cond: bool, message: str) -> None: + if not cond: + problems.append(message) + + def note(key: tuple) -> str: + text = ex.GROUP_NOTES.get(key) + need(text is not None, f"no note for {key}") + return text or "" + + local = {prefix: n or "" for prefix, _, _, n in ex.LOCAL_FAMILIES} + + obr = read("targets/sources/obr.py") + # Areas: DWP benefit rows are GB lines built without a country restriction. + rows = obr[ + obr.index("benefit_rows = {") : obr.index( + "targets = []", obr.index("benefit_rows = {") + ) + ] + names = {f"obr/{n}" for n in re.findall(r'^\s+"(\w+)": \(', rows, re.M)} + need("countries=" not in rows, "obr.py now restricts benefit_rows by country") + gb = {m for m, a in ex.AREA_BY_TARGET.items() if a == "Great Britain"} + need( + gb == names - {"obr/child_benefit"}, + f"GB benefit areas {sorted(gb)} != {sorted(names - {'obr/child_benefit'})}", + ) + for m in sorted(gb): + key = ( + "obr", + "esa_income" + if m == "obr/esa" + else ("jsa" if m == "obr/jobseekers_allowance" else m.split("/")[1]), + "national", + None, + ) + need( + "Scored on UK households" in note(key) + or "scored on UK households" in note(key), + f"{m}: no UK-scoring note", + ) + need( + '"NI domestic rates"' in obr + and ex.AREA_BY_TARGET.get("obr/domestic_rates") == "Northern Ireland", + "domestic rates area", + ) + # ESA and PIP scoring. + blm = read("targets/build_loss_matrix.py") + inc = read("targets/compute/income.py") + need( + 'if name == "obr/esa":' in blm and 'calculate("esa_contrib")' in inc, + "obr/esa no longer adds contributory ESA", + ) + need( + "income-related and contributory" + in ex.GROUP_LABELS[("obr", "esa_income", "national", None)], + "ESA label", + ) + need( + '"Disability living allowance and personal independence p"' in obr, + "obr.py PIP row label changed", + ) + need( + "Disability Living Allowance and PIP" + in ex.GROUP_LABELS[("obr", "pip", "national", None)], + "PIP label", + ) + need( + 'ctx.pe("council_tax_less_benefit")' in read("targets/compute/council_tax.py"), + "council tax scoring changed", + ) + need( + "net of reductions" in note(("obr", "council_tax", "national", None)), + "council tax note", + ) + # Independent schools. + m = re.search( + r"_PRIVATE_SCHOOL = \{y: ([\d_]+) for y in range\((\d+), (\d+)\)\}", obr + ) + need(m is not None, "obr.py private school constant changed") + if m: + n = note(("obr", "attends_private_school", "national", None)) + need( + f"{_figure(m[1]):,.0f}" in n and m[2] in n and str(int(m[3]) - 1) in n, + f"private school note: {n!r}", + ) + # Salary sacrifice headcounts. + ss = note(("obr", "pension_contributions_via_salary_sacrifice", "national", None)) + for const in ("_SS_TOTAL_USERS", "_SS_BELOW_CAP_USERS", "_SS_ABOVE_CAP_USERS"): + m = re.search(const + r" = \{\s*y: ([\d_]+) \*", obr) + need( + m is not None and _millions(_figure(m[1])) in ss, + f"{const} figure missing from note {ss!r}", + ) + m = re.search(r"_SS_HEADCOUNT_GROWTH = ([\d.]+)", obr) + need( + m is not None and f"{(float(m[1]) - 1) * 100:g}%" in ss, + "salary sacrifice growth rate", + ) + deflates = "price_adj" in inc + need( + ("2023 prices" in ss) == deflates, + "salary sacrifice £2,000 test: note and compute disagree on price basis", + ) + # SPI income bands: the projections reuse the base-year names, and the + # registry keeps the first target of each name, so 2025 fits 2023-24. + spi = read("targets/sources/hmrc_spi.py") + registry_code = read("targets/registry.py") + dropped = ( + "if target.name in seen_names:" in registry_code + and 'name = f"hmrc/{variable}_income_band_{band_label}"' + in spi[spi.index("def _read_projection_csv") :] + ) + for variable in ex.SPI_GROUPS: + n = note(("hmrc_spi", variable, "national", "total_income")) + need( + dropped == ("fitted unchanged for 2025" in n), + f"SPI {variable}: projection note {n!r}", + ) + # Mortgage, property income, capital gains, land. + m = re.search( + r"_MORTGAGE_TOTAL = ([\d_.e]+) \* 12 \* ([\d_.e]+)", + read("targets/sources/housing.py"), + ) + mort = note(("ons", "mortgage_capital_repayment", "national", None)) + need( + m is not None + and f"£{_figure(m[1]):,.0f}" in mort + and _millions(_figure(m[2])) in mort, + f"mortgage note {mort!r}", + ) + need( + ex.PUBLISHER_BY_TARGET.get("housing/total_mortgage") + == "Assumption set in the build", + "mortgage publisher", + ) + m = re.search( + r"_PROPERTY_INCOME_SCALE = ([\d.]+)", read("targets/sources/hmrc_spi.py") + ) + need( + m is not None + and f"{m[1]} times" + in note(("hmrc_spi", "property_income", "national", "total_income")), + "property income scale", + ) + cgt = read("targets/sources/hmrc_cgt.py") + need( + "float(_CGT_TAXPAYERS) for year in range" in cgt and "_project_gains(" in cgt, + "CGT construction changed", + ) + cg = note(("hmrc", "capital_gains", "national", None)) + need("gains uprated" in cg and "counts held flat" in cg, f"CGT note {cg!r}") + land = read("targets/sources/_land.py") + share_year = re.search(r"_ONS_(\d{4})_HOUSEHOLD", land) + flat = "2025: HOUSEHOLD_LAND_VALUES[2024]" in land + for key in ( + ("ons", "household_land_value", "national", None), + ("ons", "corporate_land_value", "national", None), + ): + n = note(key) + need( + share_year is not None and share_year[1] in n, + f"land share year missing from {n!r}", + ) + need(flat == ("held flat for 2025" in n), f"land carry-forward: {n!r}") + need( + 'df["dwellings"] * df["avg_house_price"]' + in read("targets/sources/mhclg_regional_land.py") + and "dwellings" in note(("ons", "household_land_value", "region", None)), + "regional land note", + ) + # Publishers from the sources' own URLs. + need( + "www.gov.scot" in read("targets/sources/voa_council_tax.py") + and ex.PUBLISHER_BY_PREFIX.get("voa/council_tax/SCOTLAND/") + == "Scottish Government", + "VOA Scotland publisher", + ) + slc = read("targets/sources/slc.py") + need( + "explore-education-statistics" in slc + and ex.PUBLISHER_BY_PREFIX.get("slc/plan_") == "Explore Education Statistics", + "SLC plan publisher", + ) + need("Borrower counts for England only" in slc, "slc.py scope changed") + for prefix in sorted( + set( + re.findall( + r'name=f?"(slc/[a-z_]+?)(?:_spend|_recipients|_borrowers|\{)', slc + ) + ) + ): + need( + ex.area_override(prefix + "_x") == "England" + or ex.area_override(prefix) == "England", + f"{prefix}: not marked England", + ) + # Local families. + uc = read("targets/sources/local_uc.py") + need("(positional order)" in uc, "local UC targets no longer positional") + if "(positional order)" in uc: + need( + "row order" in local.get("uc_households", "") + and "row order" in local.get("uc_hh_", ""), + "UC notes must flag row order", + ) + need( + 'scaled["N"] = gb_bucket_totals.copy()' in uc + and "Great Britain" in local.get("uc_hh_", ""), + "UC child split note", + ) + for area in ("constituencies", "local_authorities"): + loss = read(f"datasets/local_areas/{area}/loss.py") + m = re.search( + r"y\[col\] \*= uk_total_population / targets_total_pop \* ([\d.]+)", loss + ) + need( + m is not None and f"{float(m[1]) * 100:g}%" in local.get("age/", ""), + f"{area}: age rescale", + ) + need( + "adjustment = national_target / local_target_sum" in loss + and "Rescaled" in local.get("hmrc/employment_income/amount", ""), + f"{area}: income rescale", + ) + need( + re.search( + r'count"\] = \(\s*incomes\[f"\{income_variable\}_count"\]\.values \* adjustment', + loss, + ) + is not None + and "same factor" in local.get("hmrc/employment_income/count", ""), + f"{area}: count scaling", + ) + la = read("datasets/local_areas/local_authorities/loss.py") + need( + "VOA counts dwellings" in la + and "dwellings" in local.get("voa/council_tax/", "") + and "households" in local.get("voa/council_tax/", ""), + "VOA dwelling note", + ) + extras = read("targets/sources/local_la_extras.py") + need( + 'str.match(r"^E0[6789]")' in extras + and "England only" in local.get("rent/private_rent", ""), + "private rent scope", + ) + need( + f'VALIDATION_ONLY_LOCAL_TARGETS = ["{ex.HELD_OUT_LOCAL}"]' + in read("datasets/create_datasets.py"), + "held-out metric", + ) + return problems + + +@pytest.mark.skipif( + not DATA_SOURCE, reason="set UK_DATA_SOURCE to policyengine-uk-data at the pin" +) +def test_the_extractor_restates_the_data_package(): + assert restatement_mismatches(ex, Path(DATA_SOURCE)) == [] + + +@pytest.mark.skipif( + not DATA_SOURCE, reason="set UK_DATA_SOURCE to policyengine-uk-data at the pin" +) +@pytest.mark.parametrize( + "mutate", + [ + lambda m: m.GROUP_NOTES.pop( + ("obr", "attends_private_school", "national", None) + ), + lambda m: m.GROUP_NOTES.update( + { + ( + "obr", + "attends_private_school", + "national", + None, + ): "550,000 in every year from 2018 to 2031" + } + ), + lambda m: m.GROUP_NOTES.update( + { + ( + "ons", + "mortgage_capital_repayment", + "national", + None, + ): "£1,000 a month for 7.5 million" + } + ), + lambda m: m.PUBLISHER_BY_PREFIX.pop("voa/council_tax/SCOTLAND/"), + lambda m: m.PUBLISHER_BY_PREFIX.update({"slc/plan_": "Student Loans Company"}), + lambda m: m.AREA_BY_TARGET.pop("obr/state_pension"), + lambda m: m.AREA_BY_TARGET.pop("obr/domestic_rates"), + lambda m: m.GROUP_LABELS.update( + {("obr", "esa_income", "national", None): "Income-related ESA"} + ), + lambda m: m.GROUP_NOTES.update( + { + ( + "obr", + "pension_contributions_via_salary_sacrifice", + "national", + None, + ): "7.7 million users" + } + ), + lambda m: m.LOCAL_FAMILIES.__setitem__( + 4, m.LOCAL_FAMILIES[4][:3] + ("Universal Credit households by area",) + ), + lambda m: m.LOCAL_FAMILIES.__setitem__( + 0, m.LOCAL_FAMILIES[0][:3] + ("Rescaled to the UK population",) + ), + ], +) +def test_the_restatement_check_fails_when_a_claim_drifts(mutate): + import copy + import types + + module = types.SimpleNamespace( + **{ + name: copy.deepcopy(getattr(ex, name)) + for name in ( + "GROUP_NOTES", + "GROUP_LABELS", + "LOCAL_FAMILIES", + "AREA_BY_TARGET", + "PUBLISHER_BY_TARGET", + "PUBLISHER_BY_PREFIX", + "AREA_BY_PREFIX", + "HELD_OUT_LOCAL", + "SPI_GROUPS", + ) + } + ) + module.area_override = lambda metric: ( + module.AREA_BY_TARGET.get(metric) + or next( + (a for p, a in module.AREA_BY_PREFIX.items() if metric.startswith(p)), None + ) + ) + assert restatement_mismatches(module, Path(DATA_SOURCE)) == [] + mutate(module) + assert restatement_mismatches(module, Path(DATA_SOURCE)) != [] diff --git a/scripts/uk_obr_elasticity_grid.py b/scripts/uk_obr_elasticity_grid.py new file mode 100644 index 0000000..ac407e7 --- /dev/null +++ b/scripts/uk_obr_elasticity_grid.py @@ -0,0 +1,90 @@ +"""Print the OBR-method labour supply elasticities policyengine-uk assigns to +every family type, by running its own functions on made-up people. + + uv run --no-project --with numpy --with pandas python \\ + scripts/uk_obr_elasticity_grid.py /path/to/policyengine-uk + +It lifts calculate_labour_substitution_elasticities and +calculate_labour_net_income_elasticities out of dynamics/progression.py and +calls them with a stand-in simulation, so it needs no policyengine-uk install. +Output is JSON: one row per gender, marital status and age of the youngest +child under 18 (null for no children), with both elasticities. +src/test/ukBehavioural.test.tsx compares the page's table with it. +""" + +import ast +import json +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +FUNCTIONS = ( + "calculate_labour_substitution_elasticities", + "calculate_labour_net_income_elasticities", +) + + +def load_functions(source: Path) -> dict: + tree = ast.parse((source / "policyengine_uk/dynamics/progression.py").read_text()) + found = [ + n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in FUNCTIONS + ] + if {n.name for n in found} != set(FUNCTIONS): + raise SystemExit(f"progression.py no longer defines {FUNCTIONS}") + namespace = { + "np": np, + "Simulation": object, + "benunit_age_18_composition": lambda sim, year: sim.composition, + } + exec( + compile(ast.Module(body=found, type_ignores=[]), "progression.py", "exec"), + namespace, + ) + return {name: namespace[name] for name in FUNCTIONS} + + +class People: + """The variables the two functions read, one entry per person.""" + + def __init__(self, rows: list[dict]): + self.columns = { + "gender": np.array([r["gender"] for r in rows]), + "is_married": np.array([r["married"] for r in rows]), + } + youngest = [r["youngest"] for r in rows] + self.composition = pd.DataFrame( + { + "count_under_18": [0 if y is None else 1 for y in youngest], + "youngest_under_18_age": [-1 if y is None else y for y in youngest], + } + ) + + def calculate(self, name, year=None, map_to=None): + return self.columns[name] + + +def grid() -> list[dict]: + return [ + {"gender": gender, "married": married, "youngest": youngest} + for gender in ("FEMALE", "MALE") + for married in (True, False) + for youngest in [None, *range(18)] + ] + + +def main(source: str) -> None: + functions = load_functions(Path(source)) + rows = grid() + people = People(rows) + substitution = functions[FUNCTIONS[0]](people) + income = functions[FUNCTIONS[1]](people) + for row, s, i in zip(rows, substitution, income): + row["substitution"] = round(float(s), 6) + row["income"] = round(float(i), 6) + json.dump(rows, sys.stdout) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/scripts/uk_pay_channel_cases.py b/scripts/uk_pay_channel_cases.py new file mode 100644 index 0000000..7daa1cc --- /dev/null +++ b/scripts/uk_pay_channel_cases.py @@ -0,0 +1,197 @@ +"""Run policyengine-uk's pay-channel formulas on made-up people and print what +they return, so the behavioural page's claims about them can be checked. + + uv run --no-project --with numpy python scripts/uk_pay_channel_cases.py \\ + /path/to/policyengine-uk + +It lifts the `formula` of three variables out of their source files and calls +it with stand-in person, parameter and baseline objects, so it needs no +policyengine-uk install: +- employer_ni_fixed_employer_cost_change (pay under an employer NI reform) +- salary_sacrifice_broad_base_haircut (the cut to every worker's pay) +- salary_sacrifice_returned_to_income (contributions above the cap) +Output is JSON: one record per case. src/test/ukBehavioural.test.tsx reads it. +""" + +import ast +import json +import sys +from pathlib import Path +from types import SimpleNamespace as NS + +import numpy as np + +WEEKS_IN_YEAR = 52 +FILES = { + "employer_ni_fixed_employer_cost_change": "variables/contrib/policyengine/employer_ni/employer_ni_fixed_employer_cost_change.py", + "salary_sacrifice_broad_base_haircut": "variables/gov/hmrc/national_insurance/salary_sacrifice_broad_base_haircut.py", + "salary_sacrifice_returned_to_income": "variables/gov/hmrc/pensions/salary_sacrifice_returned_to_income.py", +} + + +def load_formula(source: Path, variable: str): + tree = ast.parse((source / "policyengine_uk" / FILES[variable]).read_text()) + cls = next( + n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == variable + ) + formula = next( + n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "formula" + ) + namespace = { + "np": np, + "max_": np.maximum, + "where": np.where, + "WEEKS_IN_YEAR": WEEKS_IN_YEAR, + "add": lambda entity, period, names: sum( + entity(name, period) for name in names + ), + } + exec( + compile(ast.Module(body=[formula], type_ignores=[]), FILES[variable], "exec"), + namespace, + ) + return namespace["formula"] + + +def parameters( + *, + rate=0.15, + threshold_weekly=96.0, + exempt=True, + incidence=1.0, + cap=np.inf, + haircut=0.0016, +): + tree = NS( + gov=NS( + hmrc=NS( + national_insurance=NS( + class_1=NS( + rates=NS(employer=rate), + thresholds=NS(secondary_threshold=threshold_weekly), + ), + salary_sacrifice_pension_cap=cap, + ) + ), + contrib=NS( + policyengine=NS( + employer_ni=NS( + employee_incidence=incidence, + exempt_employer_pension_contributions=exempt, + ) + ), + behavioral_responses=NS( + salary_sacrifice_broad_base_haircut_rate=haircut + ), + ), + ) + ) + return lambda period: tree + + +class Person: + def __init__(self, values: dict, baseline_parameters=None): + self.values = {k: np.asarray(v, dtype=float) for k, v in values.items()} + baseline = None + if baseline_parameters is not None: + baseline = NS(tax_benefit_system=NS(parameters=baseline_parameters)) + self.simulation = NS(baseline=baseline) + + def __call__(self, name, period): + return self.values.get(name, np.zeros(1)) + + +def employer_cost(pay, rate, threshold_weekly): + return pay + rate * max(pay - threshold_weekly * WEEKS_IN_YEAR, 0) + + +def main(source: str) -> None: + src = Path(source) + employer = load_formula(src, "employer_ni_fixed_employer_cost_change") + haircut = load_formula(src, "salary_sacrifice_broad_base_haircut") + returned = load_formula(src, "salary_sacrifice_returned_to_income") + cases = [] + + def employer_case(label, pay, baseline, reform): + person = Person({"employment_income_before_lsr": [pay]}, parameters(**baseline)) + change = float( + np.asarray(employer(person, 2026, parameters(**reform))).ravel()[0] + ) + new_pay = pay + change + cases.append( + { + "kind": "employer_ni", + "label": label, + "pay": pay, + "change": change, + "baselineCost": employer_cost( + pay, + baseline.get("rate", 0.15), + baseline.get("threshold_weekly", 96.0), + ), + "reformCost": employer_cost( + new_pay, + reform.get("rate", 0.15), + reform.get("threshold_weekly", 96.0), + ), + } + ) + + employer_case( + "rate rise, pay above both thresholds", 40_000, {"rate": 0.138}, {"rate": 0.15} + ) + employer_case( + "rate rise, incidence 0", + 40_000, + {"rate": 0.138, "incidence": 0.0}, + {"rate": 0.15, "incidence": 0.0}, + ) + employer_case( + "rate rise, pay below the baseline threshold", + 4_000, + {"rate": 0.138}, + {"rate": 0.15}, + ) + employer_case( + "threshold lifted past pay", + 30_000, + {"rate": 0.10, "threshold_weekly": 10_000 / WEEKS_IN_YEAR}, + {"rate": 0.10, "threshold_weekly": 50_000 / WEEKS_IN_YEAR}, + ) + employer_case("no change", 40_000, {}, {}) + + for label, cap in (("cap off", np.inf), ("cap £2,000", 2_000.0)): + person = Person({"employment_income_before_lsr": [40_000.0]}) + cases.append( + { + "kind": "haircut", + "label": label, + "pay": 40_000.0, + "change": float( + np.asarray(haircut(person, 2030, parameters(cap=cap))).ravel()[0] + ), + } + ) + for label, cap, sacrificed in ( + ("cap off", np.inf, 5_000.0), + ("cap £2,000, below it", 2_000.0, 1_500.0), + ("cap £2,000, above it", 2_000.0, 5_000.0), + ): + person = Person( + {"pension_contributions_via_salary_sacrifice_from_pay": [sacrificed]} + ) + cases.append( + { + "kind": "returned", + "label": label, + "sacrificed": sacrificed, + "change": float( + np.asarray(returned(person, 2030, parameters(cap=cap))).ravel()[0] + ), + } + ) + json.dump(cases, sys.stdout) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/scripts/variable-modules.js b/scripts/variable-modules.js new file mode 100644 index 0000000..e3e2f41 --- /dev/null +++ b/scripts/variable-modules.js @@ -0,0 +1,83 @@ +/** Read variable declarations from a GitHub commit archive without extracting it. */ + +function readString(buffer, start, length) { + return buffer.subarray(start, start + length).toString('utf8').split('\0')[0]; +} + +/** PAX records prefix each key=value line with its total byte length. */ +function paxPath(contents) { + let path; + for (let offset = 0; offset < contents.length;) { + const separator = contents.indexOf(0x20, offset); + const length = Number(contents.subarray(offset, separator).toString('ascii')); + if (separator < offset || !Number.isSafeInteger(length) || length <= separator - offset + 1 || offset + length > contents.length) { + throw new Error('Invalid PAX archive record'); + } + const record = contents.subarray(separator + 1, offset + length - 1).toString('utf8'); + if (record.startsWith('path=')) path = record.slice(5); + offset += length; + } + return path; +} + +function* tarFiles(tar) { + let nextPath; + for (let offset = 0; offset < tar.length;) { + if (offset + 512 > tar.length) throw new Error('Truncated archive header'); + const header = tar.subarray(offset, offset + 512); + if (header.every(byte => byte === 0)) return; + const sizeField = readString(header, 124, 12).trim(); + if (!/^[0-7]+$/.test(sizeField)) throw new Error('Invalid archive file size'); + const size = parseInt(sizeField, 8); + const start = offset + 512; + const end = start + size; + const paddedEnd = start + Math.ceil(size / 512) * 512; + if (!Number.isSafeInteger(end) || paddedEnd > tar.length) throw new Error('Truncated archive file'); + const contents = tar.subarray(start, end); + const type = readString(header, 156, 1); + const prefix = readString(header, 345, 155); + const name = readString(header, 0, 100); + offset = paddedEnd; + + if (type === 'x') { + nextPath = paxPath(contents) ?? nextPath; + continue; + } + if (type === 'L') { + nextPath = readString(contents, 0, contents.length).replace(/\n$/, ''); + continue; + } + if (type === 'g') continue; + const path = nextPath ?? (prefix ? `${prefix}/${name}` : name); + nextPath = undefined; + if (type === '0' || type === '') yield { path, contents }; + } +} + +/** + * Exact, case-sensitive class names, including several classes in one file and + * declarations whose Variable base is split over lines. Ambiguous names are omitted. + */ +export function extractVariableModules(country, tar) { + const packagePrefix = `policyengine_${country}/variables/`; + const paths = []; + const modules = new Map(); + for (const file of tarFiles(tar)) { + // GitHub archives place the repository contents under one top-level directory. + const packagePath = file.path.slice(file.path.indexOf('/') + 1); + if (!packagePath.startsWith(packagePrefix)) continue; + const path = packagePath.slice(packagePrefix.length); + if (!path.endsWith('.py') || path.endsWith('__init__.py')) continue; + paths.push(path); + const moduleName = path.replace(/\.py$/, '').replace(/\//g, '.'); + for (const match of file.contents.toString('utf8').matchAll(/^class\s+(\w+)\s*\(\s*Variable\s*\)\s*:/gm)) { + const name = match[1]; + modules.set(name, modules.has(name) && modules.get(name) !== moduleName ? null : moduleName); + } + } + if (!paths.length || !modules.size) throw new Error('Archive contains no variable declarations'); + return { + paths: paths.sort(), + modules: Object.fromEntries([...modules].filter(([, moduleName]) => moduleName !== null).sort(([a], [b]) => a.localeCompare(b))), + }; +} diff --git a/src/components/data/DataPipeline.tsx b/src/components/data/DataPipeline.tsx index 73d54c1..bbebcbd 100644 --- a/src/components/data/DataPipeline.tsx +++ b/src/components/data/DataPipeline.tsx @@ -5,7 +5,7 @@ import Link from 'next/link'; import { useSearchParams } from 'next/navigation'; import { colors, typography, spacing } from '../../designTokens'; import { pipelineStages } from '../../data/pipelineStages'; -import { ukPipelineStages } from '../../data/ukPipelineStages'; +import { ukPipelineStages, ukPipelineIntro, ukBranchLabels } from '../../data/ukPipelineStages'; import type { PipelineStage } from '../../data/pipelineStages'; import type { Country } from '../../hooks/useCountry'; import { usePublicBasePrefix } from '../../hooks/usePublicBasePrefix'; @@ -124,11 +124,11 @@ export default function DataPipeline({ country = 'us' }: { country?: Country }) }; const introText = country === 'uk' - ? 'PolicyEngine constructs its representative household dataset through a multi-stage pipeline, starting from the Family Resources Survey and producing two weight matrices: one for 650 parliamentary constituencies and one for 360 local authorities.' + ? ukPipelineIntro : 'PolicyEngine constructs its representative household dataset through a 14-step pipeline. Public survey data is merged, stratified, and cloned to 10 geographic variants per household. Each clone is simulated through PolicyEngine US with stochastic take-up, then calibrated via L0-regularized optimization against administrative targets at the national, state, and congressional district levels simultaneously, producing 488 geographically representative datasets.'; - const nationalLabel = country === 'uk' ? 'Constituency (national)' : 'National'; - const localLabel = country === 'uk' ? 'Local authority' : 'Geography-specific'; + const nationalLabel = country === 'uk' ? ukBranchLabels.national : 'National'; + const localLabel = country === 'uk' ? ukBranchLabels.local : 'Geography-specific'; return (
@@ -138,8 +138,8 @@ export default function DataPipeline({ country = 'us' }: { country?: Country }) {/* Pipeline flow - branching layout */}
- {/* Shared linear portion */} -
+ {/* Shared linear portion; scrolls inside itself on narrow screens */} +
{sharedStages.map((s, i) => (
{/* National branch */} -
+
{nationalLabel}
-
+
{nationalStages.map((s, i) => (
{/* Local branch */} -
+
{localLabel}
-
+
{localStages.map((s, i) => (
`${basePrefix}${path}`; + // Keep ?country= and ?embed= on the way into a child page, as the sidebar does. + const search = useSearchParams().toString(); + const href = (path: string) => `${basePrefix}${path}${search ? `?${search}` : ''}`; return (
diff --git a/src/components/microsim/ThreeIngredients.tsx b/src/components/microsim/ThreeIngredients.tsx index e3ad1cd..d63c5f6 100644 --- a/src/components/microsim/ThreeIngredients.tsx +++ b/src/components/microsim/ThreeIngredients.tsx @@ -47,14 +47,18 @@ export default function ThreeIngredients({ activeIngredient, country = 'us' }: T { title: 'Policy rules', description: - 'Tax codes, benefit formulas, and eligibility rules encoded as executable logic. Over 55 federal, state, and local programs.', + country === 'uk' + ? 'Tax codes, benefit formulas, and eligibility rules encoded as executable logic, from income tax and National Insurance to Universal Credit and Scottish Child Payment.' + : 'Tax codes, benefit formulas, and eligibility rules encoded as executable logic. Over 55 federal, state, and local programs.', icon: , key: 'policies' as const, }, { title: 'Household data', description: - 'A representative microdata sample of the population, with income, demographics, and program participation for each household.', + country === 'uk' + ? 'Family Resources Survey households, plus copies the build adds for top incomes and capital gains, each with incomes, demographics and benefit receipt, weighted towards official statistics.' + : 'A representative microdata sample of the population, with income, demographics, and program participation for each household.', icon: , key: 'households' as const, }, @@ -62,7 +66,7 @@ export default function ThreeIngredients({ activeIngredient, country = 'us' }: T title: country === 'uk' ? 'Behavioural dynamics' : 'Behavioral dynamics', description: country === 'uk' - ? 'Elasticities capturing how people adjust their labour supply and income in response to tax and benefit changes.' + ? 'Optional elasticities for how labour supply and capital gains respond to tax and benefit changes, and pay that adjusts by default under employer National Insurance and salary sacrifice reforms.' : 'Elasticities capturing how people adjust their labor supply and income in response to tax and benefit changes.', icon: , key: 'dynamics' as const, @@ -72,8 +76,9 @@ export default function ThreeIngredients({ activeIngredient, country = 'us' }: T return (

- PolicyEngine's microsimulation model combines three ingredients to estimate the impact of any - tax or benefit reform on every household in the country. + {country === 'uk' + ? 'PolicyEngine UK combines three ingredients to estimate how a tax or benefit reform it models would affect households across the UK.' + : "PolicyEngine's microsimulation model combines three ingredients to estimate the impact of any tax or benefit reform on every household in the country."}

{ingredients.map((ing, i) => ( diff --git a/src/components/microsim/Walkthrough.tsx b/src/components/microsim/Walkthrough.tsx index 8faad81..cf23906 100644 --- a/src/components/microsim/Walkthrough.tsx +++ b/src/components/microsim/Walkthrough.tsx @@ -151,7 +151,7 @@ export default function Walkthrough({ country = 'us' }: { country?: Country }) { {/* Single persistent table — only visible columns rendered */}
+

+ Illustrative example: five hypothetical households, a simplified tax schedule, and a + single elasticity. The figures are not output from the model. +

); } diff --git a/src/components/parameters/ParameterDetail.tsx b/src/components/parameters/ParameterDetail.tsx index fe00f11..d9c3b7f 100644 --- a/src/components/parameters/ParameterDetail.tsx +++ b/src/components/parameters/ParameterDetail.tsx @@ -3,6 +3,7 @@ import { motion } from 'framer-motion'; import { IconExternalLink, IconChevronDown, IconChevronRight } from '@tabler/icons-react'; import type { Parameter, ParameterLeaf } from '../../types/Variable'; import { colors, typography, spacing } from '../../designTokens'; +import { publicBasePrefixFromPath } from '../../hooks/usePublicBasePrefix'; interface ParameterDetailProps { parameter: Parameter; @@ -267,10 +268,17 @@ const treeCache = new Map | null>(); async function loadParamTree(country: string): Promise | null> { if (treeCache.has(country)) return treeCache.get(country)!; try { - const res = await fetch(`/param-tree-${country}.json`); + // Snapshots sit beside the app, so under policyengine.org/{country}/model + // they are served at that prefix, not at the site root. + const prefix = publicBasePrefixFromPath(window.location.pathname); + const res = await fetch(`${prefix}/param-tree-${country}.json`); if (!res.ok) throw new Error('not found'); - const paths: string[] = await res.json(); - const set = new Set(paths); + const tree = await res.json(); + const paths = Array.isArray(tree) ? tree : tree?.paths; + if (!Array.isArray(paths) || !paths.every((path: unknown) => typeof path === 'string')) { + throw new Error('invalid parameter tree'); + } + const set = new Set(paths); treeCache.set(country, set); return set; } catch { diff --git a/src/components/parameters/ParameterExplorer.tsx b/src/components/parameters/ParameterExplorer.tsx index 677a134..fbfaf57 100644 --- a/src/components/parameters/ParameterExplorer.tsx +++ b/src/components/parameters/ParameterExplorer.tsx @@ -6,7 +6,7 @@ import { colors, typography, spacing } from '../../designTokens'; import { useDebounce } from '../../hooks/useDebounce'; import { getLevel, getSubGroup, getSubGroupLabel, getSubGroupDescription, - LEVEL_CONFIG, LEVELS_ORDERED, + levelConfig, levelsFor, emptyLevelRecord, type Level, } from '../shared/categoryUtils'; import ParameterCard from './ParameterCard'; @@ -284,11 +284,13 @@ function StateTileGrid({ function SubGroupCardGrid({ subGroups, level, + country, levelColor, onSelect, }: { subGroups: [string, ParameterLeaf[]][]; level: Level; + country: string; levelColor: string; onSelect: (key: string) => void; }) { @@ -298,8 +300,8 @@ function SubGroupCardGrid({ style={{ gap: spacing.md }} > {subGroups.map(([key, params], i) => { - const label = getSubGroupLabel(key, level); - const desc = getSubGroupDescription(key, level); + const label = getSubGroupLabel(key, level, country); + const desc = getSubGroupDescription(key, level, country); return ( { - const seen: Record> = { federal: new Set(), state: new Set(), local: new Set(), territory: new Set(), reform: new Set(), household: new Set() }; + const seen = emptyLevelRecord(() => new Set()); for (const p of allParameters) { - seen[getLevel(p.parameter)].add(getParameterGroup(p.parameter)); + seen[getLevel(p.parameter, country)].add(getParameterGroup(p.parameter)); } return Object.fromEntries(Object.entries(seen).map(([k, v]) => [k, v.size])) as Record; - }, [allParameters]); + }, [allParameters, country]); // Filtered list const filtered = useMemo(() => { @@ -573,22 +575,22 @@ export default function ParameterExplorer({ parameters, country }: ParameterExpl }); } - if (activeLevel) result = result.filter((p) => getLevel(p.parameter) === activeLevel); + if (activeLevel) result = result.filter((p) => getLevel(p.parameter, country) === activeLevel); return result; - }, [allParameters, debouncedSearch, activeLevel]); + }, [allParameters, debouncedSearch, activeLevel, country]); // Sub-groups for the active level const subGroups = useMemo(() => { if (!activeLevel) return []; const map = new Map(); for (const p of filtered) { - const key = getSubGroup(p.parameter); + const key = getSubGroup(p.parameter, country); if (!map.has(key)) map.set(key, []); map.get(key)!.push(p); } return [...map.entries()].sort(([a], [b]) => a.localeCompare(b)); - }, [filtered, activeLevel]); + }, [filtered, activeLevel, country]); // Parameters in active sub-group const subGroupParams = useMemo(() => { @@ -701,13 +703,13 @@ export default function ParameterExplorer({ parameters, country }: ParameterExpl if (activeLevel) { items.push({ - label: LEVEL_CONFIG[activeLevel].label, + label: levelConfig(activeLevel, country).label, onClick: () => { setActiveLevel(activeLevel); setActiveSubGroup(null); setFolderStack([]); setActiveGroup(null); setSelectedParam(null); }, }); } if (activeSubGroup && activeLevel) { items.push({ - label: getSubGroupLabel(activeSubGroup, activeLevel), + label: getSubGroupLabel(activeSubGroup, activeLevel, country), onClick: () => { setFolderStack([]); setActiveGroup(null); setSelectedParam(null); }, }); } @@ -725,7 +727,7 @@ export default function ParameterExplorer({ parameters, country }: ParameterExpl }); } return items; - }, [activeLevel, activeSubGroup, folderStack, activeGroup, parameters]); + }, [activeLevel, activeSubGroup, folderStack, activeGroup, parameters, country]); // Current heading label const currentHeadingLabel = activeGroup @@ -733,7 +735,7 @@ export default function ParameterExplorer({ parameters, country }: ParameterExpl : folderStack.length > 0 ? getGroupLabel(currentFolder, parameters) : activeSubGroup && activeLevel - ? getSubGroupLabel(activeSubGroup, activeLevel) + ? getSubGroupLabel(activeSubGroup, activeLevel, country) : ''; // Param count summary for folder view (count groups, not individual leaves) @@ -772,9 +774,9 @@ export default function ParameterExplorer({ parameters, country }: ParameterExpl {isSearching && (
- {([null, ...LEVELS_ORDERED] as (Level | null)[]).map((level) => { - const label = level ? LEVEL_CONFIG[level].label : 'All'; - const clr = level ? LEVEL_CONFIG[level].color : colors.text.secondary; + {([null, ...levelsFor(country)] as (Level | null)[]).map((level) => { + const label = level ? levelConfig(level, country).label : 'All'; + const clr = level ? levelConfig(level, country).color : colors.text.secondary; const isActive = activeLevel === level; return (
{getGroupLabel(activeGroup, parameters)} diff --git a/src/components/rules/RulesOverview.tsx b/src/components/rules/RulesOverview.tsx index 17c483e..29b7000 100644 --- a/src/components/rules/RulesOverview.tsx +++ b/src/components/rules/RulesOverview.tsx @@ -16,108 +16,19 @@ import { } from '../../data/programStatus'; import type { CoverageStatus, Program } from '../../types/Program'; import { IconX, IconCalendar } from '@tabler/icons-react'; +import { + StatCard, + StatusDot, + collectAllYears, + programIncludesYear, + statusLabels, +} from './coverageParts'; +import UkCoverageTracker from './UkCoverageTracker'; import type { Country } from '../../hooks/useCountry'; -const MAX_FORWARD_YEAR = new Date().getFullYear() + 5; - -/** Parse a verifiedYears string like "2022-2026", "2022+", or "2024" into a Set of year numbers. */ -function parseYearRange(verifiedYears?: string): Set { - if (!verifiedYears) return new Set(); - const trimmed = verifiedYears.trim(); - // "2022+" means from 2022 onward (no end year — uprating covers future) - const openMatch = trimmed.match(/^(\d{4})\+$/); - if (openMatch) { - const start = parseInt(openMatch[1], 10); - const years = new Set(); - for (let y = start; y <= MAX_FORWARD_YEAR; y++) years.add(y); - return years; - } - // "2022-2026" range - const rangeMatch = trimmed.match(/^(\d{4})\s*-\s*(\d{4})$/); - if (rangeMatch) { - const start = parseInt(rangeMatch[1], 10); - const end = parseInt(rangeMatch[2], 10); - const years = new Set(); - for (let y = start; y <= end; y++) years.add(y); - return years; - } - // Single year "2024" - const singleMatch = trimmed.match(/^(\d{4})$/); - if (singleMatch) return new Set([parseInt(singleMatch[1], 10)]); - return new Set(); -} - -/** Collect all unique years across all programs' verifiedYears fields. */ -function collectAllYears(programs: Program[]): number[] { - const allYears = new Set(); - for (const p of programs) { - for (const y of parseYearRange(p.verifiedYears)) allYears.add(y); - } - return Array.from(allYears).sort((a, b) => a - b); -} - -/** Check if a program's verifiedYears includes a given year. */ -function programIncludesYear(program: Program, year: number): boolean { - return parseYearRange(program.verifiedYears).has(year); -} - -const statusLabels: Record = { - complete: 'Complete', - partial: 'Partial', - inProgress: 'In progress', - notStarted: 'Not started', -}; - -const statusIcons: Record = { - complete: '\u2713', - partial: '\u25D1', - inProgress: '\u21BB', - notStarted: '\u25CB', -}; - -function StatusDot({ status, size = 10 }: { status: CoverageStatus; size?: number }) { - return ( - - {size >= 14 ? statusIcons[status] : ''} - - ); -} - -function StatCard({ label, count, color, delay }: { label: string; count: number; color: string; delay: number }) { - return ( - -
{count}
-
{label}
- - ); -} - type ViewMode = 'programs' | 'states'; function ProgramDetailPanel({ program, onClose, allPrograms }: { program: Program; onClose: () => void; allPrograms: Program[] }) { @@ -357,7 +268,7 @@ export default function RulesOverview({ country = 'us' }: { country?: Country }) const { programs, source } = loadedPrograms; useEffect(() => { - // The UK view currently displays a static introduction, not registry data. + // The UK tracker loads its own registry. if (country === 'uk') return; let active = true; fetchProgramsWithSource(country).then(result => { @@ -425,20 +336,7 @@ export default function RulesOverview({ country = 'us' }: { country?: Country }) }).sort((a, b) => b.pct - a.pct); }, [yearFilteredPrograms]); - if (country === 'uk') { - return ( -
-

- PolicyEngine UK models over 50 tax and benefit programs across HMRC, DWP, DfE, Treasury, and devolved administrations including income tax, National Insurance, VAT, Universal Credit, pension credit, child benefit, and many more. -

-

- - View the full UK model on GitHub - -

-
- ); - } + if (country === 'uk') return ; return (
diff --git a/src/components/rules/UkCoverageTracker.tsx b/src/components/rules/UkCoverageTracker.tsx new file mode 100644 index 0000000..c04a075 --- /dev/null +++ b/src/components/rules/UkCoverageTracker.tsx @@ -0,0 +1,422 @@ +'use client'; + +import { useEffect, useMemo, useState } from 'react'; +import { AnimatePresence, motion } from 'framer-motion'; +import { IconCalendar, IconX } from '@tabler/icons-react'; +import { colors, typography, spacing, statusColors } from '../../designTokens'; +import { fetchProgramsWithSource, type ProgramsSource, type ProgramsWithSource } from '../../data/fetchPrograms'; +import { publicBasePrefixFromPath } from '../../hooks/usePublicBasePrefix'; +import type { CoverageStatus, Program } from '../../types/Program'; +import CoverageProvenance from './CoverageProvenance'; +import { + StatCard, + StatusDot, + collectAllYears, + programIncludesYear, + statusLabels, +} from './coverageParts'; + +const REPO = 'PolicyEngine/policyengine-uk'; + +type Section = 'UK government' | 'Devolved' | 'Local authorities' | 'Other'; + +/** Who runs each `agency` in policyengine-uk's programs.yaml. */ +const AGENCIES: Record = { + HMRC: { section: 'UK government', label: 'HM Revenue and Customs (HMRC)' }, + DWP: { section: 'UK government', label: 'Department for Work and Pensions (DWP)' }, + DfE: { section: 'UK government', label: 'Department for Education (DfE)' }, + DfT: { section: 'UK government', label: 'Department for Transport (DfT)' }, + DCMS: { section: 'UK government', label: 'Department for Culture, Media and Sport (DCMS)' }, + Treasury: { section: 'UK government', label: 'HM Treasury' }, + Ofgem: { section: 'UK government', label: 'Ofgem' }, + 'Revenue Scotland': { section: 'Devolved', label: 'Revenue Scotland' }, + 'Social Security Scotland': { section: 'Devolved', label: 'Social Security Scotland' }, + Local: { section: 'Local authorities', label: 'Local authorities' }, +}; +const SECTIONS: Section[] = ['UK government', 'Devolved', 'Local authorities', 'Other']; + +export function agencyOf(program: Program): { section: Section; label: string } { + return AGENCIES[program.agency ?? ''] ?? { section: 'Other', label: program.agency || 'Other' }; +} + +interface SourceTrees { + parameterPaths: Set; + variablePaths: Map; +} + +interface SourceTree { + source: { repo: string; commit: string }; + paths: string[]; + modules?: Record; +} + +interface ProgrammeLinks { + parameters?: string; + source?: string; + computationTree?: string; +} + +/** A programme's links, kept only where the pinned snapshot has the file or folder. */ +export function verifiedLinks( + program: Program, + parameterPrefix: string | undefined, + trees: SourceTrees | null, + ref: string, +): ProgrammeLinks { + if (!trees) return {}; + const links: ProgrammeLinks = {}; + if (parameterPrefix) { + const dir = parameterPrefix.replace(/\./g, '/'); + const base = `https://github.com/${REPO}`; + if (trees.parameterPaths.has(`${dir}.yaml`)) { + links.parameters = `${base}/blob/${ref}/policyengine_uk/parameters/${dir}.yaml`; + } else if ([...trees.parameterPaths].some((p) => p.startsWith(`${dir}/`))) { + links.parameters = `${base}/tree/${ref}/policyengine_uk/parameters/${dir}`; + } + } + const file = program.variable ? trees.variablePaths.get(program.variable) : null; + if (file) { + links.source = `https://github.com/${REPO}/blob/${ref}/policyengine_uk/variables/${file}`; + links.computationTree = `https://policyengine.github.io/flowchart/?variable=${program.variable}&country=UK`; + } + return links; +} + +async function loadTree(url: string, source: Extract): Promise { + try { + const response = await fetch(url); + if (!response.ok) return null; + const tree = await response.json(); + if (tree?.source?.repo !== source.repo || tree?.source?.commit !== source.commit || + !Array.isArray(tree.paths) || !tree.paths.every((path: unknown) => typeof path === 'string')) return null; + return tree; + } catch { + return null; + } +} + +/** A tree can verify links only for the same repository and commit as the registry. */ +async function loadTrees(source: ProgramsSource): Promise { + if (source.kind !== 'snapshot' || source.repo !== REPO) return null; + const prefix = publicBasePrefixFromPath(window.location.pathname); + const [params, vars] = await Promise.all([ + loadTree(`${prefix}/param-tree-uk.json`, source), + loadTree(`${prefix}/variable-tree-uk.json`, source), + ]); + if (!params && !vars) return null; + const variablePaths = new Map(); + if (vars?.modules && typeof vars.modules === 'object' && !Array.isArray(vars.modules)) { + const paths = new Set(vars.paths); + for (const [name, moduleName] of Object.entries(vars.modules)) { + if (typeof moduleName !== 'string') continue; + const path = `${moduleName.replace(/\./g, '/')}.py`; + if (paths.has(path)) variablePaths.set(name, path); + } + } + // A tree with only paths can't show that a variable exists (afip.py declares + // armed_forces_independence_payment), so it verifies no variable links. + return { parameterPaths: new Set(params?.paths ?? []), variablePaths }; +} + +function ProgrammeDetail({ + program, + links, + onClose, +}: { + program: Program; + links: ProgrammeLinks; + onClose: () => void; +}) { + const linkStyle = { + fontSize: typography.fontSize.xs, + color: colors.primary[600], + textDecoration: 'none', + padding: `${spacing.xs} ${spacing.md}`, + border: `1px solid ${colors.primary[200]}`, + borderRadius: spacing.radius.md, + }; + return ( + +
+
+

+ {program.name} +

+ {program.fullName && program.fullName !== program.name && ( +
{program.fullName}
+ )} +
+ +
+ +
+
+ + + {statusLabels[program.status]} + +
+ + {agencyOf(program).label} + + {program.coverage && ( + Applies in: {program.coverage} + )} + {program.verifiedYears && ( + Verified years: {program.verifiedYears} + )} +
+ + {program.notes && ( +
+ {program.notes.replace(/\. Years: .*$/, '').replace(/^Years: .*$/, '')} +
+ )} + +
+ {links.parameters && ( + Parameters + )} + {links.source && ( + Source code + )} + {links.computationTree && ( + + Computation tree + + )} +
+ + ); +} + +/** + * Coverage of policyengine-uk's own programme registry (`programs.yaml`), read + * from the snapshot pinned at build time. Status, coverage and verified years + * are the registry's; nothing here is typed in by hand. + */ +export default function UkCoverageTracker() { + const [loaded, setLoaded] = useState(null); + const [trees, setTrees] = useState(null); + const [selected, setSelected] = useState(null); + const [search, setSearch] = useState(''); + const [yearFilter, setYearFilter] = useState(null); + + useEffect(() => { + let active = true; + fetchProgramsWithSource('uk').then((result) => { + if (!active) return; + setLoaded(result); + loadTrees(result.source).then((result) => { if (active) setTrees(result); }); + }); + return () => { active = false; }; + }, []); + + const programs = useMemo(() => loaded?.programs ?? [], [loaded]); + const ref = loaded?.source.kind === 'snapshot' ? loaded.source.commit : 'main'; + const availableYears = useMemo(() => collectAllYears(programs), [programs]); + + const visible = useMemo(() => { + const q = search.trim().toLowerCase(); + return programs.filter((p) => + (yearFilter === null || programIncludesYear(p, yearFilter)) && + (!q || [p.name, p.fullName, p.notes ?? '', p.coverage ?? '', agencyOf(p).label] + .some((field) => field.toLowerCase().includes(q))), + ); + }, [programs, search, yearFilter]); + + const counts = useMemo(() => { + const c: Record = { complete: 0, partial: 0, inProgress: 0, notStarted: 0 }; + for (const p of visible) c[p.status]++; + return c; + }, [visible]); + + const grouped = useMemo(() => { + const bySection = new Map>(); + for (const p of visible) { + const { section, label } = agencyOf(p); + if (!bySection.has(section)) bySection.set(section, new Map()); + const departments = bySection.get(section)!; + if (!departments.has(label)) departments.set(label, []); + departments.get(label)!.push(p); + } + return SECTIONS.filter((s) => bySection.has(s)).map((s) => ({ + section: s, + departments: [...bySection.get(s)!.entries()].sort((a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0])), + })); + }, [visible]); + + if (!loaded) { + return

Loading programmes…

; + } + + if (programs.length === 0) { + return ( +

+ The UK programme registry could not be loaded. See{' '} + + programs.yaml in policyengine-uk + + . +

+ ); + } + + const selectedProgram = programs.find((p) => p.id === selected) ?? null; + + return ( +
+ +

+ Each programme below is an entry in PolicyEngine UK's programme registry, shown with the status, + coverage, verified years and notes the registry records. +

+ +
+ + + + +
+ +
+ setSearch(e.target.value)} + className="tw:flex-1" + style={{ + padding: `${spacing.sm} ${spacing.lg}`, + borderRadius: spacing.radius.lg, + border: `1px solid ${colors.border.light}`, + fontSize: typography.fontSize.sm, + fontFamily: typography.fontFamily.primary, + outline: 'none', + minWidth: '200px', + maxWidth: '400px', + }} + /> + {availableYears.length > 0 && ( +
+ + +
+ )} +
+ + + {selectedProgram && ( +
+ setSelected(null)} + /> +
+ )} + + + {grouped.length === 0 && ( +

No programmes match.

+ )} + + {grouped.map(({ section, departments }) => ( +
+

+ {section} + {departments.length === 1 && departments[0][0] === section ? ` (${departments[0][1].length})` : ''} +

+ {departments.map(([label, list]) => ( +
+ {label !== section && ( +

+ {label} ({list.length}) +

+ )} +
+ {list.map((program) => { + const isSelected = selected === program.id; + return ( + + ); + })} +
+
+ ))} +
+ ))} +
+ ); +} diff --git a/src/components/rules/coverageParts.tsx b/src/components/rules/coverageParts.tsx new file mode 100644 index 0000000..9164188 --- /dev/null +++ b/src/components/rules/coverageParts.tsx @@ -0,0 +1,100 @@ +import { motion } from 'framer-motion'; +import { colors, typography, spacing, statusColors } from '../../designTokens'; +import type { CoverageStatus, Program } from '../../types/Program'; + +const MAX_FORWARD_YEAR = new Date().getFullYear() + 5; + +/** Parse a verifiedYears string like "2022-2026", "2022+", or "2024" into a Set of year numbers. */ +export function parseYearRange(verifiedYears?: string): Set { + if (!verifiedYears) return new Set(); + const trimmed = verifiedYears.trim(); + // "2022+" means from 2022 onward (no end year — uprating covers future) + const openMatch = trimmed.match(/^(\d{4})\+$/); + if (openMatch) { + const start = parseInt(openMatch[1], 10); + const years = new Set(); + for (let y = start; y <= MAX_FORWARD_YEAR; y++) years.add(y); + return years; + } + // "2022-2026" range + const rangeMatch = trimmed.match(/^(\d{4})\s*-\s*(\d{4})$/); + if (rangeMatch) { + const start = parseInt(rangeMatch[1], 10); + const end = parseInt(rangeMatch[2], 10); + const years = new Set(); + for (let y = start; y <= end; y++) years.add(y); + return years; + } + // Single year "2024" + const singleMatch = trimmed.match(/^(\d{4})$/); + if (singleMatch) return new Set([parseInt(singleMatch[1], 10)]); + return new Set(); +} + +/** Collect all unique years across all programs' verifiedYears fields. */ +export function collectAllYears(programs: Program[]): number[] { + const allYears = new Set(); + for (const p of programs) { + for (const y of parseYearRange(p.verifiedYears)) allYears.add(y); + } + return Array.from(allYears).sort((a, b) => a - b); +} + +/** Check if a program's verifiedYears includes a given year. */ +export function programIncludesYear(program: Program, year: number): boolean { + return parseYearRange(program.verifiedYears).has(year); +} + +export const statusLabels: Record = { + complete: 'Complete', + partial: 'Partial', + inProgress: 'In progress', + notStarted: 'Not started', +}; + +const statusIcons: Record = { + complete: '\u2713', + partial: '\u25D1', + inProgress: '\u21BB', + notStarted: '\u25CB', +}; + +export function StatusDot({ status, size = 10 }: { status: CoverageStatus; size?: number }) { + return ( + + {size >= 14 ? statusIcons[status] : ''} + + ); +} + +export function StatCard({ label, count, color, delay }: { label: string; count: number; color: string; delay: number }) { + return ( + +
{count}
+
{label}
+ + ); +} diff --git a/src/components/shared/categoryUtils.ts b/src/components/shared/categoryUtils.ts index 713746f..6c9b6ff 100644 --- a/src/components/shared/categoryUtils.ts +++ b/src/components/shared/categoryUtils.ts @@ -1,7 +1,53 @@ -export type Level = 'federal' | 'state' | 'local' | 'territory' | 'household' | 'reform'; +import { colors } from '../../designTokens'; + +export type Level = + // US + | 'federal' | 'state' | 'local' | 'territory' + // UK + | 'uk_government' | 'devolved' | 'local_authority' | 'assumptions' | 'unclassified' + // Both + | 'reform' | 'household'; + +export type TaxonomyCountry = 'us' | 'uk' | string; + +// UK parameter and variable roots under gov.*, grouped by who sets them. +const UK_DEVOLVED = new Set(['revenue_scotland', 'social_security_scotland', 'scottish_government', 'wra']); +const UK_ASSUMPTIONS = new Set(['economic_assumptions', 'simulation', 'dynamic', 'indices', 'benefit_uprating_cpi']); +const UK_GOVERNMENT_TOTALS = new Set(['gov_tax', 'gov_spending', 'gov_balance']); + +function getUkLevel(path: string | null): Level { + if (!path) return 'unclassified'; + const parts = path.split('.'); + if (parts[0] === 'contrib' || (parts[0] === 'gov' && parts[1] === 'contrib')) return 'reform'; + if (parts[0] === 'household' || parts[0] === 'input') return 'household'; + if (parts[0] !== 'gov') return 'unclassified'; + if (!parts[1]) return 'assumptions'; + if (parts[1] === 'local_authorities') return 'local_authority'; + if (UK_DEVOLVED.has(parts[1])) return 'devolved'; + if (UK_ASSUMPTIONS.has(parts[1])) return 'assumptions'; + return 'uk_government'; +} + +function getUkSubGroup(path: string | null): string { + if (!path) return 'Other'; + const parts = path.split('.'); + const level = getUkLevel(path); + if (parts[0] === 'gov' && UK_GOVERNMENT_TOTALS.has(parts[1])) return 'all_government_totals'; + if (level === 'unclassified') return 'Other'; + if (level === 'reform') return parts[0] === 'gov' ? (parts[2] ?? 'Other') : (parts[1] ?? 'Other'); + if (level === 'household') { + // household.income.* -> income; input.vehicle.* -> vehicle; input.rent -> input + if (parts[0] === 'household' || parts[0] === 'input') return parts.length > 2 ? parts[1] : parts[0]; + return 'Other'; + } + if (level === 'local_authority') return parts[2] ?? 'Other'; + if (level === 'assumptions') return parts[1] ?? 'Other'; + return parts[1]; +} /** Categorize an item by its path prefix (moduleName for variables, parameter path for parameters). */ -export function getLevel(path: string | null): Level { +export function getLevel(path: string | null, country: TaxonomyCountry = 'us'): Level { + if (country === 'uk') return getUkLevel(path); if (!path) return 'household'; if (path.startsWith('gov.local')) return 'local'; if (path.startsWith('gov.states')) return 'state'; @@ -13,7 +59,8 @@ export function getLevel(path: string | null): Level { } /** Get a sub-group key from a path. */ -export function getSubGroup(path: string | null): string { +export function getSubGroup(path: string | null, country: TaxonomyCountry = 'us'): string { + if (country === 'uk') return getUkSubGroup(path); if (!path) return 'Other'; const parts = path.split('.'); @@ -45,8 +92,81 @@ export function getSubGroup(path: string | null): string { return 'Other'; } +const humanize = (key: string) => + key.replace(/_/g, ' ').replace(/^\w/, (c) => c.toUpperCase()); + +// Department names follow GOV.UK's organisation titles. +const UK_SUBGROUP_LABELS: Record = { + hmrc: 'HM Revenue and Customs (HMRC)', + dwp: 'Department for Work and Pensions (DWP)', + dfe: 'Department for Education (DfE)', + dft: 'Department for Transport (DfT)', + dcms: 'Department for Culture, Media and Sport (DCMS)', + dhsc: 'Department of Health and Social Care (DHSC)', + treasury: 'HM Treasury', + ofgem: 'Ofgem', + boe: 'Bank of England', + revenue_scotland: 'Revenue Scotland', + social_security_scotland: 'Social Security Scotland', + scottish_government: 'Scottish Government', + wra: 'Welsh Revenue Authority', + economic_assumptions: 'Economic assumptions', + simulation: 'Simulation settings', + dynamic: 'Dynamic response settings', + all_government_totals: 'Totals across all levels of government', + indices: 'Indices', + benefit_uprating_cpi: 'Benefit uprating', + domestic_rates: 'Northern Ireland domestic rates', + england: 'England-wide schemes', + scotland: 'Scotland-wide schemes', + wales: 'Wales-wide schemes', + council_tax_reduction: 'Council Tax Reduction', + kingston_upon_thames: 'Kingston upon Thames', + demographic: 'Demographics', + income: 'Income', + consumption: 'Consumption', + wealth: 'Wealth', + vehicle: 'Vehicles', + owned_company: 'Company ownership', + input: 'Other inputs', +}; + +// Each description lists programmes that policyengine-uk's programs.yaml +// assigns to that department, or the folder's contents. +const UK_SUBGROUP_DESCRIPTIONS: Record = { + all_government_totals: 'Tax, spending, and balance totals across UK, devolved, and local government', + hmrc: 'Income tax, National Insurance, VAT, capital gains tax, duties, Child Benefit, and Tax-Free Childcare', + dwp: 'Universal Credit, State Pension, Pension Credit, disability benefits, and legacy benefits', + dfe: 'Free childcare entitlements and student support', + dft: 'Vehicle excise duty, rail, and bus', + dcms: 'The TV licence', + treasury: 'Cost of Living and energy bill support payments', + ofgem: 'The energy price cap and Energy Price Guarantee', + revenue_scotland: 'Land and Buildings Transaction Tax', + social_security_scotland: 'Scottish Child Payment, Pension Age Winter Heating Payment, and Carer Support Payment', + scottish_government: 'Funded early learning and childcare', + wra: 'Land Transaction Tax in Wales', + economic_assumptions: 'Forecasts used to uprate incomes, prices, and benefits', + simulation: 'Behavioural response elasticities and other simulation settings', + demographic: 'Age, family, disability, and household composition', + income: 'Employment, self-employment, pension, investment, and other income', + consumption: 'Spending by category, used for VAT and duties', + wealth: 'Property, financial, and pension wealth', +}; + /** Get a display label for a sub-group key. */ -export function getSubGroupLabel(key: string, level: Level): string { +export function getSubGroupLabel(key: string, level: Level, country: TaxonomyCountry = 'us'): string { + if (country === 'uk') return UK_SUBGROUP_LABELS[key] ?? humanize(key); + return getUsSubGroupLabel(key, level); +} + +/** Description for sub-groups. */ +export function getSubGroupDescription(key: string, level: Level, country: TaxonomyCountry = 'us'): string | null { + if (country === 'uk') return UK_SUBGROUP_DESCRIPTIONS[key] ?? null; + return getUsSubGroupDescription(key, level); +} + +function getUsSubGroupLabel(key: string, level: Level): string { if (level === 'federal') { const agencyNames: Record = { irs: 'Internal Revenue Service (IRS)', hhs: 'Health & Human Services (HHS)', @@ -101,8 +221,7 @@ export function getSubGroupLabel(key: string, level: Level): string { return key; } -/** Description for sub-groups. */ -export function getSubGroupDescription(key: string, level: Level): string | null { +function getUsSubGroupDescription(key: string, level: Level): string | null { if (level === 'federal') { const desc: Record = { irs: 'Income tax, credits, deductions, and filing rules', @@ -132,13 +251,51 @@ export function getSubGroupDescription(key: string, level: Level): string | null return null; } -export const LEVEL_CONFIG: Record = { +interface LevelInfo { label: string; description: string; color: string; order: number } + +export const LEVEL_CONFIG: Record = { federal: { label: 'Federal', description: 'IRS, HHS, USDA, SSA, HUD, and other federal agencies', color: '#2C7A7B', order: 0 }, state: { label: 'State', description: 'State-level tax and benefit programs across all 50 states + DC', color: '#2C7A7B', order: 1 }, local: { label: 'Local', description: 'City and county programs', color: '#2C7A7B', order: 2 }, territory: { label: 'Territories', description: 'Puerto Rico and other US territories', color: '#2C7A7B', order: 3 }, + uk_government: { label: 'UK government', description: 'HMRC, DWP, DfE, DfT, HM Treasury, and other UK departments', color: '#2C7A7B', order: 0 }, + devolved: { label: 'Devolved', description: 'Revenue Scotland, Social Security Scotland, the Scottish Government, and the Welsh Revenue Authority', color: '#2C7A7B', order: 1 }, + local_authority: { label: 'Local authorities', description: 'Council Tax Reduction schemes and Northern Ireland domestic rates', color: '#2C7A7B', order: 2 }, + assumptions: { label: 'Assumptions and settings', description: 'Economic forecasts, uprating indices, and simulation settings', color: '#475569', order: 4 }, + unclassified: { label: 'Unclassified variables', description: 'Variables whose source module is unknown or outside the recognised groups', color: colors.secondary[600], order: 6 }, reform: { label: 'Reforms', description: 'Contributed reform proposals and policy experiments', color: '#2C7A7B', order: 4 }, household: { label: 'Household inputs', description: 'Demographics, income, expenses, and geographic inputs', color: '#475569', order: 5 }, }; +/** US level order; kept for callers that predate the country-aware taxonomy. */ export const LEVELS_ORDERED: Level[] = ['federal', 'state', 'local', 'territory', 'reform', 'household']; + +const UK_LEVELS: Level[] = ['uk_government', 'devolved', 'local_authority', 'reform', 'assumptions', 'household', 'unclassified']; + +/** The levels a country's explorer shows, in display order. */ +export function levelsFor(country: TaxonomyCountry = 'us'): Level[] { + return country === 'uk' ? UK_LEVELS : LEVELS_ORDERED; +} + +/** Display config for a level, including country-specific descriptions. */ +export function levelConfig(level: Level, country: TaxonomyCountry = 'us'): LevelInfo { + if (country === 'uk' && level === 'household') { + return { + ...LEVEL_CONFIG.household, + label: 'Household variables', + description: 'Demographic, income, consumption, and wealth inputs, and calculated household results such as household net income', + }; + } + if (country === 'uk' && level === 'reform') { + return { + ...LEVEL_CONFIG.reform, + description: 'Contributed reform proposals and modelling assumptions, some enabled by default', + }; + } + return LEVEL_CONFIG[level]; +} + +/** A zeroed counter over every level. */ +export function emptyLevelRecord(make: () => T): Record { + return Object.fromEntries((Object.keys(LEVEL_CONFIG) as Level[]).map((l) => [l, make()])) as Record; +} diff --git a/src/components/theory/BehavioralResponses.tsx b/src/components/theory/BehavioralResponses.tsx index 3428e70..2507f1c 100644 --- a/src/components/theory/BehavioralResponses.tsx +++ b/src/components/theory/BehavioralResponses.tsx @@ -3,6 +3,7 @@ import { colors, typography, spacing } from '../../designTokens'; import { cboDefaults, BLOG_URL, sources } from '../../data/elasticities'; import { LineChart, Line, XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer, ReferenceLine } from 'recharts'; import type { Country } from '../../hooks/useCountry'; +import UkBehaviouralResponses from './UkBehaviouralResponses'; function generateBudgetData(substitutionElasticity: number, incomeElasticity: number) { const data = []; @@ -24,7 +25,8 @@ function generateBudgetData(substitutionElasticity: number, incomeElasticity: nu } export default function BehavioralResponses({ country = 'us' }: { country?: Country }) { - const currencySymbol = country === 'uk' ? '£' : '$'; + if (country === 'uk') return ; + const currencySymbol = '$'; const avgSubstitution = cboDefaults.substitutionElasticities.reduce((a, b) => a + b.central, 0) / cboDefaults.substitutionElasticities.length; const budgetData = generateBudgetData(avgSubstitution, cboDefaults.incomeElasticity.central); @@ -32,7 +34,7 @@ export default function BehavioralResponses({ country = 'us' }: { country?: Coun

Tax and benefit reforms change incentives to work and earn. PolicyEngine models these - {country === 'uk' ? ' behavioural' : ' behavioral'} responses using the elasticity of taxable income framework, capturing both + {' behavioral'} responses using the elasticity of taxable income framework, capturing both substitution effects (changing work effort) and income effects (adjusting for changes in after-tax income).

@@ -75,7 +77,7 @@ export default function BehavioralResponses({ country = 'us' }: { country?: Coun - +

@@ -89,7 +91,7 @@ export default function BehavioralResponses({ country = 'us' }: { country?: Coun
- {country === 'uk' ? 'Reform (behavioural)' : 'Reform (behavioral)'} + Reform (behavioral)
diff --git a/src/components/theory/UkBehaviouralResponses.tsx b/src/components/theory/UkBehaviouralResponses.tsx new file mode 100644 index 0000000..5c14cdd --- /dev/null +++ b/src/components/theory/UkBehaviouralResponses.tsx @@ -0,0 +1,148 @@ +import { colors, typography, spacing } from '../../designTokens'; +import { + UK_MODEL_VERSION, + capitalGainsParameters, + labourSupplyParameters, + obrElasticities, + payParameters, + ukSourceLinks, + type ParameterRow, +} from '../../data/ukBehaviouralResponses'; +import { DataTable, ExternalLink, bodyText, cell, sectionHeading } from '../../views/data/ukTables'; +import { ukRelease } from '../../data/ukDataRelease'; + +const fmtElasticity = (x: number) => (x === 0 ? '0' : x < 0 ? `−${Math.abs(x).toFixed(3).replace(/0+$/, '')}` : x.toFixed(3).replace(/0+$/, '')); + +function ParameterTable({ rows }: { rows: ParameterRow[] }) { + return ( + + {rows.map((r) => ( + + +
{r.label}
+
+ {r.path} +
+ + {r.value} + + ))} + + ); +} + +/** PolicyEngine UK's behavioural responses, from policyengine-uk's own parameters and code. */ +export default function UkBehaviouralResponses() { + return ( +
+

+ In policyengine-uk {UK_MODEL_VERSION}, the version policyengine.py {ukRelease.release.policyengine} pins, the labour supply and capital + gains responses below are off by default. Two modelling assumptions do change pay by default, under reforms + to employer National Insurance or to salary sacrifice. +

+ +
+

Pay under employer National Insurance and salary sacrifice reforms

+

+ When a reform changes employer National Insurance, through its rate, its threshold or its treatment of + employer pension contributions, the model sets pay with a formula meant to keep what each employer pays for + each employee the same, and by default passes the whole change through to pay. The formula assumes pay + stays above the reform’s employer threshold, so a reform that lifts the threshold past someone’s pay can + raise what their employer pays. Pay below the baseline employer threshold stays as it was. +

+

+ When a cap on salary-sacrificed pension contributions free of National Insurance applies, as the model’s + current law has it from April 2029, contributions above the cap return to pay. A separate haircut takes + 0.16% off every worker’s employment income before these adjustments, the parameter’s estimate of employers + spreading the extra National Insurance across all staff. +

+ +
+ +
+

Labour supply

+

+ Two elasticities set how employment income responds to a reform, and both are 0 by default. When either is + set, each person’s employment income changes by +

+

+ employment income before the response × (income elasticity × relative change in household net income + + substitution elasticity × relative change in the marginal wage) +

+

+ Each relative change compares the reform with the baseline and is capped at 100% either way. The marginal + wage is one minus the person’s marginal tax rate. Negative employment income counts as zero, household net + income below £1 counts as £1, and a marginal wage of zero counts as 0.01. The response feeds into the + reform’s taxes and benefits. +

+ +
+ +
+

Labour supply, the OBR’s way

+

+ A module in policyengine-uk’s Python interface (Simulation.apply_dynamics), when called, + estimates how working hours respond to a reform, with elasticities that vary by family type and the age of + the youngest child. Its parameter file cites the OBR’s note on{' '} + the labour supply effects of the Autumn 2023 National Insurance cut. + It covers the two eldest adults in each household, except people whose employment status is self-employed + or student and people aged 60 or over, and models hours only: the participation response is not yet + switched on. It writes the change in earnings into the reform simulation, so taxes and benefits calculated + afterwards include it, and reports the change in earnings and in full-time-equivalent employment. +

+ + {obrElasticities.map((r) => ( + + {r.group} + {r.youngestChild} + {fmtElasticity(r.substitution)} + {fmtElasticity(r.income)} + + ))} + +

+ Married means married or in a civil partnership, as the survey records it; without that input, the model + treats every couple as married. Children are members of the benefit unit under 18. The module gives no + elasticity to unmarried men with children. +

+
+ +
+

Capital gains

+

+ An elasticity of realised gains with respect to the retention rate sets how gains respond, and it is 0 by + default. The retention rate is the share of extra gains a household keeps: the model adds £1,000 of gains + (or 0.1% of the person’s gains, if larger) and divides the rise in household net income by the rise in + gains, for the two adults in each household with the largest gains before the response. It counts as at + least 0.001. Under a + reform, each person’s gains scale by the ratio of their reform to baseline retention rates, raised to the + elasticity. Gains that qualify for Business Asset Disposal Relief can take their own elasticity, 1.4, which + the parameter cites to the OBR’s{' '} + January 2025 capital gains tax costing; until that + switch is on, they take the main elasticity. A tax-rate elasticity instead bases the response on the change + in the marginal rate itself, and the model refuses to run with both set. +

+ +
+ +

+ Sources: policyengine-uk {UK_MODEL_VERSION}{' '} + simulation parameters and{' '} + labour supply module. +

+
+ ); +} diff --git a/src/components/variables/VariableCard.tsx b/src/components/variables/VariableCard.tsx index 24b7916..aa67f24 100644 --- a/src/components/variables/VariableCard.tsx +++ b/src/components/variables/VariableCard.tsx @@ -19,12 +19,12 @@ const entityColors: Record = { export const entityLabels: Record = { person: 'Person', - tax_unit: 'Tax Unit', - spm_unit: 'SPM Unit', + tax_unit: 'Tax unit', + spm_unit: 'SPM unit', household: 'Household', family: 'Family', - marital_unit: 'Marital Unit', - benunit: 'Benefit Unit', + marital_unit: 'Marital unit', + benunit: 'Benefit unit', }; export default function VariableCard({ variable, isSelected, onClick }: VariableCardProps) { diff --git a/src/components/variables/VariableExplorer.tsx b/src/components/variables/VariableExplorer.tsx index e415d1c..1703c02 100644 --- a/src/components/variables/VariableExplorer.tsx +++ b/src/components/variables/VariableExplorer.tsx @@ -7,7 +7,7 @@ import { useDebounce } from '../../hooks/useDebounce'; import { getLevel as getLevelFromPath, getSubGroup as getSubGroupFromPath, getSubGroupLabel, getSubGroupDescription, - LEVEL_CONFIG, LEVELS_ORDERED, + levelConfig, levelsFor, emptyLevelRecord, type Level, } from '../shared/categoryUtils'; import VariableCard from './VariableCard'; @@ -26,12 +26,12 @@ interface VariableExplorerProps { onViewFlowchart?: (varName: string) => void; } -function getLevel(v: Variable): Level { - return getLevelFromPath(v.moduleName); +function getLevel(v: Variable, country: string): Level { + return getLevelFromPath(v.moduleName, country); } -function getSubGroup(v: Variable): string { - return getSubGroupFromPath(v.moduleName); +function getSubGroup(v: Variable, country: string): string { + return getSubGroupFromPath(v.moduleName, country); } function getFolderLabel(fullPath: string): string { @@ -163,19 +163,21 @@ function StateTileGrid({ function SubGroupCardGrid({ subGroups, level, + country, levelColor, onSelect, }: { subGroups: [string, Variable[]][]; level: Level; + country: string; levelColor: string; onSelect: (key: string) => void; }) { return (
{subGroups.map(([key, vars], i) => { - const label = getSubGroupLabel(key, level); - const desc = getSubGroupDescription(key, level); + const label = getSubGroupLabel(key, level, country); + const desc = getSubGroupDescription(key, level, country); return ( { - const counts: Record = { federal: 0, state: 0, local: 0, territory: 0, reform: 0, household: 0 }; - for (const v of allVariables) counts[getLevel(v)]++; + const counts = emptyLevelRecord(() => 0); + for (const v of allVariables) counts[getLevel(v, country)]++; return counts; - }, [allVariables]); + }, [allVariables, country]); // Filtered list const filtered = useMemo(() => { @@ -359,21 +361,21 @@ export default function VariableExplorer({ variables, parameters, country, onVie return words.every((w) => haystack.includes(w)); }); } - if (activeLevel) result = result.filter((v) => getLevel(v) === activeLevel); + if (activeLevel) result = result.filter((v) => getLevel(v, country) === activeLevel); return result; - }, [allVariables, debouncedSearch, activeLevel]); + }, [allVariables, debouncedSearch, activeLevel, country]); // Sub-groups for the active level const subGroups = useMemo(() => { if (!activeLevel) return []; const map = new Map(); for (const v of filtered) { - const key = getSubGroup(v); + const key = getSubGroup(v, country); if (!map.has(key)) map.set(key, []); map.get(key)!.push(v); } return [...map.entries()].sort(([a], [b]) => a.localeCompare(b)); - }, [filtered, activeLevel]); + }, [filtered, activeLevel, country]); // Variables in active sub-group const subGroupVars = useMemo(() => { @@ -406,9 +408,11 @@ export default function VariableExplorer({ variables, parameters, country, onVie // Folder contents at current level const folderContents = useMemo(() => { - if (!activeSubGroup || !currentFolder) return { folders: [], directVars: [] }; + if (!activeSubGroup) return { folders: [], directVars: [] }; + if (country === 'uk' && activeLevel === 'unclassified') return { folders: [], directVars: subGroupVars }; + if (!currentFolder) return { folders: [], directVars: [] }; return getVarFolderContents(subGroupVars, currentFolder); - }, [subGroupVars, currentFolder, activeSubGroup]); + }, [subGroupVars, currentFolder, activeSubGroup, activeLevel, country]); // Reset folder stack when sub-group changes // eslint-disable-next-line react-hooks/set-state-in-effect -- reset navigation when sub-group changes @@ -466,13 +470,13 @@ export default function VariableExplorer({ variables, parameters, country, onVie const items: { label: string; onClick: () => void }[] = []; if (activeLevel) { items.push({ - label: LEVEL_CONFIG[activeLevel].label, + label: levelConfig(activeLevel, country).label, onClick: () => { setActiveLevel(activeLevel); setActiveSubGroup(null); setFolderStack([]); setSelectedVar(null); }, }); } if (activeSubGroup && activeLevel) { items.push({ - label: getSubGroupLabel(activeSubGroup, activeLevel), + label: getSubGroupLabel(activeSubGroup, activeLevel, country), onClick: () => { setFolderStack([]); setSelectedVar(null); }, }); } @@ -484,13 +488,13 @@ export default function VariableExplorer({ variables, parameters, country, onVie }); } return items; - }, [activeLevel, activeSubGroup, folderStack]); + }, [activeLevel, activeSubGroup, folderStack, country]); // Current heading const currentHeadingLabel = folderStack.length > 0 ? getFolderLabel(currentFolder) : activeSubGroup && activeLevel - ? getSubGroupLabel(activeSubGroup, activeLevel) + ? getSubGroupLabel(activeSubGroup, activeLevel, country) : ''; @@ -518,9 +522,9 @@ export default function VariableExplorer({ variables, parameters, country, onVie {isSearching && (
- {([null, ...LEVELS_ORDERED] as (Level | null)[]).map((level) => { - const label = level ? LEVEL_CONFIG[level].label : 'All'; - const clr = level ? LEVEL_CONFIG[level].color : colors.text.secondary; + {([null, ...levelsFor(country)] as (Level | null)[]).map((level) => { + const label = level ? levelConfig(level, country).label : 'All'; + const clr = level ? levelConfig(level, country).color : colors.text.secondary; const isActive = activeLevel === level; return ( + )} + +
+ ); +} diff --git a/src/views/data/ukTables.tsx b/src/views/data/ukTables.tsx new file mode 100644 index 0000000..ac49bd4 --- /dev/null +++ b/src/views/data/ukTables.tsx @@ -0,0 +1,110 @@ +import type { CSSProperties, ReactNode } from 'react'; +import Link from 'next/link'; +import { useSearchParams } from 'next/navigation'; +import { usePublicBasePrefix } from '../../hooks/usePublicBasePrefix'; +import { colors, typography, spacing } from '../../designTokens'; + +/** Shared table chrome for the UK data pages. Scrolls inside itself on narrow screens. */ +export function DataTable({ head, children, minWidth = 640 }: { head: string[]; children: ReactNode; minWidth?: number }) { + return ( +
+ + + + {head.map((label, i) => ( + + ))} + + + {children} +
+ {label} +
+
+ ); +} + +export const cell = (align: 'left' | 'right' = 'right', extra: CSSProperties = {}): CSSProperties => ({ + padding: `${spacing.sm} ${spacing.lg}`, + textAlign: align, + fontSize: typography.fontSize.sm, + color: colors.text.primary, + borderTop: `1px solid ${colors.border.light}`, + verticalAlign: 'top', + fontVariantNumeric: 'tabular-nums', + ...extra, +}); + +export const sectionHeading: CSSProperties = { + fontSize: typography.fontSize['2xl'], + fontWeight: typography.fontWeight.bold, + color: colors.primary[900], + margin: `0 0 ${spacing.md}`, +}; + +export const bodyText: CSSProperties = { + fontSize: typography.fontSize.base, + color: colors.text.secondary, + lineHeight: 1.7, + margin: `0 0 ${spacing.lg}`, + maxWidth: '760px', +}; + +export function StatTile({ value, label }: { value: string; label: string }) { + return ( +
+
+ {value} +
+
+ {label} +
+
+ ); +} + +export function ExternalLink({ href, children }: { href: string; children: ReactNode }) { + return ( + + {children} + + ); +} + +export const issueUrl = (repo: string, n: number) => `https://github.com/PolicyEngine/${repo}/issues/${n}`; + +/** A link to another model page that keeps the public prefix and ?country=/?embed=. */ +export function PageLink({ path, children }: { path: string; children: ReactNode }) { + const base = usePublicBasePrefix(); + const search = useSearchParams().toString(); + return ( + + {children} + + ); +}