Fix the UK model pages: pipeline, calibration, validation, behavioural responses, and rules

.gitignore

@@ -29,5 +29,7 @@ policyengine-uk-data
 
 # Metadata and registry snapshots generated at build time
 public/metadata-*.json
+public/variable-tree-*.json
+public/variable-modules-*.json
 public/param-tree-*.json
 public/programs-*.json

README.md

@@ -33,8 +33,22 @@ version is included when its metadata endpoint responds. The coverage page uses
 this provenance to show which model version the coverage reflects and, when
 available, the version served by the production API.
 
+The same run snapshots each repository's `parameters/` and `variables/` file trees
+(`public/param-tree-<country>.json`, `public/variable-tree-<country>.json`) at that
+commit. Both trees record `{ source, paths }`. The UK variable tree also records
+`modules`, an exact class-to-module map read from that commit's source archive,
+because the UK metadata API serves no module names. The UK coverage tracker
+verifies parameter, source and computation-tree links only against trees whose
+repository and commit match its programme registry snapshot.
+
+If the archive is unavailable, module enrichment falls back to unique file stems.
+UK variables whose modules remain unknown appear under “Unclassified variables”.
+The registry and both trees are cleared before each refresh so a failed fetch
+cannot reuse a stale snapshot.
+
 The program loader tries the static snapshot first, then production API metadata,
-then the bundled hardcoded programs. Snapshot URLs use the public base prefix so
+then the bundled hardcoded programs (US only; the UK page says the registry could
+not be loaded). Snapshot URLs use the public base prefix so
 they work both locally and under `/us/model`.
 
 ## Daily registry refresh
@@ -55,3 +69,47 @@ The workflow sends a POST request to the hook. If the secret is absent, it logs
 notice and exits successfully. The workflow must be on the default branch for
 scheduled runs to apply. The Vercel build must run `bun run build` so each
 deployment fetches a fresh snapshot.
+
+## UK data release pages
+
+The UK pipeline, calibration targets and validation pages quote
+`src/data/ukDataRelease.json` and `src/data/ukTargetChecks.json`, not typed-in
+numbers. Both describe the UK data release that policyengine.py pins
+(`src/policyengine/data/bundle/manifest.json`, `data_releases.uk`). When that pin
+moves, regenerate them:
+
+1. Put the pinned release's files in one folder: the bundle manifest, the
+   release manifest at the revision the bundle names, `frs_<year>.h5`,
+   `enhanced_frs_<year>.h5`, both weight files (from the gated
+   `policyengine/policyengine-uk-data-private` Hugging Face repo), and the two
+   calibration logs (GitHub Actions artifacts of the uk-data publish run:
+   `gh run download <run> -R PolicyEngine/policyengine-uk-data`).
+2. Dump the release's target registry with `scripts/dump_uk_target_registry.py`,
+   and score the saved file with the pinned model with
+   `scripts/run_uk_target_check.py` (commands in each script's docstring).
+3. Run `scripts/extract_uk_release.py --artifacts <folder> --out
+   src/data/ukDataRelease.json --rows-out src/data/ukTargetChecks.json`. It checks
+   every file's sha256 against the release manifest and refuses a release other
+   than the pin.
+   The target check records the dataset's sha256, data package version and
+   year, and the extractor refuses a check of any other file, package or year.
+4. Run `UK_RELEASE_ARTIFACTS=<folder> UK_DATA_SOURCE=<policyengine-uk-data at
+   the pin> uv run --no-project --with pytest --with hypothesis --with h5py
+   --with tables --with pandas --with numpy pytest
+   scripts/test_extract_uk_release.py`, and
+   `UK_MODEL_SOURCE=<policyengine-uk at the pin> bun run test`. The first
+   checks the extractor's labels, areas and notes against the data package's
+   code; the second runs policyengine-uk's own behavioural functions against
+   the behavioural page. Then reread the pipeline stage copy in
+   `src/data/ukPipelineStages.ts` against the new release's
+   `datasets/create_datasets.py`: the counts update themselves, the mechanisms
+   do not. `UK_DATA_VERSION_DESCRIBED` (pipeline copy) and `UK_MODEL_VERSION`
+   (behavioural page) must name the new pin before the tests pass.
+
+The household files hold UK Data Service microdata. Only the aggregates the
+extractor writes may leave that folder.
+
+The UK behavioural responses page mirrors policyengine-uk's parameters and
+`dynamics/progression.py` (`src/data/ukBehaviouralResponses.ts`); run `bun run
+test` with `UK_MODEL_SOURCE` set to a policyengine-uk checkout at the pinned
+version to compare them.

app/data/calibration/page.tsx

@@ -1,4 +1,5 @@
 import CalibrationPage from '../../../src/views/data/CalibrationPageClient';
+import UkCalibrationPage from '../../../src/views/data/UkCalibrationPage';
 import CalibrationComparison from '../../../src/components/comparison/CalibrationComparison';
 import { loadComparisonData } from '../../../src/data/comparisons';
 import { detectComparePeers } from '../../../src/components/comparison/detectComparePeers';
@@ -18,7 +19,7 @@ export default async function CalibrationRoute({
   );
 
   if (!parsed.compareMode) {
-    return <CalibrationPage country={country} />;
+    return country === 'uk' ? <UkCalibrationPage /> : <CalibrationPage country={country} />;
   }
 
   const data = loadComparisonData();

app/data/page.tsx

@@ -1,31 +1,20 @@
 import SectionOverview from '../../src/components/layout/SectionOverview';
+import { detectCountry } from '../../src/components/comparison/detectCountry';
+import { dataSection } from '../../src/data/sectionOverviews';
 
-export default function DataOverviewPage() {
+export default async function DataOverviewPage({
+  searchParams,
+}: {
+  searchParams: Promise<{ country?: string | string[] }>;
+}) {
+  const country = await detectCountry((await searchParams).country);
+  const copy = dataSection[country];
   return (
     <SectionOverview
       category="Model"
       title="Data"
-      description="How PolicyEngine builds the microdata behind its simulations — from raw survey records to calibrated, validated datasets that match official population totals."
-      children={[
-        {
-          label: 'Pipeline',
-          description:
-            'Each stage that turns raw CPS, FRS, and SCF records into PolicyEngine-ready microdata.',
-          path: '/data/pipeline',
-        },
-        {
-          label: 'Calibration targets',
-          description:
-            'The population totals (program enrolment, income aggregates, demographic counts) used to reweight the underlying survey.',
-          path: '/data/calibration',
-        },
-        {
-          label: 'Validation',
-          description:
-            'How simulated outputs are cross-checked against published official statistics.',
-          path: '/data/validation',
-        },
-      ]}
+      description={copy.description}
+      children={copy.children}
     />
   );
 }

app/data/validation/page.tsx

@@ -2,6 +2,7 @@ import ValidationPage from '../../../src/views/data/ValidationPage';
 import ValidationComparison from '../../../src/components/comparison/ValidationComparison';
 import { loadComparisonData } from '../../../src/data/comparisons';
 import { detectComparePeers } from '../../../src/components/comparison/detectComparePeers';
+import UkValidationPage from '../../../src/views/data/UkValidationPage';
 
 export default async function ValidationRoute({
   searchParams,
@@ -17,6 +18,11 @@ export default async function ValidationRoute({
     sp.country,
   );
 
+  // The UK page reports the pinned UK release; TAXSIM covers US taxes only.
+  if (!parsed.compareMode && country === 'uk') {
+    return <UkValidationPage />;
+  }
+
   if (!parsed.compareMode) {
     return <ValidationPage country={country} />;
   }

app/rules/page.tsx

@@ -1,31 +1,20 @@
 import SectionOverview from '../../src/components/layout/SectionOverview';
+import { detectCountry } from '../../src/components/comparison/detectCountry';
+import { rulesSection } from '../../src/data/sectionOverviews';
 
-export default function RulesOverviewPage() {
+export default async function RulesOverviewPage({
+  searchParams,
+}: {
+  searchParams: Promise<{ country?: string | string[] }>;
+}) {
+  const country = await detectCountry((await searchParams).country);
+  const copy = rulesSection[country];
   return (
     <SectionOverview
       category="Model"
       title="Rules"
-      description="How PolicyEngine encodes tax-and-benefit rules — which programs are covered, the legislative parameters that drive calculations, and the input and output variables exposed to households and microsimulations."
-      children={[
-        {
-          label: 'Coverage tracker',
-          description:
-            'Which federal, state, and local tax-and-benefit programs PolicyEngine and peer models implement.',
-          path: '/rules/coverage',
-        },
-        {
-          label: 'Parameters',
-          description:
-            'Browse every legislative parameter that drives the model — thresholds, rates, brackets, eligibility cut-offs.',
-          path: '/rules/parameters',
-        },
-        {
-          label: 'Variables',
-          description:
-            'Inspect the input and output variables computed at the person, family, tax unit, and household level.',
-          path: '/rules/variables',
-        },
-      ]}
+      description={copy.description}
+      children={copy.children}
     />
   );
 }

package.json

@@ -39,6 +39,7 @@
     "eslint": "^9.39.1",
     "eslint-plugin-jsx-a11y": "^6.10.2",
     "eslint-plugin-react-hooks": "^7.0.1",
+    "fast-check": "4.3.0",
     "globals": "^16.5.0",
     "jsdom": "^28.1.0",
     "typescript": "~5.9.3",

proxy.ts

@@ -100,5 +100,9 @@ export const config = {
     '/((?!_next|.*\\..*).*)',
     '/:country(us|uk)/programs-:registry(us|uk).json',
     '/:country(us|uk)/model/programs-:registry(us|uk).json',
+    '/:country(us|uk)/param-tree-:registry(us|uk).json',
+    '/:country(us|uk)/model/param-tree-:registry(us|uk).json',
+    '/:country(us|uk)/variable-tree-:registry(us|uk).json',
+    '/:country(us|uk)/model/variable-tree-:registry(us|uk).json',
   ],
 };

scripts/dump_uk_target_registry.py

@@ -0,0 +1,32 @@
+"""Dump policyengine-uk-data's calibration target registry to JSON.
+
+Run it against the data package at the release tag, with the model version
+that built the release, e.g. for policyengine-uk-data 1.58.1:
+
+    git -C policyengine-uk-data archive 1.58.1 | tar -x -C /tmp/uk-data-1.58.1
+    uv run --no-project --python 3.13 --with policyengine-uk==2.122.2 \\
+      --with policyengine-core==3.32.13 --with pydantic --with pyyaml \\
+      --with /tmp/uk-data-1.58.1 python scripts/dump_uk_target_registry.py registry_targets.json
+
+The registry reads only target statistics bundled with the package, no microdata.
+"""
+
+import json
+import sys
+
+from policyengine_uk_data.targets.registry import get_all_targets
+
+
+def main(out_path: str) -> None:
+    targets = []
+    for target in get_all_targets():
+        row = target.model_dump(exclude={"custom_compute"})
+        row["values"] = {str(year): value for year, value in row["values"].items()}
+        targets.append(row)
+    with open(out_path, "w") as f:
+        json.dump(targets, f, indent=1, default=str)
+    print(f"wrote {len(targets)} targets to {out_path}")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1])