Match ONS age targets to the FRS top code of 80

.gitignore

@@ -18,6 +18,7 @@
 !policyengine_uk_data/datasets/local_areas/**/*.csv
 **/_build
 !policyengine_uk_data/storage/*.csv
+!policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv
 **/version.json
 # Build output: household weights by local area, derived from FRS microdata
 policyengine_uk_data/storage/local_geography_weights.csv.gz
@@ -26,3 +27,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz
 
 # Hypothesis example database (property-based tests)
 .hypothesis/
+
+# Local build-job notes, caches and hand-off artefacts
+.job/

changelog.d/593-age-topcode.fixed.md

@@ -0,0 +1 @@
+Matched national and regional ONS age targets to the FRS top code with open-ended 80+ bands, including the projection's oldest age groups.

policyengine_uk_data/storage/regional_age_80_plus.csv

@@ -0,0 +1,13 @@
+name,2022,2023,2024,2025,2026,2027,2028,2029,reference_url
+east_age_80_plus,350364.000,359057.134,371659.091,383909.917,397184.443,425490.369,445444.208,459810.859,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+east_midlands_age_80_plus,252810.000,259714.072,270269.576,279796.208,288881.005,308397.351,323332.659,334586.295,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+london_age_80_plus,281480.000,283880.847,288218.147,292199.323,298451.115,312294.218,322619.866,330707.051,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+north_east_age_80_plus,141323.000,142485.609,146050.948,149741.415,153969.782,164716.911,172434.769,178458.689,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+north_west_age_80_plus,368936.000,375461.807,385752.712,394643.102,404696.948,431246.821,450009.928,463577.946,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+northern_ireland_age_80_plus,86145.000,89224.000,92507.000,95719.000,98931.000,103172.000,106713.000,109973.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z6zippedpopulationprojectionsdatafilesnorthernireland
+scotland_age_80_plus,274428.000,279070.000,285484.000,290656.000,297529.000,316051.000,330092.000,341986.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z5zippedpopulationprojectionsdatafilesscotland
+south_east_age_80_plus,517027.000,531225.098,550842.841,569679.705,590428.825,631358.154,661477.046,684046.367,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+south_west_age_80_plus,359314.000,370051.020,384900.773,399727.616,414550.838,442926.618,465134.345,481977.036,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+wales_age_80_plus,175956.000,179720.000,185492.000,191184.000,196718.000,209247.000,219315.000,226905.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z4zippedpopulationprojectionsdatafileswales
+west_midlands_age_80_plus,311947.000,319178.683,330274.556,339480.659,347714.374,366032.307,380011.894,390810.558,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1
+yorkshire_and_the_humber_age_80_plus,279307.000,284139.773,292163.361,300471.035,308655.688,329410.267,344273.286,355194.173,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1

policyengine_uk_data/targets/compute/demographics.py

@@ -26,7 +26,8 @@ def compute_regional_age(target, ctx) -> np.ndarray | None:
     region_name = name[:idx]
     age_part = name[idx + 5 :]
     lower, upper = age_part.split("_")
-    lower, upper = int(lower), int(upper)
+    lower = int(lower)
+    upper = np.inf if upper == "plus" else int(upper)
 
     pe_region = _REGION_INV.get(region_name)
     if pe_region is None:
@@ -44,7 +45,7 @@ def compute_gender_age(target, ctx) -> np.ndarray:
     parts = name.split("_")
     sex = parts[0]
     lower = int(parts[1])
-    upper = int(parts[2])
+    upper = np.inf if parts[2] == "plus" else int(parts[2])
 
     gender = ctx.sim.calculate("gender").values
     sex_match = gender == ("FEMALE" if sex == "female" else "MALE")

policyengine_uk_data/targets/sources/ons_demographics.py

@@ -3,10 +3,9 @@
 Downloads the ONS 2024-based principal population projection for the
 UK to extract total population and gender × age band targets.
 
-For regional age breakdowns (12 regions × 9 age bands), reads the
-pre-existing demographics.csv which was extracted from ONS subnational
-projections. The subnational projections don't have a stable machine-
-readable download URL, so this is the pragmatic compromise.
+Regional age breakdowns use demographics.csv below age 80 and a committed
+ONS projection extract for 80+, including the terminal age group. See
+tools/ons_age_targets for the sources and extraction method.
 
 Household type and tenure targets are from ONS families & households
 datasets (also lacking stable machine-readable URLs).
@@ -52,7 +51,7 @@
 
 _YEARS = list(range(2022, 2030))
 
-# Age band boundaries
+# FRS ages are top-coded at 80, so the last band includes every older age.
 _AGE_BANDS = [
     (0, 9),
     (10, 19),
@@ -62,16 +61,18 @@
     (50, 59),
     (60, 69),
     (70, 79),
-    (80, 89),
+    (80, None),
 ]
 
 # Five-year bands. The previous 15-year bands (0-14, 15-29, ...) pinned only
 # the band totals, leaving the age composition *within* a band unconstrained
 # nationally: the calibration could satisfy 15-29 while starving 18-24, which
 # is what collapsed the modelled young-adult population to ~3.4M against ONS's
-# ~5.4M. Regional targets already use finer bands, but they do not constrain
-# the national weights. The ONS projection is published by single year of age,
-# so this is a change of aggregation only - no new source data.
+# ~5.4M. Regional ten-year bands also constrain the shared household weights,
+# including summed local weights in dense local calibration, but provide less
+# age resolution than these five-year sex bands. The ONS projection is published
+# by single year of age, so this is a change of aggregation only - no new source
+# data.
 _GENDER_BANDS = [
     (0, 4),
     (5, 9),
@@ -89,8 +90,7 @@
     (65, 69),
     (70, 74),
     (75, 79),
-    (80, 84),
-    (85, 90),
+    (80, None),
 ]
 
 
@@ -121,12 +121,19 @@ def _find_projection_member(names: list[str]) -> str:
 
 
 def _aggregate_ages(
-    df: pd.DataFrame, sex: str, low: int, high: int, years: list[int]
+    df: pd.DataFrame, sex: str, low: int, high: int | None, years: list[int]
 ) -> dict[int, float]:
-    """Sum population for a sex and age range across years."""
+    """Sum an inclusive age band; None includes the projection's terminal age."""
     sex_filter = "Females" if sex == "female" else "Males"
-    ages = pd.to_numeric(df["Age"], errors="coerce")
-    mask = (df["Sex"] == sex_filter) & ages.between(low, high)
+    # Use each group's lower age: the 2024-based workbook ends with
+    # "105 - 109" and "110 and over"; older vintages can use "90+".
+    ages = pd.to_numeric(
+        df["Age"].astype(str).str.extract(r"^(\d+)", expand=False), errors="coerce"
+    )
+    age_mask = ages >= low
+    if high is not None:
+        age_mask &= ages <= high
+    mask = (df["Sex"] == sex_filter) & age_mask
     subset = df[mask]
     result = {}
     for y in years:
@@ -176,7 +183,7 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]:
             if values:
                 targets.append(
                     Target(
-                        name=f"ons/{sex}_{low}_{high}",
+                        name=f"ons/{sex}_{low}_{'plus' if high is None else high}",
                         variable="age",
                         source="ons",
                         unit=Unit.COUNT,
@@ -190,17 +197,19 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]:
 
 
 def _parse_regional_from_csv() -> list[Target]:
-    """Read regional age band targets from demographics.csv.
-
-    This CSV was extracted from ONS subnational projections which
-    lack a stable machine-readable download URL.
-    """
+    """Read regional bands, replacing the legacy bounded top band with 80+."""
     csv_path = STORAGE / "demographics.csv"
     if not csv_path.exists():
         logger.warning("demographics.csv not found, skipping regional")
         return []
 
     demographics = pd.read_csv(csv_path)
+    # Replace the legacy bounded 80-89 targets with direct ONS 80+ sums,
+    # including 90+, instead of relabelling totals with a different scope.
+    demographics = demographics[~demographics["name"].str.endswith("_age_80_89")]
+    top_bands = pd.read_csv(STORAGE / "regional_age_80_plus.csv")
+    top_bands["unit"] = "person"
+    demographics = pd.concat([demographics, top_bands], ignore_index=True)
     targets = []
 
     # Skip rows now handled by dedicated modules (ons_households.py,
@@ -231,8 +240,8 @@ def _parse_regional_from_csv() -> list[Target]:
         for y in _YEARS:
             col = str(y)
             if col in row.index and pd.notna(row[col]):
-                # Values in CSV are in thousands
-                values[y] = float(row[col]) * 1e3
+                scale = 1 if row["unit"] == "person" else 1e3
+                values[y] = float(row[col]) * scale
         if values:
             targets.append(
                 Target(
@@ -243,7 +252,11 @@ def _parse_regional_from_csv() -> list[Target]:
                     geographic_level=GeographicLevel.REGION,
                     values=values,
                     is_count=True,
-                    reference_url=_REF_REGION,
+                    reference_url=(
+                        row["reference_url"]
+                        if pd.notna(row.get("reference_url"))
+                        else _REF_REGION
+                    ),
                 )
             )