From 61f8f7ef00297a1344345b87e0f97e9cf5efe398 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 11:11:46 -0400 Subject: [PATCH 1/2] Match ONS age targets to the FRS top code of 80 --- .gitignore | 4 + changelog.d/593-age-topcode.fixed.md | 1 + .../storage/regional_age_80_plus.csv | 13 ++ .../targets/compute/demographics.py | 5 +- .../targets/sources/ons_demographics.py | 53 +++-- .../tests/fixtures/ons_uk_population_2025.csv | 215 ++++++++++++++++++ .../tests/test_ons_demographics_targets.py | 133 +++++++++++ tools/ons_age_targets/README.md | 56 +++++ tools/ons_age_targets/build.py | 138 +++++++++++ 9 files changed, 595 insertions(+), 23 deletions(-) create mode 100644 changelog.d/593-age-topcode.fixed.md create mode 100644 policyengine_uk_data/storage/regional_age_80_plus.csv create mode 100644 policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv create mode 100644 tools/ons_age_targets/README.md create mode 100644 tools/ons_age_targets/build.py diff --git a/.gitignore b/.gitignore index 70d8396d3..b0b3a6ab6 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ !policyengine_uk_data/datasets/local_areas/**/*.csv **/_build !policyengine_uk_data/storage/*.csv +!policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv **/version.json # Build output: household weights by local area, derived from FRS microdata policyengine_uk_data/storage/local_geography_weights.csv.gz @@ -26,3 +27,6 @@ policyengine_uk_data/storage/local_geography_weights.csv.gz # Hypothesis example database (property-based tests) .hypothesis/ + +# Local build-job notes, caches and hand-off artefacts +.job/ diff --git a/changelog.d/593-age-topcode.fixed.md b/changelog.d/593-age-topcode.fixed.md new file mode 100644 index 000000000..49bfb27ce --- /dev/null +++ b/changelog.d/593-age-topcode.fixed.md @@ -0,0 +1 @@ +Matched national and regional ONS age targets to the FRS top code with open-ended 80+ bands, including the projection's oldest age groups. diff --git a/policyengine_uk_data/storage/regional_age_80_plus.csv b/policyengine_uk_data/storage/regional_age_80_plus.csv new file mode 100644 index 000000000..4afa81cb2 --- /dev/null +++ b/policyengine_uk_data/storage/regional_age_80_plus.csv @@ -0,0 +1,13 @@ +name,2022,2023,2024,2025,2026,2027,2028,2029,reference_url +east_age_80_plus,350364.000,359057.134,371659.091,383909.917,397184.443,425490.369,445444.208,459810.859,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +east_midlands_age_80_plus,252810.000,259714.072,270269.576,279796.208,288881.005,308397.351,323332.659,334586.295,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +london_age_80_plus,281480.000,283880.847,288218.147,292199.323,298451.115,312294.218,322619.866,330707.051,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +north_east_age_80_plus,141323.000,142485.609,146050.948,149741.415,153969.782,164716.911,172434.769,178458.689,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +north_west_age_80_plus,368936.000,375461.807,385752.712,394643.102,404696.948,431246.821,450009.928,463577.946,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +northern_ireland_age_80_plus,86145.000,89224.000,92507.000,95719.000,98931.000,103172.000,106713.000,109973.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z6zippedpopulationprojectionsdatafilesnorthernireland +scotland_age_80_plus,274428.000,279070.000,285484.000,290656.000,297529.000,316051.000,330092.000,341986.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z5zippedpopulationprojectionsdatafilesscotland +south_east_age_80_plus,517027.000,531225.098,550842.841,569679.705,590428.825,631358.154,661477.046,684046.367,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +south_west_age_80_plus,359314.000,370051.020,384900.773,399727.616,414550.838,442926.618,465134.345,481977.036,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +wales_age_80_plus,175956.000,179720.000,185492.000,191184.000,196718.000,209247.000,219315.000,226905.000,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/z4zippedpopulationprojectionsdatafileswales +west_midlands_age_80_plus,311947.000,319178.683,330274.556,339480.659,347714.374,366032.307,380011.894,390810.558,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 +yorkshire_and_the_humber_age_80_plus,279307.000,284139.773,292163.361,300471.035,308655.688,329410.267,344273.286,355194.173,https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1 diff --git a/policyengine_uk_data/targets/compute/demographics.py b/policyengine_uk_data/targets/compute/demographics.py index 4d0b80923..f405f24a5 100644 --- a/policyengine_uk_data/targets/compute/demographics.py +++ b/policyengine_uk_data/targets/compute/demographics.py @@ -26,7 +26,8 @@ def compute_regional_age(target, ctx) -> np.ndarray | None: region_name = name[:idx] age_part = name[idx + 5 :] lower, upper = age_part.split("_") - lower, upper = int(lower), int(upper) + lower = int(lower) + upper = np.inf if upper == "plus" else int(upper) pe_region = _REGION_INV.get(region_name) if pe_region is None: @@ -44,7 +45,7 @@ def compute_gender_age(target, ctx) -> np.ndarray: parts = name.split("_") sex = parts[0] lower = int(parts[1]) - upper = int(parts[2]) + upper = np.inf if parts[2] == "plus" else int(parts[2]) gender = ctx.sim.calculate("gender").values sex_match = gender == ("FEMALE" if sex == "female" else "MALE") diff --git a/policyengine_uk_data/targets/sources/ons_demographics.py b/policyengine_uk_data/targets/sources/ons_demographics.py index 0d04f6315..04a1463fe 100644 --- a/policyengine_uk_data/targets/sources/ons_demographics.py +++ b/policyengine_uk_data/targets/sources/ons_demographics.py @@ -3,10 +3,9 @@ Downloads the ONS 2024-based principal population projection for the UK to extract total population and gender × age band targets. -For regional age breakdowns (12 regions × 9 age bands), reads the -pre-existing demographics.csv which was extracted from ONS subnational -projections. The subnational projections don't have a stable machine- -readable download URL, so this is the pragmatic compromise. +Regional age breakdowns use demographics.csv below age 80 and a committed +ONS projection extract for 80+, including the terminal age group. See +tools/ons_age_targets for the sources and extraction method. Household type and tenure targets are from ONS families & households datasets (also lacking stable machine-readable URLs). @@ -52,7 +51,7 @@ _YEARS = list(range(2022, 2030)) -# Age band boundaries +# FRS ages are top-coded at 80, so the last band includes every older age. _AGE_BANDS = [ (0, 9), (10, 19), @@ -62,7 +61,7 @@ (50, 59), (60, 69), (70, 79), - (80, 89), + (80, None), ] # Five-year bands. The previous 15-year bands (0-14, 15-29, ...) pinned only @@ -89,8 +88,7 @@ (65, 69), (70, 74), (75, 79), - (80, 84), - (85, 90), + (80, None), ] @@ -121,12 +119,19 @@ def _find_projection_member(names: list[str]) -> str: def _aggregate_ages( - df: pd.DataFrame, sex: str, low: int, high: int, years: list[int] + df: pd.DataFrame, sex: str, low: int, high: int | None, years: list[int] ) -> dict[int, float]: - """Sum population for a sex and age range across years.""" + """Sum an inclusive age band; None includes the projection's terminal age.""" sex_filter = "Females" if sex == "female" else "Males" - ages = pd.to_numeric(df["Age"], errors="coerce") - mask = (df["Sex"] == sex_filter) & ages.between(low, high) + # Use each group's lower age: the 2024-based workbook ends with + # "105 - 109" and "110 and over"; older vintages can use "90+". + ages = pd.to_numeric( + df["Age"].astype(str).str.extract(r"^(\d+)", expand=False), errors="coerce" + ) + age_mask = ages >= low + if high is not None: + age_mask &= ages <= high + mask = (df["Sex"] == sex_filter) & age_mask subset = df[mask] result = {} for y in years: @@ -176,7 +181,7 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]: if values: targets.append( Target( - name=f"ons/{sex}_{low}_{high}", + name=f"ons/{sex}_{low}_{'plus' if high is None else high}", variable="age", source="ons", unit=Unit.COUNT, @@ -190,17 +195,19 @@ def _parse_uk_totals(df: pd.DataFrame) -> list[Target]: def _parse_regional_from_csv() -> list[Target]: - """Read regional age band targets from demographics.csv. - - This CSV was extracted from ONS subnational projections which - lack a stable machine-readable download URL. - """ + """Read regional bands, replacing the legacy bounded top band with 80+.""" csv_path = STORAGE / "demographics.csv" if not csv_path.exists(): logger.warning("demographics.csv not found, skipping regional") return [] demographics = pd.read_csv(csv_path) + # Replace the legacy bounded 80-89 targets with direct ONS 80+ sums, + # including 90+, instead of relabelling totals with a different scope. + demographics = demographics[~demographics["name"].str.endswith("_age_80_89")] + top_bands = pd.read_csv(STORAGE / "regional_age_80_plus.csv") + top_bands["unit"] = "person" + demographics = pd.concat([demographics, top_bands], ignore_index=True) targets = [] # Skip rows now handled by dedicated modules (ons_households.py, @@ -231,8 +238,8 @@ def _parse_regional_from_csv() -> list[Target]: for y in _YEARS: col = str(y) if col in row.index and pd.notna(row[col]): - # Values in CSV are in thousands - values[y] = float(row[col]) * 1e3 + scale = 1 if row["unit"] == "person" else 1e3 + values[y] = float(row[col]) * scale if values: targets.append( Target( @@ -243,7 +250,11 @@ def _parse_regional_from_csv() -> list[Target]: geographic_level=GeographicLevel.REGION, values=values, is_count=True, - reference_url=_REF_REGION, + reference_url=( + row["reference_url"] + if pd.notna(row.get("reference_url")) + else _REF_REGION + ), ) ) diff --git a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv new file mode 100644 index 000000000..585b1c84f --- /dev/null +++ b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv @@ -0,0 +1,215 @@ +Sex,Age,2025 +Females,0,317686 +Females,1,327935 +Females,2,338279 +Females,3,358658 +Females,4,356258 +Females,5,369761 +Females,6,376378 +Females,7,381376 +Females,8,390185 +Females,9,400480 +Females,10,397930 +Females,11,399560 +Females,12,408413 +Females,13,418244 +Females,14,417732 +Females,15,413729 +Females,16,410318 +Females,17,414288 +Females,18,399447 +Females,19,398620 +Females,20,406477 +Females,21,404249 +Females,22,401940 +Females,23,408447 +Females,24,412365 +Females,25,422024 +Females,26,444655 +Females,27,458010 +Females,28,469985 +Females,29,463354 +Females,30,470982 +Females,31,484020 +Females,32,486083 +Females,33,498400 +Females,34,504074 +Females,35,503593 +Females,36,500871 +Females,37,507256 +Females,38,496049 +Females,39,489353 +Females,40,485837 +Females,41,469975 +Females,42,469221 +Females,43,463843 +Females,44,467338 +Females,45,465639 +Females,46,444476 +Females,47,412021 +Females,48,402609 +Females,49,410811 +Females,50,417510 +Females,51,422475 +Females,52,438299 +Females,53,454957 +Females,54,473232 +Females,55,460650 +Females,56,470719 +Females,57,469471 +Females,58,470280 +Females,59,469617 +Females,60,471064 +Females,61,464261 +Females,62,452458 +Females,63,439638 +Females,64,421487 +Females,65,403839 +Females,66,393032 +Females,67,380880 +Females,68,364266 +Females,69,348875 +Females,70,334931 +Females,71,333201 +Females,72,324950 +Females,73,313788 +Females,74,312458 +Females,75,314249 +Females,76,317096 +Females,77,330339 +Females,78,350379 +Females,79,263767 +Females,80,249905 +Females,81,242088 +Females,82,219106 +Females,83,189777 +Females,84,163167 +Females,85,161170 +Females,86,150872 +Females,87,137597 +Females,88,121030 +Females,89,105738 +Females,90,90358 +Females,91,73607 +Females,92,61708 +Females,93,51510 +Females,94,41269 +Females,95,32160 +Females,96,23665 +Females,97,16907 +Females,98,11918 +Females,99,8188 +Females,100,5349 +Females,101,3385 +Females,102,2064 +Females,103,1211 +Females,104,677 +Females,105 - 109,546 +Females,110 and over,6 +Males,0,333572 +Males,1,344421 +Males,2,356607 +Males,3,376143 +Males,4,373747 +Males,5,386967 +Males,6,395428 +Males,7,399866 +Males,8,409003 +Males,9,420639 +Males,10,417531 +Males,11,418996 +Males,12,428612 +Males,13,438674 +Males,14,439496 +Males,15,433546 +Males,16,431615 +Males,17,436434 +Males,18,425336 +Males,19,422999 +Males,20,430550 +Males,21,430470 +Males,22,426240 +Males,23,423436 +Males,24,426041 +Males,25,435235 +Males,26,447885 +Males,27,452338 +Males,28,459538 +Males,29,451896 +Males,30,454060 +Males,31,462487 +Males,32,460659 +Males,33,470586 +Males,34,476185 +Males,35,473059 +Males,36,467528 +Males,37,469878 +Males,38,458073 +Males,39,457277 +Males,40,454111 +Males,41,439326 +Males,42,438740 +Males,43,436382 +Males,44,439505 +Males,45,441201 +Males,46,423905 +Males,47,393520 +Males,48,386917 +Males,49,395093 +Males,50,402512 +Males,51,406800 +Males,52,422253 +Males,53,436908 +Males,54,450989 +Males,55,439963 +Males,56,449773 +Males,57,447985 +Males,58,451039 +Males,59,446161 +Males,60,447763 +Males,61,440594 +Males,62,429773 +Males,63,418061 +Males,64,402529 +Males,65,384785 +Males,66,372436 +Males,67,360028 +Males,68,341388 +Males,69,326411 +Males,70,309308 +Males,71,305067 +Males,72,296642 +Males,73,281653 +Males,74,279520 +Males,75,277095 +Males,76,279193 +Males,77,286999 +Males,78,302900 +Males,79,223985 +Males,80,208431 +Males,81,199761 +Males,82,175241 +Males,83,147423 +Males,84,122567 +Males,85,118037 +Males,86,107840 +Males,87,95378 +Males,88,80913 +Males,89,67392 +Males,90,55358 +Males,91,43439 +Males,92,34474 +Males,93,26990 +Males,94,20167 +Males,95,14236 +Males,96,9666 +Males,97,6400 +Males,98,4165 +Males,99,2573 +Males,100,1535 +Males,101,844 +Males,102,456 +Males,103,227 +Males,104,109 +Males,105 - 109,73 +Males,110 and over,0 diff --git a/policyengine_uk_data/tests/test_ons_demographics_targets.py b/policyengine_uk_data/tests/test_ons_demographics_targets.py index 433bf3e25..1df25ff45 100644 --- a/policyengine_uk_data/tests/test_ons_demographics_targets.py +++ b/policyengine_uk_data/tests/test_ons_demographics_targets.py @@ -1,9 +1,21 @@ +from pathlib import Path +from types import SimpleNamespace + +import numpy as np import pandas as pd import pytest +from hypothesis import given +from hypothesis import strategies as st +from policyengine_uk_data.targets.build_loss_matrix import _compute_column +from policyengine_uk_data.targets.schema import Target, Unit from policyengine_uk_data.targets.sources.ons_demographics import ( + _AGE_BANDS, + _GENDER_BANDS, _aggregate_ages, _find_projection_member, + _parse_regional_from_csv, + _parse_uk_totals, ) @@ -22,3 +34,124 @@ def test_aggregate_ages_accepts_string_age_values(): def test_find_projection_member_fails_loudly(): with pytest.raises(RuntimeError, match="uk_ppp_machine_readable"): _find_projection_member(["uk/readme.txt"]) + + +@pytest.mark.parametrize("bands", [_AGE_BANDS, _GENDER_BANDS]) +@given(age=st.integers(min_value=0, max_value=80)) +def test_age_bands_partition_frs_ages_and_respect_top_code(bands, age): + assert bands[-1] == (80, None) + for low, high in bands: + assert low <= 80, "No target may start above the FRS top code" + assert high is None or high < 80, "80+ must be open-ended" + assert sum(low <= age and (high is None or age <= high) for low, high in bands) == 1 + + +@pytest.mark.parametrize("sex,multiplier", [("female", 1), ("male", 10)]) +def test_80_plus_target_sums_every_projection_age(sex, multiplier): + projection = pd.DataFrame( + [ + {"Sex": label, "Age": str(age), 2025: (age + 1) * factor} + for label, factor in [("Females", 1), ("Males", 10)] + for age in range(101) + ] + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + + assert targets[f"ons/{sex}_80_plus"].values[2025] == sum( + (age + 1) * multiplier for age in range(80, 101) + ) + + +@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")]) +def test_80_plus_matches_published_ons_2025_projection(sex, label): + projection = pd.read_csv( + Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv" + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + # The fixture retains ONS's single ages 0-104 and terminal 105-109/110+. + older_ages = {str(age) for age in range(80, 105)} | { + "105 - 109", + "110 and over", + } + expected = projection.loc[ + (projection.Sex == label) & projection.Age.isin(older_ages), "2025" + ].sum() + assert targets[f"ons/{sex}_80_plus"].values[2025] == expected + + +@pytest.mark.parametrize("sex,label", [("female", "Females"), ("male", "Males")]) +@pytest.mark.parametrize( + "terminal_ages", [["89", "90+"], ["105 - 109", "110 and over"]] +) +def test_80_plus_target_includes_terminal_age_groups(sex, label, terminal_ages): + projection = pd.DataFrame( + { + "Sex": [label] * 4, + "Age": ["79", "80", *terminal_ages], + 2025: [1, 2, 4, 8], + } + ) + targets = {target.name: target for target in _parse_uk_totals(projection)} + + assert targets[f"ons/{sex}_80_plus"].values[2025] == 14 + + +def test_emitted_age_target_names_respect_frs_top_code(): + projection = pd.DataFrame( + {"Sex": ["Females", "Males"], "Age": [80, 80], 2025: [1, 1]} + ) + targets = _parse_uk_totals(projection) + _parse_regional_from_csv() + top_bands = [] + for target in targets: + if "_age_" not in target.name and not target.name.startswith( + ("ons/female_", "ons/male_") + ): + continue + low, high = target.name.rsplit("_", 2)[-2:] + assert int(low) <= 80, target.name + assert high == "plus" or int(high) < 80, target.name + if int(low) == 80: + top_bands.append(target.name) + assert len(top_bands) == 14 # Both sexes and all twelve regions. + + +@pytest.mark.parametrize( + "name,expected", + [ + ("ons/female_80_plus", [1, 1, 1, 0]), + ("ons/male_80_plus", [0, 1, 1, 1]), + ("ons/london_age_80_plus", [1, 1, 1, 1]), + ], +) +def test_80_plus_dispatch_counts_matching_household_members(name, expected): + household = np.array([0, 0, 1, 1, 2, 2, 3, 3]) + gender = pd.Series( + ["FEMALE", "FEMALE", "MALE", "FEMALE", "MALE", "FEMALE", "MALE", "MALE"] + ) + ctx = SimpleNamespace( + age=np.array([79, 80, 81, 100, 80, 80, 79, 85]), + region=pd.Series( + [ + "LONDON", + "LONDON", + "LONDON", + "WALES", + "WALES", + "LONDON", + "LONDON", + "LONDON", + ] + ), + sim=SimpleNamespace(calculate=lambda variable: gender), + household_from_person=lambda values: np.bincount(household, weights=values), + ) + target = Target( + name=name, + variable="age", + source="ons", + unit=Unit.COUNT, + values={2025: 1}, + is_count=True, + ) + + np.testing.assert_array_equal(_compute_column(target, ctx, 2025), expected) diff --git a/tools/ons_age_targets/README.md b/tools/ons_age_targets/README.md new file mode 100644 index 000000000..7d223513f --- /dev/null +++ b/tools/ons_age_targets/README.md @@ -0,0 +1,56 @@ +# ONS age target sources + +`policyengine_uk_data/storage/regional_age_80_plus.csv` contains public population +projection aggregates in persons. It contains no FRS data. Reproduce it from the +repository root with the locked environment: + +```sh +uv run python tools/ons_age_targets/build.py +``` + +Downloads are cached in `.job/`; SHA-256 checks pin the source files. The generator +uses pandas, requests and openpyxl, which are already locked dependencies. + +The English regions use the [ONS 2022-based single-year subnational +projection](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/localauthoritiesinenglandz1), +five-year migration variant, published 24 June 2025. The [ONS release bulletin](https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/bulletins/subnationalpopulationprojectionsforengland/2022based) +identifies this variant as the one aligned with the 2022-based principal national +projection. For each of +the 309 English local authorities, the extract sums ages 80 through 89 and the +terminal `90 and over` group. The committed `storage/oa_crosswalk.csv.gz` maps all +309 authorities unambiguously to the nine English regions. Summing those regions +therefore reproduces the sum of all 309 authorities, without omitting or counting +an authority twice. Published counts have three decimal places; the output retains +that precision. + +Scotland, Wales and Northern Ireland use the ONS **2022-based principal national +projections**, so all 12 regional 80+ targets have the same projection vintage and +cover 2022–2029. Each country total sums both sexes at ages 80–104 and the terminal +groups `105 - 109` and `110 and over`. The regional CSV includes the reference URL +for each row. This replaces the legacy regional top-band values; it is a source +refresh as well as a correction to the open-ended band definition. The lower +regional age bands remain in `demographics.csv`. + +Pinned downloads, relative to +`https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/`: + +| Source | Download suffix | SHA-256 | +| --- | --- | --- | +| English local authorities | `localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip` | `171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b` | +| Scotland | `z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip` | `6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d` | +| Wales | `z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip` | `32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051` | +| Northern Ireland | `z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip` | `69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489` | + +The national targets continue to use the existing **2024-based UK principal +projection**. The test fixture `policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv` preserves +its complete `Sex`, `Age` and `2025` population columns, including `105 - 109` and +`110 and over`. Its source ZIP is +`z1zippedpopulationprojectionsdatafilesuk/2024based/uk.zip`, SHA-256 +`dea79703b9caddd560f1958b5a8ca12aa817b2396d244e6a2c775b1af7f48ab1`. + +The old regional rows first appeared in commit `c98c0ccf` (9 September 2024); +`63dfba48` (10 June 2025) reformatted them without changing their values. Their +reference `ons_age_sex_region` does not identify a projection vintage or extraction +method. Their finite `80_89` names alone do not prove that their published values +excluded 90+: the replacement uses directly reproducible sums rather than adding +an estimated 90+ population to those legacy values. diff --git a/tools/ons_age_targets/build.py b/tools/ons_age_targets/build.py new file mode 100644 index 000000000..b29be1a57 --- /dev/null +++ b/tools/ons_age_targets/build.py @@ -0,0 +1,138 @@ +"""Reproduce the public ONS regional 80+ target extract (no survey data).""" + +import argparse +import hashlib +import io +import zipfile +from pathlib import Path + +import pandas as pd +import requests + +ROOT = Path(__file__).resolve().parents[2] +YEARS = [str(year) for year in range(2022, 2030)] +ONS = "https://www.ons.gov.uk/file?uri=/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/" +SOURCES = { + "regions": ( + "localauthoritiesinenglandz1/2022basedfiveyearmigrationvariantprojection/2022snpppopulationsyoa5yr.zip", + "171cd05fc753277388352c76abf8cc8b305ca537cd593a6e4467af7e8a8e4b5b", + ), + "scotland": ( + "z5zippedpopulationprojectionsdatafilesscotland/2022based/sc.zip", + "6e0dc9250a9890680754ab501be56973c722a57950e4535c364fbc4494372b3d", + ), + "wales": ( + "z4zippedpopulationprojectionsdatafileswales/2022based/wa.zip", + "32a7c2aacf9b3c945abeaa19a6d6d7aa62133f491bfca5a9700a393244a7b051", + ), + "ni": ( + "z6zippedpopulationprojectionsdatafilesnorthernireland/2022based/ni.zip", + "69b006c2741943d97d153cd024e4b21ea94b52ef766efcde6bd737a400d95489", + ), +} +REGIONS = { + "E12000001": "north_east", + "E12000002": "north_west", + "E12000003": "yorkshire_and_the_humber", + "E12000004": "east_midlands", + "E12000005": "west_midlands", + "E12000006": "east", + "E12000007": "london", + "E12000008": "south_east", + "E12000009": "south_west", +} + + +def aggregate_english_regions( + population: pd.DataFrame, crosswalk: pd.DataFrame +) -> pd.DataFrame: + """Sum ages 80 through the terminal 90+ row in every English LA.""" + lookup = crosswalk[["la_code", "region_code"]].drop_duplicates() + assert lookup.la_code.is_unique, "Local authorities have ambiguous regions" + ages = population.AGE_GROUP.astype(str).str.extract(r"^(\d+)")[0] + older = population.loc[pd.to_numeric(ages, errors="coerce") >= 80].copy() + assert set(older.AGE_GROUP) == {str(age) for age in range(80, 90)} | {"90 and over"} + older = older.merge( + lookup, + left_on="AREA_CODE", + right_on="la_code", + how="left", + validate="many_to_one", + ) + assert older.region_code.notna().all(), "Some local authorities lack a region" + assert set(older.region_code) == set(REGIONS) + assert older.AREA_CODE.nunique() == 309 + assert older.groupby("AREA_CODE").AGE_GROUP.nunique().eq(11).all() + result = older.groupby("region_code")[YEARS].sum().round(3) + assert result.sum().round(3).equals(older[YEARS].sum().round(3)) + result.index = result.index.map(REGIONS) + return result + + +def aggregate_country(population: pd.DataFrame) -> pd.Series: + """Include all single years and terminal age groups from 80 upwards.""" + ages = population.Age.astype(str).str.extract(r"^(\d+)")[0] + older = population.loc[ + (pd.to_numeric(ages, errors="coerce") >= 80) + & population.Sex.isin(["Females", "Males"]) + ] + assert {"105 - 109", "110 and over"}.issubset(set(older.Age.astype(str))) + older = older.rename(columns=lambda column: str(column)) + return older[YEARS].sum() + + +def build(source_dir: Path, output: Path) -> None: + """Download pinned public sources and save their exact published sums.""" + source_dir.mkdir(parents=True, exist_ok=True) + for name, (suffix, expected_sha256) in SOURCES.items(): + path = source_dir / f"ons-{name}-2022.zip" + if not path.exists(): + response = requests.get( + ONS + suffix, + headers={ + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" + }, + timeout=120, + ) + response.raise_for_status() + path.write_bytes(response.content) + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256 + + with zipfile.ZipFile(source_dir / "ons-regions-2022.zip") as archive: + population = pd.read_csv(archive.open("2022 SNPP Population persons.csv")) + crosswalk = pd.read_csv(ROOT / "policyengine_uk_data/storage/oa_crosswalk.csv.gz") + totals = aggregate_english_regions(population, crosswalk) + for country, prefix in [("scotland", "sc"), ("wales", "wa"), ("ni", "ni")]: + with zipfile.ZipFile(source_dir / f"ons-{country}-2022.zip") as archive: + population = pd.read_excel( + io.BytesIO( + archive.read(f"{prefix}/{prefix}_ppp_machine_readable.xlsx") + ), + sheet_name="Population", + engine="openpyxl", + ) + totals.loc["northern_ireland" if country == "ni" else country] = ( + aggregate_country(population) + ) + totals.index = [f"{region}_age_80_plus" for region in totals.index] + totals.index.name = "name" + reference_base = "https://www.ons.gov.uk/peoplepopulationandcommunity/populationandmigration/populationprojections/datasets/" + totals["reference_url"] = reference_base + "localauthoritiesinenglandz1" + for country in ["scotland", "wales", "ni"]: + region = "northern_ireland" if country == "ni" else country + totals.loc[f"{region}_age_80_plus", "reference_url"] = ( + reference_base + SOURCES[country][0].split("/")[0] + ) + totals.sort_index().to_csv(output, float_format="%.3f") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-dir", type=Path, default=ROOT / ".job") + parser.add_argument( + "--output", + type=Path, + default=ROOT / "policyengine_uk_data/storage/regional_age_80_plus.csv", + ) + args = parser.parse_args() + build(args.source_dir, args.output) From 29ca75420b885ba7cd0ee3819817a974841a5a99 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 15 Oct 2026 23:10:08 -0400 Subject: [PATCH 2/2] Fix national age-band tests for open-ended 80+ targets --- .../targets/sources/ons_demographics.py | 8 +- .../tests/fixtures/ons_uk_population_2025.csv | 430 +++++++++--------- .../tests/test_national_age_bands.py | 32 +- 3 files changed, 241 insertions(+), 229 deletions(-) diff --git a/policyengine_uk_data/targets/sources/ons_demographics.py b/policyengine_uk_data/targets/sources/ons_demographics.py index 04a1463fe..a40a0a4dd 100644 --- a/policyengine_uk_data/targets/sources/ons_demographics.py +++ b/policyengine_uk_data/targets/sources/ons_demographics.py @@ -68,9 +68,11 @@ # the band totals, leaving the age composition *within* a band unconstrained # nationally: the calibration could satisfy 15-29 while starving 18-24, which # is what collapsed the modelled young-adult population to ~3.4M against ONS's -# ~5.4M. Regional targets already use finer bands, but they do not constrain -# the national weights. The ONS projection is published by single year of age, -# so this is a change of aggregation only - no new source data. +# ~5.4M. Regional ten-year bands also constrain the shared household weights, +# including summed local weights in dense local calibration, but provide less +# age resolution than these five-year sex bands. The ONS projection is published +# by single year of age, so this is a change of aggregation only - no new source +# data. _GENDER_BANDS = [ (0, 4), (5, 9), diff --git a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv index 585b1c84f..911aae4f5 100644 --- a/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv +++ b/policyengine_uk_data/tests/fixtures/ons_uk_population_2025.csv @@ -1,215 +1,215 @@ -Sex,Age,2025 -Females,0,317686 -Females,1,327935 -Females,2,338279 -Females,3,358658 -Females,4,356258 -Females,5,369761 -Females,6,376378 -Females,7,381376 -Females,8,390185 -Females,9,400480 -Females,10,397930 -Females,11,399560 -Females,12,408413 -Females,13,418244 -Females,14,417732 -Females,15,413729 -Females,16,410318 -Females,17,414288 -Females,18,399447 -Females,19,398620 -Females,20,406477 -Females,21,404249 -Females,22,401940 -Females,23,408447 -Females,24,412365 -Females,25,422024 -Females,26,444655 -Females,27,458010 -Females,28,469985 -Females,29,463354 -Females,30,470982 -Females,31,484020 -Females,32,486083 -Females,33,498400 -Females,34,504074 -Females,35,503593 -Females,36,500871 -Females,37,507256 -Females,38,496049 -Females,39,489353 -Females,40,485837 -Females,41,469975 -Females,42,469221 -Females,43,463843 -Females,44,467338 -Females,45,465639 -Females,46,444476 -Females,47,412021 -Females,48,402609 -Females,49,410811 -Females,50,417510 -Females,51,422475 -Females,52,438299 -Females,53,454957 -Females,54,473232 -Females,55,460650 -Females,56,470719 -Females,57,469471 -Females,58,470280 -Females,59,469617 -Females,60,471064 -Females,61,464261 -Females,62,452458 -Females,63,439638 -Females,64,421487 -Females,65,403839 -Females,66,393032 -Females,67,380880 -Females,68,364266 -Females,69,348875 -Females,70,334931 -Females,71,333201 -Females,72,324950 -Females,73,313788 -Females,74,312458 -Females,75,314249 -Females,76,317096 -Females,77,330339 -Females,78,350379 -Females,79,263767 -Females,80,249905 -Females,81,242088 -Females,82,219106 -Females,83,189777 -Females,84,163167 -Females,85,161170 -Females,86,150872 -Females,87,137597 -Females,88,121030 -Females,89,105738 -Females,90,90358 -Females,91,73607 -Females,92,61708 -Females,93,51510 -Females,94,41269 -Females,95,32160 -Females,96,23665 -Females,97,16907 -Females,98,11918 -Females,99,8188 -Females,100,5349 -Females,101,3385 -Females,102,2064 -Females,103,1211 -Females,104,677 -Females,105 - 109,546 -Females,110 and over,6 -Males,0,333572 -Males,1,344421 -Males,2,356607 -Males,3,376143 -Males,4,373747 -Males,5,386967 -Males,6,395428 -Males,7,399866 -Males,8,409003 -Males,9,420639 -Males,10,417531 -Males,11,418996 -Males,12,428612 -Males,13,438674 -Males,14,439496 -Males,15,433546 -Males,16,431615 -Males,17,436434 -Males,18,425336 -Males,19,422999 -Males,20,430550 -Males,21,430470 -Males,22,426240 -Males,23,423436 -Males,24,426041 -Males,25,435235 -Males,26,447885 -Males,27,452338 -Males,28,459538 -Males,29,451896 -Males,30,454060 -Males,31,462487 -Males,32,460659 -Males,33,470586 -Males,34,476185 -Males,35,473059 -Males,36,467528 -Males,37,469878 -Males,38,458073 -Males,39,457277 -Males,40,454111 -Males,41,439326 -Males,42,438740 -Males,43,436382 -Males,44,439505 -Males,45,441201 -Males,46,423905 -Males,47,393520 -Males,48,386917 -Males,49,395093 -Males,50,402512 -Males,51,406800 -Males,52,422253 -Males,53,436908 -Males,54,450989 -Males,55,439963 -Males,56,449773 -Males,57,447985 -Males,58,451039 -Males,59,446161 -Males,60,447763 -Males,61,440594 -Males,62,429773 -Males,63,418061 -Males,64,402529 -Males,65,384785 -Males,66,372436 -Males,67,360028 -Males,68,341388 -Males,69,326411 -Males,70,309308 -Males,71,305067 -Males,72,296642 -Males,73,281653 -Males,74,279520 -Males,75,277095 -Males,76,279193 -Males,77,286999 -Males,78,302900 -Males,79,223985 -Males,80,208431 -Males,81,199761 -Males,82,175241 -Males,83,147423 -Males,84,122567 -Males,85,118037 -Males,86,107840 -Males,87,95378 -Males,88,80913 -Males,89,67392 -Males,90,55358 -Males,91,43439 -Males,92,34474 -Males,93,26990 -Males,94,20167 -Males,95,14236 -Males,96,9666 -Males,97,6400 -Males,98,4165 -Males,99,2573 -Males,100,1535 -Males,101,844 -Males,102,456 -Males,103,227 -Males,104,109 -Males,105 - 109,73 -Males,110 and over,0 +Sex,Age,2025 +Females,0,317686 +Females,1,327935 +Females,2,338279 +Females,3,358658 +Females,4,356258 +Females,5,369761 +Females,6,376378 +Females,7,381376 +Females,8,390185 +Females,9,400480 +Females,10,397930 +Females,11,399560 +Females,12,408413 +Females,13,418244 +Females,14,417732 +Females,15,413729 +Females,16,410318 +Females,17,414288 +Females,18,399447 +Females,19,398620 +Females,20,406477 +Females,21,404249 +Females,22,401940 +Females,23,408447 +Females,24,412365 +Females,25,422024 +Females,26,444655 +Females,27,458010 +Females,28,469985 +Females,29,463354 +Females,30,470982 +Females,31,484020 +Females,32,486083 +Females,33,498400 +Females,34,504074 +Females,35,503593 +Females,36,500871 +Females,37,507256 +Females,38,496049 +Females,39,489353 +Females,40,485837 +Females,41,469975 +Females,42,469221 +Females,43,463843 +Females,44,467338 +Females,45,465639 +Females,46,444476 +Females,47,412021 +Females,48,402609 +Females,49,410811 +Females,50,417510 +Females,51,422475 +Females,52,438299 +Females,53,454957 +Females,54,473232 +Females,55,460650 +Females,56,470719 +Females,57,469471 +Females,58,470280 +Females,59,469617 +Females,60,471064 +Females,61,464261 +Females,62,452458 +Females,63,439638 +Females,64,421487 +Females,65,403839 +Females,66,393032 +Females,67,380880 +Females,68,364266 +Females,69,348875 +Females,70,334931 +Females,71,333201 +Females,72,324950 +Females,73,313788 +Females,74,312458 +Females,75,314249 +Females,76,317096 +Females,77,330339 +Females,78,350379 +Females,79,263767 +Females,80,249905 +Females,81,242088 +Females,82,219106 +Females,83,189777 +Females,84,163167 +Females,85,161170 +Females,86,150872 +Females,87,137597 +Females,88,121030 +Females,89,105738 +Females,90,90358 +Females,91,73607 +Females,92,61708 +Females,93,51510 +Females,94,41269 +Females,95,32160 +Females,96,23665 +Females,97,16907 +Females,98,11918 +Females,99,8188 +Females,100,5349 +Females,101,3385 +Females,102,2064 +Females,103,1211 +Females,104,677 +Females,105 - 109,546 +Females,110 and over,6 +Males,0,333572 +Males,1,344421 +Males,2,356607 +Males,3,376143 +Males,4,373747 +Males,5,386967 +Males,6,395428 +Males,7,399866 +Males,8,409003 +Males,9,420639 +Males,10,417531 +Males,11,418996 +Males,12,428612 +Males,13,438674 +Males,14,439496 +Males,15,433546 +Males,16,431615 +Males,17,436434 +Males,18,425336 +Males,19,422999 +Males,20,430550 +Males,21,430470 +Males,22,426240 +Males,23,423436 +Males,24,426041 +Males,25,435235 +Males,26,447885 +Males,27,452338 +Males,28,459538 +Males,29,451896 +Males,30,454060 +Males,31,462487 +Males,32,460659 +Males,33,470586 +Males,34,476185 +Males,35,473059 +Males,36,467528 +Males,37,469878 +Males,38,458073 +Males,39,457277 +Males,40,454111 +Males,41,439326 +Males,42,438740 +Males,43,436382 +Males,44,439505 +Males,45,441201 +Males,46,423905 +Males,47,393520 +Males,48,386917 +Males,49,395093 +Males,50,402512 +Males,51,406800 +Males,52,422253 +Males,53,436908 +Males,54,450989 +Males,55,439963 +Males,56,449773 +Males,57,447985 +Males,58,451039 +Males,59,446161 +Males,60,447763 +Males,61,440594 +Males,62,429773 +Males,63,418061 +Males,64,402529 +Males,65,384785 +Males,66,372436 +Males,67,360028 +Males,68,341388 +Males,69,326411 +Males,70,309308 +Males,71,305067 +Males,72,296642 +Males,73,281653 +Males,74,279520 +Males,75,277095 +Males,76,279193 +Males,77,286999 +Males,78,302900 +Males,79,223985 +Males,80,208431 +Males,81,199761 +Males,82,175241 +Males,83,147423 +Males,84,122567 +Males,85,118037 +Males,86,107840 +Males,87,95378 +Males,88,80913 +Males,89,67392 +Males,90,55358 +Males,91,43439 +Males,92,34474 +Males,93,26990 +Males,94,20167 +Males,95,14236 +Males,96,9666 +Males,97,6400 +Males,98,4165 +Males,99,2573 +Males,100,1535 +Males,101,844 +Males,102,456 +Males,103,227 +Males,104,109 +Males,105 - 109,73 +Males,110 and over,0 diff --git a/policyengine_uk_data/tests/test_national_age_bands.py b/policyengine_uk_data/tests/test_national_age_bands.py index f636255a6..354929a6f 100644 --- a/policyengine_uk_data/tests/test_national_age_bands.py +++ b/policyengine_uk_data/tests/test_national_age_bands.py @@ -4,8 +4,8 @@ 15-29, ...). A band that wide pins only its own total: the calibration could satisfy 15-29 while distributing those people any way it liked inside the band, and it settled on one that starved 18-24 down to ~3.4M against ONS's -~5.4M. The finer regional bands do not help, because they constrain regional -weights rather than the national ones. +~5.4M. Regional ten-year age bands constrain the same household weights, +but provide less age detail than national five-year sex bands. These tests pin the five-year granularity and the no-overlap property, so a future edit cannot quietly widen the bands or reintroduce the superseded CSV @@ -13,13 +13,17 @@ """ import re +from pathlib import Path +import pandas as pd + +from policyengine_uk_data.targets.sources import ons_demographics from policyengine_uk_data.targets.sources.ons_demographics import ( _GENDER_BANDS, get_targets, ) -_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+)$") +_GENDER_AGE = re.compile(r"^ons/(female|male)_(\d+)_(\d+|plus)$") def _gender_age_targets(): @@ -27,9 +31,11 @@ def _gender_age_targets(): def test_bands_are_at_most_five_years_wide(): - """A wider band leaves its internal age composition unconstrained.""" - for low, high in _GENDER_BANDS: - assert high - low <= 5, f"band {low}-{high} is too wide to pin composition" + """Only the terminal top-coded band may be wider than five years.""" + assert _GENDER_BANDS[-1] == (80, None) + for low, high in _GENDER_BANDS[:-1]: + assert high is not None, "only the terminal band may be open-ended" + assert 0 <= high - low < 5, f"band {low}-{high} is too wide to pin composition" def test_bands_are_contiguous_and_non_overlapping(): @@ -41,7 +47,7 @@ def test_bands_are_contiguous_and_non_overlapping(): def test_young_adult_ages_are_covered_by_narrow_bands(): """18-24 must not sit inside one wide band, which is the failure mode.""" - covering = [b for b in _GENDER_BANDS if b[0] <= 24 and b[1] >= 18] + covering = [b for b in _GENDER_BANDS if b[0] <= 24 and (b[1] is None or b[1] >= 18)] assert len(covering) >= 2, ( "18-24 is covered by a single band; its composition would be free" ) @@ -54,9 +60,13 @@ def test_national_gender_age_targets_are_emitted_once_per_band(): assert len({t.name for t in targets}) == len(targets) -def test_national_gender_age_targets_sum_to_uk_population(): - """Double-counted bands would show up here as a doubled total.""" +def test_national_gender_age_targets_sum_to_uk_population(monkeypatch): + """Every fixture age must be counted exactly once across both sexes.""" + projection = pd.read_csv( + Path(__file__).parent / "fixtures" / "ons_uk_population_2025.csv" + ) + monkeypatch.setattr(ons_demographics, "_download_uk_projection", lambda: projection) targets = _gender_age_targets() year = 2025 - total = sum(t.values.get(year, 0) for t in targets) - assert 65e6 < total < 73e6, f"national gender-age total {total / 1e6:.1f}M" + total = sum(t.values[year] for t in targets) + assert total == projection[str(year)].sum()