diff --git a/changelog.d/483.removed.md b/changelog.d/483.removed.md new file mode 100644 index 000000000..2e221a858 --- /dev/null +++ b/changelog.d/483.removed.md @@ -0,0 +1 @@ +Drop the ons/council_tax_band_d/{code} target emission: Band D amounts are rates, not additive household controls (#483). diff --git a/policyengine_uk_data/targets/sources/la_council_tax.py b/policyengine_uk_data/targets/sources/la_council_tax.py index 9aac581c0..6358fd7f6 100644 --- a/policyengine_uk_data/targets/sources/la_council_tax.py +++ b/policyengine_uk_data/targets/sources/la_council_tax.py @@ -1,11 +1,7 @@ """Local-authority council tax calibration targets (derived proxies). -Produces three kinds of LA-level calibration target from public data: +Produces two kinds of LA-level calibration target from public data: -- ``ons/council_tax_band_d/{code}``: the average Band D council tax - (inclusive of all precepts) each household pays in billing authority - ``code``. Sourced from MHCLG, Welsh Government and Scottish - Government annual publications. - ``voa/council_tax/{code}/{band}``: the number of dwellings in band ``A``–``H`` (England) or ``A``–``I`` (Wales) for billing authority ``code``. Sourced from the VOA *Council Tax: Stock of Properties* @@ -14,6 +10,11 @@ LA (net of CTR support). England derived from MHCLG taxbase × Band D; Wales sourced directly from WG Council Tax Income (Table 3). +No target is emitted from the ``band_d_amount`` column. A Band D +amount is a **rate**, not an additive household control, and the loss +matrix is an additive objective — see the note in ``get_targets`` and +issue #483. + Data for all 360 LAs in ``local_authorities_2021.csv`` is joined from the committed canonical file ``storage/la_council_tax.csv``. Rows where a source did not provide a value are omitted so calibrators cleanly @@ -88,22 +89,17 @@ _CSV_NAME = "la_council_tax.csv" -# Latest fiscal years covered by each source. The LA Band D amounts are -# structurally single-year snapshots; callers that need longer time -# series should uprate via the existing council-tax uprating index. +# Latest fiscal years covered by each source. These are structurally +# single-year snapshots; callers that need longer time series should +# uprate via the existing council-tax uprating index. +# ``_YEAR_BAND_D_ENGLAND`` is retained because the England net +# council-tax requirement is derived from the 2026-27 Band D level. _YEAR_BAND_D_ENGLAND = 2026 -_YEAR_BAND_D_WALES = 2026 -_YEAR_BAND_D_SCOTLAND = 2025 _YEAR_BAND_COUNT = 2025 _BAND_COUNT_COLUMNS = {band: f"count_band_{band}" for band in "ABCDEFGHI"} -_ENGLAND_REF = ( - "https://www.gov.uk/government/statistics/" - "council-tax-levels-set-by-local-authorities-in-england-2026-to-2027" -) _WALES_REF = "https://www.gov.wales/council-tax-levels-april-2026-march-2027-html" -_SCOTLAND_REF = "https://www.gov.scot/publications/council-tax-datasets/" _VOA_REF = ( "https://www.gov.uk/government/statistics/council-tax-stock-of-properties-2025" ) @@ -145,50 +141,35 @@ def load_la_net_council_tax() -> pd.DataFrame: ].reset_index(drop=True) -def _year_for_band_d(country: str) -> int: - if country == "WALES": - return _YEAR_BAND_D_WALES - if country == "SCOTLAND": - return _YEAR_BAND_D_SCOTLAND - return _YEAR_BAND_D_ENGLAND - - -def _ref_for_band_d(country: str) -> str: - if country == "WALES": - return _WALES_REF - if country == "SCOTLAND": - return _SCOTLAND_REF - return _ENGLAND_REF - - def get_targets() -> list[Target]: - """Emit LA-level Band D amount + band-count targets.""" + """Emit LA-level band-count and net council-tax targets.""" df = _load_table() if df is None or df.empty: return [] targets: list[Target] = [] - # Band D amount targets — one per LA with a reported value. - for _, row in df.iterrows(): - amount = row.get("band_d_amount") - if pd.isna(amount): - continue - code = str(row["code"]) - country = str(row["country"]) - targets.append( - Target( - name=f"ons/council_tax_band_d/{code}", - variable="council_tax_band_d_amount", - source="ons", - unit=Unit.GBP, - geographic_level=GeographicLevel.LOCAL_AUTHORITY, - geo_code=code, - geo_name=str(row["name"]), - values={_year_for_band_d(country): float(amount)}, - reference_url=_ref_for_band_d(country), - ) - ) + # NO Band D amount target is emitted here. Do not re-add one. + # + # A Band D council tax amount is a *rate* (£ per Band D dwelling), + # not an additive household control. The loss matrix is an additive + # objective: every target is compared against a weighted SUM over + # households. Binding a ~£2,500 per-dwelling rate into that objective + # is category-wrong — neither as a sum (which would scale with the + # number of households in the LA) nor as a mean (which would still be + # a rate inside an additive objective). + # + # Microcosm reached the same verdict independently: its + # ``uk_data_target_parity.json`` concern ``local_council_tax_band_d_rate`` + # is ``reviewed_exclusion`` / ``non_linear``, "Band D currency amounts + # are per-rate, not additive household controls", to be kept excluded + # "until a rate-aware non-linear objective exists". + # + # ``band_d_amount`` deliberately REMAINS in ``storage/la_council_tax.csv`` + # and is loadable: it is needed as data for household liability + # computation, and upstream when building the CSV's + # ``total_council_tax_net`` column (England = MHCLG taxbase x Band D). + # Only the target emission is dropped. See PolicyEngine/policyengine-uk-data#483. # Band count targets — one per (LA, band) where VOA has a value. for _, row in df.iterrows(): diff --git a/policyengine_uk_data/tests/test_la_council_tax_targets.py b/policyengine_uk_data/tests/test_la_council_tax_targets.py index 7cdc59e7b..4cc00b14f 100644 --- a/policyengine_uk_data/tests/test_la_council_tax_targets.py +++ b/policyengine_uk_data/tests/test_la_council_tax_targets.py @@ -239,11 +239,30 @@ def test_get_targets_runs_without_network(): assert targets, "Expected get_targets() to return a non-empty list" -def test_band_d_target_count_matches_csv(la_ct_df): - targets = get_targets() - band_d_targets = [t for t in targets if "council_tax_band_d/" in t.name] - expected = int(la_ct_df["band_d_amount"].notna().sum()) - assert len(band_d_targets) == expected +def test_no_band_d_amount_target_is_emitted(): + """Band D amounts are rates, not additive household controls. + + Microcosm's ``local_council_tax_band_d_rate`` concern is a + ``reviewed_exclusion``; see issue #483. The CSV column stays, the + target must not come back. + """ + offenders = [ + t.name + for t in get_targets() + if t.name.startswith("ons/council_tax_band_d/") + or "council_tax_band_d" in t.name + ] + assert not offenders, ( + "Band D amount targets must not be emitted (rate vs additive " + f"objective, #483); found {offenders[:5]}" + ) + + +def test_band_d_amount_column_still_loadable(la_ct_df): + """The data column must survive; only the target emission is gone.""" + assert "band_d_amount" in la_ct_df.columns + # Deliberate tripwire: update this pin on each la_council_tax.csv refresh. + assert int(la_ct_df["band_d_amount"].notna().sum()) == 350 def test_band_count_target_count_matches_csv(la_ct_df): @@ -251,6 +270,10 @@ def test_band_count_target_count_matches_csv(la_ct_df): bc_targets = [t for t in targets if t.name.startswith("voa/council_tax/")] expected = int(la_ct_df[list(_BAND_COUNT_COLUMNS.values())].notna().sum().sum()) assert len(bc_targets) == expected + # Pinned: dropping the Band D emission (#483) must not disturb the + # band-count family, which Microcosm binds as ported_local_declared. + # Deliberate tripwire: update this pin on each la_council_tax.csv refresh. + assert len(bc_targets) == 2563 def test_every_target_carries_local_authority_geo_level(): @@ -259,12 +282,6 @@ def test_every_target_carries_local_authority_geo_level(): assert target.geo_code is not None -def test_band_d_targets_use_gbp_unit(): - for target in get_targets(): - if "council_tax_band_d/" in target.name: - assert target.unit == Unit.GBP - - def test_band_count_targets_use_count_unit_and_is_count_flag(): for target in get_targets(): if target.name.startswith("voa/council_tax/"):