From 3c99e59dce14a09ca54e187db14e713dbc48e4d9 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Tue, 20 Oct 2026 21:03:15 -0400 Subject: [PATCH 1/2] Use disclosure-controlled UK geography outputs for constituencies and local authorities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The executor built UK constituency and local-authority results with policyengine's deprecated compute_uk_constituency_impacts and compute_longwise_uk_geography_impacts. They apply no minimum sample, so a small area with negative-income households shows an exploding relative change (Cardiff West: -£192 average, +1372% relative), and their output is not disclosure-controlled for licensed survey microdata. Both levels now go through compute_uk_geography_disclosure in one release, nested in regions and the nation, with the constituency/local-authority overlap acknowledged. Blocking findings are resolved by withholding the suggested published cells; region and national reference cells are never withheld, and a release that cannot be resolved publishes no area. The share families, which the output does not publish, are withheld up front. Contract: existing fields keep their types. A withheld area, or one whose relative change is undefined, is left out of the list. Published records gain optional disclosure_tier and 90% interval fields. An empty list is now valid. Stage 12 keeps region_code_oa and source_household_id. The local-authority breakdown no longer strips weights, so the last recorded weight-stripping exception is removed and its baseline is 0. Fixes #859 Co-Authored-By: Claude Opus 5.5 --- changelog_entry.yaml | 2 + docs/engineering/skills/microdata-weights.md | 7 +- docs/generated/rules.md | 2 +- .../macro_output.py | 39 +- .../tests/test_macro_output_uk_geography.py | 74 +++ .../simulation_output_builder.py | 42 +- .../simulation_output_common.py | 19 +- .../simulation_output_geographic.py | 476 ++++++++++++++++-- .../stage12_runtime/output_planning.py | 10 +- .../tests/test_simulation_output_builder.py | 116 +++-- .../test_simulation_output_geographic.py | 438 +++++++++++++--- .../tests/test_stage12_output_planning.py | 7 +- .../tests/golden/openapi.json | 122 ++++- .../sim-microdata-weight-stripping.txt | 7 - rules/registry.toml | 2 +- 15 files changed, 1140 insertions(+), 223 deletions(-) create mode 100644 libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py diff --git a/changelog_entry.yaml b/changelog_entry.yaml index ffa169491..1494b45b0 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,6 +1,7 @@ - bump: patch changes: added: + - Added optional disclosure fields to UK constituency and local-authority records (disclosure_tier, reported or approximate, and the 90% interval bounds of the average and relative income change); records stored before this change still validate - Added the rule sim.boundaries.sqlalchemy-and-alembic-only, with proof from the psycopg adapter of 08ebe1ca; code connects to databases only through SQLAlchemy, never by importing a database driver, and runtime code never changes the schema, which changes only through Alembic migrations; a guard parses every file under projects/*/src, libs/*/src and .github/scripts and rejects driver imports, alembic imports outside a migrations directory, create_all and drop_all, create and drop on tables, SQLAlchemy DDL statements and DDL text in string literals, which extends the Stage 12 DDL check's pattern to the gateway and every library - Made the budget's reform-minus-baseline totals log a warning when the baseline and reform household frames carry different weights, since the change is then weighted by the reform's weights alone - Added weighted-microdata rules to the registry, with proof from the #767 household count; the executor's code never aggregates a weight column as a value, reads raw values from weighted frames only in simulation_output_common.py (the UK local-authority breakdown's plain household frame is its one recorded exception there, in a list that may only shrink), and its output-builder tests build entity frames through one shared factory with non-uniform weights, while a doc-only rule records that weights, values, labels and counts each allow different operations @@ -25,6 +26,7 @@ - Ran the Modal image smoke for Stage 12 persistence changes and added policyengine-fastapi to the PR test matrix - Ran the full PR workflow and the Modal image smoke only on the lowest unmerged pull request in a stack, behind a new CI complete gate job that fails on the others fixed: + - Built UK constituency and local-authority results with policyengine's disclosure-controlled compute_uk_geography_disclosure instead of the deprecated, unprotected functions, in one release nested in regions and the nation; an area that rests on fewer than 10 survey households, is dominated by one, would be recovered by subtraction, or has a non-positive baseline income is left out of the list, so small areas no longer show exploding relative changes such as +1372% for a £192 loss - Preserved bounded reform stop dates in UK and US simulations while retaining current-law baseline artifact reuse - Resolved PolicyEngine-managed regional datasets using bundled data versions instead of release-manifest commit hashes when loading from GCS - Added a Modal app-release bundle backfill for missing data artifact revisions diff --git a/docs/engineering/skills/microdata-weights.md b/docs/engineering/skills/microdata-weights.md index 9a84a2d87..ed430851e 100644 --- a/docs/engineering/skills/microdata-weights.md +++ b/docs/engineering/skills/microdata-weights.md @@ -82,7 +82,6 @@ Each finding needs a decision, recorded in the PR: Recorded exceptions are listed in `rules/baselines/sim-microdata-weight-stripping.txt`. That list may only shrink, so adding one is a deliberate change to the rule's baseline. Today it -holds one: `_unweighted_household_frame`, the plain household frame the UK -local-authority breakdown hands policyengine's longwise geography impact. It -goes once that breakdown hands policyengine the simulations, as the -constituency and US congressional district breakdowns already do. +is empty: the last one, `_unweighted_household_frame`, went when the UK +constituency and local-authority breakdowns moved to policyengine's +disclosure-controlled geography output, which takes the simulations. diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 7d2a09995..5593c4e0a 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -66,7 +66,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf ### Ratcheted (existing violations still tolerated) - `sim.ci.multiline-run-is-script`: 13 remaining in `rules/baselines/ci-multiline-run.txt` -- `sim.microdata.weight-stripping-confined`: 1 remaining in `rules/baselines/sim-microdata-weight-stripping.txt` +- `sim.microdata.weight-stripping-confined`: 0 remaining in `rules/baselines/sim-microdata-weight-stripping.txt` ### Exceptions and review dates diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py index a45cddec4..5fcd2bb60 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Generic, TypeVar +from typing import Generic, Literal, TypeVar from pydantic import BaseModel, ConfigDict, RootModel @@ -234,12 +234,38 @@ class CliffImpactOutput(MacroOutputModel): reform: CliffImpactInSimulation -class ConstituencyImpactRecord(MacroOutputModel): +UKGeographyDisclosureTier = Literal["reported", "approximate"] + + +class UKAreaDisclosureFields(MacroOutputModel): + """Disclosure tier and 90% intervals of a published UK area's income changes. + + UK area results come from licensed survey microdata and pass policyengine's + disclosure control. An area whose income change is withheld is left out of + the list, so every record here has values. ``disclosure_tier`` is + ``reported``, or ``approximate`` when the area's effective sample is thin + and the interval understates the uncertainty. The ``_lower`` and ``_upper`` + fields bound each change's 90% bootstrap interval, rounded outward. + + All of these fields are optional, so results stored before disclosure + control still validate; there they are null. + """ + + disclosure_tier: UKGeographyDisclosureTier | None = None + average_household_income_change_lower: float | None = None + average_household_income_change_upper: float | None = None + relative_household_income_change_lower: float | None = None + relative_household_income_change_upper: float | None = None + + +class ConstituencyImpactRecord(UKAreaDisclosureFields): """Reform impact on household net income in one UK parliamentary constituency. ``average_household_income_change`` is the average change per household and ``relative_household_income_change`` the relative change in the area's total - household net income. ``x`` and ``y`` are integer map-rendering coordinates. + household net income, both rounded to the order of their standard error. + ``population`` is the weighted number of households. ``x`` and ``y`` are + integer map-rendering coordinates. """ constituency_code: str @@ -251,7 +277,7 @@ class ConstituencyImpactRecord(MacroOutputModel): population: float -class LocalAuthorityImpactRecord(MacroOutputModel): +class LocalAuthorityImpactRecord(UKAreaDisclosureFields): """Reform impact on household net income in one UK local authority. Fields mean the same as in ``ConstituencyImpactRecord``. @@ -269,7 +295,10 @@ class LocalAuthorityImpactRecord(MacroOutputModel): class GeographicImpactOutput( MacroRootModel[list[ConstituencyImpactRecord | LocalAuthorityImpactRecord]] ): - """UK constituency or local-authority impact records.""" + """UK constituency or local-authority impact records. + + Areas whose income change disclosure control withholds are left out. + """ class CongressionalDistrictImpactRecord(MacroOutputModel): diff --git a/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py b/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py new file mode 100644 index 000000000..87d8ae48a --- /dev/null +++ b/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py @@ -0,0 +1,74 @@ +"""UK area records stay readable before and after disclosure control.""" + +from __future__ import annotations + +import pytest +from policyengine_simulation_contract.macro_output import ( + ConstituencyImpactRecord, + GeographicImpactOutput, + LocalAuthorityImpactRecord, +) +from pydantic import ValidationError + +STORED_BEFORE_DISCLOSURE = { + "constituency_code": "E14001063", + "constituency_name": "Aldershot", + "x": 56, + "y": -40, + "average_household_income_change": -150.95, + "relative_household_income_change": -0.0026, + "population": 40_530.65, +} +DISCLOSURE_FIELDS = { + "disclosure_tier": "approximate", + "average_household_income_change_lower": -170.0, + "average_household_income_change_upper": -130.0, + "relative_household_income_change_lower": -0.003, + "relative_household_income_change_upper": -0.0022, +} + + +def test_records_stored_before_disclosure_control_still_validate() -> None: + record = ConstituencyImpactRecord.model_validate(STORED_BEFORE_DISCLOSURE) + + assert record.disclosure_tier is None + assert record.average_household_income_change_lower is None + assert record.relative_household_income_change_upper is None + + +def test_records_carry_the_disclosure_tier_and_intervals() -> None: + record = LocalAuthorityImpactRecord.model_validate( + { + "local_authority_code": "E06000063", + "local_authority_name": "Cumberland", + "x": 4, + "y": 18, + "average_household_income_change": -150.0, + "relative_household_income_change": -0.0026, + "population": 120_000.0, + **DISCLOSURE_FIELDS, + } + ) + + assert record.model_dump(mode="json")["disclosure_tier"] == "approximate" + assert record.average_household_income_change_lower == -170.0 + + +def test_a_withheld_tier_is_not_a_record() -> None: + # Withheld areas are left out of the list, so no record holds that tier. + with pytest.raises(ValidationError): + ConstituencyImpactRecord.model_validate( + {**STORED_BEFORE_DISCLOSURE, "disclosure_tier": "withheld"} + ) + + +def test_published_values_stay_required() -> None: + record = {**STORED_BEFORE_DISCLOSURE, **DISCLOSURE_FIELDS} + record["average_household_income_change"] = None + + with pytest.raises(ValidationError): + ConstituencyImpactRecord.model_validate(record) + + +def test_an_empty_area_list_is_valid() -> None: + assert GeographicImpactOutput.model_validate([]).root == [] diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py index 5a86a0482..6e19ad766 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py @@ -68,6 +68,7 @@ class SimulationOutputBuilder: uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None progress: JobProgressReporter = NULL_JOB_PROGRESS _analysis: Any = field(default=None, init=False) + _uk_geography_disclosure: Any = field(default=None, init=False) def __post_init__(self) -> None: """Normalize ``country`` to lower case.""" @@ -257,18 +258,53 @@ def _build_congressional_district_impact( region_code=self.resolved_region_code, ) + def _uk_geography_disclosure_output( + self, + ) -> simulation_output_geographic.UKGeographyDisclosureOutput | None: + """Return UK area changes under disclosure control, computed once. + + Constituencies and local authorities go through one release, so the + disclosure check sees both. None for other countries. + """ + if self.country != "uk": + return None + if self._uk_geography_disclosure is None: + self._uk_geography_disclosure = ( + simulation_output_geographic.compute_uk_geography_disclosure_output( + self.country, + self.baseline, + self.reform, + year=self._simulation_year(), + ) + ) + return self._uk_geography_disclosure + + def _simulation_year(self) -> int: + """Return the simulated year, which labels the disclosure release.""" + for value in ( + self.simulation_params.get("time_period"), + self.simulation_params.get("year"), + getattr(self.dataset, "year", None), + ): + if value is None: + continue + try: + return int(value) + except (TypeError, ValueError): + continue + return 0 + def _build_uk_constituency_impact(self) -> GeographicImpactOutput | None: with self._span(ANNUAL_IMPACT_STAGES.name(Stage.OUTPUT_UK_CONSTITUENCY)): return simulation_output_geographic.build_uk_constituency_impact( - self.country, self.baseline, self.reform + self.country, self._uk_geography_disclosure_output() ) def _build_uk_local_authority_impact(self) -> GeographicImpactOutput | None: with self._span(ANNUAL_IMPACT_STAGES.name(Stage.OUTPUT_UK_LOCAL_AUTHORITY)): return simulation_output_geographic.build_uk_local_authority_impact( self.country, - self.baseline, - self.reform, + self._uk_geography_disclosure_output(), uk_local_authority_metadata=self.uk_local_authority_metadata, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py index 4ce0d2feb..ae5c73308 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py @@ -8,10 +8,10 @@ from importlib import import_module from typing import TYPE_CHECKING, Any -import pandas as pd from policyengine_simulation_contract.spm import spm_error_detail if TYPE_CHECKING: + import pandas as pd from _typeshed import ConvertibleToFloat from policyengine.core import Simulation @@ -77,23 +77,6 @@ def _count_output_entities(simulation: Simulation, entity: str, weight: str) -> return float(np.asarray(data[weight], dtype=float).sum()) -def _unweighted_household_frame(simulation: Simulation) -> pd.DataFrame: - """Return the household output as a plain DataFrame, without its weights. - - policyengine's ``compute_longwise_uk_geography_impacts`` takes plain frames - and weights each area by the baseline's ``household_weight`` column itself. - The UK local-authority breakdown calls it directly, so that it can apply - this service's boundary metadata. Handing it the weighted frame would weight - twice if it ever reduced a column directly. - - This is a recorded exception to sim.microdata.weight-stripping-confined, - listed in rules/baselines/sim-microdata-weight-stripping.txt, which may only - shrink. It goes once the breakdown hands policyengine the simulations, as - the constituency and US congressional district breakdowns already do. - """ - return pd.DataFrame(simulation.output_dataset.data.household) - - def _warn_on_mismatched_weights( baseline_data: pd.DataFrame, reform_data: pd.DataFrame, entity: str ) -> None: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index 849c6c7a1..9c978ad37 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -2,13 +2,29 @@ from __future__ import annotations +import logging +import math from collections.abc import Mapping +from dataclasses import dataclass, field from functools import lru_cache -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast +import pandas as pd from policyengine.data.uk_geography_assets import ( CONSTITUENCY_ASSET_SPEC, ) +from policyengine.disclosure import UNASSIGNED, DisclosureError +from policyengine.disclosure.areas import normalise_area_codes +from policyengine.disclosure.schema import ( + APPROXIMATE, + AVERAGE_CHANGE, + FAMILY_CHANGE, + FAMILY_GAINING, + FAMILY_LOSING, + RELATIVE_CHANGE, + REPORTED, + UNCHECKED_OVERLAP, +) from policyengine_simulation_executor.simulation_macro_output import ( CongressionalDistrictImpactOutput, @@ -16,19 +32,24 @@ GeographicImpactOutput, ) from policyengine_simulation_executor.simulation_output_common import ( + _entity_data, _number, _output_model_dump, - _output_module_function, _try_compute_output, - _unweighted_household_frame, ) if TYPE_CHECKING: - import pandas as pd from policyengine.core import Simulation from policyengine.data.uk_geography_assets import UKGeographyAssetSpec + from policyengine.disclosure import PublicTable + from policyengine.outputs.uk_geography_disclosure import UKGeographyDisclosure + from policyengine_simulation_contract.macro_output import ( + UKGeographyDisclosureTier, + ) from policyengine_simulation_contract.uk_geography import UKLocalAuthorityMetadata +logger = logging.getLogger(__name__) + def _required_uk_geography_lookup_csv_path(spec: UKGeographyAssetSpec) -> str: """Resolve one required lookup under the runtime GCP identity.""" @@ -60,10 +81,13 @@ def _complete_uk_geography_output( code_field: str, name_field: str, ) -> GeographicImpactOutput: - """Require every UK geography record to contain lookup-owned metadata.""" + """Require every UK geography record to contain lookup-owned metadata. + + An empty list is valid: disclosure control may withhold every area. + """ output = build_geographic_impact_output(value) - if output is None or not output.root: - msg = "UK geography output did not contain result records" + if output is None: + msg = "UK geography output did not contain a list of result records" raise ValueError(msg) for record in output.root: record_values = record.model_dump(mode="python") @@ -236,6 +260,77 @@ def compute_and_format() -> CongressionalDistrictImpactOutput | None: return _try_compute_output("congressional district impacts", compute_and_format) +# The household output column that holds each published UK level's area code. +UK_DISCLOSURE_AREA_COLUMNS: dict[str, str] = { + "constituency": "constituency_code_oa", + "local_authority": "la_code_oa", +} +UK_DISCLOSURE_BREAKDOWN_NAMES: dict[str, str] = { + "constituency": "constituency", + "local_authority": "local-authority", +} +# The survey household each row was copied or cloned from. The count floor +# counts support by this id, so copies of one survey household count once. +UK_SOURCE_HOUSEHOLD_ID_COLUMN = "source_household_id" +# Bootstrap replicates behind each 90% interval: the disclosure layer's +# minimum. They set the interval and the rounding, never whether a cell is +# published, and the layer's default of 1,000 would add minutes to every UK run. +UK_DISCLOSURE_REPLICATES = 200 +UK_DISCLOSURE_REFORM_LABEL = "reform" +# Each round withholds at least one more cell, so this bounds the check runs. +UK_DISCLOSURE_MAX_ROUNDS = 25 +# The output publishes the change family only. The share families are +# withheld before the check, so the check treats them as secret, which they +# are, and never asks for cells to protect shares nobody publishes. +_UNPUBLISHED_FAMILY_OUTCOMES = (FAMILY_GAINING, FAMILY_LOSING) +_FAMILY_OUTCOME = { + FAMILY_CHANGE: AVERAGE_CHANGE, + FAMILY_GAINING: FAMILY_GAINING, + FAMILY_LOSING: FAMILY_LOSING, +} + + +@dataclass(frozen=True) +class UKAreaIncomeChange: + """One published UK area's disclosure-controlled household income change.""" + + tier: UKGeographyDisclosureTier + average: float + average_lower: float + average_upper: float + relative: float + relative_lower: float + relative_upper: float + population: float + + def record_fields(self) -> dict[str, object]: + """Return the fields this change contributes to an area record.""" + return { + "average_household_income_change": self.average, + "relative_household_income_change": self.relative, + "population": self.population, + "disclosure_tier": self.tier, + "average_household_income_change_lower": self.average_lower, + "average_household_income_change_upper": self.average_upper, + "relative_household_income_change_lower": self.relative_lower, + "relative_household_income_change_upper": self.relative_upper, + } + + +@dataclass(frozen=True) +class UKGeographyDisclosureOutput: + """The published areas of each UK level, keyed by area code. + + An area the disclosure check withholds is absent. + """ + + areas: Mapping[str, Mapping[str, UKAreaIncomeChange]] = field(default_factory=dict) + + def level(self, level: str) -> Mapping[str, UKAreaIncomeChange]: + """Return the published areas of one level.""" + return self.areas.get(level, {}) + + def _require_uk_area_column( household: pd.DataFrame, column: str, breakdown: str ) -> None: @@ -254,31 +349,335 @@ def _require_uk_area_column( raise ValueError(msg) -def build_uk_constituency_impact( - country: str, baseline: Simulation, reform: Simulation -) -> GeographicImpactOutput | None: - """Compute UK parliamentary constituency impacts; None for other countries. +def _require_uk_source_household_ids(household: pd.DataFrame) -> None: + """Refuse UK area outputs when support cannot be counted by survey household.""" + if UK_SOURCE_HOUSEHOLD_ID_COLUMN not in household.columns: + msg = ( + f"UK constituency and local-authority breakdowns need " + f"{UK_SOURCE_HOUSEHOLD_ID_COLUMN} on the simulated dataset: disclosure " + "control counts the survey households behind each area by it." + ) + raise ValueError(msg) + + +def _normalised_area_code(value: object) -> str: + """Return an area code as the disclosure layer keys it.""" + return str(normalise_area_codes([value])[0]) + + +def _uk_area_household_counts(household: pd.DataFrame, column: str) -> dict[str, float]: + """Return the weighted number of households in each area of one level. + + ``household`` is the weighted output frame, so a grouped count is the sum + of household weights in each area. Areas without weight are left out, as + the disclosure layer leaves them out. + """ + counts = household.groupby(column)["household_net_income"].count() + by_code: dict[str, float] = {} + for raw_code, count in counts.items(): + code = _normalised_area_code(raw_code) + if code == UNASSIGNED: + continue + by_code[code] = by_code.get(code, 0.0) + float(cast("float", count)) + return {code: count for code, count in by_code.items() if count > 0} + + +def _withhold_unpublished_families( + impact: UKGeographyDisclosure, + area_codes: Mapping[str, Mapping[str, float]], + *, + year: int, +) -> None: + release = impact.release + for level, codes in area_codes.items(): + for code in codes: + for outcome in _UNPUBLISHED_FAMILY_OUTCOMES: + try: + release.withhold( + level=level, + area_code=code, + reform=UK_DISCLOSURE_REFORM_LABEL, + year=year, + outcome=outcome, + ) + except DisclosureError: + # Not an area of the release, for example one whose + # households all have zero weight. + continue + + +def _cells_resolving_findings( + impact: UKGeographyDisclosure, published_levels: tuple[str, ...] +) -> set[tuple[str, str, str, str, int]]: + """Return the published cells to withhold for the blocking findings. + + A finding names a cell to withhold, or else the group it is about. Only + cells of published levels are withheld: regions and the nation are + references whose numbers are public elsewhere, so withholding them here + would only hide the risk. + """ + cells: set[tuple[str, str, str, str, int]] = set() + for finding in impact.findings or []: + if not finding.blocking or finding.family is None: + continue + if finding.reform is None or finding.year is None: + continue + outcome = _FAMILY_OUTCOME.get(finding.family) + if outcome is None: + continue + if finding.suggested and finding.level in published_levels: + cells.add( + ( + finding.level, + finding.suggested, + outcome, + finding.reform, + finding.year, + ) + ) + elif finding.group and finding.group_level in published_levels: + cells.add( + ( + finding.group_level, + finding.group, + outcome, + finding.reform, + finding.year, + ) + ) + return cells + + +def _resolved_public_table( + impact: UKGeographyDisclosure, + area_codes: Mapping[str, Mapping[str, float]], + *, + year: int, +) -> PublicTable | None: + """Withhold cells until no finding blocks, and return the public table. + + Returns None when the findings cannot be resolved by withholding published + cells; then nothing at these levels may be published. + """ + published_levels = tuple(area_codes) + _withhold_unpublished_families(impact, area_codes, year=year) + public = impact.refresh() + withheld: set[tuple[str, str, str, str, int]] = set() + for _ in range(UK_DISCLOSURE_MAX_ROUNDS): + if public is not None: + return public + cells = _cells_resolving_findings(impact, published_levels) - withheld + if not cells: + break + for level, code, outcome, reform, cell_year in sorted(cells): + try: + impact.release.withhold( + level=level, + area_code=code, + reform=reform, + year=cell_year, + outcome=outcome, + ) + except DisclosureError: + # Not a cell of the release. The finding stays, so the loop + # ends without progress and nothing is published. + continue + withheld |= cells + public = impact.refresh() + if public is not None: + return public + blocking = sorted( + {finding.code for finding in impact.findings or [] if finding.blocking} + ) + # Findings carry area codes; the log names only their kinds. + logger.warning( + "UK geography disclosure findings remain after withholding %d cells " + "(%s), so no constituency or local-authority area is published", + len(withheld), + ", ".join(blocking), + ) + return None + + +def _published_area_changes( + public: PublicTable, + level: str, + populations: Mapping[str, float], +) -> dict[str, UKAreaIncomeChange]: + """Return the areas of one level whose average and relative change are public. - Downloads the constituency lookup if needed and raises when the dataset - lacks constituency codes or a record lacks lookup name or coordinates. + An area is published only when both changes are: the contract has a value + for each. A relative change is withheld on its own, for example, when the + area's baseline income is not positive. + """ + rows: dict[str, dict[str, Any]] = {} + for row in public.rows: + if row.level != level or row.outcome not in {AVERAGE_CHANGE, RELATIVE_CHANGE}: + continue + if row.tier not in {REPORTED, APPROXIMATE} or row.value is None: + continue + if row.lo is None or row.hi is None: + continue + rows.setdefault(row.area_code, {})[row.outcome] = row + changes: dict[str, UKAreaIncomeChange] = {} + for code, outcomes in rows.items(): + average = outcomes.get(AVERAGE_CHANGE) + relative = outcomes.get(RELATIVE_CHANGE) + if average is None or relative is None or code not in populations: + continue + tier: UKGeographyDisclosureTier = ( + APPROXIMATE if APPROXIMATE in {average.tier, relative.tier} else REPORTED + ) + changes[code] = UKAreaIncomeChange( + tier=tier, + average=float(average.value), + average_lower=float(average.lo), + average_upper=float(average.hi), + relative=float(relative.value), + relative_lower=float(relative.lo), + relative_upper=float(relative.hi), + population=populations[code], + ) + return changes + + +def compute_uk_geography_disclosure_output( + country: str, + baseline: Simulation, + reform: Simulation, + *, + year: int, +) -> UKGeographyDisclosureOutput | None: + """Compute disclosure-controlled constituency and local-authority changes. + + None for other countries. Both levels go through one policyengine release, + nested in regions and the nation, so the check sees what subtraction + between them recovers. Raises when the dataset lacks area codes or survey + household ids. """ if country != "uk": return None - _require_uk_area_column( - baseline.output_dataset.data.household, "constituency_code_oa", "constituency" + household = _entity_data(baseline, "household") + _entity_data(reform, "household") + for level, column in UK_DISCLOSURE_AREA_COLUMNS.items(): + _require_uk_area_column(household, column, UK_DISCLOSURE_BREAKDOWN_NAMES[level]) + _require_uk_source_household_ids(household) + + from policyengine.disclosure import ( # noqa: PLC0415 # bound per call, like this module's other policyengine.outputs imports + DisclosureSettings, + ) + from policyengine.outputs.uk_geography_disclosure import ( # noqa: PLC0415 # bound per call, like this module's other policyengine.outputs imports + compute_uk_geography_disclosure, ) - lookup_csv_path = _required_uk_geography_lookup_csv_path(CONSTITUENCY_ASSET_SPEC) - impact = _output_module_function( - "constituency_impact", "compute_uk_constituency_impacts" - )( + area_codes = { + level: _uk_area_household_counts(household, column) + for level, column in UK_DISCLOSURE_AREA_COLUMNS.items() + } + impact = compute_uk_geography_disclosure( baseline, reform, - constituency_csv_path=lookup_csv_path, - download_missing_assets=False, + year=year, + levels=tuple(UK_DISCLOSURE_AREA_COLUMNS), + reform=UK_DISCLOSURE_REFORM_LABEL, + settings=DisclosureSettings(n_replicates=UK_DISCLOSURE_REPLICATES), + # Constituencies and local authorities overlap without nesting. The + # release checks each area against the areas inside and around it; + # sim-api publishes both levels from every UK run, so it accepts that + # other differences between them stay unchecked. + acknowledge=(UNCHECKED_OVERLAP,), + ) + public = _resolved_public_table(impact, area_codes, year=year) + if public is None: + return UKGeographyDisclosureOutput() + return UKGeographyDisclosureOutput( + areas={ + level: _published_area_changes(public, level, area_codes[level]) + for level in UK_DISCLOSURE_AREA_COLUMNS + } + ) + + +def _uk_constituency_lookup( + lookup_csv_path: str, +) -> dict[str, tuple[str, int | None, int | None]]: + """Return each constituency's name and map coordinates, in lookup order.""" + lookup = pd.read_csv(lookup_csv_path) + if "code" not in lookup.columns: + msg = ( + "UK constituency lookup CSV must contain a 'code' column: " + f"{lookup_csv_path}" + ) + raise ValueError(msg) + entries: dict[str, tuple[str, int | None, int | None]] = {} + for row in lookup.to_dict("records"): + code = _normalised_area_code(row.get("code")) + if not code: + continue + name = _normalised_area_code(row.get("name")) or code + entries[code] = ( + name, + _lookup_coordinate(row.get("x")), + _lookup_coordinate(row.get("y")), + ) + return entries + + +def _lookup_coordinate(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int | float | str): + return None + try: + number = float(value) + except ValueError: + return None + return int(number) if math.isfinite(number) else None + + +def _require_uk_disclosure_output( + disclosure: UKGeographyDisclosureOutput | None, breakdown: str +) -> UKGeographyDisclosureOutput: + if disclosure is None: + msg = ( + f"UK {breakdown} output needs the disclosure-controlled UK geography output" + ) + raise ValueError(msg) + return disclosure + + +def build_uk_constituency_impact( + country: str, disclosure: UKGeographyDisclosureOutput | None +) -> GeographicImpactOutput | None: + """Return the published UK parliamentary constituencies; None for other countries. + + Areas that disclosure control withholds are left out. Downloads the + constituency lookup if needed and raises when a published area lacks a + lookup name or coordinates. + """ + if country != "uk": + return None + changes = _require_uk_disclosure_output(disclosure, "constituency").level( + "constituency" ) + lookup = _uk_constituency_lookup( + _required_uk_geography_lookup_csv_path(CONSTITUENCY_ASSET_SPEC) + ) + ordered = [code for code in lookup if code in changes] + sorted( + set(changes) - set(lookup) + ) + records: list[dict[str, object]] = [] + for code in ordered: + name, x, y = lookup.get(code, (code, None, None)) + records.append( + { + "constituency_code": code, + "constituency_name": name, + "x": x, + "y": y, + **changes[code].record_fields(), + } + ) return _complete_uk_geography_output( - getattr(impact, "constituency_results", None), + records, code_field="constituency_code", name_field="constituency_name", ) @@ -286,51 +685,35 @@ def build_uk_constituency_impact( def build_uk_local_authority_impact( country: str, - baseline: Simulation, - reform: Simulation, + disclosure: UKGeographyDisclosureOutput | None, *, uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None, ) -> GeographicImpactOutput | None: - """Compute UK local authority impacts, named and placed for the detected boundaries. + """Return the published UK local authorities, named and placed for the boundaries. None for other countries. For the UK, ``uk_local_authority_metadata`` is required, and a code outside its boundary version raises ``ValueError``. + Areas that disclosure control withholds are left out. """ if country != "uk": return None - - baseline_household = _unweighted_household_frame(baseline) - _require_uk_area_column(baseline_household, "la_code_oa", "local-authority") + changes = _require_uk_disclosure_output(disclosure, "local-authority").level( + "local_authority" + ) if uk_local_authority_metadata is None: msg = "UK local-authority boundary metadata is required" raise ValueError(msg) - from policyengine.outputs.uk_geography_impact import ( # noqa: PLC0415 # bound per call, like this module's patched policyengine.outputs import above - compute_longwise_uk_geography_impacts, - ) - from policyengine_simulation_executor.uk_local_authority_metadata import ( # noqa: PLC0415 # the UK metadata module loads only for UK local-authority output load_uk_local_authority_resources, ) - reform_household = _unweighted_household_frame(reform) - numeric_records = compute_longwise_uk_geography_impacts( - baseline_household=baseline_household, - reform_household=reform_household, - geography_column="la_code_oa", - result_key_prefix="local_authority", - lookup_csv_path=None, - ) boundary_version = uk_local_authority_metadata.boundary_version display_metadata = load_uk_local_authority_resources().metadata_for( boundary_version ) records: list[dict[str, object]] = [] - for numeric_record in numeric_records: - code = numeric_record.get("local_authority_code") - if not isinstance(code, str) or not code: - msg = "UK local-authority output contains an invalid code" - raise ValueError(msg) + for code in sorted(changes): metadata = display_metadata.get(code) if metadata is None: msg = ( @@ -340,10 +723,11 @@ def build_uk_local_authority_impact( raise ValueError(msg) records.append( { - **numeric_record, + "local_authority_code": code, "local_authority_name": metadata.name, "x": metadata.x, "y": metadata.y, + **changes[code].record_fields(), } ) return _complete_uk_geography_output( diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py index cd0cbee0b..84dda7c0a 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py @@ -27,8 +27,16 @@ import pandas as pd from policyengine_simulation_contract.stage12_bundle import CountryId +# Input-dataset columns the UK constituency and local-authority breakdowns +# read: the area codes, the region the disclosure check nests them in, and the +# survey household each row was copied from, by which it counts support. UK_GEOGRAPHIC_DATASET_VARIABLES: dict[str, tuple[str, ...]] = { - "household": ("constituency_code_oa", "la_code_oa"), + "household": ( + "constituency_code_oa", + "la_code_oa", + "region_code_oa", + "source_household_id", + ), } diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py index 9d95eab0b..321a3011c 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py @@ -33,6 +33,7 @@ UKLocalAuthorityBoundaryVersion, UKLocalAuthorityMetadata, ) +from policyengine_simulation_executor import simulation_output_geographic from policyengine_simulation_executor.job_progress import NULL_JOB_PROGRESS from policyengine_simulation_executor.release_bundle import ( BUNDLE_RECEIPT_FILENAME, @@ -81,8 +82,6 @@ ) from policyengine_simulation_observability.stages import Stage -CONSTITUENCY_CSV_PATH = "/tmp/constituencies_2024.csv" - def _params_with_spm(params, selection): """Return request parameters after the runtime resolves SPM settings.""" @@ -1771,77 +1770,82 @@ def fail_change_output_variable(*_args, **_kwargs): _simulation_output_builder("us", baseline, reform)._build_budgetary_impact() -def test_uk_constituency_impact_uses_policyengine_output_function(monkeypatch): - def coded_simulation() -> SimpleNamespace: - household = weighted_entity_frame({"constituency_code_oa": ["E14001063"]}) - return SimpleNamespace( - output_dataset=SimpleNamespace(data=SimpleNamespace(household=household)) - ) +def test_uk_area_impacts_share_one_disclosure_release(monkeypatch): + baseline = object() + reform = object() + disclosure = simulation_output_geographic.UKGeographyDisclosureOutput() + computed: list[tuple[str, object, object, int]] = [] + built: list[tuple[str, object]] = [] - baseline = coded_simulation() - reform = coded_simulation() - expected = [_constituency_impact_record().model_dump(mode="json")] + def compute(country, baseline_simulation, reform_simulation, *, year): + computed.append((country, baseline_simulation, reform_simulation, year)) + return disclosure - def fake_output_module_function(module_name, name): - assert module_name == "constituency_impact" - assert name == "compute_uk_constituency_impacts" + def build_constituencies(country, value): + built.append(("constituency", value)) + return GeographicImpactOutput.model_validate([]) - def compute(baseline_simulation, reform_simulation, **kwargs): - assert baseline_simulation is baseline - assert reform_simulation is reform - assert kwargs == { - "constituency_csv_path": CONSTITUENCY_CSV_PATH, - "download_missing_assets": False, - } - return SimpleNamespace(constituency_results=expected) - - return compute + def build_authorities(country, value, **kwargs): + built.append(("local_authority", value)) + return GeographicImpactOutput.model_validate([]) monkeypatch.setattr( - "policyengine_simulation_executor.simulation_output_geographic._output_module_function", - fake_output_module_function, + "policyengine_simulation_executor.simulation_output_geographic." + "compute_uk_geography_disclosure_output", + compute, + ) + monkeypatch.setattr( + "policyengine_simulation_executor.simulation_output_geographic." + "build_uk_constituency_impact", + build_constituencies, ) monkeypatch.setattr( "policyengine_simulation_executor.simulation_output_geographic." - "_required_uk_geography_lookup_csv_path", - lambda _spec: CONSTITUENCY_CSV_PATH, + "build_uk_local_authority_impact", + build_authorities, ) - result = _simulation_output_builder( - "uk", baseline, reform - )._build_uk_constituency_impact() - assert result is not None - assert result.model_dump(mode="json") == expected - assert ( - _simulation_output_builder( - "us", baseline, reform - )._build_uk_constituency_impact() - is None + builder = _simulation_output_builder("uk", baseline, reform) + builder.simulation_params["time_period"] = "2026" + builder._build_uk_constituency_impact() + builder._build_uk_local_authority_impact() + + # Both levels come from one release, computed once. + assert computed == [("uk", baseline, reform, 2026)] + assert built == [("constituency", disclosure), ("local_authority", disclosure)] + + +def test_uk_disclosure_is_not_computed_for_other_countries(monkeypatch): + monkeypatch.setattr( + "policyengine_simulation_executor.simulation_output_geographic." + "compute_uk_geography_disclosure_output", + lambda *_args, **_kwargs: pytest.fail("US runs need no UK disclosure"), ) + builder = _simulation_output_builder("us", object(), object()) + + assert builder._build_uk_constituency_impact() is None + assert builder._build_uk_local_authority_impact() is None def test_uk_local_authority_impact_receives_detected_boundary_metadata(monkeypatch): - baseline = object() - reform = object() expected = [_local_authority_impact_record().model_dump(mode="json")] metadata = UKLocalAuthorityMetadata( boundary_version=UKLocalAuthorityBoundaryVersion.LAD23 ) - calls: list[tuple[object, object, UKLocalAuthorityMetadata | None]] = [] + disclosure = simulation_output_geographic.UKGeographyDisclosureOutput() + calls: list[tuple[object, UKLocalAuthorityMetadata | None]] = [] - def build(country, baseline_simulation, reform_simulation, **kwargs): + def build(country, value, **kwargs): if country != "uk": return None - assert country == "uk" - calls.append( - ( - baseline_simulation, - reform_simulation, - kwargs["uk_local_authority_metadata"], - ) - ) + calls.append((value, kwargs["uk_local_authority_metadata"])) return GeographicImpactOutput.model_validate(expected) + monkeypatch.setattr( + "policyengine_simulation_executor.simulation_output_geographic." + "compute_uk_geography_disclosure_output", + lambda *_args, **_kwargs: disclosure, + ) monkeypatch.setattr( "policyengine_simulation_executor.simulation_output_geographic." "build_uk_local_authority_impact", @@ -1850,16 +1854,10 @@ def build(country, baseline_simulation, reform_simulation, **kwargs): result = _simulation_output_builder( "uk", - baseline, - reform, + object(), + object(), BuilderOptions(uk_local_authority_metadata=metadata), )._build_uk_local_authority_impact() assert result is not None assert result.model_dump(mode="json") == expected - assert calls == [(baseline, reform, metadata)] - assert ( - _simulation_output_builder( - "us", baseline, reform - )._build_uk_local_authority_impact() - is None - ) + assert calls == [(disclosure, metadata)] diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py index 5e5e20cdf..e3d8320ff 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py @@ -5,6 +5,7 @@ from contextlib import nullcontext from types import SimpleNamespace +import numpy as np import pytest from fixtures.weighted_frames import weighted_entity_frame from policyengine.data.uk_geography_assets import CONSTITUENCY_ASSET_SPEC @@ -29,22 +30,6 @@ def _complete_constituency_record() -> dict[str, object]: } -def _uk_simulation( - code: str, - income: float, -) -> SimpleNamespace: - household = weighted_entity_frame( - { - "la_code_oa": [code], - "household_net_income": [income], - "household_weight": [2.0], - } - ) - return SimpleNamespace( - output_dataset=SimpleNamespace(data=SimpleNamespace(household=household)) - ) - - def test_required_uk_lookup_uses_gcp_credentials(monkeypatch) -> None: observed: list[str] = [] @@ -104,10 +89,21 @@ def test_complete_uk_geography_output_accepts_lookup_metadata() -> None: assert record["y"] == -40 -def test_complete_uk_geography_output_rejects_empty_result() -> None: - with pytest.raises(ValueError, match="did not contain result records"): +def test_complete_uk_geography_output_accepts_an_empty_list() -> None: + # Disclosure control may withhold every area of a level. + result = simulation_output_geographic._complete_uk_geography_output( + [], + code_field="constituency_code", + name_field="constituency_name", + ) + + assert result.root == [] + + +def test_complete_uk_geography_output_rejects_a_missing_list() -> None: + with pytest.raises(ValueError, match="did not contain a list"): simulation_output_geographic._complete_uk_geography_output( - [], + None, code_field="constituency_code", name_field="constituency_name", ) @@ -138,18 +134,169 @@ def test_complete_uk_geography_output_rejects_fallback_metadata( ) -@pytest.mark.parametrize( - ("boundary_version", "code", "expected_name"), - [ - (UKLocalAuthorityBoundaryVersion.LAD22, "E07000026", "Allerdale"), - (UKLocalAuthorityBoundaryVersion.LAD23, "E06000063", "Cumberland"), - ], -) -def test_local_authority_output_uses_detected_boundary_metadata( - monkeypatch, - boundary_version: UKLocalAuthorityBoundaryVersion, - code: str, - expected_name: str, +# Disclosure-controlled UK breakdowns. The households below sit in one region +# and two LAD23 local authorities. Each published seat rests on 30 survey +# households, well above the floor of 10; THIN has 3, so it is withheld, and +# the check then withholds one more seat so that subtraction from the region +# or from its local authority cannot recover it. NEGATIVE has a negative total +# baseline income, the Cardiff West case: its relative change is undefined, so +# the seat is left out although its average change passes. +NORTH_WEST = "E12000002" +CUMBERLAND = "E06000063" +WESTMORLAND = "E06000064" +SEATS = { + "E14000001": (CUMBERLAND, 30), + "E14000002": (CUMBERLAND, 30), + "E14000003": (WESTMORLAND, 30), + "E14000004": (WESTMORLAND, 30), + "E14000005": (WESTMORLAND, 30), +} +THIN = "E14000006" +NEGATIVE = "E14000007" +LAD23 = UKLocalAuthorityMetadata(boundary_version=UKLocalAuthorityBoundaryVersion.LAD23) + + +def _uk_disclosure_columns(*, with_source_ids: bool = True) -> dict[str, list]: + """Return the households' columns, with baseline and reformed incomes.""" + rng = np.random.default_rng(7) + rows: list[tuple[str, str, float, float]] = [] + for seat, (authority, households) in SEATS.items(): + for _ in range(households): + income = float(rng.uniform(20_000, 60_000)) + rows.append((seat, authority, income, -float(rng.uniform(50, 150)))) + rows.extend((THIN, WESTMORLAND, 30_000.0, -100.0) for _ in range(3)) + for index in range(30): + income = -40_000.0 if index < 20 else 30_000.0 + rows.append((NEGATIVE, WESTMORLAND, income, -float(rng.uniform(50, 150)))) + columns: dict[str, list[object]] = { + "household_id": list(range(len(rows))), + "constituency_code_oa": [row[0] for row in rows], + "la_code_oa": [row[1] for row in rows], + "region_code_oa": [NORTH_WEST] * len(rows), + "household_weight": [float(rng.uniform(50, 150)) for _ in rows], + } + if with_source_ids: + columns["source_household_id"] = list(range(len(rows))) + columns["baseline_income"] = [row[2] for row in rows] + columns["reformed_income"] = [row[2] + row[3] for row in rows] + return columns + + +def _uk_disclosure_simulations( + *, with_source_ids: bool = True +) -> tuple[SimpleNamespace, SimpleNamespace]: + columns = _uk_disclosure_columns(with_source_ids=with_source_ids) + baseline_income = columns.pop("baseline_income") + reformed_income = columns.pop("reformed_income") + baseline = weighted_entity_frame( + {**columns, "household_net_income": baseline_income} + ) + reform = weighted_entity_frame({**columns, "household_net_income": reformed_income}) + + def simulation(household): + return SimpleNamespace( + output_dataset=SimpleNamespace(data=SimpleNamespace(household=household)) + ) + + return simulation(baseline), simulation(reform) + + +@pytest.fixture(scope="module") +def uk_disclosure() -> simulation_output_geographic.UKGeographyDisclosureOutput: + baseline, reform = _uk_disclosure_simulations() + output = simulation_output_geographic.compute_uk_geography_disclosure_output( + "uk", baseline, reform, year=2026 + ) + assert output is not None + return output + + +@pytest.fixture +def constituency_lookup(tmp_path, monkeypatch) -> None: + lookup = tmp_path / "constituencies_2024.csv" + codes = [*SEATS, THIN, NEGATIVE] + lookup.write_text( + "code,name,x,y\n" + + "".join( + f"{code},Seat {index},{index},{-index}\n" + for index, code in enumerate(codes) + ) + ) + monkeypatch.setattr( + simulation_output_geographic, + "_required_uk_geography_lookup_csv_path", + lambda spec: str(lookup), + ) + + +def test_uk_disclosure_withholds_thin_and_undefined_seats(uk_disclosure) -> None: + seats = uk_disclosure.level("constituency") + + assert THIN not in seats + assert NEGATIVE not in seats + # Complementary suppression: with THIN withheld, at least one more seat + # of its region and its local authority is withheld too. + assert len(seats) < len(SEATS) + assert set(seats) < set(SEATS) + + +def test_uk_disclosure_publishes_rounded_changes_with_intervals( + uk_disclosure, +) -> None: + seats = uk_disclosure.level("constituency") + + assert seats + for change in seats.values(): + assert change.tier in {"reported", "approximate"} + assert change.average_lower <= change.average <= change.average_upper + assert change.relative_lower <= change.relative <= change.relative_upper + # Every household loses between 50 and 150 pounds. + assert -150 <= change.average <= -50 + assert change.relative < 0 + assert change.population > 0 + + +def test_uk_disclosure_population_is_the_weighted_household_count( + uk_disclosure, +) -> None: + columns = _uk_disclosure_columns() + expected: dict[str, float] = {} + for code, weight in zip( + columns["constituency_code_oa"], columns["household_weight"], strict=True + ): + expected[code] = expected.get(code, 0.0) + weight + + for code, change in uk_disclosure.level("constituency").items(): + assert change.population == pytest.approx(expected[code]) + + +def test_uk_constituency_output_lists_only_published_seats( + uk_disclosure, constituency_lookup +) -> None: + result = simulation_output_geographic.build_uk_constituency_impact( + "uk", uk_disclosure + ) + + assert result is not None + records = [record.model_dump(mode="json") for record in result.root] + published = uk_disclosure.level("constituency") + assert [record["constituency_code"] for record in records] == [ + code for code in SEATS if code in published + ] + for record in records: + change = published[record["constituency_code"]] + assert record["constituency_name"].startswith("Seat ") + assert record["average_household_income_change"] == change.average + assert record["relative_household_income_change"] == change.relative + assert record["disclosure_tier"] == change.tier + assert record["average_household_income_change_lower"] == (change.average_lower) + assert record["relative_household_income_change_upper"] == ( + change.relative_upper + ) + + +def test_uk_local_authority_output_uses_detected_boundary_metadata( + uk_disclosure, monkeypatch ) -> None: monkeypatch.setattr( simulation_output_geographic, @@ -158,67 +305,115 @@ def test_local_authority_output_uses_detected_boundary_metadata( ) result = simulation_output_geographic.build_uk_local_authority_impact( - "uk", - _uk_simulation(code, 100.0), - _uk_simulation(code, 110.0), - uk_local_authority_metadata=UKLocalAuthorityMetadata( - boundary_version=boundary_version - ), + "uk", uk_disclosure, uk_local_authority_metadata=LAD23 ) assert result is not None - record = result.root[0].model_dump(mode="python") - assert record["local_authority_code"] == code - assert record["local_authority_name"] == expected_name - assert isinstance(record["x"], int) - assert isinstance(record["y"], int) - assert record["average_household_income_change"] == 10.0 + records = { + record.model_dump(mode="json")["local_authority_code"]: record.model_dump( + mode="json" + ) + for record in result.root + } + assert set(records) == set(uk_disclosure.level("local_authority")) + assert records + for code, record in records.items(): + assert record["local_authority_name"] in { + "Cumberland", + "Westmorland and Furness", + } + assert isinstance(record["x"], int) + assert isinstance(record["y"], int) + assert record["average_household_income_change"] < 0 + assert record["population"] == ( + uk_disclosure.level("local_authority")[code].population + ) -def test_local_authority_output_rejects_code_outside_boundary_version() -> None: +def test_uk_local_authority_output_rejects_code_outside_boundary_version( + uk_disclosure, +) -> None: with pytest.raises( ValueError, match="not part of the detected LAD22 boundary version" ): simulation_output_geographic.build_uk_local_authority_impact( "uk", - _uk_simulation("E06000063", 100.0), - _uk_simulation("E06000063", 110.0), + uk_disclosure, uk_local_authority_metadata=UKLocalAuthorityMetadata( boundary_version=UKLocalAuthorityBoundaryVersion.LAD22 ), ) -def test_local_authority_output_requires_detected_boundary_metadata( - monkeypatch, +def test_uk_local_authority_output_requires_detected_boundary_metadata( + uk_disclosure, ) -> None: - monkeypatch.setattr( - simulation_output_geographic, - "_required_uk_geography_lookup_csv_path", - lambda _: pytest.fail("local-authority output must not resolve a GCS lookup"), - ) - with pytest.raises(ValueError, match="boundary metadata is required"): simulation_output_geographic.build_uk_local_authority_impact( - "uk", - _uk_simulation("E06000063", 100.0), - _uk_simulation("E06000063", 110.0), + "uk", uk_disclosure + ) + + +def test_uk_outputs_require_the_disclosure_output() -> None: + with pytest.raises(ValueError, match="disclosure-controlled"): + simulation_output_geographic.build_uk_constituency_impact("uk", None) + with pytest.raises(ValueError, match="disclosure-controlled"): + simulation_output_geographic.build_uk_local_authority_impact( + "uk", None, uk_local_authority_metadata=LAD23 + ) + + +def test_uk_outputs_are_none_for_other_countries() -> None: + baseline, reform = _uk_disclosure_simulations() + + assert ( + simulation_output_geographic.compute_uk_geography_disclosure_output( + "us", baseline, reform, year=2026 + ) + is None + ) + assert simulation_output_geographic.build_uk_constituency_impact("us", None) is None + assert ( + simulation_output_geographic.build_uk_local_authority_impact("us", None) is None + ) + + +def test_uk_outputs_list_no_area_when_every_area_is_withheld( + constituency_lookup, +) -> None: + empty = simulation_output_geographic.UKGeographyDisclosureOutput() + + constituencies = simulation_output_geographic.build_uk_constituency_impact( + "uk", empty + ) + authorities = simulation_output_geographic.build_uk_local_authority_impact( + "uk", empty, uk_local_authority_metadata=LAD23 + ) + + assert constituencies is not None + assert constituencies.root == [] + assert authorities is not None + assert authorities.root == [] + + +def test_uk_disclosure_requires_survey_household_ids() -> None: + baseline, reform = _uk_disclosure_simulations(with_source_ids=False) + + with pytest.raises(ValueError, match="need source_household_id"): + simulation_output_geographic.compute_uk_geography_disclosure_output( + "uk", baseline, reform, year=2026 ) def test_uk_geographic_outputs_are_never_dropped_for_missing_area_codes( monkeypatch, ) -> None: - monkeypatch.setattr( - simulation_output_geographic, - "_required_uk_geography_lookup_csv_path", - lambda _: pytest.fail("a dataset without area codes needs no lookup"), - ) household = weighted_entity_frame( { - "region": ["LONDON"], - "household_net_income": [100.0], - "household_weight": [2.0], + "region": ["LONDON", "LONDON"], + "source_household_id": [1, 2], + "household_net_income": [100.0, 200.0], + "household_weight": [2.0, 3.0], } ) national = SimpleNamespace( @@ -228,15 +423,108 @@ def test_uk_geographic_outputs_are_never_dropped_for_missing_area_codes( with pytest.raises( ValueError, match="constituency breakdowns need constituency_code_oa" ): - simulation_output_geographic.build_uk_constituency_impact( - "uk", national, national + simulation_output_geographic.compute_uk_geography_disclosure_output( + "uk", national, national, year=2026 ) - with pytest.raises(ValueError, match="local-authority breakdowns need la_code_oa"): - simulation_output_geographic.build_uk_local_authority_impact( - "uk", - national, - national, - uk_local_authority_metadata=UKLocalAuthorityMetadata( - boundary_version=UKLocalAuthorityBoundaryVersion.LAD23 - ), + + +def _finding(**fields: object) -> SimpleNamespace: + return SimpleNamespace( + **{ + "blocking": True, + "code": "recoverable_remainder", + "family": "change", + "reform": "reform", + "year": 2026, + "level": "constituency", + "suggested": None, + "group": None, + "group_level": None, + **fields, + } + ) + + +class _FakeRelease: + def __init__(self) -> None: + self.withheld: list[tuple[str, str, str]] = [] + + def withhold(self, *, level, area_code, reform, year, outcome) -> None: + self.withheld.append((level, area_code, outcome)) + + +class _FakeImpact: + """A disclosure result whose findings clear once the named cells are withheld.""" + + def __init__(self, findings, *, clears_after: set[tuple[str, str]]) -> None: + self.release = _FakeRelease() + self.findings = findings + self._clears_after = clears_after + self.public = None + + def refresh(self): + withheld = { + (level, code) + for level, code, outcome in self.release.withheld + if outcome == "average_change" + } + if self._clears_after <= withheld: + self.findings = [] + self.public = SimpleNamespace(rows=[]) + return self.public + + +AREAS = {"constituency": {"S1": 1.0, "S2": 1.0}, "local_authority": {"L1": 2.0}} + + +def test_findings_are_resolved_by_withholding_published_cells() -> None: + impact = _FakeImpact( + [ + _finding(suggested="S2"), + # A containment finding without a suggestion names its group. + _finding(level="constituency", group="L1", group_level="local_authority"), + ], + clears_after={("constituency", "S2"), ("local_authority", "L1")}, + ) + + public = simulation_output_geographic._resolved_public_table( + impact, AREAS, year=2026 + ) + + assert public is not None + change_cells = [ + cell for cell in impact.release.withheld if cell[2] == "average_change" + ] + assert sorted(change_cells) == [ + ("constituency", "S2", "average_change"), + ("local_authority", "L1", "average_change"), + ] + # The shares are never published, so every area's are withheld up front. + share_cells = { + cell for cell in impact.release.withheld if cell[2] != "average_change" + } + assert share_cells == { + (level, code, outcome) + for level, codes in AREAS.items() + for code in codes + for outcome in ("share_gaining", "share_losing") + } + + +def test_reference_cells_are_never_withheld_and_publish_nothing(caplog) -> None: + # A finding that only a region cell could resolve: regions are public + # elsewhere, so the breakdowns publish no area at all. + impact = _FakeImpact( + [_finding(code="recoverable_parent", level="region", group="E12000002")], + clears_after={("region", "E12000002")}, + ) + + with caplog.at_level("WARNING"): + public = simulation_output_geographic._resolved_public_table( + impact, AREAS, year=2026 ) + + assert public is None + assert all(level != "region" for level, _, _ in impact.release.withheld) + assert "recoverable_parent" in caplog.text + assert "E12000002" not in caplog.text diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py b/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py index c0e5a196d..06e3be5fa 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py @@ -148,6 +148,8 @@ def test_uk_plan_uses_uk_defaults_without_us_budget_variables() -> None: assert _dataset_variables(plan, "household") == { "constituency_code_oa", "la_code_oa", + "region_code_oa", + "source_household_id", } assert _dataset_variables(plan, "household").issubset(_variables(plan, "household")) @@ -256,7 +258,10 @@ def test_apply_and_validate_output_plan_reject_missing_materialized_columns() -> validate_output_frames(frames, plan) -@pytest.mark.parametrize("missing", ["constituency_code_oa", "la_code_oa"]) +@pytest.mark.parametrize( + "missing", + ["constituency_code_oa", "la_code_oa", "region_code_oa", "source_household_id"], +) def test_uk_output_validation_requires_geographic_dataset_columns( missing: str, ) -> None: diff --git a/projects/policyengine-simulation-gateway/tests/golden/openapi.json b/projects/policyengine-simulation-gateway/tests/golden/openapi.json index 76d3f626b..c1a6b7a35 100644 --- a/projects/policyengine-simulation-gateway/tests/golden/openapi.json +++ b/projects/policyengine-simulation-gateway/tests/golden/openapi.json @@ -1033,6 +1033,65 @@ }, "ConstituencyImpactRecord": { "properties": { + "disclosure_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "reported", + "approximate" + ] + }, + { + "type": "null" + } + ], + "title": "Disclosure Tier" + }, + "average_household_income_change_lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Household Income Change Lower" + }, + "average_household_income_change_upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Household Income Change Upper" + }, + "relative_household_income_change_lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Household Income Change Lower" + }, + "relative_household_income_change_upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Household Income Change Upper" + }, "constituency_code": { "type": "string", "title": "Constituency Code" @@ -1088,7 +1147,7 @@ "population" ], "title": "ConstituencyImpactRecord", - "description": "Reform impact on household net income in one UK parliamentary constituency.\n\n``average_household_income_change`` is the average change per household and\n``relative_household_income_change`` the relative change in the area's total\nhousehold net income. ``x`` and ``y`` are integer map-rendering coordinates." + "description": "Reform impact on household net income in one UK parliamentary constituency.\n\n``average_household_income_change`` is the average change per household and\n``relative_household_income_change`` the relative change in the area's total\nhousehold net income, both rounded to the order of their standard error.\n``population`` is the weighted number of households. ``x`` and ``y`` are\ninteger map-rendering coordinates." }, "DecileOutput": { "properties": { @@ -1180,7 +1239,7 @@ }, "type": "array", "title": "GeographicImpactOutput", - "description": "UK constituency or local-authority impact records." + "description": "UK constituency or local-authority impact records.\n\nAreas whose income change disclosure control withholds are left out." }, "HTTPValidationError": { "properties": { @@ -1682,6 +1741,65 @@ }, "LocalAuthorityImpactRecord": { "properties": { + "disclosure_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "reported", + "approximate" + ] + }, + { + "type": "null" + } + ], + "title": "Disclosure Tier" + }, + "average_household_income_change_lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Household Income Change Lower" + }, + "average_household_income_change_upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Household Income Change Upper" + }, + "relative_household_income_change_lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Household Income Change Lower" + }, + "relative_household_income_change_upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Household Income Change Upper" + }, "local_authority_code": { "type": "string", "title": "Local Authority Code" diff --git a/rules/baselines/sim-microdata-weight-stripping.txt b/rules/baselines/sim-microdata-weight-stripping.txt index 044b1e724..b396d381b 100644 --- a/rules/baselines/sim-microdata-weight-stripping.txt +++ b/rules/baselines/sim-microdata-weight-stripping.txt @@ -4,10 +4,3 @@ # ::::: a read of raw values from a weighted # frame in simulation_output_common.py for something other than counting # entities. Reads anywhere else fail outright. -# -# The UK local-authority breakdown hands policyengine's longwise geography -# impact a plain DataFrame of the household frame, which weights by its -# household_weight column itself. It goes once the breakdown hands -# policyengine the simulations, as the constituency and US congressional -# district breakdowns already do. -projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py::_unweighted_household_frame::pd.DataFrame(simulation.output_dataset.data.household) diff --git a/rules/registry.toml b/rules/registry.toml index d1d305ea8..803e99480 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -787,7 +787,7 @@ project = "projects/policyengine-simulation-executor" test = "tests/test_microdata_weight_boundaries.py::test_rule_proof_weight_stripping_confined" [rule.baseline] file = "rules/baselines/sim-microdata-weight-stripping.txt" -count = 1 +count = 0 [[rule]] id = "sim.microdata.builder-tests-use-weighted-frames" From 8fb0b788207128f7ab276a99aca0b8f981e1a849 Mon Sep 17 00:00:00 2001 From: PavelMakarchuk Date: Tue, 20 Oct 2026 21:08:22 -0400 Subject: [PATCH 2/2] Withhold the inner areas that sum to a withheld UK area A containment finding without a suggested cell names only the outer area. When that area is withheld and the public areas of the other level inside it add up to it (Orkney and Shetland is Orkney plus Shetland), the loop withheld the outer cell again, made no progress, and published no area at all. It now withholds the inner areas that lie wholly inside the outer one, read from the household area pairs as the release reads them. Co-Authored-By: Claude Opus 5.5 --- .../simulation_output_geographic.py | 93 ++++++++++++++----- .../test_simulation_output_geographic.py | 46 ++++++++- 2 files changed, 112 insertions(+), 27 deletions(-) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index 9c978ad37..19fcdebc9 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -21,6 +21,7 @@ FAMILY_CHANGE, FAMILY_GAINING, FAMILY_LOSING, + RECOVERABLE_PARENT, RELATIVE_CHANGE, REPORTED, UNCHECKED_OVERLAP, @@ -382,6 +383,49 @@ def _uk_area_household_counts(household: pd.DataFrame, column: str) -> dict[str, return {code: count for code, count in by_code.items() if count > 0} +# For each pair of overlapping published levels (inner, outer): the inner +# areas that lie wholly inside each outer area, as the release reads them. +_UKAreasInside = Mapping[tuple[str, str], Mapping[str, tuple[str, ...]]] + + +def _uk_areas_inside(household: pd.DataFrame) -> _UKAreasInside: + """Return, for each level, the areas of the other level wholly inside each area. + + An inner area lies wholly inside an outer one when every weighted + household of it has that outer area, as the disclosure release decides. + A containment finding without a suggested cell names only the outer area, + and these are the cells that resolve it. + """ + (first, first_column), (second, second_column) = UK_DISCLOSURE_AREA_COLUMNS.items() + counts = household.groupby([first_column, second_column], dropna=False)[ + "household_net_income" + ].count() + pairs: set[tuple[str, str]] = set() + for (first_raw, second_raw), count in counts.items(): + if float(cast("float", count)) > 0: + pairs.add( + (_normalised_area_code(first_raw), _normalised_area_code(second_raw)) + ) + inside: dict[tuple[str, str], dict[str, tuple[str, ...]]] = {} + for inner_level, outer_level, inner_at, outer_at in ( + (first, second, 0, 1), + (second, first, 1, 0), + ): + outers: dict[str, set[str]] = {} + for pair in pairs: + if pair[inner_at] != UNASSIGNED: + outers.setdefault(pair[inner_at], set()).add(pair[outer_at]) + by_outer: dict[str, list[str]] = {} + for inner, met in outers.items(): + outer = next(iter(met)) if len(met) == 1 else UNASSIGNED + if outer != UNASSIGNED: + by_outer.setdefault(outer, []).append(inner) + inside[(inner_level, outer_level)] = { + outer: tuple(sorted(inners)) for outer, inners in by_outer.items() + } + return inside + + def _withhold_unpublished_families( impact: UKGeographyDisclosure, area_codes: Mapping[str, Mapping[str, float]], @@ -407,14 +451,17 @@ def _withhold_unpublished_families( def _cells_resolving_findings( - impact: UKGeographyDisclosure, published_levels: tuple[str, ...] + impact: UKGeographyDisclosure, + published_levels: tuple[str, ...], + inside: _UKAreasInside, ) -> set[tuple[str, str, str, str, int]]: """Return the published cells to withhold for the blocking findings. - A finding names a cell to withhold, or else the group it is about. Only - cells of published levels are withheld: regions and the nation are - references whose numbers are public elsewhere, so withholding them here - would only hide the risk. + A finding names a cell to withhold, or else the group it is about. When a + withheld area is the sum of the public areas of the other level inside + it, those areas are withheld. Only cells of published levels are + withheld: regions and the nation are references whose numbers are public + elsewhere, so withholding them here would only hide the risk. """ cells: set[tuple[str, str, str, str, int]] = set() for finding in impact.findings or []: @@ -425,26 +472,19 @@ def _cells_resolving_findings( outcome = _FAMILY_OUTCOME.get(finding.family) if outcome is None: continue + key = (outcome, finding.reform, finding.year) if finding.suggested and finding.level in published_levels: - cells.add( - ( - finding.level, - finding.suggested, - outcome, - finding.reform, - finding.year, - ) - ) - elif finding.group and finding.group_level in published_levels: - cells.add( - ( - finding.group_level, - finding.group, - outcome, - finding.reform, - finding.year, - ) + cells.add((finding.level, finding.suggested, *key)) + elif not finding.group or finding.group_level not in published_levels: + continue + elif finding.code == RECOVERABLE_PARENT and finding.level in published_levels: + members = inside.get((finding.level, finding.group_level), {}) + cells.update( + (finding.level, member, *key) + for member in members.get(finding.group, ()) ) + else: + cells.add((finding.group_level, finding.group, *key)) return cells @@ -453,6 +493,7 @@ def _resolved_public_table( area_codes: Mapping[str, Mapping[str, float]], *, year: int, + inside: _UKAreasInside, ) -> PublicTable | None: """Withhold cells until no finding blocks, and return the public table. @@ -466,7 +507,7 @@ def _resolved_public_table( for _ in range(UK_DISCLOSURE_MAX_ROUNDS): if public is not None: return public - cells = _cells_resolving_findings(impact, published_levels) - withheld + cells = _cells_resolving_findings(impact, published_levels, inside) - withheld if not cells: break for level, code, outcome, reform, cell_year in sorted(cells): @@ -587,7 +628,9 @@ def compute_uk_geography_disclosure_output( # other differences between them stay unchecked. acknowledge=(UNCHECKED_OVERLAP,), ) - public = _resolved_public_table(impact, area_codes, year=year) + public = _resolved_public_table( + impact, area_codes, year=year, inside=_uk_areas_inside(household) + ) if public is None: return UKGeographyDisclosureOutput() return UKGeographyDisclosureOutput( diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py index e3d8320ff..84636b533 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py @@ -488,7 +488,7 @@ def test_findings_are_resolved_by_withholding_published_cells() -> None: ) public = simulation_output_geographic._resolved_public_table( - impact, AREAS, year=2026 + impact, AREAS, year=2026, inside={} ) assert public is not None @@ -521,10 +521,52 @@ def test_reference_cells_are_never_withheld_and_publish_nothing(caplog) -> None: with caplog.at_level("WARNING"): public = simulation_output_geographic._resolved_public_table( - impact, AREAS, year=2026 + impact, AREAS, year=2026, inside={} ) assert public is None assert all(level != "region" for level, _, _ in impact.release.withheld) assert "recoverable_parent" in caplog.text assert "E12000002" not in caplog.text + + +def test_uk_areas_inside_follow_the_household_area_pairs() -> None: + baseline, _ = _uk_disclosure_simulations() + household = baseline.output_dataset.data.household + + inside = simulation_output_geographic._uk_areas_inside(household) + + # Every seat lies in one local authority; no authority lies in one seat. + assert inside[("constituency", "local_authority")] == { + CUMBERLAND: ("E14000001", "E14000002"), + WESTMORLAND: ("E14000003", "E14000004", "E14000005", THIN, NEGATIVE), + } + assert inside[("local_authority", "constituency")] == {} + + +def test_a_withheld_area_summed_by_public_areas_inside_withholds_them() -> None: + # Orkney and Shetland is the sum of two local authorities. If the seat is + # withheld, the two authorities inside it must be withheld too. + inside = {("local_authority", "constituency"): {"S1": ("L1", "L2")}} + impact = _FakeImpact( + [ + _finding( + code="recoverable_parent", + level="local_authority", + group="S1", + group_level="constituency", + ) + ], + clears_after={("local_authority", "L1"), ("local_authority", "L2")}, + ) + + public = simulation_output_geographic._resolved_public_table( + impact, AREAS, year=2026, inside=inside + ) + + assert public is not None + assert { + (level, code) + for level, code, outcome in impact.release.withheld + if outcome == "average_change" + } == {("local_authority", "L1"), ("local_authority", "L2")}