diff --git a/changelog_entry.yaml b/changelog_entry.yaml index ffa169491..1494b45b0 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,6 +1,7 @@ - bump: patch changes: added: + - Added optional disclosure fields to UK constituency and local-authority records (disclosure_tier, reported or approximate, and the 90% interval bounds of the average and relative income change); records stored before this change still validate - Added the rule sim.boundaries.sqlalchemy-and-alembic-only, with proof from the psycopg adapter of 08ebe1ca; code connects to databases only through SQLAlchemy, never by importing a database driver, and runtime code never changes the schema, which changes only through Alembic migrations; a guard parses every file under projects/*/src, libs/*/src and .github/scripts and rejects driver imports, alembic imports outside a migrations directory, create_all and drop_all, create and drop on tables, SQLAlchemy DDL statements and DDL text in string literals, which extends the Stage 12 DDL check's pattern to the gateway and every library - Made the budget's reform-minus-baseline totals log a warning when the baseline and reform household frames carry different weights, since the change is then weighted by the reform's weights alone - Added weighted-microdata rules to the registry, with proof from the #767 household count; the executor's code never aggregates a weight column as a value, reads raw values from weighted frames only in simulation_output_common.py (the UK local-authority breakdown's plain household frame is its one recorded exception there, in a list that may only shrink), and its output-builder tests build entity frames through one shared factory with non-uniform weights, while a doc-only rule records that weights, values, labels and counts each allow different operations @@ -25,6 +26,7 @@ - Ran the Modal image smoke for Stage 12 persistence changes and added policyengine-fastapi to the PR test matrix - Ran the full PR workflow and the Modal image smoke only on the lowest unmerged pull request in a stack, behind a new CI complete gate job that fails on the others fixed: + - Built UK constituency and local-authority results with policyengine's disclosure-controlled compute_uk_geography_disclosure instead of the deprecated, unprotected functions, in one release nested in regions and the nation; an area that rests on fewer than 10 survey households, is dominated by one, would be recovered by subtraction, or has a non-positive baseline income is left out of the list, so small areas no longer show exploding relative changes such as +1372% for a £192 loss - Preserved bounded reform stop dates in UK and US simulations while retaining current-law baseline artifact reuse - Resolved PolicyEngine-managed regional datasets using bundled data versions instead of release-manifest commit hashes when loading from GCS - Added a Modal app-release bundle backfill for missing data artifact revisions diff --git a/docs/engineering/skills/microdata-weights.md b/docs/engineering/skills/microdata-weights.md index 9a84a2d87..ed430851e 100644 --- a/docs/engineering/skills/microdata-weights.md +++ b/docs/engineering/skills/microdata-weights.md @@ -82,7 +82,6 @@ Each finding needs a decision, recorded in the PR: Recorded exceptions are listed in `rules/baselines/sim-microdata-weight-stripping.txt`. That list may only shrink, so adding one is a deliberate change to the rule's baseline. Today it -holds one: `_unweighted_household_frame`, the plain household frame the UK -local-authority breakdown hands policyengine's longwise geography impact. It -goes once that breakdown hands policyengine the simulations, as the -constituency and US congressional district breakdowns already do. +is empty: the last one, `_unweighted_household_frame`, went when the UK +constituency and local-authority breakdowns moved to policyengine's +disclosure-controlled geography output, which takes the simulations. diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 7d2a09995..5593c4e0a 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -66,7 +66,7 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf ### Ratcheted (existing violations still tolerated) - `sim.ci.multiline-run-is-script`: 13 remaining in `rules/baselines/ci-multiline-run.txt` -- `sim.microdata.weight-stripping-confined`: 1 remaining in `rules/baselines/sim-microdata-weight-stripping.txt` +- `sim.microdata.weight-stripping-confined`: 0 remaining in `rules/baselines/sim-microdata-weight-stripping.txt` ### Exceptions and review dates diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py index a45cddec4..5fcd2bb60 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Generic, TypeVar +from typing import Generic, Literal, TypeVar from pydantic import BaseModel, ConfigDict, RootModel @@ -234,12 +234,38 @@ class CliffImpactOutput(MacroOutputModel): reform: CliffImpactInSimulation -class ConstituencyImpactRecord(MacroOutputModel): +UKGeographyDisclosureTier = Literal["reported", "approximate"] + + +class UKAreaDisclosureFields(MacroOutputModel): + """Disclosure tier and 90% intervals of a published UK area's income changes. + + UK area results come from licensed survey microdata and pass policyengine's + disclosure control. An area whose income change is withheld is left out of + the list, so every record here has values. ``disclosure_tier`` is + ``reported``, or ``approximate`` when the area's effective sample is thin + and the interval understates the uncertainty. The ``_lower`` and ``_upper`` + fields bound each change's 90% bootstrap interval, rounded outward. + + All of these fields are optional, so results stored before disclosure + control still validate; there they are null. + """ + + disclosure_tier: UKGeographyDisclosureTier | None = None + average_household_income_change_lower: float | None = None + average_household_income_change_upper: float | None = None + relative_household_income_change_lower: float | None = None + relative_household_income_change_upper: float | None = None + + +class ConstituencyImpactRecord(UKAreaDisclosureFields): """Reform impact on household net income in one UK parliamentary constituency. ``average_household_income_change`` is the average change per household and ``relative_household_income_change`` the relative change in the area's total - household net income. ``x`` and ``y`` are integer map-rendering coordinates. + household net income, both rounded to the order of their standard error. + ``population`` is the weighted number of households. ``x`` and ``y`` are + integer map-rendering coordinates. """ constituency_code: str @@ -251,7 +277,7 @@ class ConstituencyImpactRecord(MacroOutputModel): population: float -class LocalAuthorityImpactRecord(MacroOutputModel): +class LocalAuthorityImpactRecord(UKAreaDisclosureFields): """Reform impact on household net income in one UK local authority. Fields mean the same as in ``ConstituencyImpactRecord``. @@ -269,7 +295,10 @@ class LocalAuthorityImpactRecord(MacroOutputModel): class GeographicImpactOutput( MacroRootModel[list[ConstituencyImpactRecord | LocalAuthorityImpactRecord]] ): - """UK constituency or local-authority impact records.""" + """UK constituency or local-authority impact records. + + Areas whose income change disclosure control withholds are left out. + """ class CongressionalDistrictImpactRecord(MacroOutputModel): diff --git a/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py b/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py new file mode 100644 index 000000000..87d8ae48a --- /dev/null +++ b/libs/policyengine-simulation-contract/tests/test_macro_output_uk_geography.py @@ -0,0 +1,74 @@ +"""UK area records stay readable before and after disclosure control.""" + +from __future__ import annotations + +import pytest +from policyengine_simulation_contract.macro_output import ( + ConstituencyImpactRecord, + GeographicImpactOutput, + LocalAuthorityImpactRecord, +) +from pydantic import ValidationError + +STORED_BEFORE_DISCLOSURE = { + "constituency_code": "E14001063", + "constituency_name": "Aldershot", + "x": 56, + "y": -40, + "average_household_income_change": -150.95, + "relative_household_income_change": -0.0026, + "population": 40_530.65, +} +DISCLOSURE_FIELDS = { + "disclosure_tier": "approximate", + "average_household_income_change_lower": -170.0, + "average_household_income_change_upper": -130.0, + "relative_household_income_change_lower": -0.003, + "relative_household_income_change_upper": -0.0022, +} + + +def test_records_stored_before_disclosure_control_still_validate() -> None: + record = ConstituencyImpactRecord.model_validate(STORED_BEFORE_DISCLOSURE) + + assert record.disclosure_tier is None + assert record.average_household_income_change_lower is None + assert record.relative_household_income_change_upper is None + + +def test_records_carry_the_disclosure_tier_and_intervals() -> None: + record = LocalAuthorityImpactRecord.model_validate( + { + "local_authority_code": "E06000063", + "local_authority_name": "Cumberland", + "x": 4, + "y": 18, + "average_household_income_change": -150.0, + "relative_household_income_change": -0.0026, + "population": 120_000.0, + **DISCLOSURE_FIELDS, + } + ) + + assert record.model_dump(mode="json")["disclosure_tier"] == "approximate" + assert record.average_household_income_change_lower == -170.0 + + +def test_a_withheld_tier_is_not_a_record() -> None: + # Withheld areas are left out of the list, so no record holds that tier. + with pytest.raises(ValidationError): + ConstituencyImpactRecord.model_validate( + {**STORED_BEFORE_DISCLOSURE, "disclosure_tier": "withheld"} + ) + + +def test_published_values_stay_required() -> None: + record = {**STORED_BEFORE_DISCLOSURE, **DISCLOSURE_FIELDS} + record["average_household_income_change"] = None + + with pytest.raises(ValidationError): + ConstituencyImpactRecord.model_validate(record) + + +def test_an_empty_area_list_is_valid() -> None: + assert GeographicImpactOutput.model_validate([]).root == [] diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py index 5a86a0482..6e19ad766 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py @@ -68,6 +68,7 @@ class SimulationOutputBuilder: uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None progress: JobProgressReporter = NULL_JOB_PROGRESS _analysis: Any = field(default=None, init=False) + _uk_geography_disclosure: Any = field(default=None, init=False) def __post_init__(self) -> None: """Normalize ``country`` to lower case.""" @@ -257,18 +258,53 @@ def _build_congressional_district_impact( region_code=self.resolved_region_code, ) + def _uk_geography_disclosure_output( + self, + ) -> simulation_output_geographic.UKGeographyDisclosureOutput | None: + """Return UK area changes under disclosure control, computed once. + + Constituencies and local authorities go through one release, so the + disclosure check sees both. None for other countries. + """ + if self.country != "uk": + return None + if self._uk_geography_disclosure is None: + self._uk_geography_disclosure = ( + simulation_output_geographic.compute_uk_geography_disclosure_output( + self.country, + self.baseline, + self.reform, + year=self._simulation_year(), + ) + ) + return self._uk_geography_disclosure + + def _simulation_year(self) -> int: + """Return the simulated year, which labels the disclosure release.""" + for value in ( + self.simulation_params.get("time_period"), + self.simulation_params.get("year"), + getattr(self.dataset, "year", None), + ): + if value is None: + continue + try: + return int(value) + except (TypeError, ValueError): + continue + return 0 + def _build_uk_constituency_impact(self) -> GeographicImpactOutput | None: with self._span(ANNUAL_IMPACT_STAGES.name(Stage.OUTPUT_UK_CONSTITUENCY)): return simulation_output_geographic.build_uk_constituency_impact( - self.country, self.baseline, self.reform + self.country, self._uk_geography_disclosure_output() ) def _build_uk_local_authority_impact(self) -> GeographicImpactOutput | None: with self._span(ANNUAL_IMPACT_STAGES.name(Stage.OUTPUT_UK_LOCAL_AUTHORITY)): return simulation_output_geographic.build_uk_local_authority_impact( self.country, - self.baseline, - self.reform, + self._uk_geography_disclosure_output(), uk_local_authority_metadata=self.uk_local_authority_metadata, ) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py index 4ce0d2feb..ae5c73308 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py @@ -8,10 +8,10 @@ from importlib import import_module from typing import TYPE_CHECKING, Any -import pandas as pd from policyengine_simulation_contract.spm import spm_error_detail if TYPE_CHECKING: + import pandas as pd from _typeshed import ConvertibleToFloat from policyengine.core import Simulation @@ -77,23 +77,6 @@ def _count_output_entities(simulation: Simulation, entity: str, weight: str) -> return float(np.asarray(data[weight], dtype=float).sum()) -def _unweighted_household_frame(simulation: Simulation) -> pd.DataFrame: - """Return the household output as a plain DataFrame, without its weights. - - policyengine's ``compute_longwise_uk_geography_impacts`` takes plain frames - and weights each area by the baseline's ``household_weight`` column itself. - The UK local-authority breakdown calls it directly, so that it can apply - this service's boundary metadata. Handing it the weighted frame would weight - twice if it ever reduced a column directly. - - This is a recorded exception to sim.microdata.weight-stripping-confined, - listed in rules/baselines/sim-microdata-weight-stripping.txt, which may only - shrink. It goes once the breakdown hands policyengine the simulations, as - the constituency and US congressional district breakdowns already do. - """ - return pd.DataFrame(simulation.output_dataset.data.household) - - def _warn_on_mismatched_weights( baseline_data: pd.DataFrame, reform_data: pd.DataFrame, entity: str ) -> None: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py index 849c6c7a1..19fcdebc9 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py @@ -2,13 +2,30 @@ from __future__ import annotations +import logging +import math from collections.abc import Mapping +from dataclasses import dataclass, field from functools import lru_cache -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast +import pandas as pd from policyengine.data.uk_geography_assets import ( CONSTITUENCY_ASSET_SPEC, ) +from policyengine.disclosure import UNASSIGNED, DisclosureError +from policyengine.disclosure.areas import normalise_area_codes +from policyengine.disclosure.schema import ( + APPROXIMATE, + AVERAGE_CHANGE, + FAMILY_CHANGE, + FAMILY_GAINING, + FAMILY_LOSING, + RECOVERABLE_PARENT, + RELATIVE_CHANGE, + REPORTED, + UNCHECKED_OVERLAP, +) from policyengine_simulation_executor.simulation_macro_output import ( CongressionalDistrictImpactOutput, @@ -16,19 +33,24 @@ GeographicImpactOutput, ) from policyengine_simulation_executor.simulation_output_common import ( + _entity_data, _number, _output_model_dump, - _output_module_function, _try_compute_output, - _unweighted_household_frame, ) if TYPE_CHECKING: - import pandas as pd from policyengine.core import Simulation from policyengine.data.uk_geography_assets import UKGeographyAssetSpec + from policyengine.disclosure import PublicTable + from policyengine.outputs.uk_geography_disclosure import UKGeographyDisclosure + from policyengine_simulation_contract.macro_output import ( + UKGeographyDisclosureTier, + ) from policyengine_simulation_contract.uk_geography import UKLocalAuthorityMetadata +logger = logging.getLogger(__name__) + def _required_uk_geography_lookup_csv_path(spec: UKGeographyAssetSpec) -> str: """Resolve one required lookup under the runtime GCP identity.""" @@ -60,10 +82,13 @@ def _complete_uk_geography_output( code_field: str, name_field: str, ) -> GeographicImpactOutput: - """Require every UK geography record to contain lookup-owned metadata.""" + """Require every UK geography record to contain lookup-owned metadata. + + An empty list is valid: disclosure control may withhold every area. + """ output = build_geographic_impact_output(value) - if output is None or not output.root: - msg = "UK geography output did not contain result records" + if output is None: + msg = "UK geography output did not contain a list of result records" raise ValueError(msg) for record in output.root: record_values = record.model_dump(mode="python") @@ -236,6 +261,77 @@ def compute_and_format() -> CongressionalDistrictImpactOutput | None: return _try_compute_output("congressional district impacts", compute_and_format) +# The household output column that holds each published UK level's area code. +UK_DISCLOSURE_AREA_COLUMNS: dict[str, str] = { + "constituency": "constituency_code_oa", + "local_authority": "la_code_oa", +} +UK_DISCLOSURE_BREAKDOWN_NAMES: dict[str, str] = { + "constituency": "constituency", + "local_authority": "local-authority", +} +# The survey household each row was copied or cloned from. The count floor +# counts support by this id, so copies of one survey household count once. +UK_SOURCE_HOUSEHOLD_ID_COLUMN = "source_household_id" +# Bootstrap replicates behind each 90% interval: the disclosure layer's +# minimum. They set the interval and the rounding, never whether a cell is +# published, and the layer's default of 1,000 would add minutes to every UK run. +UK_DISCLOSURE_REPLICATES = 200 +UK_DISCLOSURE_REFORM_LABEL = "reform" +# Each round withholds at least one more cell, so this bounds the check runs. +UK_DISCLOSURE_MAX_ROUNDS = 25 +# The output publishes the change family only. The share families are +# withheld before the check, so the check treats them as secret, which they +# are, and never asks for cells to protect shares nobody publishes. +_UNPUBLISHED_FAMILY_OUTCOMES = (FAMILY_GAINING, FAMILY_LOSING) +_FAMILY_OUTCOME = { + FAMILY_CHANGE: AVERAGE_CHANGE, + FAMILY_GAINING: FAMILY_GAINING, + FAMILY_LOSING: FAMILY_LOSING, +} + + +@dataclass(frozen=True) +class UKAreaIncomeChange: + """One published UK area's disclosure-controlled household income change.""" + + tier: UKGeographyDisclosureTier + average: float + average_lower: float + average_upper: float + relative: float + relative_lower: float + relative_upper: float + population: float + + def record_fields(self) -> dict[str, object]: + """Return the fields this change contributes to an area record.""" + return { + "average_household_income_change": self.average, + "relative_household_income_change": self.relative, + "population": self.population, + "disclosure_tier": self.tier, + "average_household_income_change_lower": self.average_lower, + "average_household_income_change_upper": self.average_upper, + "relative_household_income_change_lower": self.relative_lower, + "relative_household_income_change_upper": self.relative_upper, + } + + +@dataclass(frozen=True) +class UKGeographyDisclosureOutput: + """The published areas of each UK level, keyed by area code. + + An area the disclosure check withholds is absent. + """ + + areas: Mapping[str, Mapping[str, UKAreaIncomeChange]] = field(default_factory=dict) + + def level(self, level: str) -> Mapping[str, UKAreaIncomeChange]: + """Return the published areas of one level.""" + return self.areas.get(level, {}) + + def _require_uk_area_column( household: pd.DataFrame, column: str, breakdown: str ) -> None: @@ -254,31 +350,377 @@ def _require_uk_area_column( raise ValueError(msg) -def build_uk_constituency_impact( - country: str, baseline: Simulation, reform: Simulation -) -> GeographicImpactOutput | None: - """Compute UK parliamentary constituency impacts; None for other countries. +def _require_uk_source_household_ids(household: pd.DataFrame) -> None: + """Refuse UK area outputs when support cannot be counted by survey household.""" + if UK_SOURCE_HOUSEHOLD_ID_COLUMN not in household.columns: + msg = ( + f"UK constituency and local-authority breakdowns need " + f"{UK_SOURCE_HOUSEHOLD_ID_COLUMN} on the simulated dataset: disclosure " + "control counts the survey households behind each area by it." + ) + raise ValueError(msg) + + +def _normalised_area_code(value: object) -> str: + """Return an area code as the disclosure layer keys it.""" + return str(normalise_area_codes([value])[0]) + + +def _uk_area_household_counts(household: pd.DataFrame, column: str) -> dict[str, float]: + """Return the weighted number of households in each area of one level. + + ``household`` is the weighted output frame, so a grouped count is the sum + of household weights in each area. Areas without weight are left out, as + the disclosure layer leaves them out. + """ + counts = household.groupby(column)["household_net_income"].count() + by_code: dict[str, float] = {} + for raw_code, count in counts.items(): + code = _normalised_area_code(raw_code) + if code == UNASSIGNED: + continue + by_code[code] = by_code.get(code, 0.0) + float(cast("float", count)) + return {code: count for code, count in by_code.items() if count > 0} + + +# For each pair of overlapping published levels (inner, outer): the inner +# areas that lie wholly inside each outer area, as the release reads them. +_UKAreasInside = Mapping[tuple[str, str], Mapping[str, tuple[str, ...]]] + - Downloads the constituency lookup if needed and raises when the dataset - lacks constituency codes or a record lacks lookup name or coordinates. +def _uk_areas_inside(household: pd.DataFrame) -> _UKAreasInside: + """Return, for each level, the areas of the other level wholly inside each area. + + An inner area lies wholly inside an outer one when every weighted + household of it has that outer area, as the disclosure release decides. + A containment finding without a suggested cell names only the outer area, + and these are the cells that resolve it. + """ + (first, first_column), (second, second_column) = UK_DISCLOSURE_AREA_COLUMNS.items() + counts = household.groupby([first_column, second_column], dropna=False)[ + "household_net_income" + ].count() + pairs: set[tuple[str, str]] = set() + for (first_raw, second_raw), count in counts.items(): + if float(cast("float", count)) > 0: + pairs.add( + (_normalised_area_code(first_raw), _normalised_area_code(second_raw)) + ) + inside: dict[tuple[str, str], dict[str, tuple[str, ...]]] = {} + for inner_level, outer_level, inner_at, outer_at in ( + (first, second, 0, 1), + (second, first, 1, 0), + ): + outers: dict[str, set[str]] = {} + for pair in pairs: + if pair[inner_at] != UNASSIGNED: + outers.setdefault(pair[inner_at], set()).add(pair[outer_at]) + by_outer: dict[str, list[str]] = {} + for inner, met in outers.items(): + outer = next(iter(met)) if len(met) == 1 else UNASSIGNED + if outer != UNASSIGNED: + by_outer.setdefault(outer, []).append(inner) + inside[(inner_level, outer_level)] = { + outer: tuple(sorted(inners)) for outer, inners in by_outer.items() + } + return inside + + +def _withhold_unpublished_families( + impact: UKGeographyDisclosure, + area_codes: Mapping[str, Mapping[str, float]], + *, + year: int, +) -> None: + release = impact.release + for level, codes in area_codes.items(): + for code in codes: + for outcome in _UNPUBLISHED_FAMILY_OUTCOMES: + try: + release.withhold( + level=level, + area_code=code, + reform=UK_DISCLOSURE_REFORM_LABEL, + year=year, + outcome=outcome, + ) + except DisclosureError: + # Not an area of the release, for example one whose + # households all have zero weight. + continue + + +def _cells_resolving_findings( + impact: UKGeographyDisclosure, + published_levels: tuple[str, ...], + inside: _UKAreasInside, +) -> set[tuple[str, str, str, str, int]]: + """Return the published cells to withhold for the blocking findings. + + A finding names a cell to withhold, or else the group it is about. When a + withheld area is the sum of the public areas of the other level inside + it, those areas are withheld. Only cells of published levels are + withheld: regions and the nation are references whose numbers are public + elsewhere, so withholding them here would only hide the risk. + """ + cells: set[tuple[str, str, str, str, int]] = set() + for finding in impact.findings or []: + if not finding.blocking or finding.family is None: + continue + if finding.reform is None or finding.year is None: + continue + outcome = _FAMILY_OUTCOME.get(finding.family) + if outcome is None: + continue + key = (outcome, finding.reform, finding.year) + if finding.suggested and finding.level in published_levels: + cells.add((finding.level, finding.suggested, *key)) + elif not finding.group or finding.group_level not in published_levels: + continue + elif finding.code == RECOVERABLE_PARENT and finding.level in published_levels: + members = inside.get((finding.level, finding.group_level), {}) + cells.update( + (finding.level, member, *key) + for member in members.get(finding.group, ()) + ) + else: + cells.add((finding.group_level, finding.group, *key)) + return cells + + +def _resolved_public_table( + impact: UKGeographyDisclosure, + area_codes: Mapping[str, Mapping[str, float]], + *, + year: int, + inside: _UKAreasInside, +) -> PublicTable | None: + """Withhold cells until no finding blocks, and return the public table. + + Returns None when the findings cannot be resolved by withholding published + cells; then nothing at these levels may be published. + """ + published_levels = tuple(area_codes) + _withhold_unpublished_families(impact, area_codes, year=year) + public = impact.refresh() + withheld: set[tuple[str, str, str, str, int]] = set() + for _ in range(UK_DISCLOSURE_MAX_ROUNDS): + if public is not None: + return public + cells = _cells_resolving_findings(impact, published_levels, inside) - withheld + if not cells: + break + for level, code, outcome, reform, cell_year in sorted(cells): + try: + impact.release.withhold( + level=level, + area_code=code, + reform=reform, + year=cell_year, + outcome=outcome, + ) + except DisclosureError: + # Not a cell of the release. The finding stays, so the loop + # ends without progress and nothing is published. + continue + withheld |= cells + public = impact.refresh() + if public is not None: + return public + blocking = sorted( + {finding.code for finding in impact.findings or [] if finding.blocking} + ) + # Findings carry area codes; the log names only their kinds. + logger.warning( + "UK geography disclosure findings remain after withholding %d cells " + "(%s), so no constituency or local-authority area is published", + len(withheld), + ", ".join(blocking), + ) + return None + + +def _published_area_changes( + public: PublicTable, + level: str, + populations: Mapping[str, float], +) -> dict[str, UKAreaIncomeChange]: + """Return the areas of one level whose average and relative change are public. + + An area is published only when both changes are: the contract has a value + for each. A relative change is withheld on its own, for example, when the + area's baseline income is not positive. + """ + rows: dict[str, dict[str, Any]] = {} + for row in public.rows: + if row.level != level or row.outcome not in {AVERAGE_CHANGE, RELATIVE_CHANGE}: + continue + if row.tier not in {REPORTED, APPROXIMATE} or row.value is None: + continue + if row.lo is None or row.hi is None: + continue + rows.setdefault(row.area_code, {})[row.outcome] = row + changes: dict[str, UKAreaIncomeChange] = {} + for code, outcomes in rows.items(): + average = outcomes.get(AVERAGE_CHANGE) + relative = outcomes.get(RELATIVE_CHANGE) + if average is None or relative is None or code not in populations: + continue + tier: UKGeographyDisclosureTier = ( + APPROXIMATE if APPROXIMATE in {average.tier, relative.tier} else REPORTED + ) + changes[code] = UKAreaIncomeChange( + tier=tier, + average=float(average.value), + average_lower=float(average.lo), + average_upper=float(average.hi), + relative=float(relative.value), + relative_lower=float(relative.lo), + relative_upper=float(relative.hi), + population=populations[code], + ) + return changes + + +def compute_uk_geography_disclosure_output( + country: str, + baseline: Simulation, + reform: Simulation, + *, + year: int, +) -> UKGeographyDisclosureOutput | None: + """Compute disclosure-controlled constituency and local-authority changes. + + None for other countries. Both levels go through one policyengine release, + nested in regions and the nation, so the check sees what subtraction + between them recovers. Raises when the dataset lacks area codes or survey + household ids. """ if country != "uk": return None - _require_uk_area_column( - baseline.output_dataset.data.household, "constituency_code_oa", "constituency" + household = _entity_data(baseline, "household") + _entity_data(reform, "household") + for level, column in UK_DISCLOSURE_AREA_COLUMNS.items(): + _require_uk_area_column(household, column, UK_DISCLOSURE_BREAKDOWN_NAMES[level]) + _require_uk_source_household_ids(household) + + from policyengine.disclosure import ( # noqa: PLC0415 # bound per call, like this module's other policyengine.outputs imports + DisclosureSettings, + ) + from policyengine.outputs.uk_geography_disclosure import ( # noqa: PLC0415 # bound per call, like this module's other policyengine.outputs imports + compute_uk_geography_disclosure, ) - lookup_csv_path = _required_uk_geography_lookup_csv_path(CONSTITUENCY_ASSET_SPEC) - impact = _output_module_function( - "constituency_impact", "compute_uk_constituency_impacts" - )( + area_codes = { + level: _uk_area_household_counts(household, column) + for level, column in UK_DISCLOSURE_AREA_COLUMNS.items() + } + impact = compute_uk_geography_disclosure( baseline, reform, - constituency_csv_path=lookup_csv_path, - download_missing_assets=False, + year=year, + levels=tuple(UK_DISCLOSURE_AREA_COLUMNS), + reform=UK_DISCLOSURE_REFORM_LABEL, + settings=DisclosureSettings(n_replicates=UK_DISCLOSURE_REPLICATES), + # Constituencies and local authorities overlap without nesting. The + # release checks each area against the areas inside and around it; + # sim-api publishes both levels from every UK run, so it accepts that + # other differences between them stay unchecked. + acknowledge=(UNCHECKED_OVERLAP,), + ) + public = _resolved_public_table( + impact, area_codes, year=year, inside=_uk_areas_inside(household) + ) + if public is None: + return UKGeographyDisclosureOutput() + return UKGeographyDisclosureOutput( + areas={ + level: _published_area_changes(public, level, area_codes[level]) + for level in UK_DISCLOSURE_AREA_COLUMNS + } ) + + +def _uk_constituency_lookup( + lookup_csv_path: str, +) -> dict[str, tuple[str, int | None, int | None]]: + """Return each constituency's name and map coordinates, in lookup order.""" + lookup = pd.read_csv(lookup_csv_path) + if "code" not in lookup.columns: + msg = ( + "UK constituency lookup CSV must contain a 'code' column: " + f"{lookup_csv_path}" + ) + raise ValueError(msg) + entries: dict[str, tuple[str, int | None, int | None]] = {} + for row in lookup.to_dict("records"): + code = _normalised_area_code(row.get("code")) + if not code: + continue + name = _normalised_area_code(row.get("name")) or code + entries[code] = ( + name, + _lookup_coordinate(row.get("x")), + _lookup_coordinate(row.get("y")), + ) + return entries + + +def _lookup_coordinate(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int | float | str): + return None + try: + number = float(value) + except ValueError: + return None + return int(number) if math.isfinite(number) else None + + +def _require_uk_disclosure_output( + disclosure: UKGeographyDisclosureOutput | None, breakdown: str +) -> UKGeographyDisclosureOutput: + if disclosure is None: + msg = ( + f"UK {breakdown} output needs the disclosure-controlled UK geography output" + ) + raise ValueError(msg) + return disclosure + + +def build_uk_constituency_impact( + country: str, disclosure: UKGeographyDisclosureOutput | None +) -> GeographicImpactOutput | None: + """Return the published UK parliamentary constituencies; None for other countries. + + Areas that disclosure control withholds are left out. Downloads the + constituency lookup if needed and raises when a published area lacks a + lookup name or coordinates. + """ + if country != "uk": + return None + changes = _require_uk_disclosure_output(disclosure, "constituency").level( + "constituency" + ) + lookup = _uk_constituency_lookup( + _required_uk_geography_lookup_csv_path(CONSTITUENCY_ASSET_SPEC) + ) + ordered = [code for code in lookup if code in changes] + sorted( + set(changes) - set(lookup) + ) + records: list[dict[str, object]] = [] + for code in ordered: + name, x, y = lookup.get(code, (code, None, None)) + records.append( + { + "constituency_code": code, + "constituency_name": name, + "x": x, + "y": y, + **changes[code].record_fields(), + } + ) return _complete_uk_geography_output( - getattr(impact, "constituency_results", None), + records, code_field="constituency_code", name_field="constituency_name", ) @@ -286,51 +728,35 @@ def build_uk_constituency_impact( def build_uk_local_authority_impact( country: str, - baseline: Simulation, - reform: Simulation, + disclosure: UKGeographyDisclosureOutput | None, *, uk_local_authority_metadata: UKLocalAuthorityMetadata | None = None, ) -> GeographicImpactOutput | None: - """Compute UK local authority impacts, named and placed for the detected boundaries. + """Return the published UK local authorities, named and placed for the boundaries. None for other countries. For the UK, ``uk_local_authority_metadata`` is required, and a code outside its boundary version raises ``ValueError``. + Areas that disclosure control withholds are left out. """ if country != "uk": return None - - baseline_household = _unweighted_household_frame(baseline) - _require_uk_area_column(baseline_household, "la_code_oa", "local-authority") + changes = _require_uk_disclosure_output(disclosure, "local-authority").level( + "local_authority" + ) if uk_local_authority_metadata is None: msg = "UK local-authority boundary metadata is required" raise ValueError(msg) - from policyengine.outputs.uk_geography_impact import ( # noqa: PLC0415 # bound per call, like this module's patched policyengine.outputs import above - compute_longwise_uk_geography_impacts, - ) - from policyengine_simulation_executor.uk_local_authority_metadata import ( # noqa: PLC0415 # the UK metadata module loads only for UK local-authority output load_uk_local_authority_resources, ) - reform_household = _unweighted_household_frame(reform) - numeric_records = compute_longwise_uk_geography_impacts( - baseline_household=baseline_household, - reform_household=reform_household, - geography_column="la_code_oa", - result_key_prefix="local_authority", - lookup_csv_path=None, - ) boundary_version = uk_local_authority_metadata.boundary_version display_metadata = load_uk_local_authority_resources().metadata_for( boundary_version ) records: list[dict[str, object]] = [] - for numeric_record in numeric_records: - code = numeric_record.get("local_authority_code") - if not isinstance(code, str) or not code: - msg = "UK local-authority output contains an invalid code" - raise ValueError(msg) + for code in sorted(changes): metadata = display_metadata.get(code) if metadata is None: msg = ( @@ -340,10 +766,11 @@ def build_uk_local_authority_impact( raise ValueError(msg) records.append( { - **numeric_record, + "local_authority_code": code, "local_authority_name": metadata.name, "x": metadata.x, "y": metadata.y, + **changes[code].record_fields(), } ) return _complete_uk_geography_output( diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py index cd0cbee0b..84dda7c0a 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/stage12_runtime/output_planning.py @@ -27,8 +27,16 @@ import pandas as pd from policyengine_simulation_contract.stage12_bundle import CountryId +# Input-dataset columns the UK constituency and local-authority breakdowns +# read: the area codes, the region the disclosure check nests them in, and the +# survey household each row was copied from, by which it counts support. UK_GEOGRAPHIC_DATASET_VARIABLES: dict[str, tuple[str, ...]] = { - "household": ("constituency_code_oa", "la_code_oa"), + "household": ( + "constituency_code_oa", + "la_code_oa", + "region_code_oa", + "source_household_id", + ), } diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py index 9d95eab0b..321a3011c 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py @@ -33,6 +33,7 @@ UKLocalAuthorityBoundaryVersion, UKLocalAuthorityMetadata, ) +from policyengine_simulation_executor import simulation_output_geographic from policyengine_simulation_executor.job_progress import NULL_JOB_PROGRESS from policyengine_simulation_executor.release_bundle import ( BUNDLE_RECEIPT_FILENAME, @@ -81,8 +82,6 @@ ) from policyengine_simulation_observability.stages import Stage -CONSTITUENCY_CSV_PATH = "/tmp/constituencies_2024.csv" - def _params_with_spm(params, selection): """Return request parameters after the runtime resolves SPM settings.""" @@ -1771,77 +1770,82 @@ def fail_change_output_variable(*_args, **_kwargs): _simulation_output_builder("us", baseline, reform)._build_budgetary_impact() -def test_uk_constituency_impact_uses_policyengine_output_function(monkeypatch): - def coded_simulation() -> SimpleNamespace: - household = weighted_entity_frame({"constituency_code_oa": ["E14001063"]}) - return SimpleNamespace( - output_dataset=SimpleNamespace(data=SimpleNamespace(household=household)) - ) +def test_uk_area_impacts_share_one_disclosure_release(monkeypatch): + baseline = object() + reform = object() + disclosure = simulation_output_geographic.UKGeographyDisclosureOutput() + computed: list[tuple[str, object, object, int]] = [] + built: list[tuple[str, object]] = [] - baseline = coded_simulation() - reform = coded_simulation() - expected = [_constituency_impact_record().model_dump(mode="json")] + def compute(country, baseline_simulation, reform_simulation, *, year): + computed.append((country, baseline_simulation, reform_simulation, year)) + return disclosure - def fake_output_module_function(module_name, name): - assert module_name == "constituency_impact" - assert name == "compute_uk_constituency_impacts" + def build_constituencies(country, value): + built.append(("constituency", value)) + return GeographicImpactOutput.model_validate([]) - def compute(baseline_simulation, reform_simulation, **kwargs): - assert baseline_simulation is baseline - assert reform_simulation is reform - assert kwargs == { - "constituency_csv_path": CONSTITUENCY_CSV_PATH, - "download_missing_assets": False, - } - return SimpleNamespace(constituency_results=expected) - - return compute + def build_authorities(country, value, **kwargs): + built.append(("local_authority", value)) + return GeographicImpactOutput.model_validate([]) monkeypatch.setattr( - "policyengine_simulation_executor.simulation_output_geographic._output_module_function", - fake_output_module_function, + "policyengine_simulation_executor.simulation_output_geographic." + "compute_uk_geography_disclosure_output", + compute, + ) + monkeypatch.setattr( + "policyengine_simulation_executor.simulation_output_geographic." + "build_uk_constituency_impact", + build_constituencies, ) monkeypatch.setattr( "policyengine_simulation_executor.simulation_output_geographic." - "_required_uk_geography_lookup_csv_path", - lambda _spec: CONSTITUENCY_CSV_PATH, + "build_uk_local_authority_impact", + build_authorities, ) - result = _simulation_output_builder( - "uk", baseline, reform - )._build_uk_constituency_impact() - assert result is not None - assert result.model_dump(mode="json") == expected - assert ( - _simulation_output_builder( - "us", baseline, reform - )._build_uk_constituency_impact() - is None + builder = _simulation_output_builder("uk", baseline, reform) + builder.simulation_params["time_period"] = "2026" + builder._build_uk_constituency_impact() + builder._build_uk_local_authority_impact() + + # Both levels come from one release, computed once. + assert computed == [("uk", baseline, reform, 2026)] + assert built == [("constituency", disclosure), ("local_authority", disclosure)] + + +def test_uk_disclosure_is_not_computed_for_other_countries(monkeypatch): + monkeypatch.setattr( + "policyengine_simulation_executor.simulation_output_geographic." + "compute_uk_geography_disclosure_output", + lambda *_args, **_kwargs: pytest.fail("US runs need no UK disclosure"), ) + builder = _simulation_output_builder("us", object(), object()) + + assert builder._build_uk_constituency_impact() is None + assert builder._build_uk_local_authority_impact() is None def test_uk_local_authority_impact_receives_detected_boundary_metadata(monkeypatch): - baseline = object() - reform = object() expected = [_local_authority_impact_record().model_dump(mode="json")] metadata = UKLocalAuthorityMetadata( boundary_version=UKLocalAuthorityBoundaryVersion.LAD23 ) - calls: list[tuple[object, object, UKLocalAuthorityMetadata | None]] = [] + disclosure = simulation_output_geographic.UKGeographyDisclosureOutput() + calls: list[tuple[object, UKLocalAuthorityMetadata | None]] = [] - def build(country, baseline_simulation, reform_simulation, **kwargs): + def build(country, value, **kwargs): if country != "uk": return None - assert country == "uk" - calls.append( - ( - baseline_simulation, - reform_simulation, - kwargs["uk_local_authority_metadata"], - ) - ) + calls.append((value, kwargs["uk_local_authority_metadata"])) return GeographicImpactOutput.model_validate(expected) + monkeypatch.setattr( + "policyengine_simulation_executor.simulation_output_geographic." + "compute_uk_geography_disclosure_output", + lambda *_args, **_kwargs: disclosure, + ) monkeypatch.setattr( "policyengine_simulation_executor.simulation_output_geographic." "build_uk_local_authority_impact", @@ -1850,16 +1854,10 @@ def build(country, baseline_simulation, reform_simulation, **kwargs): result = _simulation_output_builder( "uk", - baseline, - reform, + object(), + object(), BuilderOptions(uk_local_authority_metadata=metadata), )._build_uk_local_authority_impact() assert result is not None assert result.model_dump(mode="json") == expected - assert calls == [(baseline, reform, metadata)] - assert ( - _simulation_output_builder( - "us", baseline, reform - )._build_uk_local_authority_impact() - is None - ) + assert calls == [(disclosure, metadata)] diff --git a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py index 5e5e20cdf..84636b533 100644 --- a/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py +++ b/projects/policyengine-simulation-executor/tests/test_simulation_output_geographic.py @@ -5,6 +5,7 @@ from contextlib import nullcontext from types import SimpleNamespace +import numpy as np import pytest from fixtures.weighted_frames import weighted_entity_frame from policyengine.data.uk_geography_assets import CONSTITUENCY_ASSET_SPEC @@ -29,22 +30,6 @@ def _complete_constituency_record() -> dict[str, object]: } -def _uk_simulation( - code: str, - income: float, -) -> SimpleNamespace: - household = weighted_entity_frame( - { - "la_code_oa": [code], - "household_net_income": [income], - "household_weight": [2.0], - } - ) - return SimpleNamespace( - output_dataset=SimpleNamespace(data=SimpleNamespace(household=household)) - ) - - def test_required_uk_lookup_uses_gcp_credentials(monkeypatch) -> None: observed: list[str] = [] @@ -104,10 +89,21 @@ def test_complete_uk_geography_output_accepts_lookup_metadata() -> None: assert record["y"] == -40 -def test_complete_uk_geography_output_rejects_empty_result() -> None: - with pytest.raises(ValueError, match="did not contain result records"): +def test_complete_uk_geography_output_accepts_an_empty_list() -> None: + # Disclosure control may withhold every area of a level. + result = simulation_output_geographic._complete_uk_geography_output( + [], + code_field="constituency_code", + name_field="constituency_name", + ) + + assert result.root == [] + + +def test_complete_uk_geography_output_rejects_a_missing_list() -> None: + with pytest.raises(ValueError, match="did not contain a list"): simulation_output_geographic._complete_uk_geography_output( - [], + None, code_field="constituency_code", name_field="constituency_name", ) @@ -138,18 +134,169 @@ def test_complete_uk_geography_output_rejects_fallback_metadata( ) -@pytest.mark.parametrize( - ("boundary_version", "code", "expected_name"), - [ - (UKLocalAuthorityBoundaryVersion.LAD22, "E07000026", "Allerdale"), - (UKLocalAuthorityBoundaryVersion.LAD23, "E06000063", "Cumberland"), - ], -) -def test_local_authority_output_uses_detected_boundary_metadata( - monkeypatch, - boundary_version: UKLocalAuthorityBoundaryVersion, - code: str, - expected_name: str, +# Disclosure-controlled UK breakdowns. The households below sit in one region +# and two LAD23 local authorities. Each published seat rests on 30 survey +# households, well above the floor of 10; THIN has 3, so it is withheld, and +# the check then withholds one more seat so that subtraction from the region +# or from its local authority cannot recover it. NEGATIVE has a negative total +# baseline income, the Cardiff West case: its relative change is undefined, so +# the seat is left out although its average change passes. +NORTH_WEST = "E12000002" +CUMBERLAND = "E06000063" +WESTMORLAND = "E06000064" +SEATS = { + "E14000001": (CUMBERLAND, 30), + "E14000002": (CUMBERLAND, 30), + "E14000003": (WESTMORLAND, 30), + "E14000004": (WESTMORLAND, 30), + "E14000005": (WESTMORLAND, 30), +} +THIN = "E14000006" +NEGATIVE = "E14000007" +LAD23 = UKLocalAuthorityMetadata(boundary_version=UKLocalAuthorityBoundaryVersion.LAD23) + + +def _uk_disclosure_columns(*, with_source_ids: bool = True) -> dict[str, list]: + """Return the households' columns, with baseline and reformed incomes.""" + rng = np.random.default_rng(7) + rows: list[tuple[str, str, float, float]] = [] + for seat, (authority, households) in SEATS.items(): + for _ in range(households): + income = float(rng.uniform(20_000, 60_000)) + rows.append((seat, authority, income, -float(rng.uniform(50, 150)))) + rows.extend((THIN, WESTMORLAND, 30_000.0, -100.0) for _ in range(3)) + for index in range(30): + income = -40_000.0 if index < 20 else 30_000.0 + rows.append((NEGATIVE, WESTMORLAND, income, -float(rng.uniform(50, 150)))) + columns: dict[str, list[object]] = { + "household_id": list(range(len(rows))), + "constituency_code_oa": [row[0] for row in rows], + "la_code_oa": [row[1] for row in rows], + "region_code_oa": [NORTH_WEST] * len(rows), + "household_weight": [float(rng.uniform(50, 150)) for _ in rows], + } + if with_source_ids: + columns["source_household_id"] = list(range(len(rows))) + columns["baseline_income"] = [row[2] for row in rows] + columns["reformed_income"] = [row[2] + row[3] for row in rows] + return columns + + +def _uk_disclosure_simulations( + *, with_source_ids: bool = True +) -> tuple[SimpleNamespace, SimpleNamespace]: + columns = _uk_disclosure_columns(with_source_ids=with_source_ids) + baseline_income = columns.pop("baseline_income") + reformed_income = columns.pop("reformed_income") + baseline = weighted_entity_frame( + {**columns, "household_net_income": baseline_income} + ) + reform = weighted_entity_frame({**columns, "household_net_income": reformed_income}) + + def simulation(household): + return SimpleNamespace( + output_dataset=SimpleNamespace(data=SimpleNamespace(household=household)) + ) + + return simulation(baseline), simulation(reform) + + +@pytest.fixture(scope="module") +def uk_disclosure() -> simulation_output_geographic.UKGeographyDisclosureOutput: + baseline, reform = _uk_disclosure_simulations() + output = simulation_output_geographic.compute_uk_geography_disclosure_output( + "uk", baseline, reform, year=2026 + ) + assert output is not None + return output + + +@pytest.fixture +def constituency_lookup(tmp_path, monkeypatch) -> None: + lookup = tmp_path / "constituencies_2024.csv" + codes = [*SEATS, THIN, NEGATIVE] + lookup.write_text( + "code,name,x,y\n" + + "".join( + f"{code},Seat {index},{index},{-index}\n" + for index, code in enumerate(codes) + ) + ) + monkeypatch.setattr( + simulation_output_geographic, + "_required_uk_geography_lookup_csv_path", + lambda spec: str(lookup), + ) + + +def test_uk_disclosure_withholds_thin_and_undefined_seats(uk_disclosure) -> None: + seats = uk_disclosure.level("constituency") + + assert THIN not in seats + assert NEGATIVE not in seats + # Complementary suppression: with THIN withheld, at least one more seat + # of its region and its local authority is withheld too. + assert len(seats) < len(SEATS) + assert set(seats) < set(SEATS) + + +def test_uk_disclosure_publishes_rounded_changes_with_intervals( + uk_disclosure, +) -> None: + seats = uk_disclosure.level("constituency") + + assert seats + for change in seats.values(): + assert change.tier in {"reported", "approximate"} + assert change.average_lower <= change.average <= change.average_upper + assert change.relative_lower <= change.relative <= change.relative_upper + # Every household loses between 50 and 150 pounds. + assert -150 <= change.average <= -50 + assert change.relative < 0 + assert change.population > 0 + + +def test_uk_disclosure_population_is_the_weighted_household_count( + uk_disclosure, +) -> None: + columns = _uk_disclosure_columns() + expected: dict[str, float] = {} + for code, weight in zip( + columns["constituency_code_oa"], columns["household_weight"], strict=True + ): + expected[code] = expected.get(code, 0.0) + weight + + for code, change in uk_disclosure.level("constituency").items(): + assert change.population == pytest.approx(expected[code]) + + +def test_uk_constituency_output_lists_only_published_seats( + uk_disclosure, constituency_lookup +) -> None: + result = simulation_output_geographic.build_uk_constituency_impact( + "uk", uk_disclosure + ) + + assert result is not None + records = [record.model_dump(mode="json") for record in result.root] + published = uk_disclosure.level("constituency") + assert [record["constituency_code"] for record in records] == [ + code for code in SEATS if code in published + ] + for record in records: + change = published[record["constituency_code"]] + assert record["constituency_name"].startswith("Seat ") + assert record["average_household_income_change"] == change.average + assert record["relative_household_income_change"] == change.relative + assert record["disclosure_tier"] == change.tier + assert record["average_household_income_change_lower"] == (change.average_lower) + assert record["relative_household_income_change_upper"] == ( + change.relative_upper + ) + + +def test_uk_local_authority_output_uses_detected_boundary_metadata( + uk_disclosure, monkeypatch ) -> None: monkeypatch.setattr( simulation_output_geographic, @@ -158,67 +305,115 @@ def test_local_authority_output_uses_detected_boundary_metadata( ) result = simulation_output_geographic.build_uk_local_authority_impact( - "uk", - _uk_simulation(code, 100.0), - _uk_simulation(code, 110.0), - uk_local_authority_metadata=UKLocalAuthorityMetadata( - boundary_version=boundary_version - ), + "uk", uk_disclosure, uk_local_authority_metadata=LAD23 ) assert result is not None - record = result.root[0].model_dump(mode="python") - assert record["local_authority_code"] == code - assert record["local_authority_name"] == expected_name - assert isinstance(record["x"], int) - assert isinstance(record["y"], int) - assert record["average_household_income_change"] == 10.0 + records = { + record.model_dump(mode="json")["local_authority_code"]: record.model_dump( + mode="json" + ) + for record in result.root + } + assert set(records) == set(uk_disclosure.level("local_authority")) + assert records + for code, record in records.items(): + assert record["local_authority_name"] in { + "Cumberland", + "Westmorland and Furness", + } + assert isinstance(record["x"], int) + assert isinstance(record["y"], int) + assert record["average_household_income_change"] < 0 + assert record["population"] == ( + uk_disclosure.level("local_authority")[code].population + ) -def test_local_authority_output_rejects_code_outside_boundary_version() -> None: +def test_uk_local_authority_output_rejects_code_outside_boundary_version( + uk_disclosure, +) -> None: with pytest.raises( ValueError, match="not part of the detected LAD22 boundary version" ): simulation_output_geographic.build_uk_local_authority_impact( "uk", - _uk_simulation("E06000063", 100.0), - _uk_simulation("E06000063", 110.0), + uk_disclosure, uk_local_authority_metadata=UKLocalAuthorityMetadata( boundary_version=UKLocalAuthorityBoundaryVersion.LAD22 ), ) -def test_local_authority_output_requires_detected_boundary_metadata( - monkeypatch, +def test_uk_local_authority_output_requires_detected_boundary_metadata( + uk_disclosure, ) -> None: - monkeypatch.setattr( - simulation_output_geographic, - "_required_uk_geography_lookup_csv_path", - lambda _: pytest.fail("local-authority output must not resolve a GCS lookup"), - ) - with pytest.raises(ValueError, match="boundary metadata is required"): simulation_output_geographic.build_uk_local_authority_impact( - "uk", - _uk_simulation("E06000063", 100.0), - _uk_simulation("E06000063", 110.0), + "uk", uk_disclosure + ) + + +def test_uk_outputs_require_the_disclosure_output() -> None: + with pytest.raises(ValueError, match="disclosure-controlled"): + simulation_output_geographic.build_uk_constituency_impact("uk", None) + with pytest.raises(ValueError, match="disclosure-controlled"): + simulation_output_geographic.build_uk_local_authority_impact( + "uk", None, uk_local_authority_metadata=LAD23 + ) + + +def test_uk_outputs_are_none_for_other_countries() -> None: + baseline, reform = _uk_disclosure_simulations() + + assert ( + simulation_output_geographic.compute_uk_geography_disclosure_output( + "us", baseline, reform, year=2026 + ) + is None + ) + assert simulation_output_geographic.build_uk_constituency_impact("us", None) is None + assert ( + simulation_output_geographic.build_uk_local_authority_impact("us", None) is None + ) + + +def test_uk_outputs_list_no_area_when_every_area_is_withheld( + constituency_lookup, +) -> None: + empty = simulation_output_geographic.UKGeographyDisclosureOutput() + + constituencies = simulation_output_geographic.build_uk_constituency_impact( + "uk", empty + ) + authorities = simulation_output_geographic.build_uk_local_authority_impact( + "uk", empty, uk_local_authority_metadata=LAD23 + ) + + assert constituencies is not None + assert constituencies.root == [] + assert authorities is not None + assert authorities.root == [] + + +def test_uk_disclosure_requires_survey_household_ids() -> None: + baseline, reform = _uk_disclosure_simulations(with_source_ids=False) + + with pytest.raises(ValueError, match="need source_household_id"): + simulation_output_geographic.compute_uk_geography_disclosure_output( + "uk", baseline, reform, year=2026 ) def test_uk_geographic_outputs_are_never_dropped_for_missing_area_codes( monkeypatch, ) -> None: - monkeypatch.setattr( - simulation_output_geographic, - "_required_uk_geography_lookup_csv_path", - lambda _: pytest.fail("a dataset without area codes needs no lookup"), - ) household = weighted_entity_frame( { - "region": ["LONDON"], - "household_net_income": [100.0], - "household_weight": [2.0], + "region": ["LONDON", "LONDON"], + "source_household_id": [1, 2], + "household_net_income": [100.0, 200.0], + "household_weight": [2.0, 3.0], } ) national = SimpleNamespace( @@ -228,15 +423,150 @@ def test_uk_geographic_outputs_are_never_dropped_for_missing_area_codes( with pytest.raises( ValueError, match="constituency breakdowns need constituency_code_oa" ): - simulation_output_geographic.build_uk_constituency_impact( - "uk", national, national + simulation_output_geographic.compute_uk_geography_disclosure_output( + "uk", national, national, year=2026 ) - with pytest.raises(ValueError, match="local-authority breakdowns need la_code_oa"): - simulation_output_geographic.build_uk_local_authority_impact( - "uk", - national, - national, - uk_local_authority_metadata=UKLocalAuthorityMetadata( - boundary_version=UKLocalAuthorityBoundaryVersion.LAD23 - ), + + +def _finding(**fields: object) -> SimpleNamespace: + return SimpleNamespace( + **{ + "blocking": True, + "code": "recoverable_remainder", + "family": "change", + "reform": "reform", + "year": 2026, + "level": "constituency", + "suggested": None, + "group": None, + "group_level": None, + **fields, + } + ) + + +class _FakeRelease: + def __init__(self) -> None: + self.withheld: list[tuple[str, str, str]] = [] + + def withhold(self, *, level, area_code, reform, year, outcome) -> None: + self.withheld.append((level, area_code, outcome)) + + +class _FakeImpact: + """A disclosure result whose findings clear once the named cells are withheld.""" + + def __init__(self, findings, *, clears_after: set[tuple[str, str]]) -> None: + self.release = _FakeRelease() + self.findings = findings + self._clears_after = clears_after + self.public = None + + def refresh(self): + withheld = { + (level, code) + for level, code, outcome in self.release.withheld + if outcome == "average_change" + } + if self._clears_after <= withheld: + self.findings = [] + self.public = SimpleNamespace(rows=[]) + return self.public + + +AREAS = {"constituency": {"S1": 1.0, "S2": 1.0}, "local_authority": {"L1": 2.0}} + + +def test_findings_are_resolved_by_withholding_published_cells() -> None: + impact = _FakeImpact( + [ + _finding(suggested="S2"), + # A containment finding without a suggestion names its group. + _finding(level="constituency", group="L1", group_level="local_authority"), + ], + clears_after={("constituency", "S2"), ("local_authority", "L1")}, + ) + + public = simulation_output_geographic._resolved_public_table( + impact, AREAS, year=2026, inside={} + ) + + assert public is not None + change_cells = [ + cell for cell in impact.release.withheld if cell[2] == "average_change" + ] + assert sorted(change_cells) == [ + ("constituency", "S2", "average_change"), + ("local_authority", "L1", "average_change"), + ] + # The shares are never published, so every area's are withheld up front. + share_cells = { + cell for cell in impact.release.withheld if cell[2] != "average_change" + } + assert share_cells == { + (level, code, outcome) + for level, codes in AREAS.items() + for code in codes + for outcome in ("share_gaining", "share_losing") + } + + +def test_reference_cells_are_never_withheld_and_publish_nothing(caplog) -> None: + # A finding that only a region cell could resolve: regions are public + # elsewhere, so the breakdowns publish no area at all. + impact = _FakeImpact( + [_finding(code="recoverable_parent", level="region", group="E12000002")], + clears_after={("region", "E12000002")}, + ) + + with caplog.at_level("WARNING"): + public = simulation_output_geographic._resolved_public_table( + impact, AREAS, year=2026, inside={} ) + + assert public is None + assert all(level != "region" for level, _, _ in impact.release.withheld) + assert "recoverable_parent" in caplog.text + assert "E12000002" not in caplog.text + + +def test_uk_areas_inside_follow_the_household_area_pairs() -> None: + baseline, _ = _uk_disclosure_simulations() + household = baseline.output_dataset.data.household + + inside = simulation_output_geographic._uk_areas_inside(household) + + # Every seat lies in one local authority; no authority lies in one seat. + assert inside[("constituency", "local_authority")] == { + CUMBERLAND: ("E14000001", "E14000002"), + WESTMORLAND: ("E14000003", "E14000004", "E14000005", THIN, NEGATIVE), + } + assert inside[("local_authority", "constituency")] == {} + + +def test_a_withheld_area_summed_by_public_areas_inside_withholds_them() -> None: + # Orkney and Shetland is the sum of two local authorities. If the seat is + # withheld, the two authorities inside it must be withheld too. + inside = {("local_authority", "constituency"): {"S1": ("L1", "L2")}} + impact = _FakeImpact( + [ + _finding( + code="recoverable_parent", + level="local_authority", + group="S1", + group_level="constituency", + ) + ], + clears_after={("local_authority", "L1"), ("local_authority", "L2")}, + ) + + public = simulation_output_geographic._resolved_public_table( + impact, AREAS, year=2026, inside=inside + ) + + assert public is not None + assert { + (level, code) + for level, code, outcome in impact.release.withheld + if outcome == "average_change" + } == {("local_authority", "L1"), ("local_authority", "L2")} diff --git a/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py b/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py index c0e5a196d..06e3be5fa 100644 --- a/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py +++ b/projects/policyengine-simulation-executor/tests/test_stage12_output_planning.py @@ -148,6 +148,8 @@ def test_uk_plan_uses_uk_defaults_without_us_budget_variables() -> None: assert _dataset_variables(plan, "household") == { "constituency_code_oa", "la_code_oa", + "region_code_oa", + "source_household_id", } assert _dataset_variables(plan, "household").issubset(_variables(plan, "household")) @@ -256,7 +258,10 @@ def test_apply_and_validate_output_plan_reject_missing_materialized_columns() -> validate_output_frames(frames, plan) -@pytest.mark.parametrize("missing", ["constituency_code_oa", "la_code_oa"]) +@pytest.mark.parametrize( + "missing", + ["constituency_code_oa", "la_code_oa", "region_code_oa", "source_household_id"], +) def test_uk_output_validation_requires_geographic_dataset_columns( missing: str, ) -> None: diff --git a/projects/policyengine-simulation-gateway/tests/golden/openapi.json b/projects/policyengine-simulation-gateway/tests/golden/openapi.json index 76d3f626b..c1a6b7a35 100644 --- a/projects/policyengine-simulation-gateway/tests/golden/openapi.json +++ b/projects/policyengine-simulation-gateway/tests/golden/openapi.json @@ -1033,6 +1033,65 @@ }, "ConstituencyImpactRecord": { "properties": { + "disclosure_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "reported", + "approximate" + ] + }, + { + "type": "null" + } + ], + "title": "Disclosure Tier" + }, + "average_household_income_change_lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Household Income Change Lower" + }, + "average_household_income_change_upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Household Income Change Upper" + }, + "relative_household_income_change_lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Household Income Change Lower" + }, + "relative_household_income_change_upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Household Income Change Upper" + }, "constituency_code": { "type": "string", "title": "Constituency Code" @@ -1088,7 +1147,7 @@ "population" ], "title": "ConstituencyImpactRecord", - "description": "Reform impact on household net income in one UK parliamentary constituency.\n\n``average_household_income_change`` is the average change per household and\n``relative_household_income_change`` the relative change in the area's total\nhousehold net income. ``x`` and ``y`` are integer map-rendering coordinates." + "description": "Reform impact on household net income in one UK parliamentary constituency.\n\n``average_household_income_change`` is the average change per household and\n``relative_household_income_change`` the relative change in the area's total\nhousehold net income, both rounded to the order of their standard error.\n``population`` is the weighted number of households. ``x`` and ``y`` are\ninteger map-rendering coordinates." }, "DecileOutput": { "properties": { @@ -1180,7 +1239,7 @@ }, "type": "array", "title": "GeographicImpactOutput", - "description": "UK constituency or local-authority impact records." + "description": "UK constituency or local-authority impact records.\n\nAreas whose income change disclosure control withholds are left out." }, "HTTPValidationError": { "properties": { @@ -1682,6 +1741,65 @@ }, "LocalAuthorityImpactRecord": { "properties": { + "disclosure_tier": { + "anyOf": [ + { + "type": "string", + "enum": [ + "reported", + "approximate" + ] + }, + { + "type": "null" + } + ], + "title": "Disclosure Tier" + }, + "average_household_income_change_lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Household Income Change Lower" + }, + "average_household_income_change_upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Average Household Income Change Upper" + }, + "relative_household_income_change_lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Household Income Change Lower" + }, + "relative_household_income_change_upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Relative Household Income Change Upper" + }, "local_authority_code": { "type": "string", "title": "Local Authority Code" diff --git a/rules/baselines/sim-microdata-weight-stripping.txt b/rules/baselines/sim-microdata-weight-stripping.txt index 044b1e724..b396d381b 100644 --- a/rules/baselines/sim-microdata-weight-stripping.txt +++ b/rules/baselines/sim-microdata-weight-stripping.txt @@ -4,10 +4,3 @@ # ::::: a read of raw values from a weighted # frame in simulation_output_common.py for something other than counting # entities. Reads anywhere else fail outright. -# -# The UK local-authority breakdown hands policyengine's longwise geography -# impact a plain DataFrame of the household frame, which weights by its -# household_weight column itself. It goes once the breakdown hands -# policyengine the simulations, as the constituency and US congressional -# district breakdowns already do. -projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py::_unweighted_household_frame::pd.DataFrame(simulation.output_dataset.data.household) diff --git a/rules/registry.toml b/rules/registry.toml index d1d305ea8..803e99480 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -787,7 +787,7 @@ project = "projects/policyengine-simulation-executor" test = "tests/test_microdata_weight_boundaries.py::test_rule_proof_weight_stripping_confined" [rule.baseline] file = "rules/baselines/sim-microdata-weight-stripping.txt" -count = 1 +count = 0 [[rule]] id = "sim.microdata.builder-tests-use-weighted-frames"