Temporarily handle missing ACS WIC participation inputs

changelog.d/560.fixed.md

@@ -0,0 +1 @@
+Add opt-in verified US state/year preparation and temporarily assume WIC claiming only for missing participation decisions on identified ACS people, preserving donor decisions and country-model eligibility.

docs/engineering/runbooks/acs-local-wic-replacement.md

@@ -0,0 +1,56 @@
+# Replace the temporary ACS WIC assumption
+
+## Temporary compatibility fix
+
+The certified local-area release
+`populace-us-2024-buildo-acs-local-767312d60-20260923T074941Z` stores donor
+participation decisions as `would_claim_wic`, with missing decisions on ACS
+people. The current model consumes the monthly person input
+`takes_up_wic_if_eligible`.
+
+The wrapper preserves all valid existing decisions and sets only missing
+decisions on people whose `person_support_channel` is `acs_2024_1yr` to `True`.
+This explicitly assumes that every eligible affected ACS person claims WIC;
+it does not change eligibility. Missing donor decisions, missing identifying
+provenance, and malformed values remain errors. Source H5 files are unchanged.
+
+The existing loader applies this before calculation and year preparation,
+including baseline and reform branches. The fix does not add partitioning,
+state/year preparation APIs, new dataset defaults, or deployment behavior.
+
+## Replacement ownership
+
+[Microcosm #1154](https://github.com/PolicyEngine/microcosm/issues/1154) tracks
+the code repair, a dataset rebuild and publication by a separate dataset owner,
+certification in this repository, and removal of the exception. Merging the
+[code repair](https://github.com/PolicyEngine/microcosm/pull/1156) does not
+publish a replacement population or complete that issue.
+
+The repair generates ACS participation after completing demographic inputs and
+before combining ACS records with donors. It reuses the existing WIC generator,
+preserves donor decisions, writes complete boolean participation inputs, and
+rejects missing decisions. The dataset owner must rebuild the affected outputs
+and satisfy the existing release checks.
+
+## Remove the exception
+
+The separate stacked [removal PR #563](https://github.com/PolicyEngine/policyengine.py/pull/563)
+must remain a draft until the replacement dataset has been published and
+certified. Do not invent its revision or hash or relax its failing check.
+
+Once the replacement is available:
+
+1. Follow the [US certification runbook](build-m-us-populace-certification.md)
+   with its actual immutable regional manifest. Keep national defaults and
+   model pins unchanged unless separately approved.
+2. Verify complete current-name participation on the replacement and record
+   its revision and hashes.
+3. Add the real certification changes to the removal PR and rebase it onto
+   `main` after the temporary compatibility fix merges.
+4. Delete only the ACS missing-value exception. Keep ordinary legacy-name
+   mapping while other certified datasets require it.
+5. Run the focused mapping and country-model integration tests and validate
+   preparation against the actual replacement dataset.
+
+This fix does not rebuild or publish data, deploy services, or introduce
+environment variables or database changes.

src/policyengine/tax_benefit_models/us/legacy_inputs.py

@@ -146,8 +146,9 @@ def apply_legacy_input_renames(
     Every mapped table is checked before anything is set. Its ``{entity}_id``
     column must list the simulation's entity IDs in the simulation's order,
     and its stored values must be complete and, for a boolean live input,
-    boolean. Otherwise this raises ``ValueError`` rather than set misaligned
-    or invented values.
+    boolean. A temporary WIC-only exception assumes claiming for missing
+    decisions on explicitly identified ACS people. Missing donor decisions
+    and malformed stored values still raise ``ValueError``.
 
     Applying the mapping again sets the same values, so it is idempotent.
 
@@ -165,7 +166,15 @@ def apply_legacy_input_renames(
                 continue
             context = f"Cannot map stored {legacy!r} onto {live!r} for {year}"
             _check_order(simulation, table, entity, context)
-            values = _live_values(table[legacy], variable, context)
+            stored = table[legacy]
+            if (
+                legacy == "would_claim_wic"
+                and live == "takes_up_wic_if_eligible"
+                and entity == "person"
+                and variable.value_type is bool
+            ):
+                stored = _temporary_acs_wic_values(table, context)
+            values = _live_values(stored, variable, context)
             periods = _periods_of_year(int(year), variable.definition_period, context)
             planned.append((legacy, live, periods, values))
 
@@ -347,6 +356,34 @@ def _check_order(simulation, table: pd.DataFrame, entity: str, context: str) ->
         )
 
 
+def _temporary_acs_wic_values(table: pd.DataFrame, context: str) -> pd.Series:
+    """Preserve stored decisions; assume claiming only for missing ACS cells."""
+    stored = table["would_claim_wic"]
+    missing = stored.isna()
+    if not missing.any():
+        return stored
+    channel = "person_support_channel"
+    if channel not in table or not (
+        table.loc[missing, channel].eq("acs_2024_1yr").fillna(False).all()
+    ):
+        raise ValueError(
+            f"{context}: the stored column has missing values outside "
+            "explicitly identified ACS people."
+        )
+    # TEMPORARY ACS WIC COMPATIBILITY — REMOVE AS SOON AS POSSIBLE.
+    # Replacement dataset: https://github.com/PolicyEngine/microcosm/issues/1154
+    # Prepared removal: https://github.com/PolicyEngine/policyengine.py/pull/563
+    # The currently certified ACS-local dataset did not generate WIC
+    # participation decisions for ACS people. For missing ACS decisions only,
+    # assume that every eligible person claims WIC.
+    # This is an explicit modelling assumption, not observed participation.
+    # Replace the dataset with a corrected Microcosm release and DELETE
+    # this exception immediately after that replacement is certified.
+    # Nullable numeric columns reject boolean replacement values. Preserve the
+    # original decisions as objects; _live_values still validates before casting.
+    return stored.astype(object).mask(missing, True)
+
+
 def _live_values(stored: pd.Series, variable: Any, context: str) -> np.ndarray:
     if stored.isna().any():
         raise ValueError(f"{context}: the stored column has missing values.")
@@ -356,6 +393,15 @@ def _live_values(stored: pd.Series, variable: Any, context: str) -> np.ndarray:
         return np.asarray(stored.to_numpy(dtype=bool))
     if pd.api.types.is_numeric_dtype(stored.dtype) and stored.isin((0, 1)).all():
         return np.asarray(stored.to_numpy(), dtype=bool)
+    # Nullable legacy H5 columns may contain genuine booleans and numeric 0/1
+    # as objects. Validate each value before casting: bool("False") is True.
+    if stored.map(
+        lambda value: (
+            isinstance(value, bool | np.bool_ | int | np.integer | float | np.floating)
+            and value in (0, 1)
+        )
+    ).all():
+        return np.asarray(stored.to_numpy(), dtype=bool)
     raise ValueError(f"{context}: the stored values are not boolean.")
 
 

tests/test_us_legacy_inputs.py

@@ -322,6 +322,103 @@ def test_a_draw_that_is_not_a_complete_boolean_is_refused(draw, message):
     assert simulation.inputs == {}
 
 
+@pytest.mark.parametrize(
+    "draw",
+    [
+        [False, np.bool_(True), None, False],
+        [0, 1.0, np.nan, np.bool_(False)],
+        pd.array([False, True, pd.NA, False], dtype="boolean"),
+    ],
+)
+def test_missing_acs_wic_decisions_assume_claiming_without_changing_stored_draws(draw):
+    """Only a missing ACS decision uses the temporary 100% take-up assumption."""
+    ids = [1, 2, 3, 4]
+    person = _person(
+        ids,
+        draw,
+        person_support_channel=[
+            "asec",
+            "puf_tax_detail",
+            "acs_2024_1yr",
+            "acs_2024_1yr",
+        ],
+    )
+    original = person.copy(deep=True)
+    simulation = FakeSimulation(_variables(LIVE), ids)
+
+    assert apply_legacy_input_renames(simulation, _tables({2024: person})) == {
+        LEGACY: LIVE
+    }
+    for month in _months(2024):
+        assert simulation.inputs[(LIVE, month)].tolist() == [False, True, True, False]
+        assert simulation.inputs[(LIVE, month)].dtype == bool
+    pd.testing.assert_frame_equal(person, original)
+
+
+@pytest.mark.parametrize("channel", ["asec", "puf_tax_detail", "acs", "unknown", None])
+def test_missing_wic_decisions_outside_identified_acs_people_are_refused(channel):
+    person = _person(
+        [1, 2], [None, None], person_support_channel=["acs_2024_1yr", channel]
+    )
+    simulation = FakeSimulation(_variables(LIVE), [1, 2])
+    with pytest.raises(ValueError, match="missing values"):
+        apply_legacy_input_renames(simulation, _tables({2024: person}))
+    assert simulation.inputs == {}
+
+
+@pytest.mark.parametrize("invalid", ["False", "True", "0", 2, -1, np.inf])
+def test_acs_missing_value_does_not_hide_an_invalid_stored_wic_decision(invalid):
+    person = _person(
+        [1, 2], [invalid, None], person_support_channel=["asec", "acs_2024_1yr"]
+    )
+    simulation = FakeSimulation(_variables(LIVE), [1, 2])
+    with pytest.raises(ValueError, match="not boolean"):
+        apply_legacy_input_renames(simulation, _tables({2024: person}))
+    assert simulation.inputs == {}
+
+
+def test_a_later_invalid_year_prevents_earlier_acs_assignments():
+    valid = _person([1], [None], person_support_channel=["acs_2024_1yr"])
+    invalid = _person([1], [None], person_support_channel=["asec"])
+    simulation = FakeSimulation(_variables(LIVE), [1])
+    with pytest.raises(ValueError, match="missing values"):
+        apply_legacy_input_renames(simulation, _tables({2024: valid, 2025: invalid}))
+    assert simulation.inputs == {}
+
+
+def test_the_temporary_acs_assumption_reaches_each_existing_branch_idempotently():
+    person = _person(
+        [1, 2], [False, None], person_support_channel=["asec", "acs_2024_1yr"]
+    )
+    dataset = _multi_year_dataset({2024: person})
+    simulation = FakeSimulation(_variables(LIVE), [1, 2], dataset=dataset)
+    baseline = FakeSimulation(_variables(LIVE), [1, 2], dataset=dataset)
+    simulation.branches["baseline"] = baseline
+    apply_legacy_input_renames_to_microsimulation(simulation)
+    before = _snapshot(simulation)
+    apply_legacy_input_renames_to_microsimulation(simulation)
+    assert _snapshot(simulation) == _snapshot(baseline) == before
+    for month in _months(2024):
+        assert simulation.inputs[(LIVE, month)].tolist() == [False, True]
+
+
+def test_current_name_wic_decisions_are_not_overridden_by_the_acs_assumption():
+    person = _person(
+        [1], [None], person_support_channel=["acs_2024_1yr"], **{LIVE: [False]}
+    )
+    simulation = FakeSimulation(_variables(LIVE), [1])
+    assert apply_legacy_input_renames(simulation, _tables({2024: person})) == {}
+    assert simulation.inputs == {}
+
+
+def test_boolean_values_in_an_object_column_are_preserved():
+    person = _person([1, 2], pd.Series([False, np.bool_(True)], dtype=object))
+    person[LEGACY] = person[LEGACY].astype(object)
+    simulation = FakeSimulation(_variables(LIVE), [1, 2])
+    apply_legacy_input_renames(simulation, _tables({2024: person}))
+    assert simulation.inputs[(LIVE, "2024-01")].tolist() == [False, True]
+
+
 def test_a_non_boolean_live_input_keeps_the_stored_values():
     variables = {LIVE: _variable(value_type=float)}
     simulation = FakeSimulation(variables, [1, 2])
@@ -654,3 +751,91 @@ def test_a_part_year_period_is_refused(period):
 def test_yearly_periods_are_accepted():
     check_yearly_periods(LEGACY, ["2024", 2025, np.int64(2026)], "source.h5")
     check_yearly_periods(LEGACY, [], "source.h5")
+
+
+@pytest.mark.parametrize("dtype", ["Int64", "Float64"])
+@pytest.mark.parametrize(
+    ("draw", "channels", "expected"),
+    [
+        ([0, pd.NA], ["asec", "acs_2024_1yr"], [False, True]),
+        (
+            [0, 1, pd.NA, 0, 1],
+            ["asec", "puf_tax_detail", "acs_2024_1yr", "acs_2024_1yr", "acs_2024_1yr"],
+            [False, True, True, False, True],
+        ),
+    ],
+)
+def test_nullable_numeric_wic_decisions_fill_only_missing_acs_cells(
+    dtype, draw, channels, expected
+):
+    """Keep nullable numeric storage and preserve existing donor/ACS decisions."""
+    ids = list(range(1, len(draw) + 1))
+    person = pd.DataFrame(
+        {
+            "person_id": ids,
+            LEGACY: pd.Series(draw, dtype=dtype),
+            "person_support_channel": channels,
+        }
+    )
+    original = person.copy(deep=True)
+    assert str(person[LEGACY].dtype) == dtype
+    simulation = FakeSimulation(_variables(LIVE), ids)
+
+    assert apply_legacy_input_renames(simulation, _tables({2024: person})) == {
+        LEGACY: LIVE
+    }
+    for month in _months(2024):
+        assert simulation.inputs[(LIVE, month)].tolist() == expected
+        assert simulation.inputs[(LIVE, month)].dtype == bool
+    pd.testing.assert_frame_equal(person, original)
+
+
+@pytest.mark.parametrize(
+    ("dtype", "invalid"),
+    [
+        ("Int64", 2),
+        ("Int64", -1),
+        ("Float64", 2),
+        ("Float64", -1),
+        ("Float64", 0.5),
+        ("Float64", np.inf),
+    ],
+)
+def test_nullable_numeric_acs_fill_does_not_hide_invalid_wic_decisions(dtype, invalid):
+    person = pd.DataFrame(
+        {
+            "person_id": [1, 2],
+            LEGACY: pd.Series([invalid, pd.NA], dtype=dtype),
+            "person_support_channel": ["asec", "acs_2024_1yr"],
+        }
+    )
+    original = person.copy(deep=True)
+    simulation = FakeSimulation(_variables(LIVE), [1, 2])
+
+    with pytest.raises(ValueError, match="not boolean"):
+        apply_legacy_input_renames(simulation, _tables({2024: person}))
+    assert simulation.inputs == {}
+    assert simulation.input_variables == []
+    pd.testing.assert_frame_equal(person, original)
+
+
+@pytest.mark.parametrize("dtype", ["Int64", "Float64"])
+@pytest.mark.parametrize("channel", ["asec", None])
+def test_nullable_numeric_missing_non_acs_wic_decisions_fail_before_writes(
+    dtype, channel
+):
+    person = pd.DataFrame(
+        {
+            "person_id": [1, 2],
+            LEGACY: pd.Series([pd.NA, pd.NA], dtype=dtype),
+            "person_support_channel": ["acs_2024_1yr", channel],
+        }
+    )
+    original = person.copy(deep=True)
+    simulation = FakeSimulation(_variables(LIVE), [1, 2])
+
+    with pytest.raises(ValueError, match="missing values"):
+        apply_legacy_input_renames(simulation, _tables({2024: person}))
+    assert simulation.inputs == {}
+    assert simulation.input_variables == []
+    pd.testing.assert_frame_equal(person, original)

tests/test_us_legacy_inputs_integration.py

@@ -177,6 +177,28 @@ def test_run_keeps_a_stored_false_draw(mapped_run):
     assert mapped_run.release_bundle["legacy_input_renames"] == RENAME
 
 
+def test_missing_acs_claim_decisions_do_not_make_ineligible_people_wic_eligible(
+    tmp_path,
+):
+    frames = _frames()
+    # Keep the donor infant's False and the ACS toddler's existing True.
+    # Only the ACS adult is missing a participation decision.
+    frames["person"][LEGACY] = pd.Series([None, False, True], dtype=object)
+    frames["person"]["person_support_channel"] = [
+        "acs_2024_1yr",
+        "asec",
+        "acs_2024_1yr",
+    ]
+    person = _person_outputs(_run(_in_memory_dataset(tmp_path, frames)))
+
+    assert person[LIVE].tolist() == [True, False, True]
+    assert not person["is_wic_eligible"].iloc[0]
+    assert person["wic"].iloc[0] == 0
+    assert person["is_wic_eligible"].iloc[INFANT]
+    assert person["wic"].iloc[INFANT] == 0
+    assert person["wic"].iloc[TODDLER] > 0
+
+
 def test_run_over_a_core_h5_keeps_a_stored_false_draw(tmp_path):
     """Path 1 over a policyengine-core ``variable/period`` file.